diff --git a/.circleci/config.yml b/.circleci/config.yml index 513ce63d4f..ad9793ad98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,6 +108,28 @@ jobs: name: Tag command: | ./.circleci/ci-docker-tag-op-geth-release.sh <>/<> $CIRCLE_TAG $CIRCLE_SHA1 + - when: + condition: + equal: [optimism, << pipeline.git.branch >>] + steps: + - gcp-oidc-authenticate: + service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL + - run: + name: Sign + command: | + git clone --branch v1.0.3 --depth 1 https://github.com/ethereum-optimism/binary_signer + cd binary_signer/signer + + IMAGE_PATH="<>/<>/<>:<>" + echo $IMAGE_PATH + pip3 install -r requirements.txt + + python3 ./sign_image.py --command="sign"\ + --attestor-project-name="$ATTESTOR_PROJECT_NAME"\ + --attestor-name="$ATTESTOR_NAME"\ + --image-path="$IMAGE_PATH"\ + --signer-logging-level="INFO"\ + --attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1" build-geth: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index faf922df01..a52ce4b85b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,22 +1 @@ -# Lines starting with '#' are comments. -# Each line is a file pattern followed by one or more owners. - -accounts/usbwallet @karalabe -accounts/scwallet @gballet -accounts/abi @gballet @MariusVanDerWijden -cmd/clef @holiman -consensus @karalabe -core/ @karalabe @holiman @rjl493456442 -eth/ @karalabe @holiman @rjl493456442 -eth/catalyst/ @gballet -eth/tracers/ @s1na -graphql/ @s1na -les/ @zsfelfoldi @rjl493456442 -light/ @zsfelfoldi @rjl493456442 -node/ @fjl -p2p/ @fjl @zsfelfoldi -rpc/ @fjl @holiman -p2p/simulations @fjl -p2p/protocols @fjl -p2p/testing @fjl -signer/ @holiman +* @ethereum-optimism/op-geth-maintainers \ No newline at end of file diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index 6973b227ce..3c45b5d339 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -16,7 +16,7 @@ jobs: fetch-depth: 1000 # make sure to fetch the old commit we diff against - name: Build forkdiff - uses: "docker://protolambda/forkdiff:latest" + uses: "docker://protolambda/forkdiff:0.1.0" with: args: -repo=/github/workspace -fork=/github/workspace/fork.yaml -out=/github/workspace/index.html diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 7c018effc6..feffce65e1 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -17,4 +17,6 @@ jobs: - uses: actions/setup-go@v4 with: go-version-file: go.mod - - run: make test + - run: | + git apply tests/0001-diff-go-ethereum.patch + make test diff --git a/.golangci.yml b/.golangci.yml index 10dee809c2..ca2f7f2a38 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,13 +1,12 @@ # This file configures github.com/golangci/golangci-lint. +# see run: timeout: 20m tests: true # default is true. Enables skipping of directories: # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs-use-default: true - skip-files: - - core/genesis_alloc.go + linters: disable-all: true @@ -40,6 +39,9 @@ linters-settings: simplify: true issues: + exclude-dirs-use-default: true + exclude-dirs: + - core/genesis_alloc.go exclude-rules: - path: crypto/bn256/cloudflare/optate.go linters: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9687ad7c82..b2b3eab21f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## v0.5.0 +This release includes code merging from the upstream version v1.101315.2 along with several fixs and improvements. Fjord fork from upstream is included. +Fjord fork is scheduled to launch on the opBNB: +Testnet: Sep-10-2024 06:00 AM +UTC +Mainnet: Sep-24-2024 06:00 AM +UTC + +### User Facing Changes +Nodes on the Testnet and Mainnet need to be upgraded to this version before the hard fork time. + +### What's Changed +* Merge upstream op-geth v1.101315.2 by @redhdx in https://github.com/bnb-chain/op-geth/pull/123 +* feature(op-geth): update opBNB qanet config by @redhdx in https://github.com/bnb-chain/op-geth/pull/138 +* fix: txpool legacy pool Pending() applies filter @andyzhang2023 in https://github.com/bnb-chain/op-geth/pull/124 +* feature(op-geth): Add extra error info when meet an unexpected el sync @krish-nr in https://github.com/bnb-chain/op-geth/pull/132 +* feat: cache lastTail in txindexer to avoid read from db by @bnoieh in https://github.com/bnb-chain/op-geth/pull/157 + + +### Docker Images +ghcr.io/bnb-chain/op-geth:v0.5.0 + +**Full Changelog**: https://github.com/bnb-chain/op-geth/compare/v0.4.6...v0.5.0 + ## v0.4.6 This is a minor release for opBNB Mainnet and Testnet. diff --git a/Makefile b/Makefile index 226bac2d1e..b823006904 100644 --- a/Makefile +++ b/Makefile @@ -8,20 +8,25 @@ GOBIN = ./build/bin GO ?= latest GORUN = go run +#? geth: Build geth geth: $(GORUN) build/ci.go install ./cmd/geth @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." +#? all: Build all packages and executables all: $(GORUN) build/ci.go install +#? test: Run the tests test: all $(GORUN) build/ci.go test +#? lint: Run certain pre-selected linters lint: ## Run linters. $(GORUN) build/ci.go lint +#? clean: Clean go cache, built executables, and the auto generated folder clean: go clean -cache rm -fr build/_workspace/pkg/ $(GOBIN)/* @@ -29,6 +34,7 @@ clean: # The devtools target installs tools required for 'go generate'. # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. +#? devtools: Install recommended developer tools devtools: env GOBIN= go install golang.org/x/tools/cmd/stringer@latest env GOBIN= go install github.com/fjl/gencodec@latest @@ -42,3 +48,9 @@ forkdiff: --mount src=$(shell pwd),target=/host-pwd,type=bind \ protolambda/forkdiff:latest \ -repo /host-pwd/ -fork /host-pwd/fork.yaml -out /host-pwd/forkdiff.html + +#? help: Get more info on make commands. +help: Makefile + @echo " Choose a command run in go-ethereum:" + @sed -n 's/^#?//p' $< | column -t -s ':' | sort | sed -e 's/^/ /' +.PHONY: help diff --git a/README.md b/README.md index 77317090c1..1e8dba8090 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ ## Go Ethereum -Official Golang execution layer implementation of the Ethereum protocol. +Golang execution layer implementation of the Ethereum protocol. [![API Reference]( -https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 +https://pkg.go.dev/badge/github.com/ethereum/go-ethereum )](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) -[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) +[![Travis](https://app.travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://app.travis-ci.com/github/ethereum/go-ethereum) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) Automated builds are available for stable releases and the unstable master branch. Binary diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 4abf298068..c7bc2b4541 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -29,7 +29,7 @@ import ( ) // The ABI holds information about a contract's context and available -// invokable methods. It will allow you to type check function calls and +// invocable methods. It will allow you to type check function calls and // packs data accordingly. type ABI struct { Constructor Method diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index 2e45e86ae2..38b3046970 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -84,6 +84,11 @@ type BlockHashContractCaller interface { // used when the user does not provide some needed values, but rather leaves it up // to the transactor to decide. type ContractTransactor interface { + ethereum.GasEstimator + ethereum.GasPricer + ethereum.GasPricer1559 + ethereum.TransactionSender + // HeaderByNumber returns a block header from the current canonical chain. If // number is nil, the latest known header is returned. HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) @@ -93,38 +98,6 @@ type ContractTransactor interface { // PendingNonceAt retrieves the current pending nonce associated with an account. PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) - - // SuggestGasPrice retrieves the currently suggested gas price to allow a timely - // execution of a transaction. - SuggestGasPrice(ctx context.Context) (*big.Int, error) - - // SuggestGasTipCap retrieves the currently suggested 1559 priority fee to allow - // a timely execution of a transaction. - SuggestGasTipCap(ctx context.Context) (*big.Int, error) - - // EstimateGas tries to estimate the gas needed to execute a specific - // transaction based on the current pending state of the backend blockchain. - // There is no guarantee that this is the true gas limit requirement as other - // transactions may be added or removed by miners, but it should provide a basis - // for setting a reasonable default. - EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) - - // SendTransaction injects the transaction into the pending pool for execution. - SendTransaction(ctx context.Context, tx *types.Transaction) error -} - -// ContractFilterer defines the methods needed to access log events using one-off -// queries or continuous event subscriptions. -type ContractFilterer interface { - // FilterLogs executes a log filter operation, blocking during execution and - // returning all the results in one batch. - // - // TODO(karalabe): Deprecate when the subscription one can return past data too. - FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) - - // SubscribeFilterLogs creates a background log filtering operation, returning - // a subscription immediately, which can be used to stream the found events. - SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) } // DeployBackend wraps the operations needed by WaitMined and WaitDeployed. @@ -133,6 +106,12 @@ type DeployBackend interface { CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) } +// ContractFilterer defines the methods needed to access log events using one-off +// queries or continuous event subscriptions. +type ContractFilterer interface { + ethereum.LogFilterer +} + // ContractBackend defines the methods needed to work with contracts on a read-write basis. type ContractBackend interface { ContractCaller diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index ad59a6bbd8..38ac9d60d0 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -18,1030 +18,44 @@ package backends import ( "context" - "errors" - "fmt" - "math/big" - "sync" - "time" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/filters" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethclient/simulated" ) -// This nil assignment ensures at compile time that SimulatedBackend implements bind.ContractBackend. -var _ bind.ContractBackend = (*SimulatedBackend)(nil) - -var ( - errBlockNumberUnsupported = errors.New("simulatedBackend cannot access blocks other than the latest block") - errBlockHashUnsupported = errors.New("simulatedBackend cannot access blocks by hash other than the latest block") - errBlockDoesNotExist = errors.New("block does not exist in blockchain") - errTransactionDoesNotExist = errors.New("transaction does not exist") -) - -// SimulatedBackend implements bind.ContractBackend, simulating a blockchain in -// the background. Its main purpose is to allow for easy testing of contract bindings. -// Simulated backend implements the following interfaces: -// ChainReader, ChainStateReader, ContractBackend, ContractCaller, ContractFilterer, ContractTransactor, -// DeployBackend, GasEstimator, GasPricer, LogFilterer, PendingContractCaller, TransactionReader, and TransactionSender +// SimulatedBackend is a simulated blockchain. +// Deprecated: use package github.com/ethereum/go-ethereum/ethclient/simulated instead. type SimulatedBackend struct { - database ethdb.Database // In memory database to store our testing data - blockchain *core.BlockChain // Ethereum blockchain to handle the consensus - - consensus consensus.Engine - - mu sync.Mutex - pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on request - pendingReceipts types.Receipts // Currently receipts for the pending block - - events *filters.EventSystem // for filtering log events live - filterSystem *filters.FilterSystem // for filtering database logs - - config *params.ChainConfig + *simulated.Backend + simulated.Client } -// NewSimulatedBackendWithDatabase creates a new binding backend based on the given database -// and uses a simulated blockchain for testing purposes. -// A simulated backend always uses chainID 1337. -func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { - return NewSimulatedBackendWithOpts(WithDatabase(database), WithAlloc(alloc), WithGasLimit(gasLimit)) +// Fork sets the head to a new block, which is based on the provided parentHash. +func (b *SimulatedBackend) Fork(ctx context.Context, parentHash common.Hash) error { + return b.Backend.Fork(parentHash) } // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. -// A simulated backend always uses chainID 1337. -func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { - return NewSimulatedBackendWithOpts(WithGasLimit(gasLimit), WithAlloc(alloc)) -} - -type simulatedBackendConfig struct { - genesis core.Genesis - cacheConfig *core.CacheConfig - database ethdb.Database - vmConfig vm.Config - consensus consensus.Engine -} - -type SimulatedBackendOpt func(s *simulatedBackendConfig) - -func WithDatabase(database ethdb.Database) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.database = database - } -} - -func WithGasLimit(gasLimit uint64) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.genesis.GasLimit = gasLimit - } -} - -func WithAlloc(alloc core.GenesisAlloc) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.genesis.Alloc = alloc - } -} - -func WithCacheConfig(cacheConfig *core.CacheConfig) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.cacheConfig = cacheConfig - } -} - -func WithGenesis(genesis core.Genesis) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.genesis = genesis - } -} - -func WithVMConfig(vmConfig vm.Config) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.vmConfig = vmConfig - } -} - -func WithConsensus(consensus consensus.Engine) SimulatedBackendOpt { - return func(s *simulatedBackendConfig) { - s.consensus = consensus - } -} - -// NewSimulatedBackendWithOpts creates a new binding backend based on the given database -// and uses a simulated blockchain for testing purposes. It exposes additional configuration -// options that are useful to -func NewSimulatedBackendWithOpts(opts ...SimulatedBackendOpt) *SimulatedBackend { - config := &simulatedBackendConfig{ - genesis: core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: 100000000, Alloc: make(core.GenesisAlloc)}, - database: rawdb.NewMemoryDatabase(), - consensus: ethash.NewFaker(), - } - - for _, opt := range opts { - opt(config) - } - - config.genesis.MustCommit(config.database, trie.NewDatabase(config.database, trie.HashDefaults)) - blockchain, _ := core.NewBlockChain(config.database, config.cacheConfig, &config.genesis, nil, config.consensus, config.vmConfig, nil, nil) - - backend := &SimulatedBackend{ - database: config.database, - blockchain: blockchain, - config: config.genesis.Config, - consensus: config.consensus, - } - - filterBackend := &filterBackend{config.database, blockchain, backend} - backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) - backend.events = filters.NewEventSystem(backend.filterSystem, false) - - header := backend.blockchain.CurrentBlock() - block := backend.blockchain.GetBlock(header.Hash(), header.Number.Uint64()) - - backend.rollback(block) - return backend -} - -// Close terminates the underlying blockchain's update loop. -func (b *SimulatedBackend) Close() error { - b.blockchain.Stop() - return nil -} - -// Commit imports all the pending transactions as a single block and starts a -// fresh new state. -func (b *SimulatedBackend) Commit() common.Hash { - b.mu.Lock() - defer b.mu.Unlock() - - if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil { - panic(err) // This cannot happen unless the simulator is wrong, fail in that case - } - // Don't wait for the async tx indexing - rawdb.WriteTxLookupEntriesByBlock(b.database, b.pendingBlock) - blockHash := b.pendingBlock.Hash() - - // Using the last inserted block here makes it possible to build on a side - // chain after a fork. - b.rollback(b.pendingBlock) - - return blockHash -} - -// Rollback aborts all pending transactions, reverting to the last committed state. -func (b *SimulatedBackend) Rollback() { - b.mu.Lock() - defer b.mu.Unlock() - - header := b.blockchain.CurrentBlock() - block := b.blockchain.GetBlock(header.Hash(), header.Number.Uint64()) - - b.rollback(block) -} - -func (b *SimulatedBackend) rollback(parent *types.Block) { - blocks, _ := core.GenerateChain(b.config, parent, b.consensus, b.database, 1, func(int, *core.BlockGen) {}) - - b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), b.blockchain.StateCache(), nil) -} - -// Fork creates a side-chain that can be used to simulate reorgs. -// -// This function should be called with the ancestor block where the new side -// chain should be started. Transactions (old and new) can then be applied on -// top and Commit-ed. // -// Note, the side-chain will only become canonical (and trigger the events) when -// it becomes longer. Until then CallContract will still operate on the current -// canonical chain. -// -// There is a % chance that the side chain becomes canonical at the same length -// to simulate live network behavior. -func (b *SimulatedBackend) Fork(ctx context.Context, parent common.Hash) error { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.pendingBlock.Transactions()) != 0 { - return errors.New("pending block dirty") - } - block, err := b.blockByHash(ctx, parent) - if err != nil { - return err - } - b.rollback(block) - return nil -} - -// stateByBlockNumber retrieves a state by a given blocknumber. -func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *big.Int) (*state.StateDB, error) { - if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number) == 0 { - return b.blockchain.State() - } - block, err := b.blockByNumber(ctx, blockNumber) - if err != nil { - return nil, err - } - return b.blockchain.StateAt(block.Root()) -} - -// CodeAt returns the code associated with a certain account in the blockchain. -func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - stateDB, err := b.stateByBlockNumber(ctx, blockNumber) - if err != nil { - return nil, err - } - return stateDB.GetCode(contract), nil -} - -// CodeAtHash returns the code associated with a certain account in the blockchain. -func (b *SimulatedBackend) CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - header, err := b.headerByHash(blockHash) - if err != nil { - return nil, err - } - - stateDB, err := b.blockchain.StateAt(header.Root) - if err != nil { - return nil, err - } - - return stateDB.GetCode(contract), nil -} - -// BalanceAt returns the wei balance of a certain account in the blockchain. -func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Address, blockNumber *big.Int) (*big.Int, error) { - b.mu.Lock() - defer b.mu.Unlock() - - stateDB, err := b.stateByBlockNumber(ctx, blockNumber) - if err != nil { - return nil, err - } - return stateDB.GetBalance(contract), nil -} - -// NonceAt returns the nonce of a certain account in the blockchain. -func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, blockNumber *big.Int) (uint64, error) { - b.mu.Lock() - defer b.mu.Unlock() - - stateDB, err := b.stateByBlockNumber(ctx, blockNumber) - if err != nil { - return 0, err - } - return stateDB.GetNonce(contract), nil -} - -// StorageAt returns the value of key in the storage of an account in the blockchain. -func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Address, key common.Hash, blockNumber *big.Int) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - stateDB, err := b.stateByBlockNumber(ctx, blockNumber) - if err != nil { - return nil, err - } - val := stateDB.GetState(contract, key) - return val[:], nil -} - -// TransactionReceipt returns the receipt of a transaction. -func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - b.mu.Lock() - defer b.mu.Unlock() - - receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash, b.config) - if receipt == nil { - return nil, ethereum.NotFound - } - return receipt, nil -} - -// TransactionByHash checks the pool of pending transactions in addition to the -// blockchain. The isPending return value indicates whether the transaction has been -// mined yet. Note that the transaction may not be part of the canonical chain even if -// it's not pending. -func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (*types.Transaction, bool, error) { - b.mu.Lock() - defer b.mu.Unlock() - - tx := b.pendingBlock.Transaction(txHash) - if tx != nil { - return tx, true, nil - } - tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash) - if tx != nil { - return tx, false, nil - } - return nil, false, ethereum.NotFound -} - -// BlockByHash retrieves a block based on the block hash. -func (b *SimulatedBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - b.mu.Lock() - defer b.mu.Unlock() - - return b.blockByHash(ctx, hash) -} - -// blockByHash retrieves a block based on the block hash without Locking. -func (b *SimulatedBackend) blockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - if hash == b.pendingBlock.Hash() { - return b.pendingBlock, nil - } - - block := b.blockchain.GetBlockByHash(hash) - if block != nil { - return block, nil - } - - return nil, errBlockDoesNotExist -} - -// BlockByNumber retrieves a block from the database by number, caching it -// (associated with its hash) if found. -func (b *SimulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - b.mu.Lock() - defer b.mu.Unlock() - - return b.blockByNumber(ctx, number) -} - -// blockByNumber retrieves a block from the database by number, caching it -// (associated with its hash) if found without Lock. -func (b *SimulatedBackend) blockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - if number == nil || number.Cmp(b.pendingBlock.Number()) == 0 { - return b.blockByHash(ctx, b.blockchain.CurrentBlock().Hash()) - } - - block := b.blockchain.GetBlockByNumber(uint64(number.Int64())) - if block == nil { - return nil, errBlockDoesNotExist - } - - return block, nil -} - -// HeaderByHash returns a block header from the current canonical chain. -func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.headerByHash(hash) -} - -// headerByHash retrieves a header from the database by hash without Lock. -func (b *SimulatedBackend) headerByHash(hash common.Hash) (*types.Header, error) { - if hash == b.pendingBlock.Hash() { - return b.pendingBlock.Header(), nil - } - - header := b.blockchain.GetHeaderByHash(hash) - if header == nil { - return nil, errBlockDoesNotExist - } - - return header, nil -} - -// HeaderByNumber returns a block header from the current canonical chain. If number is -// nil, the latest known header is returned. -func (b *SimulatedBackend) HeaderByNumber(ctx context.Context, block *big.Int) (*types.Header, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if block == nil || block.Cmp(b.pendingBlock.Number()) == 0 { - return b.blockchain.CurrentHeader(), nil - } - - return b.blockchain.GetHeaderByNumber(uint64(block.Int64())), nil -} - -// TransactionCount returns the number of transactions in a given block. -func (b *SimulatedBackend) TransactionCount(ctx context.Context, blockHash common.Hash) (uint, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if blockHash == b.pendingBlock.Hash() { - return uint(b.pendingBlock.Transactions().Len()), nil - } - - block := b.blockchain.GetBlockByHash(blockHash) - if block == nil { - return uint(0), errBlockDoesNotExist - } - - return uint(block.Transactions().Len()), nil -} - -// TransactionInBlock returns the transaction for a specific block at a specific index. -func (b *SimulatedBackend) TransactionInBlock(ctx context.Context, blockHash common.Hash, index uint) (*types.Transaction, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if blockHash == b.pendingBlock.Hash() { - transactions := b.pendingBlock.Transactions() - if uint(len(transactions)) < index+1 { - return nil, errTransactionDoesNotExist - } - - return transactions[index], nil - } - - block := b.blockchain.GetBlockByHash(blockHash) - if block == nil { - return nil, errBlockDoesNotExist - } - - transactions := block.Transactions() - if uint(len(transactions)) < index+1 { - return nil, errTransactionDoesNotExist - } - - return transactions[index], nil -} - -// PendingCodeAt returns the code associated with an account in the pending state. -func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - return b.pendingState.GetCode(contract), nil -} - -func newRevertError(result *core.ExecutionResult) *revertError { - reason, errUnpack := abi.UnpackRevert(result.Revert()) - err := errors.New("execution reverted") - if errUnpack == nil { - err = fmt.Errorf("execution reverted: %v", reason) - } - return &revertError{ - error: err, - reason: hexutil.Encode(result.Revert()), - } -} - -// revertError is an API error that encompasses an EVM revert with JSON error -// code and a binary data blob. -type revertError struct { - error - reason string // revert reason hex encoded -} - -// ErrorCode returns the JSON error code for a revert. -// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal -func (e *revertError) ErrorCode() int { - return 3 -} - -// ErrorData returns the hex encoded revert reason. -func (e *revertError) ErrorData() interface{} { - return e.reason -} - -// CallContract executes a contract call. -func (b *SimulatedBackend) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number) != 0 { - return nil, errBlockNumberUnsupported - } - return b.callContractAtHead(ctx, call) -} - -// CallContractAtHash executes a contract call on a specific block hash. -func (b *SimulatedBackend) CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if blockHash != b.blockchain.CurrentBlock().Hash() { - return nil, errBlockHashUnsupported - } - return b.callContractAtHead(ctx, call) -} - -// callContractAtHead executes a contract call against the latest block state. -func (b *SimulatedBackend) callContractAtHead(ctx context.Context, call ethereum.CallMsg) ([]byte, error) { - stateDB, err := b.blockchain.State() - if err != nil { - return nil, err - } - res, err := b.callContract(ctx, call, b.blockchain.CurrentBlock(), stateDB) - if err != nil { - return nil, err - } - // If the result contains a revert reason, try to unpack and return it. - if len(res.Revert()) > 0 { - return nil, newRevertError(res) - } - return res.Return(), res.Err -} - -// PendingCallContract executes a contract call on the pending state. -func (b *SimulatedBackend) PendingCallContract(ctx context.Context, call ethereum.CallMsg) ([]byte, error) { - b.mu.Lock() - defer b.mu.Unlock() - defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot()) - - res, err := b.callContract(ctx, call, b.pendingBlock.Header(), b.pendingState) - if err != nil { - return nil, err - } - // If the result contains a revert reason, try to unpack and return it. - if len(res.Revert()) > 0 { - return nil, newRevertError(res) - } - return res.Return(), res.Err -} - -// PendingNonceAt implements PendingStateReader.PendingNonceAt, retrieving -// the nonce currently pending for the account. -func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { - b.mu.Lock() - defer b.mu.Unlock() - - return b.pendingState.GetOrNewStateObject(account).Nonce(), nil -} - -// SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated -// chain doesn't have miners, we just return a gas price of 1 for any call. -func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.pendingBlock.Header().BaseFee != nil { - return b.pendingBlock.Header().BaseFee, nil - } - return big.NewInt(1), nil -} - -// SuggestGasTipCap implements ContractTransactor.SuggestGasTipCap. Since the simulated -// chain doesn't have miners, we just return a gas tip of 1 for any call. -func (b *SimulatedBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - return big.NewInt(1), nil -} - -// EstimateGas executes the requested code against the currently pending block/state and -// returns the used amount of gas. -func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) { - b.mu.Lock() - defer b.mu.Unlock() - - // Determine the lowest and highest possible gas limits to binary search in between - var ( - lo uint64 = params.TxGas - 1 - hi uint64 - cap uint64 - ) - if call.Gas >= params.TxGas { - hi = call.Gas - } else { - hi = b.pendingBlock.GasLimit() - } - // Normalize the max fee per gas the call is willing to spend. - var feeCap *big.Int - if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { - return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } else if call.GasPrice != nil { - feeCap = call.GasPrice - } else if call.GasFeeCap != nil { - feeCap = call.GasFeeCap - } else { - feeCap = common.Big0 - } - // Recap the highest gas allowance with account's balance. - if feeCap.BitLen() != 0 { - balance := b.pendingState.GetBalance(call.From) // from can't be nil - available := new(big.Int).Set(balance) - if call.Value != nil { - if call.Value.Cmp(available) >= 0 { - return 0, core.ErrInsufficientFundsForTransfer - } - available.Sub(available, call.Value) - } - allowance := new(big.Int).Div(available, feeCap) - if allowance.IsUint64() && hi > allowance.Uint64() { - transfer := call.Value - if transfer == nil { - transfer = new(big.Int) - } - log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, - "sent", transfer, "feecap", feeCap, "fundable", allowance) - hi = allowance.Uint64() - } - } - cap = hi - - // Create a helper to check if a gas allowance results in an executable transaction - executable := func(gas uint64) (bool, *core.ExecutionResult, error) { - call.Gas = gas - - snapshot := b.pendingState.Snapshot() - res, err := b.callContract(ctx, call, b.pendingBlock.Header(), b.pendingState) - b.pendingState.RevertToSnapshot(snapshot) - - if err != nil { - if errors.Is(err, core.ErrIntrinsicGas) { - return true, nil, nil // Special case, raise gas limit - } - return true, nil, err // Bail out - } - return res.Failed(), res, nil - } - // Execute the binary search and hone in on an executable gas limit - for lo+1 < hi { - mid := (hi + lo) / 2 - failed, _, err := executable(mid) - - // If the error is not nil(consensus error), it means the provided message - // call or transaction will never be accepted no matter how much gas it is - // assigned. Return the error directly, don't struggle any more - if err != nil { - return 0, err - } - if failed { - lo = mid - } else { - hi = mid - } - } - // Reject the transaction as invalid if it still fails at the highest allowance - if hi == cap { - failed, result, err := executable(hi) - if err != nil { - return 0, err - } - if failed { - if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) { - if len(result.Revert()) > 0 { - return 0, newRevertError(result) - } - return 0, result.Err - } - // Otherwise, the specified gas cap is too low - return 0, fmt.Errorf("gas required exceeds allowance (%d)", cap) - } - } - return hi, nil -} - -// callContract implements common code between normal and pending contract calls. -// state is modified during execution, make sure to copy it if necessary. -func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, header *types.Header, stateDB *state.StateDB) (*core.ExecutionResult, error) { - // Gas prices post 1559 need to be initialized - if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { - return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } - if !b.blockchain.Config().IsLondon(header.Number) { - // If there's no basefee, then it must be a non-1559 execution - if call.GasPrice == nil { - call.GasPrice = new(big.Int) - } - call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice - } else { - // A basefee is provided, necessitating 1559-type execution - if call.GasPrice != nil { - // User specified the legacy gas field, convert to 1559 gas typing - call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice - } else { - // User specified 1559 gas fields (or none), use those - if call.GasFeeCap == nil { - call.GasFeeCap = new(big.Int) - } - if call.GasTipCap == nil { - call.GasTipCap = new(big.Int) - } - // Backfill the legacy gasPrice for EVM execution, unless we're all zeroes - call.GasPrice = new(big.Int) - if call.GasFeeCap.BitLen() > 0 || call.GasTipCap.BitLen() > 0 { - call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, header.BaseFee), call.GasFeeCap) - } - } - } - // Ensure message is initialized properly. - if call.Gas == 0 { - call.Gas = 10 * header.GasLimit - } - if call.Value == nil { - call.Value = new(big.Int) - } - - // Set infinite balance to the fake caller account. - from := stateDB.GetOrNewStateObject(call.From) - from.SetBalance(math.MaxBig256) - - // Execute the call. - msg := &core.Message{ - From: call.From, - To: call.To, - Value: call.Value, - GasLimit: call.Gas, - GasPrice: call.GasPrice, - GasFeeCap: call.GasFeeCap, - GasTipCap: call.GasTipCap, - Data: call.Data, - AccessList: call.AccessList, - SkipAccountChecks: true, - } - - // Create a new environment which holds all relevant information - // about the transaction and calling mechanisms. - txContext := core.NewEVMTxContext(msg) - evmContext := core.NewEVMBlockContext(header, b.blockchain, nil, b.config, stateDB) - vmEnv := vm.NewEVM(evmContext, txContext, stateDB, b.config, vm.Config{NoBaseFee: true}) - gasPool := new(core.GasPool).AddGas(math.MaxUint64) - - return core.ApplyMessage(vmEnv, msg, gasPool) -} - -// SendTransaction updates the pending block to include the given transaction. -func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error { - b.mu.Lock() - defer b.mu.Unlock() - - // Get the last block - block, err := b.blockByHash(ctx, b.pendingBlock.ParentHash()) - if err != nil { - return errors.New("could not fetch parent") - } - // Check transaction validity - signer := types.MakeSigner(b.blockchain.Config(), block.Number(), block.Time()) - sender, err := types.Sender(signer, tx) - if err != nil { - return fmt.Errorf("invalid transaction: %v", err) - } - nonce := b.pendingState.GetNonce(sender) - if tx.Nonce() != nonce { - return fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce) - } - // Include tx in chain - blocks, receipts := core.GenerateChain(b.config, block, b.consensus, b.database, 1, func(number int, block *core.BlockGen) { - for _, tx := range b.pendingBlock.Transactions() { - block.AddTxWithChain(b.blockchain, tx) - } - block.AddTxWithChain(b.blockchain, tx) - }) - stateDB, err := b.blockchain.State() - if err != nil { - return err - } - b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) - b.pendingReceipts = receipts[0] - return nil -} - -// FilterLogs executes a log filter operation, blocking during execution and -// returning all the results in one batch. +// A simulated backend always uses chainID 1337. // -// TODO(karalabe): Deprecate when the subscription one can return past data too. -func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) { - var filter *filters.Filter - if query.BlockHash != nil { - // Block filter requested, construct a single-shot filter - filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) - } else { - // Initialize unset filter boundaries to run from genesis to chain head - from := int64(0) - if query.FromBlock != nil { - from = query.FromBlock.Int64() - } - to := int64(-1) - if query.ToBlock != nil { - to = query.ToBlock.Int64() - } - // Construct the range filter - filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) - } - // Run the filter and return all the logs - logs, err := filter.Logs(ctx) - if err != nil { - return nil, err +// Deprecated: please use simulated.Backend from package +// github.com/ethereum/go-ethereum/ethclient/simulated instead. +func NewSimulatedBackend(alloc types.GenesisAlloc, gasLimit uint64) *SimulatedBackend { + b := simulated.NewBackend(alloc, simulated.WithBlockGasLimit(gasLimit)) + return &SimulatedBackend{ + Backend: b, + Client: b.Client(), } - res := make([]types.Log, len(logs)) - for i, nLog := range logs { - res[i] = *nLog - } - return res, nil } -// SubscribeFilterLogs creates a background log filtering operation, returning a -// subscription immediately, which can be used to stream the found events. -func (b *SimulatedBackend) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - // Subscribe to contract events - sink := make(chan []*types.Log) - - sub, err := b.events.SubscribeLogs(query, sink) - if err != nil { - return nil, err +func NewSimulatedBackendFromConfig(cfg ethconfig.Config) *SimulatedBackend { + b := simulated.NewBackendFromConfig(cfg) + return &SimulatedBackend{ + Backend: b, + Client: b.Client(), } - // Since we're getting logs in batches, we need to flatten them into a plain stream - return event.NewSubscription(func(quit <-chan struct{}) error { - defer sub.Unsubscribe() - for { - select { - case logs := <-sink: - for _, nlog := range logs { - select { - case ch <- *nlog: - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - } - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - } - }), nil -} - -// SubscribeNewHead returns an event subscription for a new header. -func (b *SimulatedBackend) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { - // subscribe to a new head - sink := make(chan *types.Header) - sub := b.events.SubscribeNewHeads(sink) - - return event.NewSubscription(func(quit <-chan struct{}) error { - defer sub.Unsubscribe() - for { - select { - case head := <-sink: - select { - case ch <- head: - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - } - }), nil -} - -// AdjustTime adds a time shift to the simulated clock. -// It can only be called on empty blocks. -func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.pendingBlock.Transactions()) != 0 { - return errors.New("could not adjust time on non-empty block") - } - // Get the last block - block := b.blockchain.GetBlockByHash(b.pendingBlock.ParentHash()) - if block == nil { - return errors.New("could not find parent") - } - - blocks, _ := core.GenerateChain(b.config, block, b.consensus, b.database, 1, func(number int, block *core.BlockGen) { - block.OffsetTime(int64(adjustment.Seconds())) - }) - stateDB, err := b.blockchain.State() - if err != nil { - return err - } - b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) - return nil -} - -// Blockchain returns the underlying blockchain. -func (b *SimulatedBackend) Blockchain() *core.BlockChain { - return b.blockchain -} - -// filterBackend implements filters.Backend to support filtering for logs without -// taking bloom-bits acceleration structures into account. -type filterBackend struct { - db ethdb.Database - bc *core.BlockChain - backend *SimulatedBackend -} - -func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } - -func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") } - -func (fb *filterBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { - switch number { - case rpc.PendingBlockNumber: - if block := fb.backend.pendingBlock; block != nil { - return block.Header(), nil - } - return nil, nil - case rpc.LatestBlockNumber: - return fb.bc.CurrentHeader(), nil - case rpc.FinalizedBlockNumber: - return fb.bc.CurrentFinalBlock(), nil - case rpc.SafeBlockNumber: - return fb.bc.CurrentSafeBlock(), nil - default: - return fb.bc.GetHeaderByNumber(uint64(number.Int64())), nil - } -} - -func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { - return fb.bc.GetHeaderByHash(hash), nil -} - -func (fb *filterBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) { - if body := fb.bc.GetBody(hash); body != nil { - return body, nil - } - return nil, errors.New("block body not found") -} - -func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return fb.backend.pendingBlock, fb.backend.pendingReceipts -} - -func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - number := rawdb.ReadHeaderNumber(fb.db, hash) - if number == nil { - return nil, nil - } - header := rawdb.ReadHeader(fb.db, hash, *number) - if header == nil { - return nil, nil - } - return rawdb.ReadReceipts(fb.db, hash, *number, header.Time, fb.bc.Config()), nil -} - -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { - logs := rawdb.ReadLogs(fb.db, hash, number) - return logs, nil -} - -func (fb *filterBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return nullSubscription() -} - -func (fb *filterBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { - return fb.bc.SubscribeChainEvent(ch) -} - -func (fb *filterBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { - return fb.bc.SubscribeRemovedLogsEvent(ch) -} - -func (fb *filterBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { - return fb.bc.SubscribeLogsEvent(ch) -} - -func (fb *filterBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return nullSubscription() -} - -func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 } - -func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) { - panic("not supported") -} - -func (fb *filterBackend) ChainConfig() *params.ChainConfig { - panic("not supported") -} - -func (fb *filterBackend) CurrentHeader() *types.Header { - panic("not supported") -} - -func nullSubscription() event.Subscription { - return event.NewSubscription(func(quit <-chan struct{}) error { - <-quit - return nil - }) } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go deleted file mode 100644 index a2acf7ead5..0000000000 --- a/accounts/abi/bind/backends/simulated_test.go +++ /dev/null @@ -1,1483 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package backends - -import ( - "bytes" - "context" - "errors" - "math/big" - "math/rand" - "reflect" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" -) - -func TestSimulatedBackend(t *testing.T) { - t.Parallel() - var gasLimit uint64 = 8000029 - key, _ := crypto.GenerateKey() // nolint: gosec - auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - genAlloc := make(core.GenesisAlloc) - genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)} - - sim := NewSimulatedBackend(genAlloc, gasLimit) - defer sim.Close() - - // should return an error if the tx is not found - txHash := common.HexToHash("2") - _, isPending, err := sim.TransactionByHash(context.Background(), txHash) - - if isPending { - t.Fatal("transaction should not be pending") - } - if err != ethereum.NotFound { - t.Fatalf("err should be `ethereum.NotFound` but received %v", err) - } - - // generate a transaction and confirm you can retrieve it - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - code := `6060604052600a8060106000396000f360606040526008565b00` - var gas uint64 = 3000000 - tx := types.NewContractCreation(0, big.NewInt(0), gas, gasPrice, common.FromHex(code)) - tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key) - - err = sim.SendTransaction(context.Background(), tx) - if err != nil { - t.Fatal("error sending transaction") - } - - txHash = tx.Hash() - _, isPending, err = sim.TransactionByHash(context.Background(), txHash) - if err != nil { - t.Fatalf("error getting transaction with hash: %v", txHash.String()) - } - if !isPending { - t.Fatal("transaction should have pending status") - } - - sim.Commit() - _, isPending, err = sim.TransactionByHash(context.Background(), txHash) - if err != nil { - t.Fatalf("error getting transaction with hash: %v", txHash.String()) - } - if isPending { - t.Fatal("transaction should not have pending status") - } -} - -var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - -// the following is based on this contract: -// -// contract T { -// event received(address sender, uint amount, bytes memo); -// event receivedAddr(address sender); -// -// function receive(bytes calldata memo) external payable returns (string memory res) { -// emit received(msg.sender, msg.value, memo); -// emit receivedAddr(msg.sender); -// return "hello world"; -// } -// } -const abiJSON = `[ { "constant": false, "inputs": [ { "name": "memo", "type": "bytes" } ], "name": "receive", "outputs": [ { "name": "res", "type": "string" } ], "payable": true, "stateMutability": "payable", "type": "function" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" }, { "indexed": false, "name": "amount", "type": "uint256" }, { "indexed": false, "name": "memo", "type": "bytes" } ], "name": "received", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" } ], "name": "receivedAddr", "type": "event" } ]` -const abiBin = `0x608060405234801561001057600080fd5b506102a0806100206000396000f3fe60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` -const deployedCode = `60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` - -// expected return value contains "hello world" -var expectedReturn = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - -func simTestBackend(testAddr common.Address) *SimulatedBackend { - return NewSimulatedBackend( - core.GenesisAlloc{ - testAddr: {Balance: big.NewInt(10000000000000000)}, - }, 10000000, - ) -} - -func TestNewSimulatedBackend(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - expectedBal := big.NewInt(10000000000000000) - sim := simTestBackend(testAddr) - defer sim.Close() - - if sim.config != params.AllEthashProtocolChanges { - t.Errorf("expected sim config to equal params.AllEthashProtocolChanges, got %v", sim.config) - } - - if sim.blockchain.Config() != params.AllEthashProtocolChanges { - t.Errorf("expected sim blockchain config to equal params.AllEthashProtocolChanges, got %v", sim.config) - } - - stateDB, _ := sim.blockchain.State() - bal := stateDB.GetBalance(testAddr) - if bal.Cmp(expectedBal) != 0 { - t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal) - } -} - -func TestAdjustTime(t *testing.T) { - t.Parallel() - sim := NewSimulatedBackend( - core.GenesisAlloc{}, 10000000, - ) - defer sim.Close() - - prevTime := sim.pendingBlock.Time() - if err := sim.AdjustTime(time.Second); err != nil { - t.Error(err) - } - newTime := sim.pendingBlock.Time() - - if newTime-prevTime != uint64(time.Second.Seconds()) { - t.Errorf("adjusted time not equal to a second. prev: %v, new: %v", prevTime, newTime) - } -} - -func TestNewAdjustTimeFail(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.blockchain.Stop() - - // Create tx and send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - sim.SendTransaction(context.Background(), signedTx) - // AdjustTime should fail on non-empty block - if err := sim.AdjustTime(time.Second); err == nil { - t.Error("Expected adjust time to error on non-empty block") - } - sim.Commit() - - prevTime := sim.pendingBlock.Time() - if err := sim.AdjustTime(time.Minute); err != nil { - t.Error(err) - } - newTime := sim.pendingBlock.Time() - if newTime-prevTime != uint64(time.Minute.Seconds()) { - t.Errorf("adjusted time not equal to a minute. prev: %v, new: %v", prevTime, newTime) - } - // Put a transaction after adjusting time - tx2 := types.NewTransaction(1, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx2, err := types.SignTx(tx2, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - sim.SendTransaction(context.Background(), signedTx2) - sim.Commit() - newTime = sim.pendingBlock.Time() - if newTime-prevTime >= uint64(time.Minute.Seconds()) { - t.Errorf("time adjusted, but shouldn't be: prev: %v, new: %v", prevTime, newTime) - } -} - -func TestBalanceAt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - expectedBal := big.NewInt(10000000000000000) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - bal, err := sim.BalanceAt(bgCtx, testAddr, nil) - if err != nil { - t.Error(err) - } - - if bal.Cmp(expectedBal) != 0 { - t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal) - } -} - -func TestBlockByHash(t *testing.T) { - t.Parallel() - sim := NewSimulatedBackend( - core.GenesisAlloc{}, 10000000, - ) - defer sim.Close() - bgCtx := context.Background() - - block, err := sim.BlockByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - blockByHash, err := sim.BlockByHash(bgCtx, block.Hash()) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - - if block.Hash() != blockByHash.Hash() { - t.Errorf("did not get expected block") - } -} - -func TestBlockByNumber(t *testing.T) { - t.Parallel() - sim := NewSimulatedBackend( - core.GenesisAlloc{}, 10000000, - ) - defer sim.Close() - bgCtx := context.Background() - - block, err := sim.BlockByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - if block.NumberU64() != 0 { - t.Errorf("did not get most recent block, instead got block number %v", block.NumberU64()) - } - - // create one block - sim.Commit() - - block, err = sim.BlockByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - if block.NumberU64() != 1 { - t.Errorf("did not get most recent block, instead got block number %v", block.NumberU64()) - } - - blockByNumber, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) - if err != nil { - t.Errorf("could not get block by number: %v", err) - } - if blockByNumber.Hash() != block.Hash() { - t.Errorf("did not get the same block with height of 1 as before") - } -} - -func TestNonceAt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - nonce, err := sim.NonceAt(bgCtx, testAddr, big.NewInt(0)) - if err != nil { - t.Errorf("could not get nonce for test addr: %v", err) - } - - if nonce != uint64(0) { - t.Errorf("received incorrect nonce. expected 0, got %v", nonce) - } - - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(nonce, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - sim.Commit() - - newNonce, err := sim.NonceAt(bgCtx, testAddr, big.NewInt(1)) - if err != nil { - t.Errorf("could not get nonce for test addr: %v", err) - } - - if newNonce != nonce+uint64(1) { - t.Errorf("received incorrect nonce. expected 1, got %v", nonce) - } - // create some more blocks - sim.Commit() - // Check that we can get data for an older block/state - newNonce, err = sim.NonceAt(bgCtx, testAddr, big.NewInt(1)) - if err != nil { - t.Fatalf("could not get nonce for test addr: %v", err) - } - if newNonce != nonce+uint64(1) { - t.Fatalf("received incorrect nonce. expected 1, got %v", nonce) - } -} - -func TestSendTransaction(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - sim.Commit() - - block, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) - if err != nil { - t.Errorf("could not get block at height 1: %v", err) - } - - if signedTx.Hash() != block.Transactions()[0].Hash() { - t.Errorf("did not commit sent transaction. expected hash %v got hash %v", block.Transactions()[0].Hash(), signedTx.Hash()) - } -} - -func TestTransactionByHash(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := NewSimulatedBackend( - core.GenesisAlloc{ - testAddr: {Balance: big.NewInt(10000000000000000)}, - }, 10000000, - ) - defer sim.Close() - bgCtx := context.Background() - - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - - // ensure tx is committed pending - receivedTx, pending, err := sim.TransactionByHash(bgCtx, signedTx.Hash()) - if err != nil { - t.Errorf("could not get transaction by hash %v: %v", signedTx.Hash(), err) - } - if !pending { - t.Errorf("expected transaction to be in pending state") - } - if receivedTx.Hash() != signedTx.Hash() { - t.Errorf("did not received committed transaction. expected hash %v got hash %v", signedTx.Hash(), receivedTx.Hash()) - } - - sim.Commit() - - // ensure tx is not and committed pending - receivedTx, pending, err = sim.TransactionByHash(bgCtx, signedTx.Hash()) - if err != nil { - t.Errorf("could not get transaction by hash %v: %v", signedTx.Hash(), err) - } - if pending { - t.Errorf("expected transaction to not be in pending state") - } - if receivedTx.Hash() != signedTx.Hash() { - t.Errorf("did not received committed transaction. expected hash %v got hash %v", signedTx.Hash(), receivedTx.Hash()) - } -} - -func TestEstimateGas(t *testing.T) { - t.Parallel() - /* - pragma solidity ^0.6.4; - contract GasEstimation { - function PureRevert() public { revert(); } - function Revert() public { revert("revert reason");} - function OOG() public { for (uint i = 0; ; i++) {}} - function Assert() public { assert(false);} - function Valid() public {} - } - */ - const contractAbi = "[{\"inputs\":[],\"name\":\"Assert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"OOG\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PureRevert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Revert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Valid\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]" - const contractBin = "0x60806040523480156100115760006000fd5b50610017565b61016e806100266000396000f3fe60806040523480156100115760006000fd5b506004361061005c5760003560e01c806350f6fe3414610062578063aa8b1d301461006c578063b9b046f914610076578063d8b9839114610080578063e09fface1461008a5761005c565b60006000fd5b61006a610094565b005b6100746100ad565b005b61007e6100b5565b005b6100886100c2565b005b610092610135565b005b6000600090505b5b808060010191505061009b565b505b565b60006000fd5b565b600015156100bf57fe5b5b565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252600d8152602001807f72657665727420726561736f6e0000000000000000000000000000000000000081526020015060200191505060405180910390fd5b565b5b56fea2646970667358221220345bbcbb1a5ecf22b53a78eaebf95f8ee0eceff6d10d4b9643495084d2ec934a64736f6c63430006040033" - - key, _ := crypto.GenerateKey() - addr := crypto.PubkeyToAddress(key.PublicKey) - opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - - sim := NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(params.Ether)}}, 10000000) - defer sim.Close() - - parsed, _ := abi.JSON(strings.NewReader(contractAbi)) - contractAddr, _, _, _ := bind.DeployContract(opts, parsed, common.FromHex(contractBin), sim) - sim.Commit() - - var cases = []struct { - name string - message ethereum.CallMsg - expect uint64 - expectError error - expectData interface{} - }{ - {"plain transfer(valid)", ethereum.CallMsg{ - From: addr, - To: &addr, - Gas: 0, - GasPrice: big.NewInt(0), - Value: big.NewInt(1), - Data: nil, - }, params.TxGas, nil, nil}, - - {"plain transfer(invalid)", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 0, - GasPrice: big.NewInt(0), - Value: big.NewInt(1), - Data: nil, - }, 0, errors.New("execution reverted"), nil}, - - {"Revert", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 0, - GasPrice: big.NewInt(0), - Value: nil, - Data: common.Hex2Bytes("d8b98391"), - }, 0, errors.New("execution reverted: revert reason"), "0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d72657665727420726561736f6e00000000000000000000000000000000000000"}, - - {"PureRevert", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 0, - GasPrice: big.NewInt(0), - Value: nil, - Data: common.Hex2Bytes("aa8b1d30"), - }, 0, errors.New("execution reverted"), nil}, - - {"OOG", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 100000, - GasPrice: big.NewInt(0), - Value: nil, - Data: common.Hex2Bytes("50f6fe34"), - }, 0, errors.New("gas required exceeds allowance (100000)"), nil}, - - {"Assert", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 100000, - GasPrice: big.NewInt(0), - Value: nil, - Data: common.Hex2Bytes("b9b046f9"), - }, 0, errors.New("invalid opcode: INVALID"), nil}, - - {"Valid", ethereum.CallMsg{ - From: addr, - To: &contractAddr, - Gas: 100000, - GasPrice: big.NewInt(0), - Value: nil, - Data: common.Hex2Bytes("e09fface"), - }, 21275, nil, nil}, - } - for _, c := range cases { - got, err := sim.EstimateGas(context.Background(), c.message) - if c.expectError != nil { - if err == nil { - t.Fatalf("Expect error, got nil") - } - if c.expectError.Error() != err.Error() { - t.Fatalf("Expect error, want %v, got %v", c.expectError, err) - } - if c.expectData != nil { - if err, ok := err.(*revertError); !ok { - t.Fatalf("Expect revert error, got %T", err) - } else if !reflect.DeepEqual(err.ErrorData(), c.expectData) { - t.Fatalf("Error data mismatch, want %v, got %v", c.expectData, err.ErrorData()) - } - } - continue - } - if got != c.expect { - t.Fatalf("Gas estimation mismatch, want %d, got %d", c.expect, got) - } - } -} - -func TestEstimateGasWithPrice(t *testing.T) { - t.Parallel() - key, _ := crypto.GenerateKey() - addr := crypto.PubkeyToAddress(key.PublicKey) - - sim := NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(params.Ether*2 + 2e17)}}, 10000000) - defer sim.Close() - - recipient := common.HexToAddress("deadbeef") - var cases = []struct { - name string - message ethereum.CallMsg - expect uint64 - expectError error - }{ - {"EstimateWithoutPrice", ethereum.CallMsg{ - From: addr, - To: &recipient, - Gas: 0, - GasPrice: big.NewInt(0), - Value: big.NewInt(100000000000), - Data: nil, - }, 21000, nil}, - - {"EstimateWithPrice", ethereum.CallMsg{ - From: addr, - To: &recipient, - Gas: 0, - GasPrice: big.NewInt(100000000000), - Value: big.NewInt(100000000000), - Data: nil, - }, 21000, nil}, - - {"EstimateWithVeryHighPrice", ethereum.CallMsg{ - From: addr, - To: &recipient, - Gas: 0, - GasPrice: big.NewInt(1e14), // gascost = 2.1ether - Value: big.NewInt(1e17), // the remaining balance for fee is 2.1ether - Data: nil, - }, 21000, nil}, - - {"EstimateWithSuperhighPrice", ethereum.CallMsg{ - From: addr, - To: &recipient, - Gas: 0, - GasPrice: big.NewInt(2e14), // gascost = 4.2ether - Value: big.NewInt(100000000000), - Data: nil, - }, 21000, errors.New("gas required exceeds allowance (10999)")}, // 10999=(2.2ether-1000wei)/(2e14) - - {"EstimateEIP1559WithHighFees", ethereum.CallMsg{ - From: addr, - To: &addr, - Gas: 0, - GasFeeCap: big.NewInt(1e14), // maxgascost = 2.1ether - GasTipCap: big.NewInt(1), - Value: big.NewInt(1e17), // the remaining balance for fee is 2.1ether - Data: nil, - }, params.TxGas, nil}, - - {"EstimateEIP1559WithSuperHighFees", ethereum.CallMsg{ - From: addr, - To: &addr, - Gas: 0, - GasFeeCap: big.NewInt(1e14), // maxgascost = 2.1ether - GasTipCap: big.NewInt(1), - Value: big.NewInt(1e17 + 1), // the remaining balance for fee is 2.1ether - Data: nil, - }, params.TxGas, errors.New("gas required exceeds allowance (20999)")}, // 20999=(2.2ether-0.1ether-1wei)/(1e14) - } - for i, c := range cases { - got, err := sim.EstimateGas(context.Background(), c.message) - if c.expectError != nil { - if err == nil { - t.Fatalf("test %d: expect error, got nil", i) - } - if c.expectError.Error() != err.Error() { - t.Fatalf("test %d: expect error, want %v, got %v", i, c.expectError, err) - } - continue - } - if c.expectError == nil && err != nil { - t.Fatalf("test %d: didn't expect error, got %v", i, err) - } - if got != c.expect { - t.Fatalf("test %d: gas estimation mismatch, want %d, got %d", i, c.expect, got) - } - } -} - -func TestHeaderByHash(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - header, err := sim.HeaderByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - headerByHash, err := sim.HeaderByHash(bgCtx, header.Hash()) - if err != nil { - t.Errorf("could not get recent block: %v", err) - } - - if header.Hash() != headerByHash.Hash() { - t.Errorf("did not get expected block") - } -} - -func TestHeaderByNumber(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - latestBlockHeader, err := sim.HeaderByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get header for tip of chain: %v", err) - } - if latestBlockHeader == nil { - t.Errorf("received a nil block header") - } else if latestBlockHeader.Number.Uint64() != uint64(0) { - t.Errorf("expected block header number 0, instead got %v", latestBlockHeader.Number.Uint64()) - } - - sim.Commit() - - latestBlockHeader, err = sim.HeaderByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get header for blockheight of 1: %v", err) - } - - blockHeader, err := sim.HeaderByNumber(bgCtx, big.NewInt(1)) - if err != nil { - t.Errorf("could not get header for blockheight of 1: %v", err) - } - - if blockHeader.Hash() != latestBlockHeader.Hash() { - t.Errorf("block header and latest block header are not the same") - } - if blockHeader.Number.Int64() != int64(1) { - t.Errorf("did not get blockheader for block 1. instead got block %v", blockHeader.Number.Int64()) - } - - block, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) - if err != nil { - t.Errorf("could not get block for blockheight of 1: %v", err) - } - - if block.Hash() != blockHeader.Hash() { - t.Errorf("block hash and block header hash do not match. expected %v, got %v", block.Hash(), blockHeader.Hash()) - } -} - -func TestTransactionCount(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - currentBlock, err := sim.BlockByNumber(bgCtx, nil) - if err != nil || currentBlock == nil { - t.Error("could not get current block") - } - - count, err := sim.TransactionCount(bgCtx, currentBlock.Hash()) - if err != nil { - t.Error("could not get current block's transaction count") - } - - if count != 0 { - t.Errorf("expected transaction count of %v does not match actual count of %v", 0, count) - } - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - - sim.Commit() - - lastBlock, err := sim.BlockByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get header for tip of chain: %v", err) - } - - count, err = sim.TransactionCount(bgCtx, lastBlock.Hash()) - if err != nil { - t.Error("could not get current block's transaction count") - } - - if count != 1 { - t.Errorf("expected transaction count of %v does not match actual count of %v", 1, count) - } -} - -func TestTransactionInBlock(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - transaction, err := sim.TransactionInBlock(bgCtx, sim.pendingBlock.Hash(), uint(0)) - if err == nil && err != errTransactionDoesNotExist { - t.Errorf("expected a transaction does not exist error to be received but received %v", err) - } - if transaction != nil { - t.Errorf("expected transaction to be nil but received %v", transaction) - } - - // expect pending nonce to be 0 since account has not been used - pendingNonce, err := sim.PendingNonceAt(bgCtx, testAddr) - if err != nil { - t.Errorf("did not get the pending nonce: %v", err) - } - - if pendingNonce != uint64(0) { - t.Errorf("expected pending nonce of 0 got %v", pendingNonce) - } - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - - sim.Commit() - - lastBlock, err := sim.BlockByNumber(bgCtx, nil) - if err != nil { - t.Errorf("could not get header for tip of chain: %v", err) - } - - transaction, err = sim.TransactionInBlock(bgCtx, lastBlock.Hash(), uint(1)) - if err == nil && err != errTransactionDoesNotExist { - t.Errorf("expected a transaction does not exist error to be received but received %v", err) - } - if transaction != nil { - t.Errorf("expected transaction to be nil but received %v", transaction) - } - - transaction, err = sim.TransactionInBlock(bgCtx, lastBlock.Hash(), uint(0)) - if err != nil { - t.Errorf("could not get transaction in the lastest block with hash %v: %v", lastBlock.Hash().String(), err) - } - - if signedTx.Hash().String() != transaction.Hash().String() { - t.Errorf("received transaction that did not match the sent transaction. expected hash %v, got hash %v", signedTx.Hash().String(), transaction.Hash().String()) - } -} - -func TestPendingNonceAt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - // expect pending nonce to be 0 since account has not been used - pendingNonce, err := sim.PendingNonceAt(bgCtx, testAddr) - if err != nil { - t.Errorf("did not get the pending nonce: %v", err) - } - - if pendingNonce != uint64(0) { - t.Errorf("expected pending nonce of 0 got %v", pendingNonce) - } - - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - - // expect pending nonce to be 1 since account has submitted one transaction - pendingNonce, err = sim.PendingNonceAt(bgCtx, testAddr) - if err != nil { - t.Errorf("did not get the pending nonce: %v", err) - } - - if pendingNonce != uint64(1) { - t.Errorf("expected pending nonce of 1 got %v", pendingNonce) - } - - // make a new transaction with a nonce of 1 - tx = types.NewTransaction(uint64(1), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err = types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not send tx: %v", err) - } - - // expect pending nonce to be 2 since account now has two transactions - pendingNonce, err = sim.PendingNonceAt(bgCtx, testAddr) - if err != nil { - t.Errorf("did not get the pending nonce: %v", err) - } - - if pendingNonce != uint64(2) { - t.Errorf("expected pending nonce of 2 got %v", pendingNonce) - } -} - -func TestTransactionReceipt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - // create a signed transaction to send - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) - if err != nil { - t.Errorf("could not sign tx: %v", err) - } - - // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) - if err != nil { - t.Errorf("could not add tx to pending block: %v", err) - } - sim.Commit() - - receipt, err := sim.TransactionReceipt(bgCtx, signedTx.Hash()) - if err != nil { - t.Errorf("could not get transaction receipt: %v", err) - } - - if receipt.ContractAddress != testAddr && receipt.TxHash != signedTx.Hash() { - t.Errorf("received receipt is not correct: %v", receipt) - } -} - -func TestSuggestGasPrice(t *testing.T) { - t.Parallel() - sim := NewSimulatedBackend( - core.GenesisAlloc{}, - 10000000, - ) - defer sim.Close() - bgCtx := context.Background() - gasPrice, err := sim.SuggestGasPrice(bgCtx) - if err != nil { - t.Errorf("could not get gas price: %v", err) - } - if gasPrice.Uint64() != sim.pendingBlock.Header().BaseFee.Uint64() { - t.Errorf("gas price was not expected value of %v. actual: %v", sim.pendingBlock.Header().BaseFee.Uint64(), gasPrice.Uint64()) - } -} - -func TestPendingCodeAt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - code, err := sim.CodeAt(bgCtx, testAddr, nil) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) != 0 { - t.Errorf("got code for account that does not have contract code") - } - - parsed, err := abi.JSON(strings.NewReader(abiJSON)) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim) - if err != nil { - t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract) - } - - code, err = sim.PendingCodeAt(bgCtx, contractAddr) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) == 0 { - t.Errorf("did not get code for account that has contract code") - } - // ensure code received equals code deployed - if !bytes.Equal(code, common.FromHex(deployedCode)) { - t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code) - } -} - -func TestCodeAt(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - code, err := sim.CodeAt(bgCtx, testAddr, nil) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) != 0 { - t.Errorf("got code for account that does not have contract code") - } - - parsed, err := abi.JSON(strings.NewReader(abiJSON)) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim) - if err != nil { - t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract) - } - - sim.Commit() - code, err = sim.CodeAt(bgCtx, contractAddr, nil) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) == 0 { - t.Errorf("did not get code for account that has contract code") - } - // ensure code received equals code deployed - if !bytes.Equal(code, common.FromHex(deployedCode)) { - t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code) - } -} - -func TestCodeAtHash(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - code, err := sim.CodeAtHash(bgCtx, testAddr, sim.Blockchain().CurrentHeader().Hash()) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) != 0 { - t.Errorf("got code for account that does not have contract code") - } - - parsed, err := abi.JSON(strings.NewReader(abiJSON)) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim) - if err != nil { - t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract) - } - - blockHash := sim.Commit() - code, err = sim.CodeAtHash(bgCtx, contractAddr, blockHash) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - if len(code) == 0 { - t.Errorf("did not get code for account that has contract code") - } - // ensure code received equals code deployed - if !bytes.Equal(code, common.FromHex(deployedCode)) { - t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code) - } -} - -// When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt: -// -// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} -func TestPendingAndCallContract(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - parsed, err := abi.JSON(strings.NewReader(abiJSON)) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - contractAuth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - addr, _, _, err := bind.DeployContract(contractAuth, parsed, common.FromHex(abiBin), sim) - if err != nil { - t.Errorf("could not deploy contract: %v", err) - } - - input, err := parsed.Pack("receive", []byte("X")) - if err != nil { - t.Errorf("could not pack receive function on contract: %v", err) - } - - // make sure you can call the contract in pending state - res, err := sim.PendingCallContract(bgCtx, ethereum.CallMsg{ - From: testAddr, - To: &addr, - Data: input, - }) - if err != nil { - t.Errorf("could not call receive method on contract: %v", err) - } - if len(res) == 0 { - t.Errorf("result of contract call was empty: %v", res) - } - - // while comparing against the byte array is more exact, also compare against the human readable string for readability - if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") { - t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res)) - } - - blockHash := sim.Commit() - - // make sure you can call the contract - res, err = sim.CallContract(bgCtx, ethereum.CallMsg{ - From: testAddr, - To: &addr, - Data: input, - }, nil) - if err != nil { - t.Errorf("could not call receive method on contract: %v", err) - } - if len(res) == 0 { - t.Errorf("result of contract call was empty: %v", res) - } - - if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") { - t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res)) - } - - // make sure you can call the contract by hash - res, err = sim.CallContractAtHash(bgCtx, ethereum.CallMsg{ - From: testAddr, - To: &addr, - Data: input, - }, blockHash) - if err != nil { - t.Errorf("could not call receive method on contract: %v", err) - } - if len(res) == 0 { - t.Errorf("result of contract call was empty: %v", res) - } - - if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") { - t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res)) - } -} - -// This test is based on the following contract: -/* -contract Reverter { - function revertString() public pure{ - require(false, "some error"); - } - function revertNoString() public pure { - require(false, ""); - } - function revertASM() public pure { - assembly { - revert(0x0, 0x0) - } - } - function noRevert() public pure { - assembly { - // Assembles something that looks like require(false, "some error") but is not reverted - mstore(0x0, 0x08c379a000000000000000000000000000000000000000000000000000000000) - mstore(0x4, 0x0000000000000000000000000000000000000000000000000000000000000020) - mstore(0x24, 0x000000000000000000000000000000000000000000000000000000000000000a) - mstore(0x44, 0x736f6d65206572726f7200000000000000000000000000000000000000000000) - return(0x0, 0x64) - } - } -}*/ -func TestCallContractRevert(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - bgCtx := context.Background() - - reverterABI := `[{"inputs": [],"name": "noRevert","outputs": [],"stateMutability": "pure","type": "function"},{"inputs": [],"name": "revertASM","outputs": [],"stateMutability": "pure","type": "function"},{"inputs": [],"name": "revertNoString","outputs": [],"stateMutability": "pure","type": "function"},{"inputs": [],"name": "revertString","outputs": [],"stateMutability": "pure","type": "function"}]` - reverterBin := "608060405234801561001057600080fd5b506101d3806100206000396000f3fe608060405234801561001057600080fd5b506004361061004c5760003560e01c80634b409e01146100515780639b340e361461005b5780639bd6103714610065578063b7246fc11461006f575b600080fd5b610059610079565b005b6100636100ca565b005b61006d6100cf565b005b610077610145565b005b60006100c8576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401808060200182810382526000815260200160200191505060405180910390fd5b565b600080fd5b6000610143576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252600a8152602001807f736f6d65206572726f720000000000000000000000000000000000000000000081525060200191505060405180910390fd5b565b7f08c379a0000000000000000000000000000000000000000000000000000000006000526020600452600a6024527f736f6d65206572726f720000000000000000000000000000000000000000000060445260646000f3fea2646970667358221220cdd8af0609ec4996b7360c7c780bad5c735740c64b1fffc3445aa12d37f07cb164736f6c63430006070033" - - parsed, err := abi.JSON(strings.NewReader(reverterABI)) - if err != nil { - t.Errorf("could not get code at test addr: %v", err) - } - contractAuth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - addr, _, _, err := bind.DeployContract(contractAuth, parsed, common.FromHex(reverterBin), sim) - if err != nil { - t.Errorf("could not deploy contract: %v", err) - } - - inputs := make(map[string]interface{}, 3) - inputs["revertASM"] = nil - inputs["revertNoString"] = "" - inputs["revertString"] = "some error" - - call := make([]func([]byte) ([]byte, error), 2) - call[0] = func(input []byte) ([]byte, error) { - return sim.PendingCallContract(bgCtx, ethereum.CallMsg{ - From: testAddr, - To: &addr, - Data: input, - }) - } - call[1] = func(input []byte) ([]byte, error) { - return sim.CallContract(bgCtx, ethereum.CallMsg{ - From: testAddr, - To: &addr, - Data: input, - }, nil) - } - - // Run pending calls then commit - for _, cl := range call { - for key, val := range inputs { - input, err := parsed.Pack(key) - if err != nil { - t.Errorf("could not pack %v function on contract: %v", key, err) - } - - res, err := cl(input) - if err == nil { - t.Errorf("call to %v was not reverted", key) - } - if res != nil { - t.Errorf("result from %v was not nil: %v", key, res) - } - if val != nil { - rerr, ok := err.(*revertError) - if !ok { - t.Errorf("expect revert error") - } - if rerr.Error() != "execution reverted: "+val.(string) { - t.Errorf("error was malformed: got %v want %v", rerr.Error(), val) - } - } else { - // revert(0x0,0x0) - if err.Error() != "execution reverted" { - t.Errorf("error was malformed: got %v want %v", err, "execution reverted") - } - } - } - input, err := parsed.Pack("noRevert") - if err != nil { - t.Errorf("could not pack noRevert function on contract: %v", err) - } - res, err := cl(input) - if err != nil { - t.Error("call to noRevert was reverted") - } - if res == nil { - t.Errorf("result from noRevert was nil") - } - sim.Commit() - } -} - -// TestFork check that the chain length after a reorg is correct. -// Steps: -// 1. Save the current block which will serve as parent for the fork. -// 2. Mine n blocks with n ∈ [0, 20]. -// 3. Assert that the chain length is n. -// 4. Fork by using the parent block as ancestor. -// 5. Mine n+1 blocks which should trigger a reorg. -// 6. Assert that the chain length is n+1. -// Since Commit() was called 2n+1 times in total, -// having a chain length of just n+1 means that a reorg occurred. -func TestFork(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - // 1. - parent := sim.blockchain.CurrentBlock() - // 2. - n := int(rand.Int31n(21)) - for i := 0; i < n; i++ { - sim.Commit() - } - // 3. - if sim.blockchain.CurrentBlock().Number.Uint64() != uint64(n) { - t.Error("wrong chain length") - } - // 4. - sim.Fork(context.Background(), parent.Hash()) - // 5. - for i := 0; i < n+1; i++ { - sim.Commit() - } - // 6. - if sim.blockchain.CurrentBlock().Number.Uint64() != uint64(n+1) { - t.Error("wrong chain length") - } -} - -/* -Example contract to test event emission: - - pragma solidity >=0.7.0 <0.9.0; - contract Callable { - event Called(); - function Call() public { emit Called(); } - } -*/ -const callableAbi = "[{\"anonymous\":false,\"inputs\":[],\"name\":\"Called\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"Call\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]" - -const callableBin = "6080604052348015600f57600080fd5b5060998061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806334e2292114602d575b600080fd5b60336035565b005b7f81fab7a4a0aa961db47eefc81f143a5220e8c8495260dd65b1356f1d19d3c7b860405160405180910390a156fea2646970667358221220029436d24f3ac598ceca41d4d712e13ced6d70727f4cdc580667de66d2f51d8b64736f6c63430008010033" - -// TestForkLogsReborn check that the simulated reorgs -// correctly remove and reborn logs. -// Steps: -// 1. Deploy the Callable contract. -// 2. Set up an event subscription. -// 3. Save the current block which will serve as parent for the fork. -// 4. Send a transaction. -// 5. Check that the event was included. -// 6. Fork by using the parent block as ancestor. -// 7. Mine two blocks to trigger a reorg. -// 8. Check that the event was removed. -// 9. Re-send the transaction and mine a block. -// 10. Check that the event was reborn. -func TestForkLogsReborn(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - // 1. - parsed, _ := abi.JSON(strings.NewReader(callableAbi)) - auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337)) - _, _, contract, err := bind.DeployContract(auth, parsed, common.FromHex(callableBin), sim) - if err != nil { - t.Errorf("deploying contract: %v", err) - } - sim.Commit() - // 2. - logs, sub, err := contract.WatchLogs(nil, "Called") - if err != nil { - t.Errorf("watching logs: %v", err) - } - defer sub.Unsubscribe() - // 3. - parent := sim.blockchain.CurrentBlock() - // 4. - tx, err := contract.Transact(auth, "Call") - if err != nil { - t.Errorf("transacting: %v", err) - } - sim.Commit() - // 5. - log := <-logs - if log.TxHash != tx.Hash() { - t.Error("wrong event tx hash") - } - if log.Removed { - t.Error("Event should be included") - } - // 6. - if err := sim.Fork(context.Background(), parent.Hash()); err != nil { - t.Errorf("forking: %v", err) - } - // 7. - sim.Commit() - sim.Commit() - // 8. - log = <-logs - if log.TxHash != tx.Hash() { - t.Error("wrong event tx hash") - } - if !log.Removed { - t.Error("Event should be removed") - } - // 9. - if err := sim.SendTransaction(context.Background(), tx); err != nil { - t.Errorf("sending transaction: %v", err) - } - sim.Commit() - // 10. - log = <-logs - if log.TxHash != tx.Hash() { - t.Error("wrong event tx hash") - } - if log.Removed { - t.Error("Event should be included") - } -} - -// TestForkResendTx checks that re-sending a TX after a fork -// is possible and does not cause a "nonce mismatch" panic. -// Steps: -// 1. Save the current block which will serve as parent for the fork. -// 2. Send a transaction. -// 3. Check that the TX is included in block 1. -// 4. Fork by using the parent block as ancestor. -// 5. Mine a block, Re-send the transaction and mine another one. -// 6. Check that the TX is now included in block 2. -func TestForkResendTx(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - // 1. - parent := sim.blockchain.CurrentBlock() - // 2. - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - - _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) - sim.SendTransaction(context.Background(), tx) - sim.Commit() - // 3. - receipt, _ := sim.TransactionReceipt(context.Background(), tx.Hash()) - if h := receipt.BlockNumber.Uint64(); h != 1 { - t.Errorf("TX included in wrong block: %d", h) - } - // 4. - if err := sim.Fork(context.Background(), parent.Hash()); err != nil { - t.Errorf("forking: %v", err) - } - // 5. - sim.Commit() - if err := sim.SendTransaction(context.Background(), tx); err != nil { - t.Errorf("sending transaction: %v", err) - } - sim.Commit() - // 6. - receipt, _ = sim.TransactionReceipt(context.Background(), tx.Hash()) - if h := receipt.BlockNumber.Uint64(); h != 2 { - t.Errorf("TX included in wrong block: %d", h) - } -} - -func TestCommitReturnValue(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - - startBlockHeight := sim.blockchain.CurrentBlock().Number.Uint64() - - // Test if Commit returns the correct block hash - h1 := sim.Commit() - if h1 != sim.blockchain.CurrentBlock().Hash() { - t.Error("Commit did not return the hash of the last block.") - } - - // Create a block in the original chain (containing a transaction to force different block hashes) - head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) - _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) - tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) - sim.SendTransaction(context.Background(), tx) - h2 := sim.Commit() - - // Create another block in the original chain - sim.Commit() - - // Fork at the first bock - if err := sim.Fork(context.Background(), h1); err != nil { - t.Errorf("forking: %v", err) - } - - // Test if Commit returns the correct block hash after the reorg - h2fork := sim.Commit() - if h2 == h2fork { - t.Error("The block in the fork and the original block are the same block!") - } - if sim.blockchain.GetHeader(h2fork, startBlockHeight+2) == nil { - t.Error("Could not retrieve the just created block (side-chain)") - } -} - -// TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork -// block's parent rather than the canonical head's parent. -func TestAdjustTimeAfterFork(t *testing.T) { - t.Parallel() - testAddr := crypto.PubkeyToAddress(testKey.PublicKey) - sim := simTestBackend(testAddr) - defer sim.Close() - - sim.Commit() // h1 - h1 := sim.blockchain.CurrentHeader().Hash() - sim.Commit() // h2 - sim.Fork(context.Background(), h1) - sim.AdjustTime(1 * time.Second) - sim.Commit() - - head := sim.blockchain.CurrentHeader() - if head.Number == common.Big2 && head.ParentHash != h1 { - t.Errorf("failed to build block on fork") - } -} diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 1f96a85162..941aa6901c 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -289,7 +289,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -297,7 +297,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy an interaction tester contract and call a transaction on it @@ -305,6 +305,7 @@ var bindTests = []struct { if err != nil { t.Fatalf("Failed to deploy interactor contract: %v", err) } + sim.Commit() if _, err := interactor.Transact(auth, "Transact string"); err != nil { t.Fatalf("Failed to transact with interactor contract: %v", err) } @@ -344,7 +345,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -352,7 +353,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tuple tester contract and execute a structured call on it @@ -390,7 +391,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -398,7 +399,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tuple tester contract and execute a structured call on it @@ -448,7 +449,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -456,7 +457,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a slice tester contract and execute a n array call on it @@ -496,7 +497,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -504,7 +505,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a default method invoker contract and execute its default method @@ -512,6 +513,7 @@ var bindTests = []struct { if err != nil { t.Fatalf("Failed to deploy defaulter contract: %v", err) } + sim.Commit() if _, err := (&DefaulterRaw{defaulter}).Transfer(auth); err != nil { t.Fatalf("Failed to invoke default method: %v", err) } @@ -562,7 +564,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -570,7 +572,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a structs method invoker contract and execute its default method @@ -608,12 +610,12 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000)) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{}, uint64(10000000000)) defer sim.Close() nonexistent, err := NewNonExistent(common.Address{}, sim) @@ -647,12 +649,12 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000)) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{}, uint64(10000000000)) defer sim.Close() nonexistent, err := NewNonExistentStruct(common.Address{}, sim) @@ -694,7 +696,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -702,7 +704,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a funky gas pattern contract @@ -744,7 +746,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -752,7 +754,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a sender tester contract and execute a structured call on it @@ -819,7 +821,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -827,7 +829,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a underscorer tester contract and execute a structured call on it @@ -913,7 +915,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -921,7 +923,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy an eventer contract @@ -1103,7 +1105,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1111,7 +1113,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() //deploy the test contract @@ -1238,7 +1240,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, @@ -1246,7 +1248,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() _, _, contract, err := DeployTuple(auth, sim) @@ -1380,7 +1382,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1388,7 +1390,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() //deploy the test contract @@ -1446,14 +1448,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` // Initialize test accounts key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // deploy the test contract @@ -1535,7 +1537,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` // Initialize test accounts @@ -1543,7 +1545,7 @@ var bindTests = []struct { addr := crypto.PubkeyToAddress(key.PublicKey) // Deploy registrar contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1598,14 +1600,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" `, ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) // Deploy registrar contract - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1659,7 +1661,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` @@ -1667,7 +1669,7 @@ var bindTests = []struct { key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() // Deploy a tester contract and execute a structured call on it @@ -1720,14 +1722,14 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" `, ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) defer sim.Close() opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -1808,7 +1810,7 @@ var bindTests = []struct { "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1816,7 +1818,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1874,11 +1876,12 @@ var bindTests = []struct { []string{"0x6080604052348015600f57600080fd5b5060998061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063726c638214602d575b600080fd5b60336035565b005b60405163024876cd60e61b815260016004820152600260248201526003604482015260640160405180910390fdfea264697066735822122093f786a1bc60216540cd999fbb4a6109e0fef20abcff6e9107fb2817ca968f3c64736f6c63430008070033"}, []string{`[{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError","type":"error"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError1","type":"error"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"},{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError2","type":"error"},{"inputs":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"},{"internalType":"uint256","name":"c","type":"uint256"}],"name":"MyError3","type":"error"},{"inputs":[],"name":"Error","outputs":[],"stateMutability":"pure","type":"function"}]`}, ` + "context" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1886,7 +1889,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1895,7 +1898,7 @@ var bindTests = []struct { t.Fatal(err) } sim.Commit() - _, err = bind.WaitDeployed(nil, sim, tx) + _, err = bind.WaitDeployed(context.Background(), sim, tx) if err != nil { t.Error(err) } @@ -1926,11 +1929,12 @@ var bindTests = []struct { bytecode: []string{`0x608060405234801561001057600080fd5b506040516101c43803806101c48339818101604052810190610032919061014a565b50610177565b6000604051905090565b600080fd5b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6100958261004c565b810181811067ffffffffffffffff821117156100b4576100b361005d565b5b80604052505050565b60006100c7610038565b90506100d3828261008c565b919050565b6000819050919050565b6100eb816100d8565b81146100f657600080fd5b50565b600081519050610108816100e2565b92915050565b60006020828403121561012457610123610047565b5b61012e60206100bd565b9050600061013e848285016100f9565b60008301525092915050565b6000602082840312156101605761015f610042565b5b600061016e8482850161010e565b91505092915050565b603f806101856000396000f3fe6080604052600080fdfea2646970667358221220cdffa667affecefac5561f65f4a4ba914204a8d4eb859d8cd426fb306e5c12a364736f6c634300080a0033`}, abi: []string{`[{"inputs":[{"components":[{"internalType":"uint256","name":"field","type":"uint256"}],"internalType":"struct ConstructorWithStructParam.StructType","name":"st","type":"tuple"}],"stateMutability":"nonpayable","type":"constructor"}]`}, imports: ` + "context" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1938,7 +1942,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1948,7 +1952,7 @@ var bindTests = []struct { } sim.Commit() - if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) } @@ -1974,11 +1978,12 @@ var bindTests = []struct { bytecode: []string{"0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033"}, abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, imports: ` + "context" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -1986,7 +1991,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() @@ -1996,7 +2001,7 @@ var bindTests = []struct { } sim.Commit() - if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) } @@ -2014,11 +2019,12 @@ var bindTests = []struct { bytecode: []string{"0x608060405234801561001057600080fd5b5060dc8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063527a119f14602d575b600080fd5b60436004803603810190603f9190605b565b6045565b005b50565b6000813590506055816092565b92915050565b600060208284031215606e57606d608d565b5b6000607a848285016048565b91505092915050565b6000819050919050565b600080fd5b6099816083565b811460a357600080fd5b5056fea2646970667358221220d4f4525e2615516394055d369fb17df41c359e5e962734f27fd683ea81fd9db164736f6c63430008070033"}, abi: []string{`[{"inputs":[{"internalType":"uint256","name":"range","type":"uint256"}],"name":"functionWithKeywordParameter","outputs":[],"stateMutability":"pure","type":"function"}]`}, imports: ` + "context" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" `, @@ -2026,7 +2032,7 @@ var bindTests = []struct { var ( key, _ = crypto.GenerateKey() user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) _, tx, _, err := DeployRangeKeyword(user, sim) if err != nil { @@ -2034,7 +2040,7 @@ var bindTests = []struct { } sim.Commit() - if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Errorf("error deploying the contract: %v", err) } `, diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 826426632c..592465f2ac 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -24,11 +24,11 @@ import ( "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient/simulated" + "github.com/ethereum/go-ethereum/params" ) var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -55,20 +55,19 @@ var waitDeployedTests = map[string]struct { func TestWaitDeployed(t *testing.T) { t.Parallel() for name, test := range waitDeployedTests { - backend := backends.NewSimulatedBackend( - core.GenesisAlloc{ + backend := simulated.NewBackend( + types.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, - 10000000, ) defer backend.Close() // Create the transaction - head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough - gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + head, _ := backend.Client().HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) tx := types.NewContractCreation(0, big.NewInt(0), test.gas, gasPrice, common.FromHex(test.code)) - tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + tx, _ = types.SignTx(tx, types.LatestSignerForChainID(big.NewInt(1337)), testKey) // Wait for it to get mined in the background. var ( @@ -78,12 +77,12 @@ func TestWaitDeployed(t *testing.T) { ctx = context.Background() ) go func() { - address, err = bind.WaitDeployed(ctx, backend, tx) + address, err = bind.WaitDeployed(ctx, backend.Client(), tx) close(mined) }() // Send and mine the transaction. - backend.SendTransaction(ctx, tx) + backend.Client().SendTransaction(ctx, tx) backend.Commit() select { @@ -101,42 +100,40 @@ func TestWaitDeployed(t *testing.T) { } func TestWaitDeployedCornerCases(t *testing.T) { - t.Parallel() - backend := backends.NewSimulatedBackend( - core.GenesisAlloc{ + backend := simulated.NewBackend( + types.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, - 10000000, ) defer backend.Close() - head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + head, _ := backend.Client().HeaderByNumber(context.Background(), nil) // Should be child's, good enough gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) // Create a transaction to an account. code := "6060604052600a8060106000396000f360606040526008565b00" tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, gasPrice, common.FromHex(code)) - tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + tx, _ = types.SignTx(tx, types.LatestSigner(params.AllDevChainProtocolChanges), testKey) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - backend.SendTransaction(ctx, tx) + backend.Client().SendTransaction(ctx, tx) backend.Commit() notContractCreation := errors.New("tx is not contract creation") - if _, err := bind.WaitDeployed(ctx, backend, tx); err.Error() != notContractCreation.Error() { + if _, err := bind.WaitDeployed(ctx, backend.Client(), tx); err.Error() != notContractCreation.Error() { t.Errorf("error mismatch: want %q, got %q, ", notContractCreation, err) } // Create a transaction that is not mined. tx = types.NewContractCreation(1, big.NewInt(0), 3000000, gasPrice, common.FromHex(code)) - tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) + tx, _ = types.SignTx(tx, types.LatestSigner(params.AllDevChainProtocolChanges), testKey) go func() { contextCanceled := errors.New("context canceled") - if _, err := bind.WaitDeployed(ctx, backend, tx); err.Error() != contextCanceled.Error() { + if _, err := bind.WaitDeployed(ctx, backend.Client(), tx); err.Error() != contextCanceled.Error() { t.Errorf("error mismatch: want %q, got %q, ", contextCanceled, err) } }() - backend.SendTransaction(ctx, tx) + backend.Client().SendTransaction(ctx, tx) cancel() } diff --git a/accounts/abi/topics.go b/accounts/abi/topics.go index 60c71d88b2..7ce9b7273c 100644 --- a/accounts/abi/topics.go +++ b/accounts/abi/topics.go @@ -24,6 +24,7 @@ import ( "reflect" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" ) @@ -41,8 +42,7 @@ func MakeTopics(query ...[]interface{}) ([][]common.Hash, error) { case common.Address: copy(topic[common.HashLength-common.AddressLength:], rule[:]) case *big.Int: - blob := rule.Bytes() - copy(topic[common.HashLength-len(blob):], blob) + copy(topic[:], math.U256Bytes(rule)) case bool: if rule { topic[common.HashLength-1] = 1 diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index b31f58fba3..9e1efd3821 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -17,6 +17,7 @@ package abi import ( + "math" "math/big" "reflect" "testing" @@ -55,9 +56,27 @@ func TestMakeTopics(t *testing.T) { false, }, { - "support *big.Int types in topics", - args{[][]interface{}{{big.NewInt(1).Lsh(big.NewInt(2), 254)}}}, - [][]common.Hash{{common.Hash{128}}}, + "support positive *big.Int types in topics", + args{[][]interface{}{ + {big.NewInt(1)}, + {big.NewInt(1).Lsh(big.NewInt(2), 254)}, + }}, + [][]common.Hash{ + {common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001")}, + {common.Hash{128}}, + }, + false, + }, + { + "support negative *big.Int types in topics", + args{[][]interface{}{ + {big.NewInt(-1)}, + {big.NewInt(math.MinInt64)}, + }}, + [][]common.Hash{ + {common.MaxHash}, + {common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffff8000000000000000")}, + }, false, }, { diff --git a/accounts/keystore/passphrase.go b/accounts/keystore/passphrase.go index 8d6ed2b14e..e7a7f8d0cb 100644 --- a/accounts/keystore/passphrase.go +++ b/accounts/keystore/passphrase.go @@ -136,7 +136,7 @@ func (ks keyStorePassphrase) JoinPath(filename string) string { return filepath.Join(ks.keysDirPath, filename) } -// Encryptdata encrypts the data given as 'data' with the password 'auth'. +// EncryptDataV3 encrypts the data given as 'data' with the password 'auth'. func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) { salt := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, salt); err != nil { diff --git a/accounts/scwallet/hub.go b/accounts/scwallet/hub.go index f9dcf58e19..5f1f369ca2 100644 --- a/accounts/scwallet/hub.go +++ b/accounts/scwallet/hub.go @@ -241,7 +241,7 @@ func (hub *Hub) refreshWallets() { card.Disconnect(pcsc.LeaveCard) continue } - // Card connected, start tracking in amongs the wallets + // Card connected, start tracking among the wallets hub.wallets[reader] = wallet events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived}) } diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 723df0f2b3..d0cb93e74e 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -279,7 +279,7 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er } hexstr := reply[1 : 1+int(reply[0])] - // Decode the hex sting into an Ethereum address and return + // Decode the hex string into an Ethereum address and return var address common.Address if _, err = hex.Decode(address[:], hexstr); err != nil { return common.Address{}, err diff --git a/accounts/usbwallet/trezor/trezor.go b/accounts/usbwallet/trezor/trezor.go index 7e756e609b..cdca6b4e0b 100644 --- a/accounts/usbwallet/trezor/trezor.go +++ b/accounts/usbwallet/trezor/trezor.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Trezor hardware // wallets. The wire protocol spec can be found on the SatoshiLabs website: -// https://wiki.trezor.io/Developers_guide-Message_Workflows +// https://docs.trezor.io/trezor-firmware/common/message-workflows.html // !!! STAHP !!! // diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 487693ea18..9a3ea8d077 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -26,6 +26,16 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// PayloadVersion denotes the version of PayloadAttributes used to request the +// building of the payload to commence. +type PayloadVersion byte + +var ( + PayloadV1 PayloadVersion = 0x1 + PayloadV2 PayloadVersion = 0x2 + PayloadV3 PayloadVersion = 0x3 +) + //go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go // PayloadAttributes describes the environment context in which a block should @@ -129,6 +139,21 @@ type TransitionConfigurationV1 struct { // PayloadID is an identifier of the payload build process type PayloadID [8]byte +// Version returns the payload version associated with the identifier. +func (b PayloadID) Version() PayloadVersion { + return PayloadVersion(b[0]) +} + +// Is returns whether the identifier matches any of provided payload versions. +func (b PayloadID) Is(versions ...PayloadVersion) bool { + for _, v := range versions { + if v == b.Version() { + return true + } + } + return false +} + func (b PayloadID) String() string { return hexutil.Encode(b[:]) } @@ -298,3 +323,21 @@ type ExecutionPayloadBodyV1 struct { TransactionData []hexutil.Bytes `json:"transactions"` Withdrawals []*types.Withdrawal `json:"withdrawals"` } + +// Client identifiers to support ClientVersionV1. +const ( + ClientCode = "GE" + ClientName = "go-ethereum" +) + +// ClientVersionV1 contains information which identifies a client implementation. +type ClientVersionV1 struct { + Code string `json:"code"` + Name string `json:"clientName"` + Version string `json:"version"` + Commit string `json:"commit"` +} + +func (v *ClientVersionV1) String() string { + return fmt.Sprintf("%s-%s-%s-%s", v.Code, v.Name, v.Version, v.Commit) +} diff --git a/build/checksums.txt b/build/checksums.txt index 96815ff791..03a53946df 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,9 +1,9 @@ # This file contains sha256 checksums of optional build dependencies. -# version:spec-tests 1.0.6 +# version:spec-tests 2.1.0 # https://github.com/ethereum/execution-spec-tests/releases -# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/ -485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1 fixtures_develop.tar.gz +# https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.0/ +ca89c76851b0900bfcc3cbb9a26cbece1f3d7c64a3bed38723e914713290df6c fixtures_develop.tar.gz # version:golang 1.21.6 # https://go.dev/dl/ diff --git a/build/ci.go b/build/ci.go index 2ba3824bb7..0acc416635 100644 --- a/build/ci.go +++ b/build/ci.go @@ -122,14 +122,14 @@ var ( // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: the following Ubuntu releases have been officially deprecated on Launchpad: // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish, - // kinetic + // kinetic, lunar debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", // EOL: 04/2024 - "xenial": "golang-go", // EOL: 04/2026 - "bionic": "golang-go", // EOL: 04/2028 - "focal": "golang-go", // EOL: 04/2030 - "jammy": "golang-go", // EOL: 04/2032 - "lunar": "golang-go", // EOL: 01/2024 + "trusty": "golang-1.11", // 14.04, EOL: 04/2024 + "xenial": "golang-go", // 16.04, EOL: 04/2026 + "bionic": "golang-go", // 18.04, EOL: 04/2028 + "focal": "golang-go", // 20.04, EOL: 04/2030 + "jammy": "golang-go", // 22.04, EOL: 04/2032 + "mantic": "golang-go", // 23.10, EOL: 07/2024 } debGoBootPaths = map[string]string{ @@ -289,7 +289,7 @@ func doTest(cmdline []string) { coverage = flag.Bool("coverage", false, "Whether to record code coverage") verbose = flag.Bool("v", false, "Whether to log verbosely") race = flag.Bool("race", false, "Execute the race detector") - short = flag.Bool("short", false, "Pass the 'short'-flag to go test") + short = flag.Bool("short", false, "Pass the 'short'-flag to go test") cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads") ) flag.CommandLine.Parse(cmdline) @@ -837,10 +837,10 @@ func makeWorkdir(wdflag string) string { } func isUnstableBuild(env build.Environment) bool { - if env.Tag != "" { - return false + if env.Tag == "untagged" || env.Tag == "" { + return true } - return true + return false } type debPackage struct { diff --git a/build/nsis.geth.nsi b/build/nsis.geth.nsi index 1034f30235..03710dd95d 100644 --- a/build/nsis.geth.nsi +++ b/build/nsis.geth.nsi @@ -20,7 +20,7 @@ # - NSIS Large Strings build, http://nsis.sourceforge.net/Special_Builds # - SFP, http://nsis.sourceforge.net/NSIS_Simple_Firewall_Plugin (put dll in NSIS\Plugins\x86-ansi) # -# After intalling NSIS extra the NSIS Large Strings build zip and replace the makensis.exe and the +# After installing NSIS extra the NSIS Large Strings build zip and replace the makensis.exe and the # files found in Stub. # # based on: http://nsis.sourceforge.net/A_simple_installer_with_start_menu_shortcut_and_uninstaller diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 3a43db8c95..cf09265136 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | -| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| -| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | -| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | -| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| +| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | +| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| diff --git a/cmd/clef/datatypes.md b/cmd/clef/datatypes.md index dd8cda5846..8456edfa35 100644 --- a/cmd/clef/datatypes.md +++ b/cmd/clef/datatypes.md @@ -75,7 +75,7 @@ Example: }, { "type": "Info", - "message": "User should see this aswell" + "message": "User should see this as well" } ], "meta": { diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index 2d36ccb423..ba3c0585fd 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -166,7 +166,7 @@ func (c *Conn) ReadEth() (any, error) { case eth.TransactionsMsg: msg = new(eth.TransactionsPacket) case eth.NewPooledTransactionHashesMsg: - msg = new(eth.NewPooledTransactionHashesPacket68) + msg = new(eth.NewPooledTransactionHashesPacket) case eth.GetPooledTransactionsMsg: msg = new(eth.GetPooledTransactionsPacket) case eth.PooledTransactionsMsg: diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index dd42ec7f7f..d9efe26244 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -64,23 +64,23 @@ func NewSuite(dest *enode.Node, chainDir, engineURL, jwt string) (*Suite, error) func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "TestStatus", Fn: s.TestStatus}, + {Name: "Status", Fn: s.TestStatus}, // get block headers - {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, - {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, - {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, - {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, + {Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "SimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "SameRequestID", Fn: s.TestSameRequestID}, + {Name: "ZeroRequestID", Fn: s.TestZeroRequestID}, // get block bodies - {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, + {Name: "GetBlockBodies", Fn: s.TestGetBlockBodies}, // // malicious handshakes + status - {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, - {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, + {Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "MaliciousStatus", Fn: s.TestMaliciousStatus}, // test transactions - {Name: "TestTransaction", Fn: s.TestTransaction}, - {Name: "TestInvalidTxs", Fn: s.TestInvalidTxs}, - {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest}, - {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, - {Name: "TestBlobViolations", Fn: s.TestBlobViolations}, + {Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true}, + {Name: "Transaction", Fn: s.TestTransaction}, + {Name: "InvalidTxs", Fn: s.TestInvalidTxs}, + {Name: "NewPooledTxs", Fn: s.TestNewPooledTxs}, + {Name: "BlobViolations", Fn: s.TestBlobViolations}, } } @@ -94,9 +94,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -// TestStatus attempts to connect to the given node and exchange a status -// message with it on the eth protocol. func (s *Suite) TestStatus(t *utesting.T) { + t.Log(`This test is just a sanity check. It performs an eth protocol handshake.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -112,9 +112,9 @@ func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } -// TestGetBlockHeaders tests whether the given node can respond to an eth -// `GetBlockHeaders` request and that the response is accurate. func (s *Suite) TestGetBlockHeaders(t *utesting.T) { + t.Log(`This test requests block headers from the node.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -154,10 +154,10 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { } } -// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests -// from the same connection with different request IDs and checks to make sure -// the node responds with the correct headers per request. func (s *Suite) TestSimultaneousRequests(t *utesting.T) { + t.Log(`This test requests blocks headers from the node, performing two requests +concurrently, with different request IDs.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -228,9 +228,10 @@ func (s *Suite) TestSimultaneousRequests(t *utesting.T) { } } -// TestSameRequestID sends two requests with the same request ID to a single -// node. func (s *Suite) TestSameRequestID(t *utesting.T) { + t.Log(`This test requests block headers, performing two concurrent requests with the +same request ID. The node should handle the request by responding to both requests.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -298,9 +299,10 @@ func (s *Suite) TestSameRequestID(t *utesting.T) { } } -// TestZeroRequestID checks that a message with a request ID of zero is still handled -// by the node. func (s *Suite) TestZeroRequestID(t *utesting.T) { + t.Log(`This test sends a GetBlockHeaders message with a request-id of zero, +and expects a response.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -333,9 +335,9 @@ func (s *Suite) TestZeroRequestID(t *utesting.T) { } } -// TestGetBlockBodies tests whether the given node can respond to a -// `GetBlockBodies` request and that the response is accurate. func (s *Suite) TestGetBlockBodies(t *utesting.T) { + t.Log(`This test sends GetBlockBodies requests to the node for known blocks in the test chain.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -376,12 +378,12 @@ func randBuf(size int) []byte { return buf } -// TestMaliciousHandshake tries to send malicious data during the handshake. func (s *Suite) TestMaliciousHandshake(t *utesting.T) { - key, _ := crypto.GenerateKey() + t.Log(`This test tries to send malicious data during the devp2p handshake, in various ways.`) // Write hello to client. var ( + key, _ = crypto.GenerateKey() pub0 = crypto.FromECDSAPub(&key.PublicKey)[1:] version = eth.ProtocolVersions[0] ) @@ -451,8 +453,9 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) { } } -// TestMaliciousStatus sends a status package with a large total difficulty. func (s *Suite) TestMaliciousStatus(t *utesting.T) { + t.Log(`This test sends a malicious eth Status message to the node and expects a disconnect.`) + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -486,9 +489,10 @@ func (s *Suite) TestMaliciousStatus(t *utesting.T) { } } -// TestTransaction sends a valid transaction to the node and checks if the -// transaction gets propagated. func (s *Suite) TestTransaction(t *utesting.T) { + t.Log(`This test sends a valid transaction to the node and checks if the +transaction gets propagated.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -507,15 +511,16 @@ func (s *Suite) TestTransaction(t *utesting.T) { if err != nil { t.Fatalf("failed to sign tx: %v", err) } - if err := s.sendTxs([]*types.Transaction{tx}); err != nil { + if err := s.sendTxs(t, []*types.Transaction{tx}); err != nil { t.Fatal(err) } s.chain.IncNonce(from, 1) } -// TestInvalidTxs sends several invalid transactions and tests whether -// the node will propagate them. func (s *Suite) TestInvalidTxs(t *utesting.T) { + t.Log(`This test sends several kinds of invalid transactions and checks that the node +does not propagate them.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -534,7 +539,7 @@ func (s *Suite) TestInvalidTxs(t *utesting.T) { if err != nil { t.Fatalf("failed to sign tx: %v", err) } - if err := s.sendTxs([]*types.Transaction{tx}); err != nil { + if err := s.sendTxs(t, []*types.Transaction{tx}); err != nil { t.Fatalf("failed to send txs: %v", err) } s.chain.IncNonce(from, 1) @@ -590,14 +595,15 @@ func (s *Suite) TestInvalidTxs(t *utesting.T) { } txs = append(txs, tx) } - if err := s.sendInvalidTxs(txs); err != nil { + if err := s.sendInvalidTxs(t, txs); err != nil { t.Fatalf("failed to send invalid txs: %v", err) } } -// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions -// request. func (s *Suite) TestLargeTxRequest(t *utesting.T) { + t.Log(`This test first send ~2000 transactions to the node, then requests them +on another peer connection using GetPooledTransactions.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) @@ -630,7 +636,7 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) { s.chain.IncNonce(from, uint64(count)) // Send txs. - if err := s.sendTxs(txs); err != nil { + if err := s.sendTxs(t, txs); err != nil { t.Fatalf("failed to send txs: %v", err) } @@ -667,13 +673,15 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) { } } -// TestNewPooledTxs tests whether a node will do a GetPooledTransactions request -// upon receiving a NewPooledTransactionHashes announcement. func (s *Suite) TestNewPooledTxs(t *utesting.T) { + t.Log(`This test announces transaction hashes to the node and expects it to fetch +the transactions using a GetPooledTransactions request.`) + // Nudge client out of syncing mode to accept pending txs. if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("failed to send next block: %v", err) } + var ( count = 50 from, nonce = s.chain.GetSender(1) @@ -710,7 +718,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { } // Send announcement. - ann := eth.NewPooledTransactionHashesPacket68{Types: txTypes, Sizes: sizes, Hashes: hashes} + ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes} err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann) if err != nil { t.Fatalf("failed to write to connection: %v", err) @@ -728,7 +736,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest)) } return - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: continue case *eth.TransactionsPacket: continue @@ -762,7 +770,7 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra from, nonce := s.chain.GetSender(5) for i := 0; i < count; i++ { // Make blob data, max of 2 blobs per tx. - blobdata := make([]byte, blobs%2) + blobdata := make([]byte, blobs%3) for i := range blobdata { blobdata[i] = discriminator blobs -= 1 @@ -787,21 +795,23 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra } func (s *Suite) TestBlobViolations(t *utesting.T) { + t.Log(`This test sends some invalid blob tx announcements and expects the node to disconnect.`) + if err := s.engine.sendForkchoiceUpdated(); err != nil { t.Fatalf("send fcu failed: %v", err) } - // Create blob txs for each tests with unqiue tx hashes. + // Create blob txs for each tests with unique tx hashes. var ( t1 = s.makeBlobTxs(2, 3, 0x1) t2 = s.makeBlobTxs(2, 3, 0x2) ) for _, test := range []struct { - ann eth.NewPooledTransactionHashesPacket68 + ann eth.NewPooledTransactionHashesPacket resp eth.PooledTransactionsResponse }{ // Invalid tx size. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.BlobTxType, types.BlobTxType}, Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)}, Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()}, @@ -810,7 +820,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { }, // Wrong tx type. { - ann: eth.NewPooledTransactionHashesPacket68{ + ann: eth.NewPooledTransactionHashesPacket{ Types: []byte{types.DynamicFeeTxType, types.BlobTxType}, Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())}, Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()}, diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 79146c8aba..ad73bc9f90 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -63,6 +63,9 @@ func TestEthSuite(t *testing.T) { } for _, test := range suite.EthTests() { t.Run(test.Name, func(t *testing.T) { + if test.Slow && testing.Short() { + t.Skipf("%s: skipping in -short mode", test.Name) + } result := utesting.RunTests([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { t.Fatal() diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index e6ce37aae3..80b5d80745 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -25,11 +25,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/internal/utesting" ) // sendTxs sends the given transactions to the node and // expects the node to accept and propagate them. -func (s *Suite) sendTxs(txs []*types.Transaction) error { +func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { // Open sending conn. sendConn, err := s.dial() if err != nil { @@ -70,10 +71,19 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error { for _, tx := range *msg { got[tx.Hash()] = true } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { got[hash] = true } + case *eth.GetBlockHeadersPacket: + headers, err := s.chain.GetHeaders(msg) + if err != nil { + t.Logf("invalid GetBlockHeaders request: %v", err) + } + recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ + RequestId: msg.RequestId, + BlockHeadersRequest: headers, + }) default: return fmt.Errorf("unexpected eth wire msg: %s", pretty.Sdump(msg)) } @@ -95,7 +105,7 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error { return fmt.Errorf("timed out waiting for txs") } -func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { +func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error { // Open sending conn. sendConn, err := s.dial() if err != nil { @@ -128,7 +138,7 @@ func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { invalids[tx.Hash()] = struct{}{} } - // Get repsonses. + // Get responses. recvConn.SetReadDeadline(time.Now().Add(timeout)) for { msg, err := recvConn.ReadEth() @@ -146,12 +156,21 @@ func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error { return fmt.Errorf("received bad tx: %s", tx.Hash()) } } - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: for _, hash := range msg.Hashes { if _, ok := invalids[hash]; ok { return fmt.Errorf("received bad tx: %s", hash) } } + case *eth.GetBlockHeadersPacket: + headers, err := s.chain.GetHeaders(msg) + if err != nil { + t.Logf("invalid GetBlockHeaders request: %v", err) + } + recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ + RequestId: msg.RequestId, + BlockHeadersRequest: headers, + }) default: return fmt.Errorf("unexpected eth message: %v", pretty.Sdump(msg)) } diff --git a/cmd/devp2p/internal/v4test/discv4tests.go b/cmd/devp2p/internal/v4test/discv4tests.go index 3afcfd0698..ca556851b4 100644 --- a/cmd/devp2p/internal/v4test/discv4tests.go +++ b/cmd/devp2p/internal/v4test/discv4tests.go @@ -497,7 +497,7 @@ func FindnodeAmplificationWrongIP(t *utesting.T) { // If we receive a NEIGHBORS response, the attack worked and the test fails. reply, _, _ := te.read(te.l2) if reply != nil { - t.Error("Got NEIGHORS response for FINDNODE from wrong IP") + t.Error("Got NEIGHBORS response for FINDNODE from wrong IP") } } diff --git a/cmd/era/main.go b/cmd/era/main.go new file mode 100644 index 0000000000..d913958a0e --- /dev/null +++ b/cmd/era/main.go @@ -0,0 +1,329 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/urfave/cli/v2" +) + +var app = flags.NewApp("go-ethereum era tool") + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Usage: "directory storing all relevant era1 files", + Value: "eras", + } + networkFlag = &cli.StringFlag{ + Name: "network", + Usage: "network name associated with era1 files", + Value: "mainnet", + } + eraSizeFlag = &cli.IntFlag{ + Name: "size", + Usage: "number of blocks per era", + Value: era.MaxEra1Size, + } + txsFlag = &cli.BoolFlag{ + Name: "txs", + Usage: "print full transaction values", + } +) + +var ( + blockCommand = &cli.Command{ + Name: "block", + Usage: "get block data", + ArgsUsage: "", + Action: block, + Flags: []cli.Flag{ + txsFlag, + }, + } + infoCommand = &cli.Command{ + Name: "info", + ArgsUsage: "", + Usage: "get epoch information", + Action: info, + } + verifyCommand = &cli.Command{ + Name: "verify", + ArgsUsage: "", + Usage: "verifies each era1 against expected accumulator root", + Action: verify, + } +) + +func init() { + app.Commands = []*cli.Command{ + blockCommand, + infoCommand, + verifyCommand, + } + app.Flags = []cli.Flag{ + dirFlag, + networkFlag, + eraSizeFlag, + } +} + +func main() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// block prints the specified block from an era1 store. +func block(ctx *cli.Context) error { + num, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid block number: %w", err) + } + e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) + if err != nil { + return fmt.Errorf("error opening era1: %w", err) + } + defer e.Close() + // Read block with number. + block, err := e.GetBlockByNumber(num) + if err != nil { + return fmt.Errorf("error reading block %d: %w", num, err) + } + // Convert block to JSON and print. + // TODO: without a proper Eth API Backend, this tool isn't expected to function correctly + val, err := ethapi.RPCMarshalBlock(ctx.Context, block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig, ð.EthAPIBackend{}) + if err != nil { + return fmt.Errorf("error marshaling block: %w", err) + } + b, err := json.MarshalIndent(val, "", " ") + if err != nil { + return fmt.Errorf("error marshaling json: %w", err) + } + fmt.Println(string(b)) + return nil +} + +// info prints some high-level information about the era1 file. +func info(ctx *cli.Context) error { + epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid epoch number: %w", err) + } + e, err := open(ctx, epoch) + if err != nil { + return err + } + defer e.Close() + acc, err := e.Accumulator() + if err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + td, err := e.InitialTD() + if err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + info := struct { + Accumulator common.Hash `json:"accumulator"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` + }{ + acc, td, e.Start(), e.Count(), + } + b, _ := json.MarshalIndent(info, "", " ") + fmt.Println(string(b)) + return nil +} + +// open opens an era1 file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + ) + entries, err := era.ReadDir(dir, network) + if err != nil { + return nil, fmt.Errorf("error reading era dir: %w", err) + } + if epoch >= uint64(len(entries)) { + return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) + } + return era.Open(path.Join(dir, entries[epoch])) +} + +// verify checks each era1 file in a directory to ensure it is well-formed and +// that the accumulator matches the expected value. +func verify(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + return fmt.Errorf("missing accumulators file") + } + + roots, err := readHashes(ctx.Args().First()) + if err != nil { + return fmt.Errorf("unable to read expected roots file: %w", err) + } + + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + start = time.Now() + reported = time.Now() + ) + + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + + if len(entries) != len(roots) { + return fmt.Errorf("number of era1 files should match the number of accumulator hashes") + } + + // Verify each epoch matches the expected root. + for i, want := range roots { + // Wrap in function so defers don't stack. + err := func() error { + name := entries[i] + e, err := era.Open(path.Join(dir, name)) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + defer e.Close() + // Read accumulator and check against expected. + if got, err := e.Accumulator(); err != nil { + return fmt.Errorf("error retrieving accumulator for %s: %w", name, err) + } else if got != want { + return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want) + } + // Recompute accumulator. + if err := checkAccumulator(e); err != nil { + return fmt.Errorf("error verify era1 file %s: %w", name, err) + } + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +// checkAccumulator verifies the accumulator matches the data in the Era. +func checkAccumulator(e *era.Era) error { + var ( + err error + want common.Hash + td *big.Int + tds = make([]*big.Int, 0) + hashes = make([]common.Hash, 0) + ) + if want, err = e.Accumulator(); err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + if td, err = e.InitialTD(); err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era iterator: %w", err) + } + // To fully verify an era the following attributes must be checked: + // 1) the block index is constructed correctly + // 2) the tx root matches the value in the block + // 3) the receipts root matches the value in the block + // 4) the starting total difficulty value is correct + // 5) the accumulator is correct by recomputing it locally, which verifies + // the blocks are all correct (via hash) + // + // The attributes 1), 2), and 3) are checked for each block. 4) and 5) require + // accumulation across the entire set and are verified at the end. + for it.Next() { + // 1) next() walks the block index, so we're able to implicitly verify it. + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + block, receipts, err := it.BlockAndReceipts() + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + // 2) recompute tx root and verify against header. + tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)) + if tr != block.TxHash() { + return fmt.Errorf("tx root in block %d mismatch: want %s, got %s", block.NumberU64(), block.TxHash(), tr) + } + // 3) recompute receipt root and check value against block. + rr := types.DeriveSha(receipts, trie.NewStackTrie(nil)) + if rr != block.ReceiptHash() { + return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) + } + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } + // 4+5) Verify accumulator and total difficulty. + got, err := era.ComputeAccumulator(hashes, tds) + if err != nil { + return fmt.Errorf("error computing accumulator: %w", err) + } + if got != want { + return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want) + } + return nil +} + +// readHashes reads a file of newline-delimited hashes. +func readHashes(f string) ([]common.Hash, error) { + b, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("unable to open accumulators file") + } + s := strings.Split(string(b), "\n") + // Remove empty last element, if present. + if s[len(s)-1] == "" { + s = s[:len(s)-1] + } + // Convert to hashes. + r := make([]common.Hash, len(s)) + for i := range s { + r[i] = common.HexToHash(s[i]) + } + return r, nil +} diff --git a/cmd/evm/README.md b/cmd/evm/README.md index 41d8ced278..25647c18a9 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -214,7 +214,7 @@ exitcode:3 OK The chain configuration to be used for a transition is specified via the `--state.fork` CLI flag. A list of possible values and configurations can be -found in [`tests/init.go`](tests/init.go). +found in [`tests/init.go`](../../tests/init.go). #### Examples ##### Basic usage diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 86e42211bb..498c887fba 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -36,12 +36,14 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) type Prestate struct { - Env stEnv `json:"env"` - Pre core.GenesisAlloc `json:"pre"` + Env stEnv `json:"env"` + Pre types.GenesisAlloc `json:"pre"` } // ExecutionResult contains the execution status after running a state test, any @@ -140,6 +142,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, rejectedTxs []*rejectedTx includedTxs types.Transactions gasUsed = uint64(0) + blobGasUsed = uint64(0) receipts = make(types.Receipts, 0) txIndex = 0 ) @@ -192,7 +195,6 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } - var blobGasUsed uint64 for i := 0; txIt.Next(); i++ { tx, err := txIt.Tx() @@ -213,15 +215,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) continue } + txBlobGas := uint64(0) if tx.Type() == types.BlobTxType { - txBlobGas := uint64(params.BlobTxBlobGasPerBlob * len(tx.BlobHashes())) + txBlobGas = uint64(params.BlobTxBlobGasPerBlob * len(tx.BlobHashes())) if used, max := blobGasUsed+txBlobGas, uint64(params.MaxBlobGasPerBlock); used > max { err := fmt.Errorf("blob gas (%d) would exceed maximum allowance %d", used, max) log.Warn("rejected tx", "index", i, "err", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) continue } - blobGasUsed += txBlobGas } tracer, err := getTracerFn(txIndex, tx.Hash()) if err != nil { @@ -250,6 +252,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, if hashError != nil { return nil, nil, nil, NewError(ErrorMissingBlockhash, hashError) } + blobGasUsed += txBlobGas gasUsed += msgResult.UsedGas // Receipt: @@ -310,15 +313,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) - statedb.AddBalance(ommer.Address, reward) + statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward)) } - statedb.AddBalance(pre.Env.Coinbase, minerReward) + statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward)) } // Apply withdrawals for _, w := range pre.Env.Withdrawals { // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) - statedb.AddBalance(w.Address, amount) + statedb.AddBalance(w.Address, uint256.MustFromBig(amount)) } // Commit block root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber)) @@ -355,13 +358,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return statedb, execRs, body, nil } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { + sdb := state.NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) statedb, _ := state.New(types.EmptyRootHash, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, a.Balance) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance)) for k, v := range a.Storage { statedb.SetState(addr, k, v) } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 837e2938e7..1059586f39 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -74,10 +73,10 @@ var ( ) type input struct { - Alloc core.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs []*txWithKey `json:"txs,omitempty"` - TxRlp string `json:"txsRlp,omitempty"` + Alloc types.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Transition(ctx *cli.Context) error { @@ -188,7 +187,7 @@ func Transition(ctx *cli.Context) error { if err != nil { return err } - // Dump the excution result + // Dump the execution result collector := make(Alloc) s.DumpToCollector(collector, nil) return dispatchOutput(ctx, baseDir, result, collector, body) @@ -272,7 +271,7 @@ func applyCancunChecks(env *stEnv, chainConfig *params.ChainConfig) error { return nil } -type Alloc map[common.Address]core.GenesisAccount +type Alloc map[common.Address]types.Account func (g Alloc) OnRoot(common.Hash) {} @@ -280,7 +279,7 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { if addr == nil { return } - balance, _ := new(big.Int).SetString(dumpAccount.Balance, 10) + balance, _ := new(big.Int).SetString(dumpAccount.Balance, 0) var storage map[common.Hash]common.Hash if dumpAccount.Storage != nil { storage = make(map[common.Hash]common.Hash) @@ -288,7 +287,7 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { storage[k] = common.HexToHash(v) } } - genesisAccount := core.GenesisAccount{ + genesisAccount := types.Account{ Code: dumpAccount.Code, Storage: storage, Balance: balance, diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index c65fdc8077..d7a90c6866 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -38,8 +38,8 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" "github.com/urfave/cli/v2" ) @@ -148,7 +148,7 @@ func runCmd(ctx *cli.Context) error { } db := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(db, &trie.Config{ + triedb := triedb.NewDatabase(db, &triedb.Config{ Preimages: preimages, HashDB: hashdb.Defaults, }) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6e751b630f..458d809ad8 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/tests" @@ -90,26 +89,27 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { if err != nil { return err } - var tests map[string]tests.StateTest - if err := json.Unmarshal(src, &tests); err != nil { + var testsByName map[string]tests.StateTest + if err := json.Unmarshal(src, &testsByName); err != nil { return err } + // Iterate over all the tests, run them and aggregate the results - results := make([]StatetestResult, 0, len(tests)) - for key, test := range tests { + results := make([]StatetestResult, 0, len(testsByName)) + for key, test := range testsByName { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, statedb *state.StateDB) { + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { var root common.Hash - if statedb != nil { - root = statedb.IntermediateRoot(false) + if tstate.StateDB != nil { + root = tstate.StateDB.IntermediateRoot(false) result.Root = &root if jsonOut { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) } if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, statedb.Database(), nil) + cpy, _ := state.New(root, tstate.StateDB.Database(), nil) dump := cpy.RawDump(nil) result.State = &dump } diff --git a/cmd/evm/transition-test.sh b/cmd/evm/transition-test.sh index 8cc6aa41de..2ddda2d473 100644 --- a/cmd/evm/transition-test.sh +++ b/cmd/evm/transition-test.sh @@ -103,7 +103,7 @@ type Env struct { CurrentTimestamp uint64 `json:"currentTimestamp"` Withdrawals []*Withdrawal `json:"withdrawals"` // optional - CurrentDifficulty *big.Int `json:"currentDifficuly"` + CurrentDifficulty *big.Int `json:"currentDifficulty"` CurrentRandom *big.Int `json:"currentRandom"` CurrentBaseFee *big.Int `json:"currentBaseFee"` ParentDifficulty *big.Int `json:"parentDifficulty"` diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 0733a29392..7421b21622 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,10 +35,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" ) @@ -122,6 +124,33 @@ Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, + } + importHistoryCommand = &cli.Command{ + Action: importHistory, + Name: "import-history", + Usage: "Import an Era archive", + ArgsUsage: "", + Flags: flags.Merge([]cli.Flag{ + utils.TxLookupLimitFlag, + }, + utils.DatabaseFlags, + utils.NetworkFlags, + ), + Description: ` +The import-history command will import blocks and their corresponding receipts +from Era archives. +`, + } + exportHistoryCommand = &cli.Command{ + Action: exportHistory, + Name: "export-history", + Usage: "Export blockchain history to Era archives", + ArgsUsage: " ", + Flags: flags.Merge(utils.DatabaseFlags), + Description: ` +The export-history command will export blocks and their corresponding receipts +into Era archives. Eras are typically packaged in steps of 8192 blocks. +`, } importPreimagesCommand = &cli.Command{ Action: importPreimages, @@ -364,7 +393,97 @@ func exportChain(ctx *cli.Context) error { } err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last)) } + if err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} + +func importHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + chain, db := utils.MakeChain(ctx, stack, false) + defer db.Close() + + var ( + start = time.Now() + dir = ctx.Args().Get(0) + network string + ) + + // Determine network. + if utils.IsNetworkPreset(ctx) { + switch { + case ctx.Bool(utils.MainnetFlag.Name): + network = "mainnet" + case ctx.Bool(utils.SepoliaFlag.Name): + network = "sepolia" + case ctx.Bool(utils.GoerliFlag.Name): + network = "goerli" + } + } else { + // No network flag set, try to determine network based on files + // present in directory. + var networks []string + for _, n := range params.NetworkNames { + entries, err := era.ReadDir(dir, n) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + if len(entries) > 0 { + networks = append(networks, n) + } + } + if len(networks) == 0 { + return fmt.Errorf("no era1 files found in %s", dir) + } + if len(networks) > 1 { + return fmt.Errorf("multiple networks found, use a network flag to specify desired network") + } + network = networks[0] + } + + if err := utils.ImportHistory(chain, db, dir, network); err != nil { + return err + } + fmt.Printf("Import done in %v\n", time.Since(start)) + return nil +} + +// exportHistory exports chain history in Era archives at a specified +// directory. +func exportHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 3 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, _ := utils.MakeChain(ctx, stack, true) + start := time.Now() + + var ( + dir = ctx.Args().Get(0) + first, ferr = strconv.ParseInt(ctx.Args().Get(1), 10, 64) + last, lerr = strconv.ParseInt(ctx.Args().Get(2), 10, 64) + ) + if ferr != nil || lerr != nil { + utils.Fatalf("Export error in parsing parameters: block number not an integer\n") + } + if first < 0 || last < 0 { + utils.Fatalf("Export error: block number must be greater than 0\n") + } + if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { + utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) + } + err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) if err != nil { utils.Fatalf("Export error: %v\n", err) } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 34987c7d5f..27496113d7 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -204,6 +204,11 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { cfg.Eth.OverrideOptimismEcotone = &v } + if ctx.IsSet(utils.OverrideOptimismFjord.Name) { + v := ctx.Uint64(utils.OverrideOptimismFjord.Name) + cfg.Eth.OverrideOptimismFjord = &v + } + if ctx.IsSet(utils.OverrideOptimismInterop.Name) { v := ctx.Uint64(utils.OverrideOptimismInterop.Name) cfg.Eth.OverrideOptimismInterop = &v diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 7835cc5048..c0746f8963 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -39,18 +39,29 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" ) var ( + removeStateDataFlag = &cli.BoolFlag{ + Name: "remove.state", + Usage: "If set, selects the state data for removal", + } + removeChainDataFlag = &cli.BoolFlag{ + Name: "remove.chain", + Usage: "If set, selects the state data for removal", + } + removedbCommand = &cli.Command{ Action: removeDB, Name: "removedb", Usage: "Remove blockchain and state databases", ArgsUsage: "", - Flags: utils.DatabaseFlags, + Flags: flags.Merge(utils.DatabaseFlags, + []cli.Flag{removeStateDataFlag, removeChainDataFlag}), Description: ` Remove blockchain and state databases`, } @@ -277,11 +288,11 @@ func removeDB(ctx *cli.Context) error { } // Delete state data statePaths := []string{rootDir, filepath.Join(ancientDir, rawdb.StateFreezerName)} - confirmAndRemoveDB(statePaths, "state data") + confirmAndRemoveDB(statePaths, "state data", ctx, removeStateDataFlag.Name) // Delete ancient chain chainPaths := []string{filepath.Join(ancientDir, rawdb.ChainFreezerName)} - confirmAndRemoveDB(chainPaths, "ancient chain") + confirmAndRemoveDB(chainPaths, "ancient chain", ctx, removeChainDataFlag.Name) return nil } @@ -304,14 +315,26 @@ func removeFolder(dir string) { // confirmAndRemoveDB prompts the user for a last confirmation and removes the // list of folders if accepted. -func confirmAndRemoveDB(paths []string, kind string) { +func confirmAndRemoveDB(paths []string, kind string, ctx *cli.Context, removeFlagName string) { + var ( + confirm bool + err error + ) msg := fmt.Sprintf("Location(s) of '%s': \n", kind) for _, path := range paths { msg += fmt.Sprintf("\t- %s\n", path) } fmt.Println(msg) - - confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove '%s'?", kind)) + if ctx.IsSet(removeFlagName) { + confirm = ctx.Bool(removeFlagName) + if confirm { + fmt.Printf("Remove '%s'? [y/n] y\n", kind) + } else { + fmt.Printf("Remove '%s'? [y/n] n\n", kind) + } + } else { + confirm, err = prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove '%s'?", kind)) + } switch { case err != nil: utils.Fatalf("%v", err) @@ -395,16 +418,16 @@ func inspectTrie(ctx *cli.Context) error { fmt.Printf("ReadBlockHeader, root: %v, block number: %v\n", trieRootHash, blockNumber) dbScheme := rawdb.ReadStateScheme(db) - var config *trie.Config + var config *triedb.Config if dbScheme == rawdb.PathScheme { - config = &trie.Config{ + config = &triedb.Config{ PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly), } } else if dbScheme == rawdb.HashScheme { - config = trie.HashDefaults + config = triedb.HashDefaults } - triedb := trie.NewDatabase(db, config) + triedb := triedb.NewDatabase(db, config) theTrie, err := trie.New(trie.TrieID(trieRootHash), triedb) if err != nil { fmt.Printf("Failed to new trie tree, err: %v, root hash: %v\n", err, trieRootHash.String()) @@ -1041,8 +1064,8 @@ func hbss2pbss(ctx *cli.Context) error { db.Sync() defer db.Close() - config := trie.HashDefaults - triedb := trie.NewDatabase(db, config) + config := triedb.HashDefaults + triedb := triedb.NewDatabase(db, config) triedb.Cap(0) log.Info("hbss2pbss triedb", "scheme", triedb.Scheme()) defer triedb.Close() diff --git a/cmd/geth/logtestcmd_active.go b/cmd/geth/logtestcmd_active.go index 5cce1ec6ab..f2a2c5ded5 100644 --- a/cmd/geth/logtestcmd_active.go +++ b/cmd/geth/logtestcmd_active.go @@ -26,7 +26,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/holiman/uint256" "github.com/urfave/cli/v2" @@ -51,9 +50,6 @@ func (c customQuotedStringer) String() string { // logTest is an entry point which spits out some logs. This is used by testing // to verify expected outputs func logTest(ctx *cli.Context) error { - // clear field padding map - debug.ResetLogging() - { // big.Int ba, _ := new(big.Int).SetString("111222333444555678999", 10) // "111,222,333,444,555,678,999" bb, _ := new(big.Int).SetString("-111222333444555678999", 10) // "-111,222,333,444,555,678,999" diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 23761964c4..c8ad9de1a2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// geth is the official command-line client for Ethereum. +// geth is a command-line client for Ethereum. package main import ( @@ -69,6 +69,7 @@ var ( utils.OverrideVerkle, utils.OverrideOptimismCanyon, utils.OverrideOptimismEcotone, + utils.OverrideOptimismFjord, utils.OverrideOptimismInterop, utils.EnablePersonal, utils.TxPoolLocalsFlag, @@ -131,6 +132,7 @@ var ( utils.MaxPendingPeersFlag, utils.MiningEnabledFlag, utils.MinerGasLimitFlag, + utils.MinerEffectiveGasLimitFlag, utils.MinerGasPriceFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, @@ -228,12 +230,13 @@ var app = flags.NewApp("the go-ethereum command line interface") func init() { // Initialize the CLI app and start Geth app.Action = geth - app.Copyright = "Copyright 2013-2023 The go-ethereum Authors" app.Commands = []*cli.Command{ // See chaincmd.go: initCommand, importCommand, exportCommand, + importHistoryCommand, + exportHistoryCommand, importPreimagesCommand, removedbCommand, dumpCommand, diff --git a/cmd/geth/testdata/logging/logtest-json.txt b/cmd/geth/testdata/logging/logtest-json.txt index 3bfe718660..d2bd0ad91a 100644 --- a/cmd/geth/testdata/logging/logtest-json.txt +++ b/cmd/geth/testdata/logging/logtest-json.txt @@ -29,7 +29,7 @@ {"t":"2023-11-22T15:42:00.408237+08:00","lvl":"info","msg":"repeated-key 2","xx":"short","xx":"longer"} {"t":"2023-11-22T15:42:00.408241+08:00","lvl":"info","msg":"log at level info"} {"t":"2023-11-22T15:42:00.408244+08:00","lvl":"warn","msg":"log at level warn"} -{"t":"2023-11-22T15:42:00.408247+08:00","lvl":"eror","msg":"log at level error"} +{"t":"2023-11-22T15:42:00.408247+08:00","lvl":"error","msg":"log at level error"} {"t":"2023-11-22T15:42:00.408251+08:00","lvl":"info","msg":"test","bar":"short","a":"aligned left"} {"t":"2023-11-22T15:42:00.408254+08:00","lvl":"info","msg":"test","bar":"a long message","a":1} {"t":"2023-11-22T15:42:00.408258+08:00","lvl":"info","msg":"test","bar":"short","a":"aligned right"} diff --git a/cmd/geth/testdata/logging/logtest-logfmt.txt b/cmd/geth/testdata/logging/logtest-logfmt.txt index f20d66635d..5c5316b7d9 100644 --- a/cmd/geth/testdata/logging/logtest-logfmt.txt +++ b/cmd/geth/testdata/logging/logtest-logfmt.txt @@ -29,7 +29,7 @@ t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="repeated-key 1" foo=alpha foo=beta t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="repeated-key 2" xx=short xx=longer t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="log at level info" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=warn msg="log at level warn" -t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=eror msg="log at level error" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=error msg="log at level error" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar=short a="aligned left" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar="a long message" a=1 t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar=short a="aligned right" diff --git a/cmd/rlpdump/main.go b/cmd/rlpdump/main.go index 70337749ae..7e1d314d49 100644 --- a/cmd/rlpdump/main.go +++ b/cmd/rlpdump/main.go @@ -25,7 +25,9 @@ import ( "flag" "fmt" "io" + "math" "os" + "strconv" "strings" "github.com/ethereum/go-ethereum/common" @@ -37,6 +39,7 @@ var ( reverseMode = flag.Bool("reverse", false, "convert ASCII to rlp") noASCII = flag.Bool("noascii", false, "don't print ASCII strings readably") single = flag.Bool("single", false, "print only the first element, discard the rest") + showpos = flag.Bool("pos", false, "display element byte posititions") ) func init() { @@ -52,17 +55,17 @@ If the filename is omitted, data is read from stdin.`) func main() { flag.Parse() - var r io.Reader + var r *inStream switch { case *hexMode != "": data, err := hex.DecodeString(strings.TrimPrefix(*hexMode, "0x")) if err != nil { die(err) } - r = bytes.NewReader(data) + r = newInStream(bytes.NewReader(data), int64(len(data))) case flag.NArg() == 0: - r = os.Stdin + r = newInStream(bufio.NewReader(os.Stdin), 0) case flag.NArg() == 1: fd, err := os.Open(flag.Arg(0)) @@ -70,13 +73,19 @@ func main() { die(err) } defer fd.Close() - r = fd + var size int64 + finfo, err := fd.Stat() + if err == nil { + size = finfo.Size() + } + r = newInStream(bufio.NewReader(fd), size) default: fmt.Fprintln(os.Stderr, "Error: too many arguments") flag.Usage() os.Exit(2) } + out := os.Stdout if *reverseMode { data, err := textToRlp(r) @@ -93,10 +102,10 @@ func main() { } } -func rlpToText(r io.Reader, out io.Writer) error { - s := rlp.NewStream(r, 0) +func rlpToText(in *inStream, out io.Writer) error { + stream := rlp.NewStream(in, 0) for { - if err := dump(s, 0, out); err != nil { + if err := dump(in, stream, 0, out); err != nil { if err != io.EOF { return err } @@ -110,7 +119,10 @@ func rlpToText(r io.Reader, out io.Writer) error { return nil } -func dump(s *rlp.Stream, depth int, out io.Writer) error { +func dump(in *inStream, s *rlp.Stream, depth int, out io.Writer) error { + if *showpos { + fmt.Fprintf(out, "%s: ", in.posLabel()) + } kind, size, err := s.Kind() if err != nil { return err @@ -137,7 +149,7 @@ func dump(s *rlp.Stream, depth int, out io.Writer) error { if i > 0 { fmt.Fprint(out, ",\n") } - if err := dump(s, depth+1, out); err == rlp.EOL { + if err := dump(in, s, depth+1, out); err == rlp.EOL { break } else if err != nil { return err @@ -208,3 +220,36 @@ func textToRlp(r io.Reader) ([]byte, error) { data, err := rlp.EncodeToBytes(obj[0]) return data, err } + +type inStream struct { + br rlp.ByteReader + pos int + columns int +} + +func newInStream(br rlp.ByteReader, totalSize int64) *inStream { + col := int(math.Ceil(math.Log10(float64(totalSize)))) + return &inStream{br: br, columns: col} +} + +func (rc *inStream) Read(b []byte) (n int, err error) { + n, err = rc.br.Read(b) + rc.pos += n + return n, err +} + +func (rc *inStream) ReadByte() (byte, error) { + b, err := rc.br.ReadByte() + if err == nil { + rc.pos++ + } + return b, err +} + +func (rc *inStream) posLabel() string { + l := strconv.FormatInt(int64(rc.pos), 10) + if len(l) < rc.columns { + l = strings.Repeat(" ", rc.columns-len(l)) + l + } + return l +} diff --git a/cmd/rlpdump/rlpdump_test.go b/cmd/rlpdump/rlpdump_test.go index 8d55f4200a..4b0ae680ac 100644 --- a/cmd/rlpdump/rlpdump_test.go +++ b/cmd/rlpdump/rlpdump_test.go @@ -34,7 +34,8 @@ func TestRoundtrip(t *testing.T) { "0xc780c0c1c0825208", } { var out strings.Builder - err := rlpToText(bytes.NewReader(common.FromHex(want)), &out) + in := newInStream(bytes.NewReader(common.FromHex(want)), 0) + err := rlpToText(in, &out) if err != nil { t.Fatal(err) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 8b571be1ef..4b57164665 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -19,12 +19,15 @@ package utils import ( "bufio" + "bytes" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" "os" "os/signal" + "path" "runtime" "strings" "syscall" @@ -39,8 +42,10 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" ) @@ -228,6 +233,105 @@ func ImportChain(chain *core.BlockChain, fn string) error { return nil } +func readList(filename string) ([]string, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return strings.Split(string(b), "\n"), nil +} + +// ImportHistory imports Era1 files containing historical block information, +// starting from genesis. +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { + if chain.CurrentSnapBlock().Number.BitLen() != 0 { + return fmt.Errorf("history import only supported when starting from genesis") + } + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + checksums, err := readList(path.Join(dir, "checksums.txt")) + if err != nil { + return fmt.Errorf("unable to read checksums.txt: %w", err) + } + if len(checksums) != len(entries) { + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + } + var ( + start = time.Now() + reported = time.Now() + imported = 0 + forker = core.NewForkChoice(chain, nil) + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + for i, filename := range entries { + err := func() error { + f, err := os.Open(path.Join(dir, filename)) + if err != nil { + return fmt.Errorf("unable to open era: %w", err) + } + defer f.Close() + + // Validate checksum. + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to recalculate checksum: %w", err) + } + if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { + return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + } + h.Reset() + buf.Reset() + + // Import all block data from Era1. + e, err := era.From(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era reader: %w", err) + } + for it.Next() { + block, err := it.Block() + if err != nil { + return fmt.Errorf("error reading block %d: %w", it.Number(), err) + } + if block.Number().BitLen() == 0 { + continue // skip genesis + } + receipts, err := it.Receipts() + if err != nil { + return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) + } + if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil { + return fmt.Errorf("error inserting header %d: %w", it.Number(), err) + } else if status != core.CanonStatTy { + return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status) + } + if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil { + return fmt.Errorf("error inserting body %d: %w", it.Number(), err) + } + imported += 1 + + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + imported = 0 + reported = time.Now() + } + } + return nil + }() + if err != nil { + return err + } + } + + return nil +} + func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { head := chain.CurrentBlock() for i, block := range blocks { @@ -297,6 +401,93 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return nil } +// ExportHistory exports blockchain history into the specified directory, +// following the Era format. +func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { + log.Info("Exporting blockchain history", "dir", dir) + if head := bc.CurrentBlock().Number.Uint64(); head < last { + log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) + last = head + } + network := "unknown" + if name, ok := params.NetworkNames[bc.Config().ChainID.String()]; ok { + network = name + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating output directory: %w", err) + } + var ( + start = time.Now() + reported = time.Now() + h = sha256.New() + buf = bytes.NewBuffer(nil) + checksums []string + ) + for i := first; i <= last; i += step { + err := func() error { + filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{})) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("could not create era file: %w", err) + } + defer f.Close() + + w := era.NewBuilder(f) + for j := uint64(0); j < step && j <= last-i; j++ { + var ( + n = i + j + block = bc.GetBlockByNumber(n) + ) + if block == nil { + return fmt.Errorf("export failed on #%d: not found", n) + } + receipts := bc.GetReceiptsByHash(block.Hash()) + if receipts == nil { + return fmt.Errorf("export failed on #%d: receipts not found", n) + } + td := bc.GetTd(block.Hash(), block.NumberU64()) + if td == nil { + return fmt.Errorf("export failed on #%d: total difficulty not found", n) + } + if err := w.Add(block, receipts, td); err != nil { + return err + } + } + root, err := w.Finalize() + if err != nil { + return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + } + // Set correct filename with root. + os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root))) + + // Compute checksum of entire Era1. + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("unable to calculate checksum: %w", err) + } + checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) + h.Reset() + buf.Reset() + return nil + }() + if err != nil { + return err + } + if time.Since(reported) >= 8*time.Second { + log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + } + + os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + + log.Info("Exported blockchain to", "dir", dir) + + return nil +} + // ImportPreimages imports a batch of exported hash preimages into the database. // It's a part of the deprecated functionality, should be removed in the future. func ImportPreimages(db ethdb.Database, fn string) error { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8d1a1fe1d6..7e44853681 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -75,9 +75,9 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // These are all the command line flags we support. @@ -295,6 +295,11 @@ var ( Usage: "Manually specify the Optimism Ecotone fork timestamp, overriding the bundled setting", Category: flags.EthCategory, } + OverrideOptimismFjord = &cli.Uint64Flag{ + Name: "override.fjord", + Usage: "Manually specify the Optimism Fjord fork timestamp, overriding the bundled setting", + Category: flags.EthCategory, + } OverrideOptimismInterop = &cli.Uint64Flag{ Name: "override.interop", Usage: "Manually specify the Optimsim Interop feature-set fork timestamp, overriding the bundled setting", @@ -538,6 +543,12 @@ var ( Value: ethconfig.Defaults.Miner.GasCeil, Category: flags.MinerCategory, } + MinerEffectiveGasLimitFlag = &cli.Uint64Flag{ + Name: "miner.effectivegaslimit", + Usage: "If non-zero, an effective gas limit to apply in addition to the block header gaslimit.", + Value: 0, + Category: flags.MinerCategory, + } MinerGasPriceFlag = &flags.BigFlag{ Name: "miner.gasprice", Usage: "Minimum gas price for mining a transaction", @@ -1197,13 +1208,6 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { if ctx.Uint64(NetworkIdFlag.Name) == params.OpBNBTestnet { urls = params.OpBNBTestnetBootnodes } - case ctx.IsSet(OPNetworkFlag.Name): - network := ctx.String(OPNetworkFlag.Name) - if strings.Contains(strings.ToLower(network), "mainnet") { - urls = params.OPMainnetBootnodes - } else { - urls = params.OPSepoliaBootnodes - } } } cfg.BootstrapNodes = mustParseBootnodes(urls) @@ -1236,9 +1240,9 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { case ctx.IsSet(OPNetworkFlag.Name): network := ctx.String(OPNetworkFlag.Name) if strings.Contains(strings.ToLower(network), "mainnet") { - urls = append(urls, params.OPMainnetBootnodes...) + urls = params.V5OPBootnodes } else { - urls = append(urls, params.OPSepoliaBootnodes...) + urls = params.V5OPTestnetBootnodes } case ctx.Bool(OpBNBTestnetFlag.Name): urls = params.OpBNBTestnetBootnodes @@ -1715,6 +1719,11 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) { if ctx.IsSet(TxPoolLifetimeFlag.Name) { cfg.Lifetime = ctx.Duration(TxPoolLifetimeFlag.Name) } + if ctx.IsSet(MinerEffectiveGasLimitFlag.Name) { + // While technically this is a miner config parameter, we also want the txpool to enforce + // it to avoid accepting transactions that can never be included in a block. + cfg.EffectiveGasCeil = ctx.Uint64(MinerEffectiveGasLimitFlag.Name) + } if ctx.IsSet(TxPoolReannounceTimeFlag.Name) { cfg.ReannounceTime = ctx.Duration(TxPoolReannounceTimeFlag.Name) } @@ -1736,6 +1745,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.IsSet(MinerGasLimitFlag.Name) { cfg.GasCeil = ctx.Uint64(MinerGasLimitFlag.Name) } + if ctx.IsSet(MinerEffectiveGasLimitFlag.Name) { + cfg.EffectiveGasCeil = ctx.Uint64(MinerEffectiveGasLimitFlag.Name) + } if ctx.IsSet(MinerGasPriceFlag.Name) { cfg.GasPrice = flags.GlobalBig(ctx, MinerGasPriceFlag.Name) } @@ -2496,8 +2508,8 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } // MakeTrieDatabase constructs a trie database based on the configured scheme. -func MakeTrieDatabase(ctx *cli.Context, stack *node.Node, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *trie.Database { - config := &trie.Config{ +func MakeTrieDatabase(ctx *cli.Context, stack *node.Node, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *triedb.Database { + config := &triedb.Config{ Preimages: preimage, IsVerkle: isVerkle, } @@ -2510,7 +2522,7 @@ func MakeTrieDatabase(ctx *cli.Context, stack *node.Node, disk ethdb.Database, p // ignore the parameter silently. TODO(rjl493456442) // please config it if read mode is implemented. config.HashDB = hashdb.Defaults - return trie.NewDatabase(disk, config) + return triedb.NewDatabase(disk, config) } if readOnly { config.PathDB = pathdb.ReadOnly @@ -2518,5 +2530,5 @@ func MakeTrieDatabase(ctx *cli.Context, stack *node.Node, disk ethdb.Database, p config.PathDB = pathdb.Defaults } config.PathDB.JournalFilePath = fmt.Sprintf("%s/%s", stack.ResolvePath("chaindata"), eth.JournalFileName) - return trie.NewDatabase(disk, config) + return triedb.NewDatabase(disk, config) } diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go new file mode 100644 index 0000000000..9b7f1797d8 --- /dev/null +++ b/cmd/utils/history_test.go @@ -0,0 +1,185 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import ( + "bytes" + "crypto/sha256" + "io" + "math/big" + "os" + "path" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" +) + +var ( + count uint64 = 128 + step uint64 = 16 +) + +func TestHistoryImportAndExport(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) + + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) + + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error insterting chain: %v", err) + } + + // Make temp directory for era files. + dir, err := os.MkdirTemp("", "history-export-test") + if err != nil { + t.Fatalf("error creating temp test directory: %v", err) + } + defer os.RemoveAll(dir) + + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, step); err != nil { + t.Fatalf("error exporting history: %v", err) + } + + // Read checksums. + b, err := os.ReadFile(path.Join(dir, "checksums.txt")) + if err != nil { + t.Fatalf("failed to read checksums: %v", err) + } + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(path.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := era.From(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := era.NewIterator(e) + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() + } + + // Now import Era. + freezer := t.TempDir() + db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false) + if err != nil { + panic(err) + } + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, triedb.NewDatabase(db, triedb.HashDefaults)) + imported, err := core.NewBlockChain(db2, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil { + t.Fatalf("failed to import chain: %v", err) + } + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } +} diff --git a/common/big.go b/common/big.go index 65d4377bf7..cbb562a28e 100644 --- a/common/big.go +++ b/common/big.go @@ -16,7 +16,11 @@ package common -import "math/big" +import ( + "math/big" + + "github.com/holiman/uint256" +) // Common big integers often used var ( @@ -27,4 +31,6 @@ var ( Big32 = big.NewInt(32) Big256 = big.NewInt(256) Big257 = big.NewInt(257) + + U2560 = uint256.NewInt(0) ) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 1db9d81a1d..ba4980b8d4 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) // Proof-of-stake protocol constants. @@ -371,8 +372,8 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // Withdrawals processing. for _, w := range withdrawals { // Convert amount from gwei to wei. - amount := new(big.Int).SetUint64(w.Amount) - amount = amount.Mul(amount, big.NewInt(params.GWei)) + amount := new(uint256.Int).SetUint64(w.Amount) + amount = amount.Mul(amount, uint256.NewInt(params.GWei)) state.AddBalance(w.Address, amount) } // No block reward which is issued by consensus layer instead. diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 7cd5919c5e..8ef8dbffa9 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -47,7 +47,7 @@ func TestReimportMirroredState(t *testing.T) { genspec := &core.Genesis{ Config: params.AllCliqueProtocolChanges, ExtraData: make([]byte, extraVanity+common.AddressLength+extraSeal), - Alloc: map[common.Address]core.GenesisAccount{ + Alloc: map[common.Address]types.Account{ addr: {Balance: big.NewInt(10000000000000000)}, }, BaseFee: big.NewInt(params.InitialBaseFee), diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 130dfdf213..c2936fd4b3 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -33,16 +33,17 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = uint256.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = uint256.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + ConstantinopleBlockReward = uint256.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks // calcDifficultyEip5133 is the difficulty adjustment algorithm as specified by EIP 5133. // It offsets the bomb a total of 11.4M blocks. @@ -562,8 +563,8 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { // Some weird constants to avoid constant memory allocs for them. var ( - big8 = big.NewInt(8) - big32 = big.NewInt(32) + u256_8 = uint256.NewInt(8) + u256_32 = uint256.NewInt(32) ) // AccumulateRewards credits the coinbase of the given block with the mining @@ -579,16 +580,18 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header blockReward = ConstantinopleBlockReward } // Accumulate the rewards for the miner and any included uncles - reward := new(big.Int).Set(blockReward) - r := new(big.Int) + reward := new(uint256.Int).Set(blockReward) + r := new(uint256.Int) + hNum, _ := uint256.FromBig(header.Number) for _, uncle := range uncles { - r.Add(uncle.Number, big8) - r.Sub(r, header.Number) + uNum, _ := uint256.FromBig(uncle.Number) + r.AddUint64(uNum, 8) + r.Sub(r, hNum) r.Mul(r, blockReward) - r.Div(r, big8) + r.Div(r, u256_8) state.AddBalance(uncle.Coinbase, r) - r.Div(blockReward, big32) + r.Div(blockReward, u256_32) reward.Add(reward, r) } state.AddBalance(header.Coinbase, reward) diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index 96995616de..e21a44f63d 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var ( @@ -81,6 +82,6 @@ func ApplyDAOHardFork(statedb *state.StateDB) { // Move every DAO account and extra-balance account funds into the refund contract for _, addr := range params.DAODrainList() { statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr)) - statedb.SetBalance(addr, new(big.Int)) + statedb.SetBalance(addr, new(uint256.Int)) } } diff --git a/core/bench_test.go b/core/bench_test.go index c5991f10e8..97713868a5 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -189,7 +189,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // generator function. gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, + Alloc: types.GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, } _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), b.N, gen) @@ -243,7 +243,7 @@ func BenchmarkChainWrite_full_500k(b *testing.B) { // makeChainForBench writes a given number of headers or empty blocks/receipts // into a database. -func makeChainForBench(db ethdb.Database, full bool, count uint64) { +func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uint64) { var hash common.Hash for n := uint64(0); n < count; n++ { header := &types.Header{ @@ -255,6 +255,9 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { TxHash: types.EmptyTxsHash, ReceiptHash: types.EmptyReceiptsHash, } + if n == 0 { + header = genesis.ToBlock().Header() + } hash = header.Hash() rawdb.WriteHeader(db, header) @@ -262,7 +265,7 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1))) if n == 0 { - rawdb.WriteChainConfig(db, hash, params.AllEthashProtocolChanges) + rawdb.WriteChainConfig(db, hash, genesis.Config) } rawdb.WriteHeadHeaderHash(db, hash) @@ -276,13 +279,14 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { } func benchWriteChain(b *testing.B, full bool, count uint64) { + genesis := &Genesis{Config: params.AllEthashProtocolChanges} for i := 0; i < b.N; i++ { dir := b.TempDir() db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - makeChainForBench(db, full, count) + makeChainForBench(db, genesis, full, count) db.Close() } } @@ -294,7 +298,8 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - makeChainForBench(db, full, count) + genesis := &Genesis{Config: params.AllEthashProtocolChanges} + makeChainForBench(db, genesis, full, count) db.Close() cacheConfig := *defaultCacheConfig cacheConfig.TrieDirtyDisabled = true @@ -307,7 +312,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, &cacheConfig, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, &cacheConfig, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 48bdceff62..385c0afd9d 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -106,7 +106,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { gspec = &Genesis{ Config: &config, ExtraData: make([]byte, 32+common.AddressLength+crypto.SignatureLength), - Alloc: map[common.Address]GenesisAccount{ + Alloc: map[common.Address]types.Account{ addr: {Balance: big.NewInt(1)}, }, BaseFee: big.NewInt(params.InitialBaseFee), diff --git a/core/blockchain.go b/core/blockchain.go index 2250ccd6c6..7e4b81b153 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,8 +28,6 @@ import ( "sync/atomic" "time" - "golang.org/x/exp/slices" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/common/mclock" @@ -49,9 +47,10 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "golang.org/x/exp/slices" ) var ( @@ -172,8 +171,8 @@ type CacheConfig struct { } // triedbConfig derives the configures for trie database. -func (c *CacheConfig) triedbConfig(keepFunc pathdb.NotifyKeepFunc) *trie.Config { - config := &trie.Config{ +func (c *CacheConfig) triedbConfig(keepFunc pathdb.NotifyKeepFunc) *triedb.Config { + config := &triedb.Config{ Preimages: c.Preimages, NoTries: c.NoTries, } @@ -216,6 +215,13 @@ func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { return &config } +// txLookup is wrapper over transaction lookup along with the corresponding +// transaction object. +type txLookup struct { + lookup *rawdb.LegacyTxLookupEntry + transaction *types.Transaction +} + // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -240,16 +246,10 @@ type BlockChain struct { gcproc time.Duration // Accumulates canonical block processing for trie dumping lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state - triedb *trie.Database // The database handler for maintaining trie nodes. + triedb *triedb.Database // The database handler for maintaining trie nodes. stateCache state.Database // State database to reuse between imports (contains state cache) proofKeeper *ProofKeeper // Store/Query op-proposal proof to ensure consistent. - - // txLookupLimit is the maximum number of blocks from head whose tx indices - // are reserved: - // * 0: means no limit and regenerate any missing indexes - // * N: means N block limit [HEAD-N+1, HEAD] and delete extra indexes - // * nil: disable tx reindexer/deleter, but still index new blocks - txLookupLimit uint64 + txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain rmLogsFeed event.Feed @@ -274,7 +274,7 @@ type BlockChain struct { bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] receiptsCache *lru.Cache[common.Hash, []*types.Receipt] blockCache *lru.Cache[common.Hash, *types.Block] - txLookupCache *lru.Cache[common.Hash, *rawdb.LegacyTxLookupEntry] + txLookupCache *lru.Cache[common.Hash, txLookup] miningReceiptsCache *lru.Cache[common.Hash, []*types.Receipt] miningTxLogsCache *lru.Cache[common.Hash, []*types.Log] @@ -283,10 +283,10 @@ type BlockChain struct { // future blocks are blocks added for later processing futureBlocks *lru.Cache[common.Hash, *types.Block] - wg sync.WaitGroup // - quit chan struct{} // shutdown signal, closed in Stop. - stopping atomic.Bool // false if chain is running, true when stopped - procInterrupt atomic.Bool // interrupt signaler for block processing + wg sync.WaitGroup + quit chan struct{} // shutdown signal, closed in Stop. + stopping atomic.Bool // false if chain is running, true when stopped + procInterrupt atomic.Bool // interrupt signaler for block processing engine consensus.Engine validator Validator // Block and state validator interface @@ -312,7 +312,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis proofKeeper := newProofKeeper(opts) // Open trie database with provided config trieConfig := cacheConfig.triedbConfig(proofKeeper.GetNotifyKeepRecordFunc()) - triedb := trie.NewDatabase(db, trieConfig) + triedb := triedb.NewDatabase(db, trieConfig) // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the @@ -345,7 +345,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - txLookupCache: lru.NewCache[common.Hash, *rawdb.LegacyTxLookupEntry](txLookupCacheLimit), + txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), miningReceiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), miningTxLogsCache: lru.NewCache[common.Hash, []*types.Log](txLogsCacheLimit), miningStateCache: lru.NewCache[common.Hash, *state.StateDB](miningStateCacheLimit), @@ -526,12 +526,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } rawdb.WriteChainConfig(db, genesisHash, chainConfig) } - // Start tx indexer/unindexer if required. + // Start tx indexer if it's enabled. if txLookupLimit != nil { - bc.txLookupLimit = *txLookupLimit - - bc.wg.Add(1) - go bc.maintainTxIndex() + bc.txIndexer = newTxIndexer(*txLookupLimit, bc) } return bc, nil } @@ -1024,7 +1021,10 @@ func (bc *BlockChain) stopWithoutSaving() { if !bc.stopping.CompareAndSwap(false, true) { return } - + // Signal shutdown tx indexer. + if bc.txIndexer != nil { + bc.txIndexer.close() + } // Unsubscribe all subscriptions registered from blockchain. bc.scope.Close() @@ -1246,14 +1246,13 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Ensure genesis is in ancients. if first.NumberU64() == 1 { if frozen, _ := bc.db.Ancients(); frozen == 0 { - b := bc.genesisBlock td := bc.genesisBlock.Difficulty() - writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{b}, []types.Receipts{nil}, td) - size += writeSize + writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}, td) if err != nil { log.Error("Error writing genesis to ancients", "err", err) return 0, err } + size += writeSize log.Info("Wrote genesis to ancients") } } @@ -1267,44 +1266,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Write all chain data to ancients. td := bc.GetTd(first.Hash(), first.NumberU64()) writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain, td) - size += writeSize if err != nil { log.Error("Error importing chain data to ancients", "err", err) return 0, err } - - // Write tx indices if any condition is satisfied: - // * If user requires to reserve all tx indices(txlookuplimit=0) - // * If all ancient tx indices are required to be reserved(txlookuplimit is even higher than ancientlimit) - // * If block number is large enough to be regarded as a recent block - // It means blocks below the ancientLimit-txlookupLimit won't be indexed. - // - // But if the `TxIndexTail` is not nil, e.g. Geth is initialized with - // an external ancient database, during the setup, blockchain will start - // a background routine to re-indexed all indices in [ancients - txlookupLimit, ancients) - // range. In this case, all tx indices of newly imported blocks should be - // generated. - batch := bc.db.NewBatch() - for i, block := range blockChain { - if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit { - rawdb.WriteTxLookupEntriesByBlock(batch, block) - } else if rawdb.ReadTxIndexTail(bc.db) != nil { - rawdb.WriteTxLookupEntriesByBlock(batch, block) - } - stats.processed++ - - if batch.ValueSize() > ethdb.IdealBatchSize || i == len(blockChain)-1 { - size += int64(batch.ValueSize()) - if err = batch.Write(); err != nil { - snapBlock := bc.CurrentSnapBlock().Number.Uint64() - if _, err := bc.db.TruncateHead(snapBlock + 1); err != nil { - log.Error("Can't truncate ancient store after failed insert", "err", err) - } - return 0, err - } - batch.Reset() - } - } + size += writeSize // Sync the ancient store explicitly to ensure all data has been flushed to disk. if err := bc.db.Sync(); err != nil { @@ -1322,8 +1288,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Delete block data from the main database. - batch.Reset() - canonHashes := make(map[common.Hash]struct{}) + var ( + batch = bc.db.NewBatch() + canonHashes = make(map[common.Hash]struct{}) + ) for _, block := range blockChain { canonHashes[block.Hash()] = struct{}{} if block.NumberU64() == 0 { @@ -1341,13 +1309,16 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if err := batch.Write(); err != nil { return 0, err } + stats.processed += int32(len(blockChain)) return 0, nil } // writeLive writes blockchain and corresponding receipt chain into active store. writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { - skipPresenceCheck := false - batch := bc.db.NewBatch() + var ( + skipPresenceCheck = false + batch = bc.db.NewBatch() + ) for i, block := range blockChain { // Short circuit insertion if shutting down or processing failed if bc.insertStopped() { @@ -1372,11 +1343,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // Write all the data out into the database rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i]) - rawdb.WriteTxLookupEntriesByBlock(batch, block) // Always write tx indices for live blocks, we assume they are needed // Write everything belongs to the blocks into the database. So that - // we can ensure all components of body is completed(body, receipts, - // tx indexes) + // we can ensure all components of body is completed(body, receipts) + // except transaction indexes(will be created once sync is finished). if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return 0, err @@ -1408,19 +1378,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return n, err } } - // Write the tx index tail (block number from where we index) before write any live blocks - if len(liveBlocks) > 0 && liveBlocks[0].NumberU64() == ancientLimit+1 { - // The tx index tail can only be one of the following two options: - // * 0: all ancient blocks have been indexed - // * ancient-limit: the indices of blocks before ancient-limit are ignored - if tail := rawdb.ReadTxIndexTail(bc.db); tail == nil { - if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit { - rawdb.WriteTxIndexTail(bc.db, 0) - } else { - rawdb.WriteTxIndexTail(bc.db, ancientLimit-bc.txLookupLimit) - } - } - } if len(liveBlocks) > 0 { if n, err := writeLive(liveBlocks, liveReceipts); err != nil { if err == errInsertionInterrupted { @@ -1429,13 +1386,14 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return n, err } } - - head := blockChain[len(blockChain)-1] - context := []interface{}{ - "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "number", head.Number(), "hash", head.Hash(), "age", common.PrettyAge(time.Unix(int64(head.Time()), 0)), - "size", common.StorageSize(size), - } + var ( + head = blockChain[len(blockChain)-1] + context = []interface{}{ + "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), + "number", head.Number(), "hash", head.Hash(), "age", common.PrettyAge(time.Unix(int64(head.Time()), 0)), + "size", common.StorageSize(size), + } + ) if stats.ignored > 0 { context = append(context, []interface{}{"ignored", stats.ignored}...) } @@ -1451,7 +1409,6 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e if bc.insertStopped() { return errInsertionInterrupted } - batch := bc.db.NewBatch() rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td) rawdb.WriteBlock(batch, block) @@ -1829,7 +1786,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) // The chain importer is starting and stopping trie prefetchers. If a bad // block or other error is hit however, an early return may not properly // terminate the background threads. This defer ensures that we clean up - // and dangling prefetcher, without defering each and holding on live refs. + // and dangling prefetcher, without deferring each and holding on live refs. if activeState != nil { activeState.StopPrefetcher() } @@ -2369,6 +2326,12 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // rewind the canonical chain to a lower point. log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "oldblocks", len(oldChain), "newnum", newBlock.Number(), "newhash", newBlock.Hash(), "newblocks", len(newChain)) } + // Reset the tx lookup cache in case to clear stale txlookups. + // This is done before writing any new chain data to avoid the + // weird scenario that canonical chain is changed while the + // stale lookups are still cached. + bc.txLookupCache.Purge() + // Insert the new chain(except the head block(reverse order)), // taking care of the proper incremental order. for i := len(newChain) - 1; i >= 1; i-- { @@ -2383,11 +2346,13 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Block) error { // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. - indexesBatch := bc.db.NewBatch() - for _, tx := range types.HashDifference(deletedTxs, addedTxs) { + var ( + indexesBatch = bc.db.NewBatch() + diffs = types.HashDifference(deletedTxs, addedTxs) + ) + for _, tx := range diffs { rawdb.DeleteTxLookupEntry(indexesBatch, tx) } - // Delete all hash markers that are not part of the new canonical chain. // Because the reorg function does not handle new chain head, all hash // markers greater than or equal to new chain head should be deleted. @@ -2562,102 +2527,6 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { return false } -// indexBlocks reindexes or unindexes transactions depending on user configuration -func (bc *BlockChain) indexBlocks(tail *uint64, head uint64, done chan struct{}) { - defer func() { close(done) }() - - // If head is 0, it means the chain is just initialized and no blocks are inserted, - // so don't need to indexing anything. - if head == 0 { - return - } - - // The tail flag is not existent, it means the node is just initialized - // and all blocks(may from ancient store) are not indexed yet. - if tail == nil { - from := uint64(0) - if bc.txLookupLimit != 0 && head >= bc.txLookupLimit { - from = head - bc.txLookupLimit + 1 - } - rawdb.IndexTransactions(bc.db, from, head+1, bc.quit) - return - } - // The tail flag is existent, but the whole chain is required to be indexed. - if bc.txLookupLimit == 0 || head < bc.txLookupLimit { - if *tail > 0 { - // It can happen when chain is rewound to a historical point which - // is even lower than the indexes tail, recap the indexing target - // to new head to avoid reading non-existent block bodies. - end := *tail - if end > head+1 { - end = head + 1 - } - rawdb.IndexTransactions(bc.db, 0, end, bc.quit) - } - return - } - // Update the transaction index to the new chain state - if head-bc.txLookupLimit+1 < *tail { - // Reindex a part of missing indices and rewind index tail to HEAD-limit - rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail, bc.quit) - } else { - // Unindex a part of stale indices and forward index tail to HEAD-limit - rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1, bc.quit) - } -} - -// maintainTxIndex is responsible for the construction and deletion of the -// transaction index. -// -// User can use flag `txlookuplimit` to specify a "recentness" block, below -// which ancient tx indices get deleted. If `txlookuplimit` is 0, it means -// all tx indices will be reserved. -// -// The user can adjust the txlookuplimit value for each launch after sync, -// Geth will automatically construct the missing indices or delete the extra -// indices. -func (bc *BlockChain) maintainTxIndex() { - defer bc.wg.Done() - - // Listening to chain events and manipulate the transaction indexes. - var ( - done chan struct{} // Non-nil if background unindexing or reindexing routine is active. - headCh = make(chan ChainHeadEvent, 1) // Buffered to avoid locking up the event feed - ) - sub := bc.SubscribeChainHeadEvent(headCh) - if sub == nil { - return - } - defer sub.Unsubscribe() - log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit()) - - // Launch the initial processing if chain is not empty. This step is - // useful in these scenarios that chain has no progress and indexer - // is never triggered. - if head := rawdb.ReadHeadBlock(bc.db); head != nil { - done = make(chan struct{}) - go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.NumberU64(), done) - } - - for { - select { - case head := <-headCh: - if done == nil { - done = make(chan struct{}) - go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.Block.NumberU64(), done) - } - case <-done: - done = nil - case <-bc.quit: - if done != nil { - log.Info("Waiting background transaction indexer to exit") - <-done - } - return - } - } -} - // reportBlock logs a bad block error. func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, err error) { rawdb.WriteBadBlock(bc.db, block) @@ -2724,7 +2593,7 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) { bc.flushInterval.Store(int64(interval)) } -// GetTrieFlushInterval gets the in-memory tries flush interval +// GetTrieFlushInterval gets the in-memory tries flushAlloc interval func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 716ee2fccc..c2374f14b7 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // CurrentHeader retrieves the current head header of the canonical chain. The @@ -255,20 +255,46 @@ func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, max return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) } -// GetTransactionLookup retrieves the lookup associate with the given transaction -// hash from the cache or database. -func (bc *BlockChain) GetTransactionLookup(hash common.Hash) *rawdb.LegacyTxLookupEntry { +// GetTransactionLookup retrieves the lookup along with the transaction +// itself associate with the given transaction hash. +// +// An error will be returned if the transaction is not found, and background +// indexing for transactions is still in progress. The transaction might be +// reachable shortly once it's indexed. +// +// A null will be returned in the transaction is not found and background +// transaction indexing is already finished. The transaction is not existent +// from the node's perspective. +func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction, error) { // Short circuit if the txlookup already in the cache, retrieve otherwise - if lookup, exist := bc.txLookupCache.Get(hash); exist { - return lookup + if item, exist := bc.txLookupCache.Get(hash); exist { + return item.lookup, item.transaction, nil } tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(bc.db, hash) if tx == nil { - return nil + progress, err := bc.TxIndexProgress() + if err != nil { + return nil, nil, nil + } + // The transaction indexing is not finished yet, returning an + // error to explicitly indicate it. + if !progress.Done() { + return nil, nil, errors.New("transaction indexing still in progress") + } + // The transaction is already indexed, the transaction is either + // not existent or not in the range of index, returning null. + return nil, nil, nil + } + lookup := &rawdb.LegacyTxLookupEntry{ + BlockHash: blockHash, + BlockIndex: blockNumber, + Index: txIndex, } - lookup := &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} - bc.txLookupCache.Add(hash, lookup) - return lookup + bc.txLookupCache.Add(hash, txLookup{ + lookup: lookup, + transaction: tx, + }) + return lookup, tx, nil } // GetTd retrieves a block's total difficulty in the canonical chain from the @@ -395,23 +421,24 @@ func (bc *BlockChain) GetVMConfig() *vm.Config { return &bc.vmConfig } -// SetTxLookupLimit is responsible for updating the txlookup limit to the -// original one stored in db if the new mismatches with the old one. -func (bc *BlockChain) SetTxLookupLimit(limit uint64) { - bc.txLookupLimit = limit -} - -// TxLookupLimit retrieves the txlookup limit used by blockchain to prune -// stale transaction indices. -func (bc *BlockChain) TxLookupLimit() uint64 { - return bc.txLookupLimit +// TxIndexProgress returns the transaction indexing progress. +func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { + if bc.txIndexer == nil { + return TxIndexProgress{}, errors.New("tx indexer is not enabled") + } + return bc.txIndexer.txIndexProgress() } // TrieDB retrieves the low level trie database used for data storage. -func (bc *BlockChain) TrieDB() *trie.Database { +func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb } +// HeaderChain returns the underlying header chain. +func (bc *BlockChain) HeaderChain() *HeaderChain { + return bc.hc +} + // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index fa739f924f..1504c74e0e 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -34,9 +34,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // rewindTest is a test case for chain rollback upon user request. @@ -2033,13 +2033,13 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme } // Reopen the trie database without persisting in-memory dirty nodes. chain.triedb.Close() - dbconfig := &trie.Config{} + dbconfig := &triedb.Config{} if scheme == rawdb.PathScheme { dbconfig.PathDB = pathdb.Defaults } else { dbconfig.HashDB = hashdb.Defaults } - chain.triedb = trie.NewDatabase(chain.db, dbconfig) + chain.triedb = triedb.NewDatabase(chain.db, dbconfig) chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb) // Force run a freeze cycle diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 134465aeb6..22db20a23e 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) // So we can deterministically seed different blockchains @@ -839,7 +840,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -972,7 +973,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } ) @@ -1092,7 +1093,7 @@ func testChainTxReorgs(t *testing.T, scheme string) { gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: big.NewInt(1000000000000000)}, addr2: {Balance: big.NewInt(1000000000000000)}, addr3: {Balance: big.NewInt(1000000000000000)}, @@ -1207,7 +1208,7 @@ func testLogReorgs(t *testing.T, scheme string) { // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) ) @@ -1264,7 +1265,7 @@ func testLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) @@ -1346,7 +1347,7 @@ func testSideLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) @@ -1443,7 +1444,7 @@ func testReorgSideEvent(t *testing.T, scheme string) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, } signer = types.LatestSigner(gspec.Config) ) @@ -1586,7 +1587,7 @@ func testEIP155Transition(t *testing.T, scheme string) { EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int), }, - Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } ) genDb, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, func(i int, block *BlockGen) { @@ -1701,7 +1702,7 @@ func testEIP161AccountRemoval(t *testing.T, scheme string) { EIP150Block: new(big.Int), EIP158Block: big.NewInt(2), }, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, } ) _, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, block *BlockGen) { @@ -1932,7 +1933,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) { key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: types.GenesisAlloc{address: {Balance: funds}}} ) height := uint64(1024) _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil) @@ -2137,7 +2138,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon gspec = &Genesis{ Config: &chainConfig, - Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, + Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -2723,191 +2724,6 @@ func testReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T, scheme strin } } -func TestTransactionIndices(t *testing.T) { - // Configure and generate a sample block chain - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(100000000000000000) - gspec = &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - signer = types.LatestSigner(gspec.Config) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) - if err != nil { - panic(err) - } - block.AddTx(tx) - }) - - check := func(tail *uint64, chain *BlockChain) { - stored := rawdb.ReadTxIndexTail(chain.db) - if tail == nil && stored != nil { - t.Fatalf("Oldest indexded block mismatch, want nil, have %d", *stored) - } - if tail != nil && *stored != *tail { - t.Fatalf("Oldest indexded block mismatch, want %d, have %d", *tail, *stored) - } - if tail != nil { - for i := *tail; i <= chain.CurrentBlock().Number.Uint64(); i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index == nil { - t.Fatalf("Miss transaction indice, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - for i := uint64(0); i < *tail; i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil { - t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - } - } - // Init block chain with external ancients, check all needed indices has been indexed. - limit := []uint64{0, 32, 64, 128} - for _, l := range limit { - frdir := t.TempDir() - ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - - l := l - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) - - var tail uint64 - if l != 0 { - tail = uint64(128) - l + 1 - } - check(&tail, chain) - chain.Stop() - ancientDb.Close() - os.RemoveAll(frdir) - } - - // Reconstruct a block chain which only reserves HEAD-64 tx indices - ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - defer ancientDb.Close() - - rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} - for _, l := range limit { - l := l - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - var tail uint64 - if l != 0 { - tail = uint64(128) - l + 1 - } - chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) - check(&tail, chain) - chain.Stop() - } -} - -func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { - testSkipStaleTxIndicesInSnapSync(t, rawdb.HashScheme) - testSkipStaleTxIndicesInSnapSync(t, rawdb.PathScheme) -} - -func testSkipStaleTxIndicesInSnapSync(t *testing.T, scheme string) { - // Configure and generate a sample block chain - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(100000000000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - signer = types.LatestSigner(gspec.Config) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) - if err != nil { - panic(err) - } - block.AddTx(tx) - }) - - check := func(tail *uint64, chain *BlockChain) { - stored := rawdb.ReadTxIndexTail(chain.db) - if tail == nil && stored != nil { - t.Fatalf("Oldest indexded block mismatch, want nil, have %d", *stored) - } - if tail != nil && *stored != *tail { - t.Fatalf("Oldest indexded block mismatch, want %d, have %d", *tail, *stored) - } - if tail != nil { - for i := *tail; i <= chain.CurrentBlock().Number.Uint64(); i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index == nil { - t.Fatalf("Miss transaction indice, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - for i := uint64(0); i < *tail; i++ { - block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i) - if block.Transactions().Len() == 0 { - continue - } - for _, tx := range block.Transactions() { - if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil { - t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex()) - } - } - } - } - } - - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } - defer ancientDb.Close() - - // Import all blocks into ancient db, only HEAD-32 indices are kept. - l := uint64(32) - chain, err := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - defer chain.Stop() - - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := chain.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } - // The indices before ancient-N(32) should be ignored. After that all blocks should be indexed. - if n, err := chain.InsertReceiptChain(blocks, receipts, 64); err != nil { - t.Fatalf("block %d: failed to insert into chain: %v", n, err) - } - tail := uint64(32) - check(&tail, chain) -} - // Benchmarks large blocks with value transfers to non-existing accounts func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) { var ( @@ -2917,7 +2733,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in bankFunds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ testBankAddress: {Balance: bankFunds}, common.HexToAddress("0xc0de"): { Code: []byte{0x60, 0x01, 0x50}, @@ -3095,7 +2911,7 @@ func testDeleteCreateRevert(t *testing.T, scheme string) { funds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3219,7 +3035,7 @@ func testDeleteRecreateSlots(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3305,7 +3121,7 @@ func testDeleteRecreateAccount(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3426,7 +3242,7 @@ func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { t.Logf("Destination address: %x\n", aa) gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAAA selfdestructs if called aa: { @@ -3621,7 +3437,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address aa has some funds aa: {Balance: big.NewInt(100000)}, @@ -3653,7 +3469,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { defer chain.Stop() statedb, _ := chain.State() - if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 { + if got, exp := statedb.GetBalance(aa), uint256.NewInt(100000); got.Cmp(exp) != 0 { t.Fatalf("Genesis err, got %v exp %v", got, exp) } // First block tries to create, but fails @@ -3663,7 +3479,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) } statedb, _ = chain.State() - if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 { + if got, exp := statedb.GetBalance(aa), uint256.NewInt(100000); got.Cmp(exp) != 0 { t.Fatalf("block %d: got %v exp %v", block.NumberU64(), got, exp) } } @@ -3696,7 +3512,7 @@ func testEIP2718Transition(t *testing.T, scheme string) { funds = big.NewInt(1000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 aa: { @@ -3781,7 +3597,7 @@ func testEIP1559Transition(t *testing.T, scheme string) { config = *params.AllEthashProtocolChanges gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 @@ -3849,17 +3665,17 @@ func testEIP1559Transition(t *testing.T, scheme string) { state, _ := chain.State() // 3: Ensure that miner received only the tx's tip. - actual := state.GetBalance(block.Coinbase()) + actual := state.GetBalance(block.Coinbase()).ToBig() expected := new(big.Int).Add( new(big.Int).SetUint64(block.GasUsed()*block.Transactions()[0].GasTipCap().Uint64()), - ethash.ConstantinopleBlockReward, + ethash.ConstantinopleBlockReward.ToBig(), ) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (tip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr1)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr1).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (block.Transactions()[0].GasTipCap().Uint64() + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) @@ -3889,17 +3705,17 @@ func testEIP1559Transition(t *testing.T, scheme string) { effectiveTip := block.Transactions()[0].GasTipCap().Uint64() - block.BaseFee().Uint64() // 6+5: Ensure that miner received only the tx's effective tip. - actual = state.GetBalance(block.Coinbase()) + actual = state.GetBalance(block.Coinbase()).ToBig() expected = new(big.Int).Add( new(big.Int).SetUint64(block.GasUsed()*effectiveTip), - ethash.ConstantinopleBlockReward, + ethash.ConstantinopleBlockReward.ToBig(), ) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (effectiveTip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr2)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr2).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (effectiveTip + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) @@ -3922,7 +3738,7 @@ func testSetCanonical(t *testing.T, scheme string) { funds = big.NewInt(100000000000000000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } signer = types.LatestSigner(gspec.Config) @@ -4039,7 +3855,7 @@ func testCanonicalHashMarker(t *testing.T, scheme string) { var ( gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{}, + Alloc: types.GenesisAlloc{}, BaseFee: big.NewInt(params.InitialBaseFee), } engine = ethash.NewFaker() @@ -4104,212 +3920,6 @@ func testCanonicalHashMarker(t *testing.T, scheme string) { } } -// TestTxIndexer tests the tx indexes are updated correctly. -func TestTxIndexer(t *testing.T) { - var ( - testBankKey, _ = crypto.GenerateKey() - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1000000000000000000) - - gspec = &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - engine = ethash.NewFaker() - nonce = uint64(0) - ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 128, func(i int, gen *BlockGen) { - tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) - gen.AddTx(tx) - nonce += 1 - }) - - // verifyIndexes checks if the transaction indexes are present or not - // of the specified block. - verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { - if number == 0 { - return - } - block := blocks[number-1] - for _, tx := range block.Transactions() { - lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) - if exist && lookup == nil { - t.Fatalf("missing %d %x", number, tx.Hash().Hex()) - } - if !exist && lookup != nil { - t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) - } - } - } - // verifyRange runs verifyIndexes for a range of blocks, from and to are included. - verifyRange := func(db ethdb.Database, from, to uint64, exist bool) { - for number := from; number <= to; number += 1 { - verifyIndexes(db, number, exist) - } - } - verify := func(db ethdb.Database, expTail uint64) { - tail := rawdb.ReadTxIndexTail(db) - if tail == nil { - t.Fatal("Failed to write tx index tail") - } - if *tail != expTail { - t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) - } - if *tail != 0 { - verifyRange(db, 0, *tail-1, false) - } - verifyRange(db, *tail, 128, true) - } - - var cases = []struct { - limitA uint64 - tailA uint64 - limitB uint64 - tailB uint64 - limitC uint64 - tailC uint64 - }{ - { - // LimitA: 0 - // TailA: 0 - // - // all blocks are indexed - limitA: 0, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 64 - // TailA: 65 - // - // block [65, 128] are indexed - limitA: 64, - tailA: 65, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 127 - // TailA: 2 - // - // block [2, 128] are indexed - limitA: 127, - tailA: 2, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 128 - // TailA: 1 - // - // block [2, 128] are indexed - limitA: 128, - tailA: 1, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - { - // LimitA: 129 - // TailA: 0 - // - // block [0, 128] are indexed - limitA: 129, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, - }, - } - for _, c := range cases { - frdir := t.TempDir() - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) - - // Index the initial blocks from ancient store - chain, _ := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, &c.limitA) - chain.indexBlocks(nil, 128, make(chan struct{})) - verify(db, c.tailA) - - chain.SetTxLookupLimit(c.limitB) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, c.tailB) - - chain.SetTxLookupLimit(c.limitC) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, c.tailC) - - // Recover all indexes - chain.SetTxLookupLimit(0) - chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) - verify(db, 0) - - chain.Stop() - db.Close() - os.RemoveAll(frdir) - } -} - func TestCreateThenDeletePreByzantium(t *testing.T) { // We use Ropsten chain config instead of Testchain config, this is // deliberate: we want to use pre-byz rules where we have intermediate state roots @@ -4358,7 +3968,7 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { }...) gspec := &Genesis{ Config: config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4444,7 +4054,7 @@ func TestDeleteThenCreate(t *testing.T) { gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4556,7 +4166,7 @@ func TestTransientStorageReset(t *testing.T) { }...) gspec := &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, }, } @@ -4624,7 +4234,7 @@ func TestEIP3651(t *testing.T) { config = *params.AllEthashProtocolChanges gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, // The address 0xAAAA sloads 0x00 and 0x01 @@ -4704,14 +4314,14 @@ func TestEIP3651(t *testing.T) { state, _ := chain.State() // 3: Ensure that miner received only the tx's tip. - actual := state.GetBalance(block.Coinbase()) + actual := state.GetBalance(block.Coinbase()).ToBig() expected := new(big.Int).SetUint64(block.GasUsed() * block.Transactions()[0].GasTipCap().Uint64()) if actual.Cmp(expected) != 0 { t.Fatalf("miner balance incorrect: expected %d, got %d", expected, actual) } // 4: Ensure the tx sender paid for the gasUsed * (tip + block baseFee). - actual = new(big.Int).Sub(funds, state.GetBalance(addr1)) + actual = new(big.Int).Sub(funds, state.GetBalance(addr1).ToBig()) expected = new(big.Int).SetUint64(block.GasUsed() * (block.Transactions()[0].GasTipCap().Uint64() + block.BaseFee().Uint64())) if actual.Cmp(expected) != 0 { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) diff --git a/core/chain_makers.go b/core/chain_makers.go index 0c121a7631..abf01d4ad2 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -31,7 +31,8 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/holiman/uint256" ) // BlockGen creates blocks for testing. @@ -82,7 +83,7 @@ func (b *BlockGen) SetDifficulty(diff *big.Int) { b.header.Difficulty = diff } -// SetPos makes the header a PoS-header (0 difficulty) +// SetPoS makes the header a PoS-header (0 difficulty) func (b *BlockGen) SetPoS() { b.header.Difficulty = new(big.Int) } @@ -157,7 +158,7 @@ func (b *BlockGen) AddTxWithVMConfig(tx *types.Transaction, config vm.Config) { } // GetBalance returns the balance of the given address at the generated block. -func (b *BlockGen) GetBalance(addr common.Address) *big.Int { +func (b *BlockGen) GetBalance(addr common.Address) *uint256.Int { return b.statedb.GetBalance(addr) } @@ -311,7 +312,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } cm := newChainMaker(parent, config, engine) - genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { + genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} b.header = cm.makeHeader(parent, statedb, b.engine) @@ -364,7 +365,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) defer triedb.Close() for i := 0; i < n; i++ { @@ -409,7 +410,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) defer triedb.Close() _, err := genesis.Commit(db, triedb) if err != nil { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 84148841f5..b46b898afb 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func TestGeneratePOSChain(t *testing.T) { @@ -46,7 +46,7 @@ func TestGeneratePOSChain(t *testing.T) { asm4788 = common.Hex2Bytes("3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500") gspec = &Genesis{ Config: &config, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, params.BeaconRootsStorageAddress: {Balance: common.Big0, Code: asm4788}, }, @@ -69,19 +69,19 @@ func TestGeneratePOSChain(t *testing.T) { storage[common.Hash{0x01}] = common.Hash{0x01} storage[common.Hash{0x02}] = common.Hash{0x02} storage[common.Hash{0x03}] = common.HexToHash("0303") - gspec.Alloc[aa] = GenesisAccount{ + gspec.Alloc[aa] = types.Account{ Balance: common.Big1, Nonce: 1, Storage: storage, Code: common.Hex2Bytes("6042"), } - gspec.Alloc[bb] = GenesisAccount{ + gspec.Alloc[bb] = types.Account{ Balance: common.Big2, Nonce: 1, Storage: storage, Code: common.Hex2Bytes("600154600354"), } - genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults)) + genesis := gspec.MustCommit(gendb, triedb.NewDatabase(gendb, triedb.HashDefaults)) genchain, genreceipts := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { gen.SetParentBeaconRoot(common.Hash{byte(i + 1)}) @@ -202,9 +202,9 @@ func ExampleGenerateChain() { // Ensure that key1 has some funds in the genesis block. gspec := &Genesis{ Config: ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, - Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, } - genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults)) + genesis := gspec.MustCommit(genDb, triedb.NewDatabase(genDb, triedb.HashDefaults)) // This call generates a chain of 5 blocks. The function runs for // each block and adds different features to gen based on the diff --git a/core/error.go b/core/error.go index 43f2510680..8c691b17ff 100644 --- a/core/error.go +++ b/core/error.go @@ -105,6 +105,12 @@ var ( // blob gas fee of the block. ErrBlobFeeCapTooLow = errors.New("max fee per blob gas less than block blob gas fee") + // ErrMissingBlobHashes is returned if a blob transaction has no blob hashes. + ErrMissingBlobHashes = errors.New("blob transaction missing blob hashes") + + // ErrBlobTxCreate is returned if a blob transaction has no explicit to field. + ErrBlobTxCreate = errors.New("blob transaction of type create") + // ErrSystemTxNotSupported is returned for any deposit tx with IsSystemTx=true after the Regolith fork ErrSystemTxNotSupported = errors.New("system tx not supported") ) diff --git a/core/evm.go b/core/evm.go index b0ac3276e0..d1e826d2a3 100644 --- a/core/evm.go +++ b/core/evm.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // ChainContext supports retrieving headers and consensus parameters from the @@ -131,12 +132,12 @@ func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash // CanTransfer checks whether there are enough funds in the address' account to make a transfer. // This does not take the necessary gas in to account to make the transfer valid. -func CanTransfer(db vm.StateDB, addr common.Address, amount *big.Int) bool { +func CanTransfer(db vm.StateDB, addr common.Address, amount *uint256.Int) bool { return db.GetBalance(addr).Cmp(amount) >= 0 } // Transfer subtracts amount from sender and adds amount to recipient using the given Db -func Transfer(db vm.StateDB, sender, recipient common.Address, amount *big.Int) { +func Transfer(db vm.StateDB, sender, recipient common.Address, amount *uint256.Int) { db.SubBalance(sender, amount) db.AddBalance(recipient, amount) } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index e311c0b43f..b9d346bd90 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -74,8 +74,10 @@ func TestCreation(t *testing.T) { {15049999, 0, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block {15050000, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // First Gray Glacier block {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block - {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // First Shanghai block - {30000000, 2000000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}}, // Future Shanghai block + {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block + {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block + {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block }, }, // Goerli test cases @@ -91,8 +93,10 @@ func TestCreation(t *testing.T) { {5000000, 0, ID{Hash: checksumToBytes(0x757a1c47), Next: 5062605}}, // Last Berlin block {5062605, 0, ID{Hash: checksumToBytes(0xB8C6299D), Next: 1678832736}}, // First London block {6000000, 1678832735, ID{Hash: checksumToBytes(0xB8C6299D), Next: 1678832736}}, // Last London block - {6000001, 1678832736, ID{Hash: checksumToBytes(0xf9843abf), Next: 0}}, // First Shanghai block - {6500000, 2678832736, ID{Hash: checksumToBytes(0xf9843abf), Next: 0}}, // Future Shanghai block + {6000001, 1678832736, ID{Hash: checksumToBytes(0xf9843abf), Next: 1705473120}}, // First Shanghai block + {6500002, 1705473119, ID{Hash: checksumToBytes(0xf9843abf), Next: 1705473120}}, // Last Shanghai block + {6500003, 1705473120, ID{Hash: checksumToBytes(0x70cc14e2), Next: 0}}, // First Cancun block + {6500003, 2705473120, ID{Hash: checksumToBytes(0x70cc14e2), Next: 0}}, // Future Cancun block }, }, // Sepolia test cases @@ -104,7 +108,10 @@ func TestCreation(t *testing.T) { {1735370, 0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Last London block {1735371, 0, ID{Hash: checksumToBytes(0xb96cbd13), Next: 1677557088}}, // First MergeNetsplit block {1735372, 1677557087, ID{Hash: checksumToBytes(0xb96cbd13), Next: 1677557088}}, // Last MergeNetsplit block - {1735372, 1677557088, ID{Hash: checksumToBytes(0xf7f9bc08), Next: 0}}, // First Shanghai block + {1735372, 1677557088, ID{Hash: checksumToBytes(0xf7f9bc08), Next: 1706655072}}, // First Shanghai block + {1735372, 1706655071, ID{Hash: checksumToBytes(0xf7f9bc08), Next: 1706655072}}, // Last Shanghai block + {1735372, 1706655072, ID{Hash: checksumToBytes(0x88cf81d9), Next: 0}}, // First Cancun block + {1735372, 2706655072, ID{Hash: checksumToBytes(0x88cf81d9), Next: 0}}, // Future Cancun block }, }, // Holesky test cases @@ -112,9 +119,12 @@ func TestCreation(t *testing.T) { params.HoleskyChainConfig, core.DefaultHoleskyGenesisBlock().ToBlock(), []testcase{ - {0, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin, London, Paris block - {123, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // First MergeNetsplit block - {123, 1696000704, ID{Hash: checksumToBytes(0xfd4f016b), Next: 0}}, // Last MergeNetsplit block + {0, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin, London, Paris block + {123, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // First MergeNetsplit block + {123, 1696000704, ID{Hash: checksumToBytes(0xfd4f016b), Next: 1707305664}}, // First Shanghai block + {123, 1707305663, ID{Hash: checksumToBytes(0xfd4f016b), Next: 1707305664}}, // Last Shanghai block + {123, 1707305664, ID{Hash: checksumToBytes(0x9b192ad0), Next: 0}}, // First Cancun block + {123, 2707305664, ID{Hash: checksumToBytes(0x9b192ad0), Next: 0}}, // Future Cancun block }, }, } @@ -133,6 +143,7 @@ func TestValidation(t *testing.T) { // Config that has not timestamp enabled legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil + legacyConfig.CancunTime = nil tests := []struct { config *params.ChainConfig @@ -205,14 +216,10 @@ func TestValidation(t *testing.T) { // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 88888888, 0, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. - // - // TODO(karalabe): This testcase will fail once mainnet gets timestamped forks, make legacy chain config {&legacyConfig, 7279999, 0, ID{Hash: checksumToBytes(0xa00bc324), Next: 7279999}, ErrLocalIncompatibleOrStale}, //------------------------------------ @@ -289,34 +296,25 @@ func TestValidation(t *testing.T) { // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, but it's not yet aware of Cancun (e.g. non updated node before the fork). // In this case we don't know if Cancun passed yet or not. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of Cancun (e.g. updated node before the fork). We // don't know if Cancun passed yet (will pass) or not. - // - // TODO(karalabe): Enable this when Cancun is specced and update next timestamp - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet currently in Shanghai only (so it's aware of Cancun), remote announces // also Shanghai, and it's also aware of some random fork (e.g. misconfigured Cancun). As // neither forks passed at neither nodes, they may mismatch, but we still connect for now. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0x71147644), Next: math.MaxUint64}, nil}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(0xdce96c2d), Next: math.MaxUint64}, nil}, // Local is mainnet exactly on Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - // {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Cancun, remote announces Shanghai + knowledge about Cancun. Remote // is simply out of sync, accept. - // TODO(karalabe): Enable this when Cancun is specced, update local head and time, next timestamp - //{params.MainnetChainConfig, 21123456, 1678123456, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, nil}, + {params.MainnetChainConfig, 21123456, 1710338136, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}, nil}, // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. @@ -325,9 +323,7 @@ func TestValidation(t *testing.T) { //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. @@ -337,9 +333,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. - // - // TODO(karalabe): Enable this when Cancun is specced, update local head and time - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x71147644), Next: 0}, ErrRemoteStale}, + {params.MainnetChainConfig, 21000000, 1710338135, ID{Hash: checksumToBytes(0xdce96c2d), Next: 0}, ErrRemoteStale}, // Local is mainnet Shanghai, and isn't aware of more forks. Remote announces Shanghai + // 0xffffffff. Local needs software update, reject. @@ -347,24 +341,20 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, and is aware of Cancun. Remote announces Cancun + // 0xffffffff. Local needs software update, reject. - // - // TODO(karalabe): Enable this when Cancun is specced, update remote checksum - //{params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x00000000, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20000000, 1668000000, ID{Hash: checksumToBytes(checksumUpdate(0x9f3d2254, math.MaxUint64)), Next: 0}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Shanghai, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xdce96c2d), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. - // - // TODO(karalabe): Enable this when Cancun is specced - //{params.MainnetChainConfig, 20999999, 1677999999, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 20999999, 1699999999, ID{Hash: checksumToBytes(0x71147644), Next: 1700000000}, ErrLocalIncompatibleOrStale}, } genesis := core.DefaultGenesisBlock().ToBlock() for i, tt := range tests { diff --git a/core/gen_genesis.go b/core/gen_genesis.go index bd2a069b54..4c0be7ed4d 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" ) @@ -18,22 +19,22 @@ var _ = (*genesisSpecMarshaling)(nil) // MarshalJSON marshals as JSON. func (g Genesis) MarshalJSON() ([]byte, error) { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce math.HexOrDecimal64 `json:"nonce"` - Timestamp math.HexOrDecimal64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash common.Hash `json:"mixHash"` - Coinbase common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number math.HexOrDecimal64 `json:"number"` - GasUsed math.HexOrDecimal64 `json:"gasUsed"` - ParentHash common.Hash `json:"parentHash"` - BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` - ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` - BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` - StateHash *common.Hash `json:"stateHash,omitempty"` + Config *params.ChainConfig `json:"config"` + Nonce math.HexOrDecimal64 `json:"nonce"` + Timestamp math.HexOrDecimal64 `json:"timestamp"` + ExtraData hexutil.Bytes `json:"extraData"` + GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash common.Hash `json:"mixHash"` + Coinbase common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]types.Account `json:"alloc" gencodec:"required"` + Number math.HexOrDecimal64 `json:"number"` + GasUsed math.HexOrDecimal64 `json:"gasUsed"` + ParentHash common.Hash `json:"parentHash"` + BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` + BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + StateHash *common.Hash `json:"stateHash,omitempty"` } var enc Genesis enc.Config = g.Config @@ -45,7 +46,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.Mixhash = g.Mixhash enc.Coinbase = g.Coinbase if g.Alloc != nil { - enc.Alloc = make(map[common.UnprefixedAddress]GenesisAccount, len(g.Alloc)) + enc.Alloc = make(map[common.UnprefixedAddress]types.Account, len(g.Alloc)) for k, v := range g.Alloc { enc.Alloc[common.UnprefixedAddress(k)] = v } @@ -113,7 +114,7 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { if dec.Alloc == nil { return errors.New("missing required field 'alloc' for Genesis") } - g.Alloc = make(GenesisAlloc, len(dec.Alloc)) + g.Alloc = make(types.GenesisAlloc, len(dec.Alloc)) for k, v := range dec.Alloc { g.Alloc[common.Address(k)] = v } diff --git a/core/genesis.go b/core/genesis.go index bb6c82da37..8434bafd7e 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -19,7 +19,6 @@ package core import ( "bytes" "embed" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -41,17 +40,24 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" ) //go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go -//go:generate go run github.com/fjl/gencodec -type GenesisAccount -field-override genesisAccountMarshaling -out gen_genesis_account.go //go:embed assets/mainnet assets/testnet assets/qanet var assetFS embed.FS var errGenesisNoConfig = errors.New("genesis has no chain configuration") +// Deprecated: use types.GenesisAccount instead. +type GenesisAccount = types.Account + +// Deprecated: use types.GenesisAlloc instead. +type GenesisAlloc = types.GenesisAlloc + // Genesis specifies the header fields, state of a genesis block. It also defines hard // fork switch-over blocks through the chain configuration. type Genesis struct { @@ -63,7 +69,7 @@ type Genesis struct { Difficulty *big.Int `json:"difficulty" gencodec:"required"` Mixhash common.Hash `json:"mixHash"` Coinbase common.Address `json:"coinbase"` - Alloc GenesisAlloc `json:"alloc" gencodec:"required"` + Alloc types.GenesisAlloc `json:"alloc" gencodec:"required"` // These fields are used for consensus tests. Please don't use them // in actual genesis blocks. @@ -122,29 +128,14 @@ func ReadGenesis(db ethdb.Database) (*Genesis, error) { return &genesis, nil } -// GenesisAlloc specifies the initial state that is part of the genesis block. -type GenesisAlloc map[common.Address]GenesisAccount - -func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { - m := make(map[common.UnprefixedAddress]GenesisAccount) - if err := json.Unmarshal(data, &m); err != nil { - return err - } - *ga = make(GenesisAlloc) - for addr, a := range m { - (*ga)[common.Address(addr)] = a - } - return nil -} - -// hash computes the state root according to the genesis specification. -func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { +// hashAlloc computes the state root according to the genesis specification. +func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { // If a genesis-time verkle trie is requested, create a trie config // with the verkle trie enabled so that the tree can be initialized // as such. - var config *trie.Config + var config *triedb.Config if isVerkle { - config = &trie.Config{ + config = &triedb.Config{ PathDB: pathdb.Defaults, IsVerkle: true, } @@ -158,7 +149,7 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, account.Balance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance)) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -169,10 +160,10 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { return statedb.Commit(0, false) } -// flush is very similar with hash, but the main difference is all the generated +// flushAlloc is very similar with hash, but the main difference is all the generated // states will be persisted into the given database. Also, the genesis state // specification will be flushed as well. -func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { +func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database, blockhash common.Hash) error { triedbConfig := triedb.Config() if triedbConfig != nil { triedbConfig.NoTries = false @@ -184,7 +175,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, account.Balance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance)) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -211,15 +202,6 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas return nil } -// GenesisAccount is an account in the state of the genesis block. -type GenesisAccount struct { - Code []byte `json:"code,omitempty"` - Storage map[common.Hash]common.Hash `json:"storage,omitempty"` - Balance *big.Int `json:"balance" gencodec:"required"` - Nonce uint64 `json:"nonce,omitempty"` - PrivateKey []byte `json:"secretKey,omitempty"` // for tests -} - // field type overrides for gencodec type genesisSpecMarshaling struct { Nonce math.HexOrDecimal64 @@ -229,40 +211,12 @@ type genesisSpecMarshaling struct { GasUsed math.HexOrDecimal64 Number math.HexOrDecimal64 Difficulty *math.HexOrDecimal256 - Alloc map[common.UnprefixedAddress]GenesisAccount + Alloc map[common.UnprefixedAddress]types.Account BaseFee *math.HexOrDecimal256 ExcessBlobGas *math.HexOrDecimal64 BlobGasUsed *math.HexOrDecimal64 } -type genesisAccountMarshaling struct { - Code hexutil.Bytes - Balance *math.HexOrDecimal256 - Nonce math.HexOrDecimal64 - Storage map[storageJSON]storageJSON - PrivateKey hexutil.Bytes -} - -// storageJSON represents a 256 bit byte array, but allows less than 256 bits when -// unmarshaling from hex. -type storageJSON common.Hash - -func (h *storageJSON) UnmarshalText(text []byte) error { - text = bytes.TrimPrefix(text, []byte("0x")) - if len(text) > 64 { - return fmt.Errorf("too many hex characters in storage key/value %q", text) - } - offset := len(h) - len(text)/2 // pad on the left - if _, err := hex.Decode(h[offset:], text); err != nil { - return fmt.Errorf("invalid hex storage key/value %q", text) - } - return nil -} - -func (h storageJSON) MarshalText() ([]byte, error) { - return hexutil.Bytes(h[:]).MarshalText() -} - // GenesisMismatchError is raised when trying to overwrite an existing // genesis block with an incompatible one. type GenesisMismatchError struct { @@ -280,6 +234,7 @@ type ChainOverrides struct { // optimism OverrideOptimismCanyon *uint64 OverrideOptimismEcotone *uint64 + OverrideOptimismFjord *uint64 ApplySuperchainUpgrades bool OverrideOptimismInterop *uint64 } @@ -297,11 +252,11 @@ type ChainOverrides struct { // error is a *params.ConfigCompatError and the new, unwritten config is returned. // // The returned chain configuration is never nil. -func SetupGenesisBlock(db ethdb.Database, triedb *trie.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { return SetupGenesisBlockWithOverride(db, triedb, genesis, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -345,6 +300,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen config.EcotoneTime = overrides.OverrideOptimismEcotone config.CancunTime = overrides.OverrideOptimismEcotone } + if overrides != nil && overrides.OverrideOptimismFjord != nil { + config.FjordTime = overrides.OverrideOptimismFjord + } if overrides != nil && overrides.OverrideOptimismInterop != nil { config.InteropTime = overrides.OverrideOptimismInterop } @@ -479,6 +437,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return g.Config case ghash == params.MainnetGenesisHash: return params.MainnetChainConfig + case ghash == params.HoleskyGenesisHash: + return params.HoleskyChainConfig case ghash == params.SepoliaGenesisHash: return params.SepoliaChainConfig case ghash == params.GoerliGenesisHash: @@ -510,7 +470,7 @@ func (g *Genesis) ToBlock() *types.Block { "and non-empty state-allocation", *g.StateHash)) } root = *g.StateHash - } else if root, err = g.Alloc.hash(g.IsVerkle()); err != nil { + } else if root, err = hashAlloc(&g.Alloc, g.IsVerkle()); err != nil { panic(err) } head := &types.Header{ @@ -568,7 +528,7 @@ func (g *Genesis) ToBlock() *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block, error) { +func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") @@ -583,10 +543,10 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - // All the checks has passed, flush the states derived from the genesis + // All the checks has passed, flushAlloc the states derived from the genesis // specification as well as the specification itself into the provided // database. - if err := g.Alloc.flush(db, triedb, block.Hash()); err != nil { + if err := flushAlloc(&g.Alloc, db, triedb, block.Hash()); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -602,7 +562,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. -func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { +func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block { block, err := g.Commit(db, triedb) if err != nil { panic(err) @@ -670,7 +630,7 @@ func DeveloperGenesisBlock(gasLimit uint64, faucet *common.Address) *Genesis { GasLimit: gasLimit, BaseFee: big.NewInt(params.InitialBaseFee), Difficulty: big.NewInt(1), - Alloc: map[common.Address]GenesisAccount{ + Alloc: map[common.Address]types.Account{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD @@ -683,12 +643,12 @@ func DeveloperGenesisBlock(gasLimit uint64, faucet *common.Address) *Genesis { }, } if faucet != nil { - genesis.Alloc[*faucet] = GenesisAccount{Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))} + genesis.Alloc[*faucet] = types.Account{Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))} } return genesis } -func decodePrealloc(data string) GenesisAlloc { +func decodePrealloc(data string) types.GenesisAlloc { var p []struct { Addr *big.Int Balance *big.Int @@ -704,9 +664,9 @@ func decodePrealloc(data string) GenesisAlloc { if err := rlp.NewStream(strings.NewReader(data), 0).Decode(&p); err != nil { panic(err) } - ga := make(GenesisAlloc, len(p)) + ga := make(types.GenesisAlloc, len(p)) for _, account := range p { - acc := GenesisAccount{Balance: account.Balance} + acc := types.Account{Balance: account.Balance} if account.Misc != nil { acc.Nonce = account.Misc.Nonce acc.Code = account.Misc.Code diff --git a/core/genesis_test.go b/core/genesis_test.go index 1d85b510ca..61be0bd252 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -27,18 +27,19 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) func TestInvalidCliqueConfig(t *testing.T) { block := DefaultGoerliGenesisBlock() block.ExtraData = []byte{} db := rawdb.NewMemoryDatabase() - if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { + if _, err := block.Commit(db, triedb.NewDatabase(db, nil)); err == nil { t.Fatal("Expected error on invalid clique config") } } @@ -53,7 +54,7 @@ func testSetupGenesis(t *testing.T, scheme string) { customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customg = Genesis{ Config: ¶ms.ChainConfig{HomesteadBlock: big.NewInt(3)}, - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, }, } @@ -71,7 +72,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "genesis without ChainConfig", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis)) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), new(Genesis)) }, wantErr: errGenesisNoConfig, wantConfig: params.AllEthashProtocolChanges, @@ -79,7 +80,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "no block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -87,8 +88,8 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "mainnet block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme))) - return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) + DefaultGenesisBlock().MustCommit(db, triedb.NewDatabase(db, newDbConfig(scheme))) + return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -96,7 +97,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, nil) }, @@ -106,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "custom block in DB, genesis == goerli", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock()) }, @@ -117,7 +118,7 @@ func testSetupGenesis(t *testing.T, scheme string) { { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) oldcustomg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, &customg) }, @@ -129,7 +130,7 @@ func testSetupGenesis(t *testing.T, scheme string) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. - tdb := trie.NewDatabase(db, newDbConfig(scheme)) + tdb := triedb.NewDatabase(db, newDbConfig(scheme)) oldcustomg.Commit(db, tdb) bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) @@ -188,7 +189,7 @@ func TestGenesisHashes(t *testing.T) { } { // Test via MustCommit db := rawdb.NewMemoryDatabase() - if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want { + if have := c.genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)).Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock @@ -206,7 +207,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) + genesisBlock := genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") @@ -228,16 +229,16 @@ func TestGenesis_Commit(t *testing.T) { func TestReadWriteGenesisAlloc(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - alloc = &GenesisAlloc{ + alloc = &types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash, _ = alloc.hash(false) + hash, _ = hashAlloc(alloc, false) ) blob, _ := json.Marshal(alloc) rawdb.WriteGenesisStateSpec(db, hash, blob) - var reload GenesisAlloc + var reload types.GenesisAlloc err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) @@ -256,11 +257,11 @@ func TestReadWriteGenesisAlloc(t *testing.T) { } } -func newDbConfig(scheme string) *trie.Config { +func newDbConfig(scheme string) *triedb.Config { if scheme == rawdb.HashScheme { - return trie.HashDefaults + return triedb.HashDefaults } - return &trie.Config{PathDB: pathdb.Defaults} + return &triedb.Config{PathDB: pathdb.Defaults} } func TestVerkleGenesisCommit(t *testing.T) { @@ -298,7 +299,7 @@ func TestVerkleGenesisCommit(t *testing.T) { Config: verkleConfig, Timestamp: verkleTime, Difficulty: big.NewInt(0), - Alloc: GenesisAlloc{ + Alloc: types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, }, } @@ -310,7 +311,7 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(db, &trie.Config{IsVerkle: true, PathDB: pathdb.Defaults}) + triedb := triedb.NewDatabase(db, &triedb.Config{IsVerkle: true, PathDB: pathdb.Defaults}) block := genesis.MustCommit(db, triedb) if !bytes.Equal(block.Root().Bytes(), expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 2c0323e6f7..25d9bfffcb 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func verifyUnbrokenCanonchain(hc *HeaderChain) error { @@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, trie.NewDatabase(db, nil)) + gspec.Commit(db, triedb.NewDatabase(db, nil)) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/core/proof_keeper.go b/core/proof_keeper.go index 720e808d1c..61123f3101 100644 --- a/core/proof_keeper.go +++ b/core/proof_keeper.go @@ -17,7 +17,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" trie2 "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) const ( @@ -217,7 +217,7 @@ func (keeper *ProofKeeper) getInnerProof(kRecord *pathdb.KeepRecord) (*proofData StateRoot: kRecord.StateRoot, Address: l2ToL1MessagePasserAddr, AccountProof: accountProof, - Balance: stateDB.GetBalance(l2ToL1MessagePasserAddr), + Balance: stateDB.GetBalance(l2ToL1MessagePasserAddr).ToBig(), CodeHash: stateDB.GetCodeHash(l2ToL1MessagePasserAddr), Nonce: stateDB.GetNonce(l2ToL1MessagePasserAddr), StorageHash: stateDB.GetStorageRoot(l2ToL1MessagePasserAddr), diff --git a/core/proof_keeper_test.go b/core/proof_keeper_test.go index a081f58b1a..0aea4574c3 100644 --- a/core/proof_keeper_test.go +++ b/core/proof_keeper_test.go @@ -8,7 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/stretchr/testify/assert" ) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 40d60236c1..2b201f3290 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -278,23 +278,6 @@ func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) { } } -// ReadFastTxLookupLimit retrieves the tx lookup limit used in fast sync. -func ReadFastTxLookupLimit(db ethdb.KeyValueReader) *uint64 { - data, _ := db.Get(fastTxLookupLimitKey) - if len(data) != 8 { - return nil - } - number := binary.BigEndian.Uint64(data) - return &number -} - -// WriteFastTxLookupLimit stores the txlookup limit used in fast sync into database. -func WriteFastTxLookupLimit(db ethdb.KeyValueWriter, number uint64) { - if err := db.Put(fastTxLookupLimitKey, encodeBlockNumber(number)); err != nil { - log.Crit("Failed to store transaction lookup limit for fast sync", "err", err) - } -} - // ReadHeaderRange returns the rlp-encoded headers, starting at 'number', and going // backwards towards genesis. This method assumes that the caller already has // placed a cap on count, to prevent DoS issues. @@ -333,8 +316,8 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu if count == 0 { return rlpHeaders } - // read remaining from ancients - data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0) + // read remaining from ancients, cap at 2M + data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024) if err != nil { log.Error("Failed to read headers from freezer", "err", err) return rlpHeaders diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 56bb15b718..c2343f937f 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -178,7 +178,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { +func indexTransactions(db ethdb.Database, from uint64, to uint64, lastTailCh chan *uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) { // short circuit for invalid range if from >= to { return @@ -188,13 +188,13 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan batch = db.NewBatch() start = time.Now() logged = start.Add(-7 * time.Second) + // Since we iterate in reverse, we expect the first number to come // in to be [to-1]. Therefore, setting lastNum to means that the - // prqueue gap-evaluation will work correctly - lastNum = to - queue = prque.New[int64, *blockTxHashes](nil) - // for stats reporting - blocks, txs = 0, 0 + // queue gap-evaluation will work correctly + lastNum = to + queue = prque.New[int64, *blockTxHashes](nil) + blocks, txs = 0, 0 // for stats reporting ) for chanDelivery := range hashesCh { // Push the delivery into the queue and process contiguous ranges. @@ -240,11 +240,18 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan log.Crit("Failed writing batch to db", "error", err) return } + if lastTailCh != nil { + lastTailCh <- &lastNum + } + logger := log.Debug + if report { + logger = log.Info + } select { case <-interrupt: - log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) } } @@ -257,20 +264,20 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { - indexTransactions(db, from, to, interrupt, nil) +func IndexTransactions(db ethdb.Database, from uint64, to uint64, lastTailCh chan *uint64, interrupt chan struct{}, report bool) { + indexTransactions(db, from, to, lastTailCh, interrupt, nil, report) } // indexTransactionsForTesting is the internal debug version with an additional hook. func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { - indexTransactions(db, from, to, interrupt, hook) + indexTransactions(db, from, to, nil, interrupt, hook, false) } // unindexTransactions removes txlookup indices of the specified block range. // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { +func unindexTransactions(db ethdb.Database, from uint64, to uint64, lastTailCh chan *uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) { // short circuit for invalid range if from >= to { return @@ -280,12 +287,12 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch batch = db.NewBatch() start = time.Now() logged = start.Add(-7 * time.Second) + // we expect the first number to come in to be [from]. Therefore, setting - // nextNum to from means that the prqueue gap-evaluation will work correctly - nextNum = from - queue = prque.New[int64, *blockTxHashes](nil) - // for stats reporting - blocks, txs = 0, 0 + // nextNum to from means that the queue gap-evaluation will work correctly + nextNum = from + queue = prque.New[int64, *blockTxHashes](nil) + blocks, txs = 0, 0 // for stats reporting ) // Otherwise spin up the concurrent iterator and unindexer for delivery := range hashesCh { @@ -332,11 +339,18 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch log.Crit("Failed writing batch to db", "error", err) return } + if lastTailCh != nil { + lastTailCh <- &nextNum + } + logger := log.Debug + if report { + logger = log.Info + } select { case <-interrupt: - log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) } } @@ -345,11 +359,11 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch // // There is a passed channel, the whole procedure will be interrupted if any // signal received. -func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { - unindexTransactions(db, from, to, interrupt, nil) +func UnindexTransactions(db ethdb.Database, from uint64, to uint64, lastTailCh chan *uint64, interrupt chan struct{}, report bool) { + unindexTransactions(db, from, to, lastTailCh, interrupt, nil, report) } // unindexTransactionsForTesting is the internal debug version with an additional hook. func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { - unindexTransactions(db, from, to, interrupt, hook) + unindexTransactions(db, from, to, nil, interrupt, hook, false) } diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 9580cd92a8..7ec23d6885 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -162,18 +162,18 @@ func TestIndexTransactions(t *testing.T) { t.Fatalf("Transaction tail mismatch") } } - IndexTransactions(chainDb, 5, 11, nil) + IndexTransactions(chainDb, 5, 11, nil, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - IndexTransactions(chainDb, 0, 5, nil) + IndexTransactions(chainDb, 0, 5, nil, nil, false) verify(0, 11, true, 0) - UnindexTransactions(chainDb, 0, 5, nil) + UnindexTransactions(chainDb, 0, 5, nil, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - UnindexTransactions(chainDb, 5, 11, nil) + UnindexTransactions(chainDb, 5, 11, nil, nil, false) verify(0, 11, false, 11) // Testing corner cases @@ -190,7 +190,7 @@ func TestIndexTransactions(t *testing.T) { }) verify(9, 11, true, 9) verify(0, 9, false, 9) - IndexTransactions(chainDb, 0, 9, nil) + IndexTransactions(chainDb, 0, 9, nil, nil, false) signal = make(chan struct{}) var once2 sync.Once diff --git a/core/rawdb/database.go b/core/rawdb/database.go index ba222a6b46..9b0a5885c0 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -679,7 +679,6 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { {"snapshotRecoveryNumber", pp(ReadSnapshotRecoveryNumber(db))}, {"snapshotRoot", fmt.Sprintf("%v", ReadSnapshotRoot(db))}, {"txIndexTail", pp(ReadTxIndexTail(db))}, - {"fastTxLookupLimit", pp(ReadFastTxLookupLimit(db))}, } if b := ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 4471463932..91b4943e59 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -894,7 +894,7 @@ func getChunk(size int, b int) []byte { } // TODO (?) -// - test that if we remove several head-files, aswell as data last data-file, +// - test that if we remove several head-files, as well as data last data-file, // the index is truncated accordingly // Right now, the freezer would fail on these conditions: // 1. have data files d0, d1, d2, d3 diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 4fd1d3fcc9..32457eeb5b 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -80,6 +80,8 @@ var ( txIndexTailKey = []byte("TransactionIndexTail") // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. + // This flag is deprecated, it's kept to avoid reporting errors when inspect + // database. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") // badBlockKey tracks the list of bad blocks seen by local diff --git a/core/rlp_test.go b/core/rlp_test.go index a2fb4937f8..bc37408537 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -41,7 +41,7 @@ func getBlock(transactions int, uncles int, dataSize int) *types.Block { funds = big.NewInt(1_000_000_000_000_000_000) gspec = &Genesis{ Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, } ) // We need to generate as many blocks +1 as uncles diff --git a/core/state/database.go b/core/state/database.go index 1a753fdfc6..61e778ca73 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/triedb" ) const ( @@ -67,7 +68,7 @@ type Database interface { DiskDB() ethdb.KeyValueStore // TrieDB returns the underlying trie database for managing trie nodes. - TrieDB() *trie.Database + TrieDB() *triedb.Database // NoTries returns whether the database has tries storage. NoTries() bool @@ -153,18 +154,18 @@ func NewDatabase(db ethdb.Database) Database { // NewDatabaseWithConfig creates a backing store for state. The returned database // is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a // large memory cache. -func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { +func NewDatabaseWithConfig(db ethdb.Database, config *triedb.Config) Database { return &cachingDB{ disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: trie.NewDatabase(db, config), + triedb: triedb.NewDatabase(db, config), noTries: config != nil && config.NoTries, } } // NewDatabaseWithNodeDB creates a state database with an already initialized node database. -func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { +func NewDatabaseWithNodeDB(db ethdb.Database, triedb *triedb.Database) Database { return &cachingDB{ disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), @@ -178,7 +179,7 @@ type cachingDB struct { disk ethdb.KeyValueStore codeSizeCache *lru.Cache[common.Hash, int] codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - triedb *trie.Database + triedb *triedb.Database noTries bool } @@ -275,7 +276,7 @@ func (db *cachingDB) DiskDB() ethdb.KeyValueStore { } // TrieDB retrieves any intermediate trie-node caching layer. -func (db *cachingDB) TrieDB() *trie.Database { +func (db *cachingDB) TrieDB() *triedb.Database { return db.triedb } diff --git a/core/state/journal.go b/core/state/journal.go index 137ec76395..6cdc1fc868 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -17,9 +17,8 @@ package state import ( - "math/big" - "github.com/ethereum/go-ethereum/common" + "github.com/holiman/uint256" ) // journalEntry is a modification entry in the state change journal that can be @@ -103,13 +102,13 @@ type ( selfDestructChange struct { account *common.Address prev bool // whether account had already self-destructed - prevbalance *big.Int + prevbalance *uint256.Int } // Changes to individual accounts. balanceChange struct { account *common.Address - prev *big.Int + prev *uint256.Int } nonceChange struct { account *common.Address diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 9f068eaf2d..dad2b5b2a8 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -27,17 +27,10 @@ import ( bloomfilter "github.com/holiman/bloomfilter/v2" ) -// stateBloomHasher is a wrapper around a byte blob to satisfy the interface API -// requirements of the bloom library used. It's used to convert a trie hash or -// contract code hash into a 64 bit mini hash. -type stateBloomHasher []byte - -func (f stateBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (f stateBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (f stateBloomHasher) Reset() { panic("not implemented") } -func (f stateBloomHasher) BlockSize() int { panic("not implemented") } -func (f stateBloomHasher) Size() int { return 8 } -func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } +// stateBloomHash is used to convert a trie hash or contract code hash into a 64 bit mini hash. +func stateBloomHash(f []byte) uint64 { + return binary.BigEndian.Uint64(f) +} // stateBloom is a bloom filter used during the state conversion(snapshot->state). // The keys of all generated entries will be recorded here so that in the pruning @@ -113,10 +106,10 @@ func (bloom *stateBloom) Put(key []byte, value []byte) error { if !isCode { return errors.New("invalid entry") } - bloom.bloom.Add(stateBloomHasher(codeKey)) + bloom.bloom.AddHash(stateBloomHash(codeKey)) return nil } - bloom.bloom.Add(stateBloomHasher(key)) + bloom.bloom.AddHash(stateBloomHash(key)) return nil } @@ -128,5 +121,5 @@ func (bloom *stateBloom) Delete(key []byte) error { panic("not supported") } // - If it says yes, the key may be contained // - If it says no, the key is definitely not contained. func (bloom *stateBloom) Contain(key []byte) bool { - return bloom.bloom.Contains(stateBloomHasher(key)) + return bloom.bloom.ContainsHash(stateBloomHash(key)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 5431512db3..2d0724187b 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "github.com/holiman/uint256" "math" "os" "path/filepath" @@ -37,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) const ( @@ -165,7 +167,7 @@ func pruneAll(pruneDB ethdb.Database, g *core.Genesis) error { } statedb, _ := state.New(common.Hash{}, state.NewDatabase(pruneDB), nil) for addr, account := range g.Alloc { - statedb.AddBalance(addr, account.Balance) + statedb.AddBalance(addr, uint256.MustFromBig(account.Balance)) statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) for key, value := range account.Storage { @@ -186,7 +188,7 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { return nil, errors.New("failed to load head block") } // Offline pruning is only supported in legacy hash based scheme. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) snapconfig := snapshot.Config{ CacheSize: 256, @@ -221,7 +223,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta // the trie nodes(and codes) belong to the active state will be filtered // out. A very small part of stale tries will also be filtered because of // the false-positive rate of bloom filter. But the assumption is held here - // that the false-positive is low enough(~0.05%). The probablity of the + // that the false-positive is low enough(~0.05%). The probability of the // dangling node is the state root is super low. So the dangling nodes in // theory will never ever be visited again. var ( @@ -459,7 +461,7 @@ func RecoverPruning(datadir string, db ethdb.Database) error { AsyncBuild: false, } // Offline pruning is only supported in legacy hash based scheme. - triedb := trie.NewDatabase(db, trie.HashDefaults) + triedb := triedb.NewDatabase(db, triedb.HashDefaults) snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root()) if err != nil { return err // The relevant snapshot(s) might not exist @@ -502,7 +504,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), triedb.NewDatabase(db, triedb.HashDefaults)) if err != nil { return err } @@ -526,7 +528,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { } if acc.Root != types.EmptyRootHash { id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults)) + storageTrie, err := trie.NewStateTrie(id, triedb.NewDatabase(db, triedb.HashDefaults)) if err != nil { return err } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 681be7ebc0..8a0fd1989a 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -362,15 +362,15 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou } func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) { - options := trie.NewStackTrieOptions() + var onTrieNode trie.OnTrieNode if db != nil { - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { + onTrieNode = func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme) - }) + } } - t := trie.NewStackTrie(options) + t := trie.NewStackTrie(onTrieNode) for leaf := range in { t.Update(leaf.key[:], leaf.value) } - out <- t.Commit() + out <- t.Hash() } diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index b6aca599c5..70c9f44189 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -43,7 +43,7 @@ var ( aggregatorMemoryLimit = uint64(4 * 1024 * 1024) // aggregatorItemLimit is an approximate number of items that will end up - // in the agregator layer before it's flushed out to disk. A plain account + // in the aggregator layer before it's flushed out to disk. A plain account // weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot // 0B (+hash). Slots are mostly set/unset in lockstep, so that average at // 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a @@ -124,47 +124,20 @@ type diffLayer struct { lock sync.RWMutex } -// destructBloomHasher is a wrapper around a common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert a destruct -// event into a 64 bit mini hash. -type destructBloomHasher common.Hash - -func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h destructBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h destructBloomHasher) Reset() { panic("not implemented") } -func (h destructBloomHasher) BlockSize() int { panic("not implemented") } -func (h destructBloomHasher) Size() int { return 8 } -func (h destructBloomHasher) Sum64() uint64 { +// destructBloomHash is used to convert a destruct event into a 64 bit mini hash. +func destructBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) } -// accountBloomHasher is a wrapper around a common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert an account -// hash into a 64 bit mini hash. -type accountBloomHasher common.Hash - -func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h accountBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h accountBloomHasher) Reset() { panic("not implemented") } -func (h accountBloomHasher) BlockSize() int { panic("not implemented") } -func (h accountBloomHasher) Size() int { return 8 } -func (h accountBloomHasher) Sum64() uint64 { +// accountBloomHash is used to convert an account hash into a 64 bit mini hash. +func accountBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) } -// storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface -// API requirements of the bloom library used. It's used to convert an account -// hash into a 64 bit mini hash. -type storageBloomHasher [2]common.Hash - -func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } -func (h storageBloomHasher) Sum(b []byte) []byte { panic("not implemented") } -func (h storageBloomHasher) Reset() { panic("not implemented") } -func (h storageBloomHasher) BlockSize() int { panic("not implemented") } -func (h storageBloomHasher) Size() int { return 8 } -func (h storageBloomHasher) Sum64() uint64 { - return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^ - binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) +// storageBloomHash is used to convert an account hash and a storage hash into a 64 bit mini hash. +func storageBloomHash(h0, h1 common.Hash) uint64 { + return binary.BigEndian.Uint64(h0[bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^ + binary.BigEndian.Uint64(h1[bloomStorageHasherOffset:bloomStorageHasherOffset+8]) } // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low @@ -233,14 +206,14 @@ func (dl *diffLayer) rebloom(origin *diskLayer) { } // Iterate over all the accounts and storage slots and index them for hash := range dl.destructSet { - dl.diffed.Add(destructBloomHasher(hash)) + dl.diffed.AddHash(destructBloomHash(hash)) } for hash := range dl.accountData { - dl.diffed.Add(accountBloomHasher(hash)) + dl.diffed.AddHash(accountBloomHash(hash)) } for accountHash, slots := range dl.storageData { for storageHash := range slots { - dl.diffed.Add(storageBloomHasher{accountHash, storageHash}) + dl.diffed.AddHash(storageBloomHash(accountHash, storageHash)) } } // Calculate the current false positive rate and update the error rate meter. @@ -301,9 +274,9 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - hit := dl.diffed.Contains(accountBloomHasher(hash)) + hit := dl.diffed.ContainsHash(accountBloomHash(hash)) if !hit { - hit = dl.diffed.Contains(destructBloomHasher(hash)) + hit = dl.diffed.ContainsHash(destructBloomHash(hash)) } var origin *diskLayer if !hit { @@ -372,9 +345,9 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro dl.lock.RUnlock() return nil, ErrSnapshotStale } - hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash}) + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) if !hit { - hit = dl.diffed.Contains(destructBloomHasher(accountHash)) + hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) } var origin *diskLayer if !hit { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index d563b67ca4..f5518a204c 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -26,13 +26,13 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // diskLayer is a low level persistent snapshot built on top of a key-value store. type diskLayer struct { diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot - triedb *trie.Database // Trie node cache for reconstruction purposes + triedb *triedb.Database // Trie node cache for reconstruction purposes cache *fastcache.Cache // Cache to avoid hitting the disk for direct access root common.Hash // Root hash of the base snapshot diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index f95b798515..168458c405 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -139,7 +139,7 @@ func TestDiskMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("update not flattend into the disk layer") + t.Fatalf("update not flattened into the disk layer") } // assertAccount ensures that an account matches the given blob. @@ -362,7 +362,7 @@ func TestDiskPartialMerge(t *testing.T) { // Retrieve all the data through the disk layer and validate it base = snaps.Snapshot(diffRoot) if _, ok := base.(*diskLayer); !ok { - t.Fatalf("test %d: update not flattend into the disk layer", i) + t.Fatalf("test %d: update not flattened into the disk layer", i) } assertAccount(accNoModNoCache, accNoModNoCache[:]) assertAccount(accNoModCache, accNoModCache[:]) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f455a6db3f..8de4b134d3 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -55,7 +56,7 @@ var ( // generateSnapshot regenerates a brand new snapshot based on an existing state // database and head block asynchronously. The snapshot is returned immediately // and generation is continued in the background until done. -func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *diskLayer { +func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, cache int, root common.Hash) *diskLayer { // Create a new disk layer with an initialized state marker at zero var ( stats = &generatorStats{start: time.Now()} @@ -353,7 +354,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi var resolver trie.NodeResolver if len(result.keys) > 0 { mdb := rawdb.NewMemoryDatabase() - tdb := trie.NewDatabase(mdb, trie.HashDefaults) + tdb := triedb.NewDatabase(mdb, triedb.HashDefaults) defer tdb.Close() snapTrie := trie.NewEmpty(tdb) for i, key := range result.keys { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index c25f3e7e8b..da93ebc875 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -18,7 +18,6 @@ package snapshot import ( "fmt" - "math/big" "os" "testing" "time" @@ -30,9 +29,11 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -58,9 +59,9 @@ func testGeneration(t *testing.T, scheme string) { var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -97,16 +98,16 @@ func testGenerateExistentState(t *testing.T, scheme string) { var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) root, snap := helper.CommitAndGenerate() @@ -155,20 +156,20 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { type testHelper struct { diskdb ethdb.Database - triedb *trie.Database + triedb *triedb.Database accTrie *trie.StateTrie nodes *trienode.MergedNodeSet } func newHelper(scheme string) *testHelper { diskdb := rawdb.NewMemoryDatabase() - config := &trie.Config{} + config := &triedb.Config{} if scheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{} // disable caching } else { config.HashDB = &hashdb.Config{} // disable caching } - triedb := trie.NewDatabase(diskdb, config) + triedb := triedb.NewDatabase(diskdb, config) accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb) return &testHelper{ diskdb: diskdb, @@ -259,28 +260,28 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { helper := newHelper(scheme) // Account one, empty root but non-empty database - helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account two, non empty root but empty database stRoot := helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Miss slots { // Account three, non empty root but misses slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) // Account four, non empty root but misses slots in the middle helper.makeStorageTrie(hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-4", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) // Account five, non empty root but misses slots in the end helper.makeStorageTrie(hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-5", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } @@ -288,22 +289,22 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { { // Account six, non empty root but wrong slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-6", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) // Account seven, non empty root but wrong slots in the middle helper.makeStorageTrie(hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-7", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) // Account eight, non empty root but wrong slots in the end helper.makeStorageTrie(hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-8", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-8", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) // Account 9, non empty root but rotated slots helper.makeStorageTrie(hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-9", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-9", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } @@ -311,17 +312,17 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { { // Account 10, non empty root but extra slots in the beginning helper.makeStorageTrie(hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-10", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-10", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) // Account 11, non empty root but extra slots in the middle helper.makeStorageTrie(hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-11", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-11", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) // Account 12, non empty root but extra slots in the end helper.makeStorageTrie(hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-12", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-12", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } @@ -366,25 +367,25 @@ func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) { // Missing accounts, only in the trie { - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning - helper.addTrieAccount("acc-4", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle - helper.addTrieAccount("acc-6", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning + helper.addTrieAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle + helper.addTrieAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End } // Wrong accounts { - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) } // Extra accounts, only in the snap { - helper.addSnapAccount("acc-0", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning - helper.addSnapAccount("acc-5", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: common.Hex2Bytes("0x1234")}) // Middle - helper.addSnapAccount("acc-7", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // after the end + helper.addSnapAccount("acc-0", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning + helper.addSnapAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: common.Hex2Bytes("0x1234")}) // Middle + helper.addSnapAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // after the end } root, snap := helper.CommitAndGenerate() @@ -418,9 +419,9 @@ func testGenerateCorruptAccountTrie(t *testing.T, scheme string) { // without any storage slots to keep the test smaller. helper := newHelper(scheme) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 @@ -462,11 +463,11 @@ func testGenerateMissingStorageTrie(t *testing.T, scheme string) { acc3 = hashData([]byte("acc-3")) helper = newHelper(scheme) ) - stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -502,11 +503,11 @@ func testGenerateCorruptStorageTrie(t *testing.T, scheme string) { // two of which also has the same 3-slot storage trie attached. helper := newHelper(scheme) - stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -546,7 +547,7 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -566,7 +567,7 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -622,7 +623,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { []string{"val-1", "val-2", "val-3"}, true, ) - acc := &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -636,7 +637,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { { // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { - acc := &types.StateAccount{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -678,7 +679,7 @@ func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) { } helper := newHelper(scheme) { - acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate(common.HexToHash("0x03").Bytes(), val) helper.accTrie.MustUpdate(common.HexToHash("0x07").Bytes(), val) @@ -720,7 +721,7 @@ func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) { } helper := newHelper(scheme) { - acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.MustUpdate(common.HexToHash("0x03").Bytes(), val) @@ -764,7 +765,7 @@ func testGenerateFromEmptySnap(t *testing.T, scheme string) { for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), - &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) } root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 @@ -806,7 +807,7 @@ func testGenerateWithIncompleteStorage(t *testing.T, scheme string) { for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) stRoot := helper.makeStorageTrie(hashData([]byte(accKey)), stKeys, stVals, true) - helper.addAccount(accKey, &types.StateAccount{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount(accKey, &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) var moddedKeys []string var moddedVals []string for ii := 0; ii < 8; ii++ { @@ -903,11 +904,11 @@ func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addAccount("acc-2", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -943,11 +944,11 @@ func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) populateDangling(helper.diskdb) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 696f831a36..fa71e81836 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) const journalVersion uint64 = 0 @@ -120,7 +120,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool, withoutTrie bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, root common.Hash, cache int, recovery bool, noBuild bool, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 759298ccb5..3077468b48 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -169,7 +169,7 @@ type Config struct { type Tree struct { config Config // Snapshots configurations diskdb ethdb.KeyValueStore // Persistent database to store the snapshot - triedb *trie.Database // In-memory cache to access the trie through + triedb *triedb.Database // In-memory cache to access the trie through layers map[common.Hash]snapshot // Collection of all known layers lock sync.RWMutex @@ -193,7 +193,7 @@ type Tree struct { // state trie. // - otherwise, the entire snapshot is considered invalid and will be recreated on // a background thread. -func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) { +func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, root common.Hash) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ config: config, @@ -259,6 +259,14 @@ func (t *Tree) Disable() { for _, layer := range t.layers { switch layer := layer.(type) { case *diskLayer: + + layer.lock.RLock() + generating := layer.genMarker != nil + layer.lock.RUnlock() + if !generating { + // Generator is already aborted or finished + break + } // If the base layer is generating, abort it if layer.genAbort != nil { abort := make(chan *generatorStats) diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index b66799757e..a9ab3eaea3 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -20,7 +20,6 @@ import ( crand "crypto/rand" "encoding/binary" "fmt" - "math/big" "math/rand" "testing" "time" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) // randomHash generates a random blob of data and returns it as a hash. @@ -44,7 +44,7 @@ func randomHash() common.Hash { // randomAccount generates a random account and returns it RLP encoded. func randomAccount() []byte { a := &types.StateAccount{ - Balance: big.NewInt(rand.Int63()), + Balance: uint256.NewInt(rand.Uint64()), Nonce: rand.Uint64(), Root: randomHash(), CodeHash: types.EmptyCodeHash[:], diff --git a/core/state/state_object.go b/core/state/state_object.go index 52df7db225..8696557845 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "io" - "math/big" "sync" "time" @@ -32,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" ) type Code []byte @@ -96,7 +96,7 @@ type stateObject struct { // empty returns whether the account is considered empty. func (s *stateObject) empty() bool { - return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) + return s.data.Nonce == 0 && s.data.Balance.IsZero() && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes()) } // newObject creates a state object. @@ -439,36 +439,36 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { // AddBalance adds amount to s's balance. // It is used to add funds to the destination account of a transfer. -func (s *stateObject) AddBalance(amount *big.Int) { +func (s *stateObject) AddBalance(amount *uint256.Int) { // EIP161: We must check emptiness for the objects such that the account // clearing (0,0,0 objects) can take effect. - if amount.Sign() == 0 { + if amount.IsZero() { if s.empty() { s.touch() } return } - s.SetBalance(new(big.Int).Add(s.Balance(), amount)) + s.SetBalance(new(uint256.Int).Add(s.Balance(), amount)) } // SubBalance removes amount from s's balance. // It is used to remove funds from the origin account of a transfer. -func (s *stateObject) SubBalance(amount *big.Int) { - if amount.Sign() == 0 { +func (s *stateObject) SubBalance(amount *uint256.Int) { + if amount.IsZero() { return } - s.SetBalance(new(big.Int).Sub(s.Balance(), amount)) + s.SetBalance(new(uint256.Int).Sub(s.Balance(), amount)) } -func (s *stateObject) SetBalance(amount *big.Int) { +func (s *stateObject) SetBalance(amount *uint256.Int) { s.db.journal.append(balanceChange{ account: &s.address, - prev: new(big.Int).Set(s.data.Balance), + prev: new(uint256.Int).Set(s.data.Balance), }) s.setBalance(amount) } -func (s *stateObject) setBalance(amount *big.Int) { +func (s *stateObject) setBalance(amount *uint256.Int) { s.data.Balance = amount } @@ -568,7 +568,7 @@ func (s *stateObject) CodeHash() []byte { return s.data.CodeHash } -func (s *stateObject) Balance() *big.Int { +func (s *stateObject) Balance() *uint256.Int { return s.data.Balance } diff --git a/core/state/state_test.go b/core/state/state_test.go index 2f45ba44b4..9be610f962 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "encoding/json" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -27,7 +26,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/holiman/uint256" ) type stateEnv struct { @@ -43,17 +43,17 @@ func newStateEnv() *stateEnv { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateEnv{db: db, state: sdb} // generate a few entries - obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(big.NewInt(22)) - obj2 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) + obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) + obj1.AddBalance(uint256.NewInt(22)) + obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) - obj3 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(big.NewInt(44)) + obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) + obj3.SetBalance(uint256.NewInt(44)) // write some of them to the trie s.state.updateStateObject(obj1) @@ -100,19 +100,19 @@ func TestDump(t *testing.T) { func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateEnv{db: db, state: sdb} // generate a few entries - obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) - obj1.AddBalance(big.NewInt(22)) - obj2 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) + obj1 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01})) + obj1.AddBalance(uint256.NewInt(22)) + obj2 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) - obj3 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x02})) - obj3.SetBalance(big.NewInt(44)) - obj4 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x00})) - obj4.AddBalance(big.NewInt(1337)) + obj3 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x02})) + obj3.SetBalance(uint256.NewInt(44)) + obj4 := s.state.getOrNewStateObject(common.BytesToAddress([]byte{0x00})) + obj4.AddBalance(uint256.NewInt(1337)) // write some of them to the trie s.state.updateStateObject(obj1) @@ -208,7 +208,7 @@ func TestSnapshot2(t *testing.T) { // db, trie are already non-empty values so0 := state.getStateObject(stateobjaddr0) - so0.SetBalance(big.NewInt(42)) + so0.SetBalance(uint256.NewInt(42)) so0.SetNonce(43) so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}) so0.selfDestructed = false @@ -220,7 +220,7 @@ func TestSnapshot2(t *testing.T) { // and one with deleted == true so1 := state.getStateObject(stateobjaddr1) - so1.SetBalance(big.NewInt(52)) + so1.SetBalance(uint256.NewInt(52)) so1.SetNonce(53) so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}) so1.selfDestructed = true diff --git a/core/state/statedb.go b/core/state/statedb.go index 41f988e3ae..f5464eb23c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -19,7 +19,6 @@ package state import ( "fmt" - "math/big" "runtime" "sort" "sync" @@ -38,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/holiman/uint256" ) const ( @@ -336,12 +336,12 @@ func (s *StateDB) Empty(addr common.Address) bool { } // GetBalance retrieves the balance from the given address or 0 if object not found -func (s *StateDB) GetBalance(addr common.Address) *big.Int { +func (s *StateDB) GetBalance(addr common.Address) *uint256.Int { stateObject := s.getStateObject(addr) if stateObject != nil { return stateObject.Balance() } - return common.Big0 + return common.U2560 } // GetNonce retrieves the nonce from the given address or 0 if object not found @@ -387,10 +387,10 @@ func (s *StateDB) GetCodeSize(addr common.Address) int { func (s *StateDB) GetCodeHash(addr common.Address) common.Hash { stateObject := s.getStateObject(addr) - if stateObject == nil { - return common.Hash{} + if stateObject != nil { + return common.BytesToHash(stateObject.CodeHash()) } - return common.BytesToHash(stateObject.CodeHash()) + return common.Hash{} } // GetState retrieves a value from the given account's storage trie. @@ -429,44 +429,44 @@ func (s *StateDB) HasSelfDestructed(addr common.Address) bool { */ // AddBalance adds amount to the account associated with addr. -func (s *StateDB) AddBalance(addr common.Address, amount *big.Int) { - stateObject := s.GetOrNewStateObject(addr) +func (s *StateDB) AddBalance(addr common.Address, amount *uint256.Int) { + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.AddBalance(amount) } } // SubBalance subtracts amount from the account associated with addr. -func (s *StateDB) SubBalance(addr common.Address, amount *big.Int) { - stateObject := s.GetOrNewStateObject(addr) +func (s *StateDB) SubBalance(addr common.Address, amount *uint256.Int) { + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SubBalance(amount) } } -func (s *StateDB) SetBalance(addr common.Address, amount *big.Int) { - stateObject := s.GetOrNewStateObject(addr) +func (s *StateDB) SetBalance(addr common.Address, amount *uint256.Int) { + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetBalance(amount) } } func (s *StateDB) SetNonce(addr common.Address, nonce uint64) { - stateObject := s.GetOrNewStateObject(addr) + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetNonce(nonce) } } func (s *StateDB) SetCode(addr common.Address, code []byte) { - stateObject := s.GetOrNewStateObject(addr) + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetCode(crypto.Keccak256Hash(code), code) } } func (s *StateDB) SetState(addr common.Address, key, value common.Hash) { - stateObject := s.GetOrNewStateObject(addr) + stateObject := s.getOrNewStateObject(addr) if stateObject != nil { stateObject.SetState(key, value) } @@ -487,7 +487,7 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common if _, ok := s.stateObjectsDestruct[addr]; !ok { s.stateObjectsDestruct[addr] = nil } - stateObject := s.GetOrNewStateObject(addr) + stateObject := s.getOrNewStateObject(addr) for k, v := range storage { stateObject.SetState(k, v) } @@ -506,10 +506,10 @@ func (s *StateDB) SelfDestruct(addr common.Address) { s.journal.append(selfDestructChange{ account: &addr, prev: stateObject.selfDestructed, - prevbalance: new(big.Int).Set(stateObject.Balance()), + prevbalance: new(uint256.Int).Set(stateObject.Balance()), }) stateObject.markSelfdestructed() - stateObject.data.Balance = new(big.Int) + stateObject.data.Balance = new(uint256.Int) } func (s *StateDB) Selfdestruct6780(addr common.Address) { @@ -677,8 +677,8 @@ func (s *StateDB) setStateObject(object *stateObject) { s.stateObjects[object.Address()] = object } -// GetOrNewStateObject retrieves a state object or create a new state object if nil. -func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { +// getOrNewStateObject retrieves a state object or create a new state object if nil. +func (s *StateDB) getOrNewStateObject(addr common.Address) *stateObject { stateObject := s.getStateObject(addr) if stateObject == nil { stateObject, _ = s.createObject(addr) @@ -1081,12 +1081,10 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo nodes = trienode.NewNodeSet(addrHash) slots = make(map[common.Hash][]byte) ) - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { + stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeleted()) size += common.StorageSize(len(path)) }) - stack := trie.NewStackTrie(options) for iter.Next() { if size > storageDeleteLimit { return true, size, nil, nil, nil diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index c4704257c7..b416bcf1f3 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "math/big" "math/rand" "reflect" "strings" @@ -36,8 +35,10 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" ) // A stateTest checks that the state changes are correctly captured. Instances @@ -60,7 +61,7 @@ func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, big.NewInt(a.args[0])) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, @@ -181,7 +182,7 @@ func (test *stateTest) run() bool { storageList = append(storageList, copy2DSet(states.Storages)) } disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, &trie.Config{PathDB: pathdb.Defaults}) + tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) @@ -252,7 +253,7 @@ func (test *stateTest) run() bool { // - the account was indeed not present in trie // - the account is present in new trie, nil->nil is regarded as invalid // - the slots transition is correct -func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -303,7 +304,7 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database // - the account was indeed present in trie // - the account in old trie matches the provided value // - the slots transition is correct -func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -357,7 +358,7 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, return nil } -func (test *stateTest) verify(root common.Hash, next common.Hash, db *trie.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { +func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { otr, err := trie.New(trie.StateTrieID(root), db) if err != nil { return err diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 8936edbb3a..e71c984f12 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "math/big" "math/rand" "reflect" "strings" @@ -30,8 +29,6 @@ import ( "testing" "testing/quick" - "github.com/holiman/uint256" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -39,9 +36,11 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" ) // Tests that updating a state trie does not leak any database writes prior to @@ -50,14 +49,14 @@ func TestUpdateLeaks(t *testing.T) { // Create an empty state database var ( db = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(db, nil) + tdb = triedb.NewDatabase(db, nil) ) state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil) // Update it with some accounts for i := byte(0); i < 255; i++ { addr := common.BytesToAddress([]byte{i}) - state.AddBalance(addr, big.NewInt(int64(11*i))) + state.AddBalance(addr, uint256.NewInt(uint64(11*i))) state.SetNonce(addr, uint64(42*i)) if i%2 == 0 { state.SetState(addr, common.BytesToHash([]byte{i, i, i}), common.BytesToHash([]byte{i, i, i, i})) @@ -86,13 +85,13 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase() - transNdb := trie.NewDatabase(transDb, nil) - finalNdb := trie.NewDatabase(finalDb, nil) + transNdb := triedb.NewDatabase(transDb, nil) + finalNdb := triedb.NewDatabase(finalDb, nil) transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil) finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) modify := func(state *StateDB, addr common.Address, i, tweak byte) { - state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) + state.SetBalance(addr, uint256.NewInt(uint64(11*i)+uint64(tweak))) state.SetNonce(addr, uint64(42*i+tweak)) if i%2 == 0 { state.SetState(addr, common.Hash{i, i, i, 0}, common.Hash{}) @@ -167,8 +166,8 @@ func TestCopy(t *testing.T) { orig, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) for i := byte(0); i < 255; i++ { - obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - obj.AddBalance(big.NewInt(int64(i))) + obj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) + obj.AddBalance(uint256.NewInt(uint64(i))) orig.updateStateObject(obj) } orig.Finalise(false) @@ -181,13 +180,13 @@ func TestCopy(t *testing.T) { // modify all in memory for i := byte(0); i < 255; i++ { - origObj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - copyObj := copy.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - ccopyObj := ccopy.GetOrNewStateObject(common.BytesToAddress([]byte{i})) + origObj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) + copyObj := copy.getOrNewStateObject(common.BytesToAddress([]byte{i})) + ccopyObj := ccopy.getOrNewStateObject(common.BytesToAddress([]byte{i})) - origObj.AddBalance(big.NewInt(2 * int64(i))) - copyObj.AddBalance(big.NewInt(3 * int64(i))) - ccopyObj.AddBalance(big.NewInt(4 * int64(i))) + origObj.AddBalance(uint256.NewInt(2 * uint64(i))) + copyObj.AddBalance(uint256.NewInt(3 * uint64(i))) + ccopyObj.AddBalance(uint256.NewInt(4 * uint64(i))) orig.updateStateObject(origObj) copy.updateStateObject(copyObj) @@ -209,17 +208,17 @@ func TestCopy(t *testing.T) { // Verify that the three states have been updated independently for i := byte(0); i < 255; i++ { - origObj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - copyObj := copy.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - ccopyObj := ccopy.GetOrNewStateObject(common.BytesToAddress([]byte{i})) + origObj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) + copyObj := copy.getOrNewStateObject(common.BytesToAddress([]byte{i})) + ccopyObj := ccopy.getOrNewStateObject(common.BytesToAddress([]byte{i})) - if want := big.NewInt(3 * int64(i)); origObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(3 * uint64(i)); origObj.Balance().Cmp(want) != 0 { t.Errorf("orig obj %d: balance mismatch: have %v, want %v", i, origObj.Balance(), want) } - if want := big.NewInt(4 * int64(i)); copyObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(4 * uint64(i)); copyObj.Balance().Cmp(want) != 0 { t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, copyObj.Balance(), want) } - if want := big.NewInt(5 * int64(i)); ccopyObj.Balance().Cmp(want) != 0 { + if want := uint256.NewInt(5 * uint64(i)); ccopyObj.Balance().Cmp(want) != 0 { t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, ccopyObj.Balance(), want) } } @@ -267,14 +266,14 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { { name: "SetBalance", fn: func(a testAction, s *StateDB) { - s.SetBalance(addr, big.NewInt(a.args[0])) + s.SetBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, { name: "AddBalance", fn: func(a testAction, s *StateDB) { - s.AddBalance(addr, big.NewInt(a.args[0])) + s.AddBalance(addr, uint256.NewInt(uint64(a.args[0]))) }, args: make([]int64, 1), }, @@ -532,12 +531,12 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { func TestTouchDelete(t *testing.T) { s := newStateEnv() - s.state.GetOrNewStateObject(common.Address{}) + s.state.getOrNewStateObject(common.Address{}) root, _ := s.state.Commit(0, false) s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() - s.state.AddBalance(common.Address{}, new(big.Int)) + s.state.AddBalance(common.Address{}, new(uint256.Int)) if len(s.state.journal.dirties) != 1 { t.Fatal("expected one dirty state object") @@ -553,7 +552,7 @@ func TestTouchDelete(t *testing.T) { func TestCopyOfCopy(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.HexToAddress("aaaa") - state.SetBalance(addr, big.NewInt(42)) + state.SetBalance(addr, uint256.NewInt(42)) if got := state.Copy().GetBalance(addr).Uint64(); got != 42 { t.Fatalf("1st copy fail, expected 42, got %v", got) @@ -576,11 +575,11 @@ func TestCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -594,7 +593,7 @@ func TestCopyCommitCopy(t *testing.T) { } // Copy the non-committed state database and check pre/post commit balance copyOne := state.Copy() - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyOne.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("first copy pre-commit balance mismatch: have %v, want %v", balance, 42) } if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -608,7 +607,7 @@ func TestCopyCommitCopy(t *testing.T) { } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyTwo.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("second copy balance mismatch: have %v, want %v", balance, 42) } if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -623,7 +622,7 @@ func TestCopyCommitCopy(t *testing.T) { // Commit state, ensure states can be loaded from disk root, _ := state.Commit(0, false) state, _ = New(root, tdb, nil) - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -649,11 +648,11 @@ func TestCopyCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -667,7 +666,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the non-committed state database and check pre/post commit balance copyOne := state.Copy() - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyOne.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("first copy balance mismatch: have %v, want %v", balance, 42) } if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -681,7 +680,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyTwo.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("second copy pre-commit balance mismatch: have %v, want %v", balance, 42) } if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -695,7 +694,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // Copy the copy-copy and check the balance once more copyThree := copyTwo.Copy() - if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := copyThree.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("third copy balance mismatch: have %v, want %v", balance, 42) } if code := copyThree.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -718,11 +717,11 @@ func TestCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie - if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { @@ -737,7 +736,7 @@ func TestCommitCopy(t *testing.T) { // Copy the committed state database, the copied one is not functional. state.Commit(0, true) copied := state.Copy() - if balance := copied.GetBalance(addr); balance.Cmp(big.NewInt(0)) != 0 { + if balance := copied.GetBalance(addr); balance.Cmp(uint256.NewInt(0)) != 0 { t.Fatalf("unexpected balance: have %v", balance) } if code := copied.GetCode(addr); code != nil { @@ -767,7 +766,7 @@ func TestDeleteCreateRevert(t *testing.T) { state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.BytesToAddress([]byte("so")) - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) root, _ := state.Commit(0, false) state, _ = New(root, state.db, state.snaps) @@ -777,7 +776,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.Finalise(true) id := state.Snapshot() - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) state.RevertToSnapshot(id) // Commit the entire state and make sure we don't crash and have the correct state @@ -800,34 +799,34 @@ func TestMissingTrieNodes(t *testing.T) { func testMissingTrieNodes(t *testing.T, scheme string) { // Create an initial state with a few accounts var ( - triedb *trie.Database - memDb = rawdb.NewMemoryDatabase() + tdb *triedb.Database + memDb = rawdb.NewMemoryDatabase() ) if scheme == rawdb.PathScheme { - triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{ + tdb = triedb.NewDatabase(memDb, &triedb.Config{PathDB: &pathdb.Config{ CleanCacheSize: 0, DirtyCacheSize: 0, }}) // disable caching } else { - triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{ + tdb = triedb.NewDatabase(memDb, &triedb.Config{HashDB: &hashdb.Config{ CleanCacheSize: 0, }}) // disable caching } - db := NewDatabaseWithNodeDB(memDb, triedb) + db := NewDatabaseWithNodeDB(memDb, tdb) var root common.Hash state, _ := New(types.EmptyRootHash, db, nil) addr := common.BytesToAddress([]byte("so")) { - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.SetCode(addr, []byte{1, 2, 3}) a2 := common.BytesToAddress([]byte("another")) - state.SetBalance(a2, big.NewInt(100)) + state.SetBalance(a2, uint256.NewInt(100)) state.SetCode(a2, []byte{1, 2, 4}) root, _ = state.Commit(0, false) t.Logf("root: %x", root) // force-flush - triedb.Commit(root, false) + tdb.Commit(root, false) } // Create a new state on the old root state, _ = New(root, db, nil) @@ -847,7 +846,7 @@ func testMissingTrieNodes(t *testing.T, scheme string) { t.Errorf("expected %d, got %d", exp, got) } // Modify the state - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) root, err := state.Commit(0, false) if err == nil { t.Fatalf("expected error, got root :%x", root) @@ -1034,7 +1033,7 @@ func TestFlushOrderDataLoss(t *testing.T) { // Create a state trie with many accounts and slots var ( memdb = rawdb.NewMemoryDatabase() - triedb = trie.NewDatabase(memdb, nil) + triedb = triedb.NewDatabase(memdb, nil) statedb = NewDatabaseWithNodeDB(memdb, triedb) state, _ = New(types.EmptyRootHash, statedb, nil) ) @@ -1106,7 +1105,7 @@ func TestStateDBTransientStorage(t *testing.T) { func TestResetObject(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, nil) + tdb = triedb.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) state, _ = New(types.EmptyRootHash, db, snaps) @@ -1115,13 +1114,13 @@ func TestResetObject(t *testing.T) { slotB = common.HexToHash("0x2") ) // Initialize account with balance and storage in first transaction. - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.SetState(addr, slotA, common.BytesToHash([]byte{0x1})) state.IntermediateRoot(true) // Reset account and mutate balance and storages state.CreateAccount(addr) - state.SetBalance(addr, big.NewInt(2)) + state.SetBalance(addr, uint256.NewInt(2)) state.SetState(addr, slotB, common.BytesToHash([]byte{0x2})) root := state.IntermediateRoot(true) state.SetExpectedStateRoot(root) @@ -1142,14 +1141,14 @@ func TestResetObject(t *testing.T) { func TestDeleteStorage(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk, nil) + tdb = triedb.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) state, _ = New(types.EmptyRootHash, db, snaps) addr = common.HexToAddress("0x1") ) // Initialize account and populate storage - state.SetBalance(addr, big.NewInt(1)) + state.SetBalance(addr, uint256.NewInt(1)) state.CreateAccount(addr) for i := 0; i < 1000; i++ { slot := common.Hash(uint256.NewInt(uint64(i)).Bytes32()) @@ -1161,7 +1160,7 @@ func TestDeleteStorage(t *testing.T) { fastState, _ := New(root, db, snaps) slowState, _ := New(root, db, nil) - obj := fastState.GetOrNewStateObject(addr) + obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) diff --git a/core/state/sync.go b/core/state/sync.go index d6775e8896..411b54eab0 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// NewStateSync create a new state trie download scheduler. +// NewStateSync creates a new state trie download scheduler. func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { // Register the storage slot callback if the external callback is specified. var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 6196e77817..052c166578 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -18,7 +18,6 @@ package state import ( "bytes" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -28,40 +27,42 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" ) // testAccount is the data associated with an account used by the state tests. type testAccount struct { address common.Address - balance *big.Int + balance *uint256.Int nonce uint64 code []byte } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) { +func makeTestState(scheme string) (ethdb.Database, Database, *triedb.Database, common.Hash, []*testAccount) { // Create an empty state - config := &trie.Config{Preimages: true} + config := &triedb.Config{Preimages: true} if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } else { config.HashDB = hashdb.Defaults } db := rawdb.NewMemoryDatabase() - nodeDb := trie.NewDatabase(db, config) + nodeDb := triedb.NewDatabase(db, config) sdb := NewDatabaseWithNodeDB(db, nodeDb) state, _ := New(types.EmptyRootHash, sdb, nil) // Fill it with some arbitrary data var accounts []*testAccount for i := byte(0); i < 96; i++ { - obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) + obj := state.getOrNewStateObject(common.BytesToAddress([]byte{i})) acc := &testAccount{address: common.BytesToAddress([]byte{i})} - obj.AddBalance(big.NewInt(int64(11 * i))) - acc.balance = big.NewInt(int64(11 * i)) + obj.AddBalance(uint256.NewInt(uint64(11 * i))) + acc.balance = uint256.NewInt(uint64(11 * i)) obj.SetNonce(uint64(42 * i)) acc.nonce = uint64(42 * i) @@ -87,7 +88,7 @@ func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, com // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) { - var config trie.Config + var config triedb.Config if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } @@ -114,7 +115,7 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root com // checkStateConsistency checks that all data of a state root is present. func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error { - config := &trie.Config{Preimages: true} + config := &triedb.Config{Preimages: true} if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } @@ -130,8 +131,8 @@ func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) e // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { - dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) - dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults}) + dbA := triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbB := triedb.NewDatabase(rawdb.NewMemoryDatabase(), &triedb.Config{PathDB: pathdb.Defaults}) sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme()) if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { @@ -237,7 +238,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) stTrie, err := trie.New(id, ndb) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + t.Fatalf("failed to retrieve storage trie for path %x: %v", node.syncPath[1], err) } data, _, err := stTrie.GetNode(node.syncPath[1]) if err != nil { diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index b190567e92..711ec83250 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) func filledStateDB() *StateDB { @@ -34,9 +35,9 @@ func filledStateDB() *StateDB { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, uint256.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie for i := 0; i < 100; i++ { sk := common.BigToHash(big.NewInt(int64(i))) state.SetState(addr, sk, sk) // Change the storage trie diff --git a/core/state_processor.go b/core/state_processor.go index 4dd2072adc..c9df98536c 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -218,6 +218,6 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *stat } vmenv.Reset(NewEVMTxContext(msg), statedb) statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.Big0) + _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) statedb.Finalise(true) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 5bfb5947a7..77efaede58 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -117,12 +117,12 @@ func TestStateProcessorErrors(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: config, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, }, - common.HexToAddress("0xfd0810DD14796680f72adf1a371963d0745BCc64"): GenesisAccount{ + common.HexToAddress("0xfd0810DD14796680f72adf1a371963d0745BCc64"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: math.MaxUint64, }, @@ -232,7 +232,7 @@ func TestStateProcessorErrors(t *testing.T) { txs: []*types.Transaction{ mkDynamicTx(0, common.Address{}, params.TxGas, bigNumber, bigNumber), }, - want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 1000000000000000000 want 2431633873983640103894990685182446064918669677978451844828609264166175722438635000", + want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 required balance exceeds 256 bits", }, { // ErrMaxInitCodeSizeExceeded txs: []*types.Transaction{ @@ -281,8 +281,8 @@ func TestStateProcessorErrors(t *testing.T) { IstanbulBlock: big.NewInt(0), MuirGlacierBlock: big.NewInt(0), }, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, }, @@ -319,8 +319,8 @@ func TestStateProcessorErrors(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: config, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): types.Account{ Balance: big.NewInt(1000000000000000000), // 1 ether Nonce: 0, Code: common.FromHex("0xB0B0FACE"), diff --git a/core/state_transition.go b/core/state_transition.go index 898ddc31a6..a23a26468e 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -17,7 +17,6 @@ package core import ( - "errors" "fmt" "math" "math/big" @@ -27,7 +26,9 @@ import ( cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // TODO delete after debug performance metrics @@ -70,7 +71,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { @@ -280,7 +281,11 @@ func (st *StateTransition) buyGas() error { mgval.Add(mgval, blobFee) } } - if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { + balanceCheckU256, overflow := uint256.FromBig(balanceCheck) + if overflow { + return fmt.Errorf("%w: address %v required balance exceeds 256 bits", ErrInsufficientFunds, st.msg.From.Hex()) + } + if have, want := st.state.GetBalance(st.msg.From), balanceCheckU256; have.Cmp(want) < 0 { return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) } if err := st.gp.SubGas(st.msg.GasLimit); err != nil { @@ -289,7 +294,8 @@ func (st *StateTransition) buyGas() error { st.gasRemaining += st.msg.GasLimit st.initialGas = st.msg.GasLimit - st.state.SubBalance(st.msg.From, mgval) + mgvalU256, _ := uint256.FromBig(mgval) + st.state.SubBalance(st.msg.From, mgvalU256) return nil } @@ -358,13 +364,18 @@ func (st *StateTransition) preCheck() error { } // Check the blob version validity if msg.BlobHashes != nil { + // The to field of a blob tx type is mandatory, and a `BlobTx` transaction internally + // has it as a non-nillable value, so any msg derived from blob transaction has it non-nil. + // However, messages created through RPC (eth_call) don't have this restriction. + if msg.To == nil { + return ErrBlobTxCreate + } if len(msg.BlobHashes) == 0 { - return errors.New("blob transaction missing blob hashes") + return ErrMissingBlobHashes } for i, hash := range msg.BlobHashes { - if hash[0] != params.BlobTxHashVersion { - return fmt.Errorf("blob %d hash version mismatch (have %d, supported %d)", - i, hash[0], params.BlobTxHashVersion) + if !kzg4844.IsValidVersionedHash(hash[:]) { + return fmt.Errorf("blob %d has invalid hash version", i) } } } @@ -398,7 +409,11 @@ func (st *StateTransition) preCheck() error { // nil evm execution result. func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if mint := st.msg.Mint; mint != nil { - st.state.AddBalance(st.msg.From, mint) + mintU256, overflow := uint256.FromBig(mint) + if overflow { + return nil, fmt.Errorf("mint value exceeds uint256: %d", mintU256) + } + st.state.AddBalance(st.msg.From, mintU256) } snap := st.state.Snapshot() @@ -446,7 +461,11 @@ func (st *StateTransition) innerTransitionDb() (*ExecutionResult, error) { if tracer := st.evm.Config.Tracer; tracer != nil { tracer.CaptureTxStart(st.initialGas) defer func() { - tracer.CaptureTxEnd(st.gasRemaining) + if st.msg.IsDepositTx { + tracer.CaptureTxEnd(0) + } else { + tracer.CaptureTxEnd(st.gasRemaining) + } }() } @@ -468,7 +487,11 @@ func (st *StateTransition) innerTransitionDb() (*ExecutionResult, error) { st.gasRemaining -= gas // Check clause 6 - if msg.Value.Sign() > 0 && !st.evm.Context.CanTransfer(st.state, msg.From, msg.Value) { + value, overflow := uint256.FromBig(msg.Value) + if overflow { + return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From.Hex()) + } + if !value.IsZero() && !st.evm.Context.CanTransfer(st.state, msg.From, value) { return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From.Hex()) } @@ -488,11 +511,11 @@ func (st *StateTransition) innerTransitionDb() (*ExecutionResult, error) { ) start := time.Now() if contractCreation { - ret, _, st.gasRemaining, vmerr = st.evm.Create(sender, msg.Data, st.gasRemaining, msg.Value) + ret, _, st.gasRemaining, vmerr = st.evm.Create(sender, msg.Data, st.gasRemaining, value) } else { // Increment the nonce for the next transaction st.state.SetNonce(msg.From, st.state.GetNonce(sender.Address())+1) - ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, msg.Value) + ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, value) } DebugInnerExecutionDuration += time.Since(start) @@ -535,29 +558,35 @@ func (st *StateTransition) innerTransitionDb() (*ExecutionResult, error) { if rules.IsLondon { effectiveTip = cmath.BigMin(msg.GasTipCap, new(big.Int).Sub(msg.GasFeeCap, st.evm.Context.BaseFee)) } + effectiveTipU256, _ := uint256.FromBig(effectiveTip) if st.evm.Config.NoBaseFee && msg.GasFeeCap.Sign() == 0 && msg.GasTipCap.Sign() == 0 { // Skip fee payment when NoBaseFee is set and the fee fields // are 0. This avoids a negative effectiveTip being applied to // the coinbase when simulating calls. } else { - fee := new(big.Int).SetUint64(st.gasUsed()) - fee.Mul(fee, effectiveTip) + fee := new(uint256.Int).SetUint64(st.gasUsed()) + fee.Mul(fee, effectiveTipU256) st.state.AddBalance(st.evm.Context.Coinbase, fee) } // Check that we are post bedrock to enable op-geth to be able to create pseudo pre-bedrock blocks (these are pre-bedrock, but don't follow l2 geth rules) // Note optimismConfig will not be nil if rules.IsOptimismBedrock is true if optimismConfig := st.evm.ChainConfig().Optimism; optimismConfig != nil && rules.IsOptimismBedrock && !st.msg.IsDepositTx { - st.state.AddBalance(params.OptimismBaseFeeRecipient, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.evm.Context.BaseFee)) - var l1Cost *big.Int - if st.msg.GasPrice.Cmp(big.NewInt(0)) == 0 && st.evm.ChainConfig().IsWright(st.evm.Context.Time) { - l1Cost = big.NewInt(0) - } else { - l1Cost = st.evm.Context.L1CostFunc(st.msg.RollupCostData, st.evm.Context.Time) + gasCost := new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.evm.Context.BaseFee) + amtU256, overflow := uint256.FromBig(gasCost) + if overflow { + return nil, fmt.Errorf("optimism gas cost overflows U256: %d", gasCost) } - if l1Cost != nil { - st.state.AddBalance(params.OptimismL1FeeRecipient, l1Cost) + st.state.AddBalance(params.OptimismBaseFeeRecipient, amtU256) + if st.msg.GasPrice.Cmp(big.NewInt(0)) == 0 && st.evm.ChainConfig().IsWright(st.evm.Context.Time) { + st.state.AddBalance(params.OptimismL1FeeRecipient, uint256.NewInt(0)) + } else if l1Cost := st.evm.Context.L1CostFunc(st.msg.RollupCostData, st.evm.Context.Time); l1Cost != nil { + amtU256, overflow = uint256.FromBig(l1Cost) + if overflow { + return nil, fmt.Errorf("optimism l1 cost overflows U256: %d", l1Cost) + } + st.state.AddBalance(params.OptimismL1FeeRecipient, amtU256) } } @@ -578,7 +607,8 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { st.gasRemaining += refund // Return ETH for remaining gas, exchanged at the original rate. - remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gasRemaining), st.msg.GasPrice) + remaining := uint256.NewInt(st.gasRemaining) + remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) st.state.AddBalance(st.msg.From, remaining) // Also return remaining gas to the block gas counter so it is diff --git a/core/superchain.go b/core/superchain.go index 96f68cedcf..47b160c519 100644 --- a/core/superchain.go +++ b/core/superchain.go @@ -27,19 +27,21 @@ func LoadOPStackGenesis(chainID uint64) (*Genesis, error) { } genesis := &Genesis{ - Config: cfg, - Nonce: gen.Nonce, - Timestamp: gen.Timestamp, - ExtraData: gen.ExtraData, - GasLimit: gen.GasLimit, - Difficulty: (*big.Int)(gen.Difficulty), - Mixhash: common.Hash(gen.Mixhash), - Coinbase: common.Address(gen.Coinbase), - Alloc: make(GenesisAlloc), - Number: gen.Number, - GasUsed: gen.GasUsed, - ParentHash: common.Hash(gen.ParentHash), - BaseFee: (*big.Int)(gen.BaseFee), + Config: cfg, + Nonce: gen.Nonce, + Timestamp: gen.Timestamp, + ExtraData: gen.ExtraData, + GasLimit: gen.GasLimit, + Difficulty: (*big.Int)(gen.Difficulty), + Mixhash: common.Hash(gen.Mixhash), + Coinbase: common.Address(gen.Coinbase), + Alloc: make(GenesisAlloc), + Number: gen.Number, + GasUsed: gen.GasUsed, + ParentHash: common.Hash(gen.ParentHash), + BaseFee: (*big.Int)(gen.BaseFee), + ExcessBlobGas: gen.ExcessBlobGas, + BlobGasUsed: gen.BlobGasUsed, } for addr, acc := range gen.Alloc { @@ -93,7 +95,7 @@ func LoadOPStackGenesis(chainID uint64) (*Genesis, error) { } } if expectedHash != genesisBlockHash { - return nil, fmt.Errorf("produced genesis with hash %s but expected %s", genesisBlockHash, expectedHash) + return nil, fmt.Errorf("chainID=%d: produced genesis with hash %s but expected %s", chainID, genesisBlockHash, expectedHash) } return genesis, nil } diff --git a/core/superchain_test.go b/core/superchain_test.go index 9959374bdd..4f7ea71ac5 100644 --- a/core/superchain_test.go +++ b/core/superchain_test.go @@ -6,16 +6,15 @@ import ( "github.com/ethereum-optimism/superchain-registry/superchain" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func TestOPStackGenesis(t *testing.T) { for id := range superchain.OPChains { - gen, err := LoadOPStackGenesis(id) + _, err := LoadOPStackGenesis(id) if err != nil { - t.Fatal(err) + t.Error(err) } - t.Logf("chain: %d, genesis block hash: %s", id, gen.ToBlock().Hash()) } } @@ -32,7 +31,7 @@ func TestRegistryChainConfigOverride(t *testing.T) { genesis.Config.RegolithTime = nil // initialize the DB - tdb := trie.NewDatabase(db, newDbConfig(rawdb.PathScheme)) + tdb := triedb.NewDatabase(db, newDbConfig(rawdb.PathScheme)) genesis.MustCommit(db, tdb) bl := genesis.ToBlock() rawdb.WriteCanonicalHash(db, bl.Hash(), 0) diff --git a/core/txindexer.go b/core/txindexer.go new file mode 100644 index 0000000000..77b518dca1 --- /dev/null +++ b/core/txindexer.go @@ -0,0 +1,224 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package core + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// TxIndexProgress is the struct describing the progress for transaction indexing. +type TxIndexProgress struct { + Indexed uint64 // number of blocks whose transactions are indexed + Remaining uint64 // number of blocks whose transactions are not indexed yet +} + +// Done returns an indicator if the transaction indexing is finished. +func (progress TxIndexProgress) Done() bool { + return progress.Remaining == 0 +} + +// txIndexer is the module responsible for maintaining transaction indexes +// according to the configured indexing range by users. +type txIndexer struct { + // limit is the maximum number of blocks from head whose tx indexes + // are reserved: + // * 0: means the entire chain should be indexed + // * N: means the latest N blocks [HEAD-N+1, HEAD] should be indexed + // and all others shouldn't. + limit uint64 + db ethdb.Database + progress chan chan TxIndexProgress + term chan chan struct{} + closed chan struct{} +} + +// newTxIndexer initializes the transaction indexer. +func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { + indexer := &txIndexer{ + limit: limit, + db: chain.db, + progress: make(chan chan TxIndexProgress), + term: make(chan chan struct{}), + closed: make(chan struct{}), + } + go indexer.loop(chain) + + var msg string + if limit == 0 { + msg = "entire chain" + } else { + msg = fmt.Sprintf("last %d blocks", limit) + } + log.Info("Initialized transaction indexer", "range", msg) + + return indexer +} + +// run executes the scheduled indexing/unindexing task in a separate thread. +// If the stop channel is closed, the task should be terminated as soon as +// possible, the done channel will be closed once the task is finished. +func (indexer *txIndexer) run(tail *uint64, head uint64, lastTailCh chan *uint64, stop chan struct{}, done chan struct{}) { + defer func() { close(done) }() + + // Short circuit if chain is empty and nothing to index. + if head == 0 { + return + } + // The tail flag is not existent, it means the node is just initialized + // and all blocks in the chain (part of them may from ancient store) are + // not indexed yet, index the chain according to the configured limit. + if tail == nil { + from := uint64(0) + if indexer.limit != 0 && head >= indexer.limit { + from = head - indexer.limit + 1 + } + rawdb.IndexTransactions(indexer.db, from, head+1, lastTailCh, stop, true) + return + } + // The tail flag is existent (which means indexes in [tail, head] should be + // present), while the whole chain are requested for indexing. + if indexer.limit == 0 || head < indexer.limit { + if *tail > 0 { + // It can happen when chain is rewound to a historical point which + // is even lower than the indexes tail, recap the indexing target + // to new head to avoid reading non-existent block bodies. + end := *tail + if end > head+1 { + end = head + 1 + } + rawdb.IndexTransactions(indexer.db, 0, end, lastTailCh, stop, true) + } + return + } + // The tail flag is existent, adjust the index range according to configured + // limit and the latest chain head. + if head-indexer.limit+1 < *tail { + // Reindex a part of missing indices and rewind index tail to HEAD-limit + rawdb.IndexTransactions(indexer.db, head-indexer.limit+1, *tail, lastTailCh, stop, true) + } else { + // Unindex a part of stale indices and forward index tail to HEAD-limit + rawdb.UnindexTransactions(indexer.db, *tail, head-indexer.limit+1, lastTailCh, stop, false) + } +} + +// loop is the scheduler of the indexer, assigning indexing/unindexing tasks depending +// on the received chain event. +func (indexer *txIndexer) loop(chain *BlockChain) { + defer close(indexer.closed) + + // Listening to chain events and manipulate the transaction indexes. + var ( + stop chan struct{} // Non-nil if background routine is active. + done chan struct{} // Non-nil if background routine is active. + lastHead uint64 // The latest announced chain head (whose tx indexes are assumed created) + lastTail = rawdb.ReadTxIndexTail(indexer.db) // The oldest indexed block, nil means nothing indexed + lastTailCh = make(chan *uint64, 1) // Pass updated lastTail after indexing work, to avoid read from db + + headCh = make(chan ChainHeadEvent) + sub = chain.SubscribeChainHeadEvent(headCh) + ) + defer sub.Unsubscribe() + + // Launch the initial processing if chain is not empty (head != genesis). + // This step is useful in these scenarios that chain has no progress. + if head := rawdb.ReadHeadBlock(indexer.db); head != nil && head.Number().Uint64() != 0 { + stop = make(chan struct{}) + done = make(chan struct{}) + lastHead = head.Number().Uint64() + go indexer.run(lastTail, head.NumberU64(), lastTailCh, stop, done) + } + for { + select { + case head := <-headCh: + if done == nil { + stop = make(chan struct{}) + done = make(chan struct{}) + go indexer.run(lastTail, head.Block.NumberU64(), lastTailCh, stop, done) + } + lastHead = head.Block.NumberU64() + case <-done: + select { + case lastTail = <-lastTailCh: + default: + } + stop = nil + done = nil + case ch := <-indexer.progress: + ch <- indexer.report(lastHead, lastTail) + case ch := <-indexer.term: + if stop != nil { + close(stop) + } + if done != nil { + log.Info("Waiting background transaction indexer to exit") + <-done + } + close(lastTailCh) + close(ch) + return + } + } +} + +// report returns the tx indexing progress. +func (indexer *txIndexer) report(head uint64, tail *uint64) TxIndexProgress { + total := indexer.limit + if indexer.limit == 0 || total > head { + total = head + 1 // genesis included + } + var indexed uint64 + if tail != nil { + indexed = head - *tail + 1 + } + // The value of indexed might be larger than total if some blocks need + // to be unindexed, avoiding a negative remaining. + var remaining uint64 + if indexed < total { + remaining = total - indexed + } + return TxIndexProgress{ + Indexed: indexed, + Remaining: remaining, + } +} + +// txIndexProgress retrieves the tx indexing progress, or an error if the +// background tx indexer is already stopped. +func (indexer *txIndexer) txIndexProgress() (TxIndexProgress, error) { + ch := make(chan TxIndexProgress, 1) + select { + case indexer.progress <- ch: + return <-ch, nil + case <-indexer.closed: + return TxIndexProgress{}, errors.New("indexer is closed") + } +} + +// close shutdown the indexer. Safe to be called for multiple times. +func (indexer *txIndexer) close() { + ch := make(chan struct{}) + select { + case indexer.term <- ch: + <-ch + case <-indexer.closed: + } +} diff --git a/core/txindexer_test.go b/core/txindexer_test.go new file mode 100644 index 0000000000..d2570da673 --- /dev/null +++ b/core/txindexer_test.go @@ -0,0 +1,243 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package core + +import ( + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// TestTxIndexer tests the functionalities for managing transaction indexes. +func TestTxIndexer(t *testing.T) { + var ( + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFaker() + nonce = uint64(0) + chainHead = uint64(128) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) + gen.AddTx(tx) + nonce += 1 + }) + + // verifyIndexes checks if the transaction indexes are present or not + // of the specified block. + verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { + if number == 0 { + return + } + block := blocks[number-1] + for _, tx := range block.Transactions() { + lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) + if exist && lookup == nil { + t.Fatalf("missing %d %x", number, tx.Hash().Hex()) + } + if !exist && lookup != nil { + t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) + } + } + } + verify := func(db ethdb.Database, expTail uint64, indexer *txIndexer) { + tail := rawdb.ReadTxIndexTail(db) + if tail == nil { + t.Fatal("Failed to write tx index tail") + } + if *tail != expTail { + t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) + } + if *tail != 0 { + for number := uint64(0); number < *tail; number += 1 { + verifyIndexes(db, number, false) + } + } + for number := *tail; number <= chainHead; number += 1 { + verifyIndexes(db, number, true) + } + progress := indexer.report(chainHead, tail) + if !progress.Done() { + t.Fatalf("Expect fully indexed") + } + } + + var cases = []struct { + limitA uint64 + tailA uint64 + limitB uint64 + tailB uint64 + limitC uint64 + tailC uint64 + }{ + { + // LimitA: 0 + // TailA: 0 + // + // all blocks are indexed + limitA: 0, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 64 + // TailA: 65 + // + // block [65, 128] are indexed + limitA: 64, + tailA: 65, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 127 + // TailA: 2 + // + // block [2, 128] are indexed + limitA: 127, + tailA: 2, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 128 + // TailA: 1 + // + // block [2, 128] are indexed + limitA: 128, + tailA: 1, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 129 + // TailA: 0 + // + // block [0, 128] are indexed + limitA: 129, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + } + for _, c := range cases { + frdir := t.TempDir() + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) + + // Index the initial blocks from ancient store + indexer := &txIndexer{ + limit: c.limitA, + db: db, + progress: make(chan chan TxIndexProgress), + } + indexer.run(nil, 128, nil, make(chan struct{}), make(chan struct{})) + verify(db, c.tailA, indexer) + + indexer.limit = c.limitB + indexer.run(rawdb.ReadTxIndexTail(db), 128, nil, make(chan struct{}), make(chan struct{})) + verify(db, c.tailB, indexer) + + indexer.limit = c.limitC + indexer.run(rawdb.ReadTxIndexTail(db), 128, nil, make(chan struct{}), make(chan struct{})) + verify(db, c.tailC, indexer) + + // Recover all indexes + indexer.limit = 0 + indexer.run(rawdb.ReadTxIndexTail(db), 128, nil, make(chan struct{}), make(chan struct{})) + verify(db, 0, indexer) + + db.Close() + os.RemoveAll(frdir) + } +} diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 499f8f1312..b613d0acc5 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -268,7 +268,7 @@ func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { // going up, crossing the smaller positive jump counter). As such, the pool // cares only about the min of the two delta values for eviction priority. // -// priority = min(delta-basefee, delta-blobfee) +// priority = min(deltaBasefee, deltaBlobfee) // // - The above very aggressive dimensionality and noise reduction should result // in transaction being grouped into a small number of buckets, the further @@ -280,7 +280,7 @@ func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { // with high fee caps since it could enable pool wars. As such, any positive // priority will be grouped together. // -// priority = min(delta-basefee, delta-blobfee, 0) +// priority = min(deltaBasefee, deltaBlobfee, 0) // // Optimisation tradeoffs: // @@ -342,7 +342,7 @@ func (p *BlobPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. -func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.AddressReserver) error { +func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { p.reserve = reserve var ( @@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } } // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). state, err := p.chain.StateAt(head.Root) if err != nil { @@ -371,14 +371,14 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } p.head, p.state = head, state - // Index all transactions on disk and delete anything inprocessable + // Index all transactions on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, blob []byte) { if p.parseTransaction(id, size, blob) != nil { fails = append(fails, id) } } - store, err := billy.Open(billy.Options{Path: queuedir}, newSlotter(), index) + store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, newSlotter(), index) if err != nil { return err } @@ -386,6 +386,8 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr if len(fails) > 0 { log.Warn("Dropping invalidated blob transactions", "ids", fails) + dropInvalidMeter.Mark(int64(len(fails))) + for _, id := range fails { if err := p.store.Delete(id); err != nil { p.Close() @@ -400,7 +402,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr } var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head, p.head.Time+1)) - blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) ) if p.head.ExcessBlobGas != nil { blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) @@ -418,7 +420,7 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr basefeeGauge.Update(int64(basefee.Uint64())) blobfeeGauge.Update(int64(blobfee.Uint64())) - p.SetGasTip(gasTip) + p.SetGasTip(new(big.Int).SetUint64(gasTip)) // Since the user might have modified their pool's capacity, evict anything // above the current allowance @@ -434,8 +436,10 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr // Close closes down the underlying persistent store. func (p *BlobPool) Close() error { var errs []error - if err := p.limbo.Close(); err != nil { - errs = append(errs, err) + if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set + if err := p.limbo.Close(); err != nil { + errs = append(errs, err) + } } if err := p.store.Close(); err != nil { errs = append(errs, err) @@ -456,7 +460,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { tx := new(types.Transaction) if err := rlp.DecodeBytes(blob, tx); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob pool entry", "id", id, "err", err) return err @@ -467,11 +471,17 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { } meta := newBlobTxMeta(id, size, tx) - + if _, exists := p.lookup[meta.hash]; exists { + // This path is only possible after a crash, where deleted items are not + // removed via the normal shutdown-startup procedure and thus may get + // partially resurrected. + log.Error("Rejecting duplicate blob pool entry", "id", id, "hash", tx.Hash()) + return errors.New("duplicate blob entry") + } sender, err := p.signer.Sender(tx) if err != nil { // This path is impossible unless the signature validity changes across - // restarts. For that ever unprobable case, recover gracefully by ignoring + // restarts. For that ever improbable case, recover gracefully by ignoring // this data entry. log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err) return err @@ -530,15 +540,17 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) if gapped { log.Warn("Dropping dangling blob transactions", "from", addr, "missing", next, "drop", nonces, "ids", ids) + dropDanglingMeter.Mark(int64(len(ids))) } else { log.Trace("Dropping filled blob transactions", "from", addr, "filled", nonces, "ids", ids) + dropFilledMeter.Mark(int64(len(ids))) } for _, id := range ids { if err := p.store.Delete(id); err != nil { @@ -569,6 +581,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[1:] } log.Trace("Dropping overlapped blob transactions", "from", addr, "overlapped", nonces, "ids", ids, "left", len(txs)) + dropOverlappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -583,7 +597,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs[0].evictionBlobFeeJumps = txs[0].blobfeeJumps for i := 1; i < len(txs); i++ { - // If there's no nonce gap, initialize the evicion thresholds as the + // If there's no nonce gap, initialize the eviction thresholds as the // minimum between the cumulative thresholds and the current tx fees if txs[i].nonce == txs[i-1].nonce+1 { txs[i].evictionExecTip = txs[i-1].evictionExecTip @@ -600,10 +614,30 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } continue } - // Sanity check that there's no double nonce. This case would be a coding - // error, but better know about it + // Sanity check that there's no double nonce. This case would generally + // be a coding error, so better know about it. + // + // Also, Billy behind the blobpool does not journal deletes. A process + // crash would result in previously deleted entities being resurrected. + // That could potentially cause a duplicate nonce to appear. if txs[i].nonce == txs[i-1].nonce { - log.Error("Duplicate nonce blob transaction", "from", addr, "nonce", txs[i].nonce) + id := p.lookup[txs[i].hash] + + log.Error("Dropping repeat nonce blob transaction", "from", addr, "nonce", txs[i].nonce, "id", id) + dropRepeatedMeter.Mark(1) + + p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) + p.stored -= uint64(txs[i].size) + delete(p.lookup, txs[i].hash) + + if err := p.store.Delete(id); err != nil { + log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) + } + txs = append(txs[:i], txs[i+1:]...) + p.index[addr] = txs + + i-- + continue } // Otherwise if there's a nonce gap evict all later transactions var ( @@ -621,6 +655,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 txs = txs[:i] log.Error("Dropping gapped blob transactions", "from", addr, "missing", txs[i-1].nonce+1, "drop", nonces, "ids", ids) + dropGappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -632,7 +668,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 // Ensure that there's no over-draft, this is expected to happen when some // transactions get included without publishing on the network var ( - balance = uint256.MustFromBig(p.state.GetBalance(addr)) + balance = p.state.GetBalance(addr) spent = p.spent[addr] ) if spent.Cmp(balance) > 0 { @@ -657,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if len(txs) == 0 { delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -665,6 +701,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs } log.Warn("Dropping overdrafted blob transactions", "from", addr, "balance", balance, "spent", spent, "drop", nonces, "ids", ids) + dropOverdraftedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -695,6 +733,8 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 p.index[addr] = txs log.Warn("Dropping overcapped blob transactions", "from", addr, "kept", len(txs), "drop", nonces, "ids", ids) + dropOvercappedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete blob transaction", "from", addr, "id", id, "err", err) @@ -711,7 +751,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 // offload removes a tracked blob transaction from the pool and moves it into the // limbo for tracking until finality. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. @@ -769,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } } // Recheck the account's pooled transactions to drop included and - // invalidated one + // invalidated ones p.recheck(addr, inclusions) } if len(adds) > 0 { @@ -952,7 +992,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { return err } - // Update the indixes and metrics + // Update the indices and metrics meta := newBlobTxMeta(id, p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { @@ -1019,6 +1059,8 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { } // Clear out the transactions from the data store log.Warn("Dropping underpriced blob transaction", "from", addr, "rejected", tx.nonce, "tip", tx.execTipCap, "want", tip, "drop", nonces, "ids", ids) + dropUnderpricedMeter.Mark(int64(len(ids))) + for _, id := range ids { if err := p.store.Delete(id); err != nil { log.Error("Failed to delete dropped transaction", "id", id, "err", err) @@ -1161,7 +1203,7 @@ func (p *BlobPool) Get(hash common.Hash) *types.Transaction { } // Add inserts a set of blob transactions into the pool if they pass validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error { var ( adds = make([]*types.Transaction, 0, len(txs)) @@ -1181,10 +1223,10 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error } // Add inserts a new blob transaction into the pool if it passes validation (both -// consensus validity and pool restictions). +// consensus validity and pool restrictions). func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are - // only even pulled form the network, so this method will act as the overload + // only even pulled from the network, so this method will act as the overload // protection for fetches. waitStart := time.Now() p.lock.Lock() @@ -1198,6 +1240,22 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) + switch { + case errors.Is(err, txpool.ErrUnderpriced): + addUnderpricedMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooLow): + addStaleMeter.Mark(1) + case errors.Is(err, core.ErrNonceTooHigh): + addGappedMeter.Mark(1) + case errors.Is(err, core.ErrInsufficientFunds): + addOverdraftedMeter.Mark(1) + case errors.Is(err, txpool.ErrAccountLimitExceeded): + addOvercappedMeter.Mark(1) + case errors.Is(err, txpool.ErrReplaceUnderpriced): + addNoreplaceMeter.Mark(1) + default: + addInvalidMeter.Mark(1) + } return err } // If the address is not yet known, request exclusivity to track the account @@ -1205,6 +1263,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { from, _ := types.Sender(p.signer, tx) // already validated above if _, ok := p.index[from]; !ok { if err := p.reserve(from, true); err != nil { + addNonExclusiveMeter.Mark(1) return err } defer func() { @@ -1244,6 +1303,8 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } if len(p.index[from]) > offset { // Transaction replaces a previously queued one + dropReplacedMeter.Mark(1) + prev := p.index[from][offset] if err := p.store.Delete(prev.id); err != nil { // Shitty situation, but try to recover gracefully instead of going boom @@ -1322,6 +1383,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } p.updateStorageMetrics() + addValidMeter.Mark(1) return nil } @@ -1355,7 +1417,7 @@ func (p *BlobPool) drop() { p.stored -= uint64(drop.size) delete(p.lookup, drop.hash) - // Remove the transaction from the pool's evicion heap: + // Remove the transaction from the pool's eviction heap: // - If the entire account was dropped, pop off the address // - Otherwise, if the new tail has better eviction caps, fix the heap if last { @@ -1371,7 +1433,9 @@ func (p *BlobPool) drop() { } } // Remove the transaction from the data store - log.Warn("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + log.Debug("Evicting overflown blob transaction", "from", from, "evicted", drop.nonce, "id", drop.id) + dropOverflownMeter.Mark(1) + if err := p.store.Delete(drop.id); err != nil { log.Error("Failed to drop evicted transaction", "id", drop.id, "err", err) } @@ -1379,7 +1443,15 @@ func (p *BlobPool) drop() { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. -func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +// +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only plain transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyPlainTxs { + return nil + } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. // The latter will be pretty much moot, but we've kept it to have symmetric @@ -1389,20 +1461,40 @@ func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTr pendwaitHist.Update(time.Since(pendStart).Nanoseconds()) defer p.lock.RUnlock() - defer func(start time.Time) { - pendtimeHist.Update(time.Since(start).Nanoseconds()) - }(time.Now()) + execStart := time.Now() + defer func() { + pendtimeHist.Update(time.Since(execStart).Nanoseconds()) + }() - pending := make(map[common.Address][]*txpool.LazyTransaction) + pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { - var lazies []*txpool.LazyTransaction + lazies := make([]*txpool.LazyTransaction, 0, len(txs)) for _, tx := range txs { + // If transaction filtering was requested, discard badly priced ones + if filter.MinTip != nil && filter.BaseFee != nil { + if tx.execFeeCap.Lt(filter.BaseFee) { + break // basefee too low, cannot be included, discard rest of txs from the account + } + tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee) + if tip.Gt(tx.execTipCap) { + tip = tx.execTipCap + } + if tip.Lt(filter.MinTip) { + break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account + } + } + if filter.BlobFee != nil { + if tx.blobFeeCap.Lt(filter.BlobFee) { + break // blobfee too low, cannot be included, discard rest of txs from the account + } + } + // Transaction was accepted according to the filter, append to the pending list lazies = append(lazies, &txpool.LazyTransaction{ Pool: p, Hash: tx.hash, - Time: time.Now(), // TODO(karalabe): Maybe save these and use that? - GasFeeCap: tx.execFeeCap.ToBig(), - GasTipCap: tx.execTipCap.ToBig(), + Time: execStart, // TODO(karalabe): Maybe save these and use that? + GasFeeCap: tx.execFeeCap, + GasTipCap: tx.execTipCap, Gas: tx.execGas, BlobGas: tx.blobGas, }) @@ -1462,7 +1554,7 @@ func (p *BlobPool) updateStorageMetrics() { } // updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes -// // them out as metrics. +// them out as metrics. func (p *BlobPool) updateLimboMetrics() { stats := p.limbo.store.Infos() diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index b877be5614..92e2f05809 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -51,21 +51,9 @@ var ( emptyBlob = kzg4844.Blob{} emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) - emptyBlobVHash = blobHash(emptyBlobCommit) + emptyBlobVHash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) ) -func blobHash(commit kzg4844.Commitment) common.Hash { - hasher := sha256.New() - hasher.Write(commit[:]) - hash := hasher.Sum(nil) - - var vhash common.Hash - vhash[0] = params.BlobTxHashVersion - copy(vhash[1:], hash[1:]) - - return vhash -} - // Chain configuration with Cancun enabled. // // TODO(karalabe): replace with params.MainnetChainConfig after Cancun. @@ -197,7 +185,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx) } -// makeUnsignedTx is a utility method to construct a random blob tranasaction +// makeUnsignedTx is a utility method to construct a random blob transaction // without signing it. func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx { return &types.BlobTx{ @@ -317,7 +305,16 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { // - 1. A transaction that cannot be decoded must be dropped // - 2. A transaction that cannot be recovered (bad signature) must be dropped // - 3. All transactions after a nonce gap must be dropped -// - 4. All transactions after an underpriced one (including it) must be dropped +// - 4. All transactions after an already included nonce must be dropped +// - 5. All transactions after an underpriced one (including it) must be dropped +// - 6. All transactions after an overdrafting sequence must be dropped +// - 7. All transactions exceeding the per-account limit must be dropped +// +// Furthermore, some strange corner-cases can also occur after a crash, as Billy's +// simplicity also allows it to resurrect past deleted entities: +// +// - 8. Fully duplicate transactions (matching hash) must be dropped +// - 9. Duplicate nonces from the same account must be dropped func TestOpenDrops(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) @@ -350,7 +347,7 @@ func TestOpenDrops(t *testing.T) { badsig, _ := store.Put(blob) // Insert a sequence of transactions with a nonce gap in between to verify - // that anything gapped will get evicted (case 3) + // that anything gapped will get evicted (case 3). var ( gapper, _ = crypto.GenerateKey() @@ -369,7 +366,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with a gapped starting nonce to verify - // that the entire set will get dropped. + // that the entire set will get dropped (case 3). var ( dangler, _ = crypto.GenerateKey() dangling = make(map[uint64]struct{}) @@ -382,7 +379,7 @@ func TestOpenDrops(t *testing.T) { dangling[id] = struct{}{} } // Insert a sequence of transactions with already passed nonces to veirfy - // that the entire set will get dropped. + // that the entire set will get dropped (case 4). var ( filler, _ = crypto.GenerateKey() filled = make(map[uint64]struct{}) @@ -394,8 +391,8 @@ func TestOpenDrops(t *testing.T) { id, _ := store.Put(blob) filled[id] = struct{}{} } - // Insert a sequence of transactions with partially passed nonces to veirfy - // that the included part of the set will get dropped + // Insert a sequence of transactions with partially passed nonces to verify + // that the included part of the set will get dropped (case 4). var ( overlapper, _ = crypto.GenerateKey() overlapped = make(map[uint64]struct{}) @@ -412,7 +409,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions with an underpriced first to verify that - // the entire set will get dropped (case 4). + // the entire set will get dropped (case 5). var ( underpayer, _ = crypto.GenerateKey() underpaid = make(map[uint64]struct{}) @@ -431,7 +428,7 @@ func TestOpenDrops(t *testing.T) { } // Insert a sequence of transactions with an underpriced in between to verify - // that it and anything newly gapped will get evicted (case 4). + // that it and anything newly gapped will get evicted (case 5). var ( outpricer, _ = crypto.GenerateKey() outpriced = make(map[uint64]struct{}) @@ -453,7 +450,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions fully overdrafted to verify that the - // entire set will get invalidated. + // entire set will get invalidated (case 6). var ( exceeder, _ = crypto.GenerateKey() exceeded = make(map[uint64]struct{}) @@ -471,7 +468,7 @@ func TestOpenDrops(t *testing.T) { exceeded[id] = struct{}{} } // Insert a sequence of transactions partially overdrafted to verify that part - // of the set will get invalidated. + // of the set will get invalidated (case 6). var ( overdrafter, _ = crypto.GenerateKey() overdrafted = make(map[uint64]struct{}) @@ -493,7 +490,7 @@ func TestOpenDrops(t *testing.T) { } } // Insert a sequence of transactions overflowing the account cap to verify - // that part of the set will get invalidated. + // that part of the set will get invalidated (case 7). var ( overcapper, _ = crypto.GenerateKey() overcapped = make(map[uint64]struct{}) @@ -508,21 +505,59 @@ func TestOpenDrops(t *testing.T) { overcapped[id] = struct{}{} } } + // Insert a batch of duplicated transactions to verify that only one of each + // version will remain (case 8). + var ( + duplicater, _ = crypto.GenerateKey() + duplicated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, duplicater)) + + for i := 0; i < int(nonce)+1; i++ { + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + duplicated[id] = struct{}{} + } + } + } + // Insert a batch of duplicated nonces to verify that only one of each will + // remain (case 9). + var ( + repeater, _ = crypto.GenerateKey() + repeated = make(map[uint64]struct{}) + ) + for _, nonce := range []uint64{0, 1, 2} { + for i := 0; i < int(nonce)+1; i++ { + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)) + + id, _ := store.Put(blob) + if i == 0 { + valids[id] = struct{}{} + } else { + repeated[id] = struct{}{} + } + } + } store.Close() // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), big.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), uint256.NewInt(1000000)) statedb.SetNonce(crypto.PubkeyToAddress(filler.PublicKey), 3) - statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), big.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overlapper.PublicKey), uint256.NewInt(1000000)) statedb.SetNonce(crypto.PubkeyToAddress(overlapper.PublicKey), 2) - statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), big.NewInt(1000000)) - statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), big.NewInt(10000000)) + statedb.AddBalance(crypto.PubkeyToAddress(underpayer.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(outpricer.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(exceeder.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overdrafter.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(overcapper.PublicKey), uint256.NewInt(10000000)) + statedb.AddBalance(crypto.PubkeyToAddress(duplicater.PublicKey), uint256.NewInt(1000000)) + statedb.AddBalance(crypto.PubkeyToAddress(repeater.PublicKey), uint256.NewInt(1000000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -532,7 +567,7 @@ func TestOpenDrops(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -566,6 +601,10 @@ func TestOpenDrops(t *testing.T) { t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id) } else if _, ok := overcapped[tx.id]; ok { t.Errorf("overcapped transaction remained in storage: %d", tx.id) + } else if _, ok := duplicated[tx.id]; ok { + t.Errorf("duplicated transaction remained in storage: %d", tx.id) + } else if _, ok := repeated[tx.id]; ok { + t.Errorf("repeated nonce transaction remained in storage: %d", tx.id) } else { alive[tx.id] = struct{}{} } @@ -596,7 +635,7 @@ func TestOpenDrops(t *testing.T) { // Tests that transactions loaded from disk are indexed correctly. // -// - 1. Transactions must be groupped by sender, sorted by nonce +// - 1. Transactions must be grouped by sender, sorted by nonce // - 2. Eviction thresholds are calculated correctly for the sequences // - 3. Balance usage of an account is totals across all transactions func TestOpenIndex(t *testing.T) { @@ -610,7 +649,7 @@ func TestOpenIndex(t *testing.T) { store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) // Insert a sequence of transactions with varying price points to check that - // the cumulative minimumw will be maintained. + // the cumulative minimum will be maintained. var ( key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) @@ -637,7 +676,7 @@ func TestOpenIndex(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -647,7 +686,7 @@ func TestOpenIndex(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -737,9 +776,9 @@ func TestOpenHeap(t *testing.T) { // Create a blob pool out of the pre-seeded data statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr2, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr3, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -749,7 +788,7 @@ func TestOpenHeap(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -817,9 +856,9 @@ func TestOpenCap(t *testing.T) { for _, datacap := range []uint64{2 * (txAvgSize + blobSize), 100 * (txAvgSize + blobSize)} { // Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) - statedb.AddBalance(addr1, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr2, big.NewInt(1_000_000_000)) - statedb.AddBalance(addr3, big.NewInt(1_000_000_000)) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000)) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000)) statedb.Commit(0, true) chain := &testBlockChain{ @@ -829,7 +868,7 @@ func TestOpenCap(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage, Datacap: datacap}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } // Verify that enough transactions have been dropped to get the pool's size @@ -1189,6 +1228,24 @@ func TestAdd(t *testing.T) { }, }, }, + // Blob transactions that don't meet the min blob gas price should be rejected + { + seeds: map[string]seed{ + "alice": {balance: 10000000}, + }, + adds: []addtx{ + { // New account, no previous txs, nonce 0, but blob fee cap too low + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 0), + err: txpool.ErrUnderpriced, + }, + { // Same as above but blob fee cap equals minimum, should be accepted + from: "alice", + tx: makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice), + err: nil, + }, + }, + }, } for i, tt := range tests { // Create a temporary folder for the persistent backend @@ -1209,8 +1266,8 @@ func TestAdd(t *testing.T) { keys[acc], _ = crypto.GenerateKey() addrs[acc] = crypto.PubkeyToAddress(keys[acc].PublicKey) - // Seed the state database with this acocunt - statedb.AddBalance(addrs[acc], new(big.Int).SetUint64(seed.balance)) + // Seed the state database with this account + statedb.AddBalance(addrs[acc], new(uint256.Int).SetUint64(seed.balance)) statedb.SetNonce(addrs[acc], seed.nonce) // Sign the seed transactions and store them in the data store @@ -1231,7 +1288,7 @@ func TestAdd(t *testing.T) { statedb: statedb, } pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(big.NewInt(1), chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { t.Fatalf("test %d: failed to create blob pool: %v", i, err) } verifyPoolInternals(t, pool) @@ -1249,3 +1306,65 @@ func TestAdd(t *testing.T) { pool.Close() } } + +// Benchmarks the time it takes to assemble the lazy pending transaction list +// from the pool contents. +func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) } +func BenchmarkPoolPending1GB(b *testing.B) { benchmarkPoolPending(b, 1_000_000_000) } +func BenchmarkPoolPending10GB(b *testing.B) { benchmarkPoolPending(b, 10_000_000_000) } + +func benchmarkPoolPending(b *testing.B, datacap uint64) { + // Calculate the maximum number of transaction that would fit into the pool + // and generate a set of random accounts to seed them with. + capacity := datacap / params.BlobTxBlobGasPerBlob + + var ( + basefee = uint64(1050) + blobfee = uint64(105) + signer = types.LatestSigner(testChainConfig) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + chain = &testBlockChain{ + config: testChainConfig, + basefee: uint256.NewInt(basefee), + blobfee: uint256.NewInt(blobfee), + statedb: statedb, + } + pool = New(Config{Datadir: ""}, chain) + ) + + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + b.Fatalf("failed to create blob pool: %v", err) + } + // Fill the pool up with one random transaction from each account with the + // same price and everything to maximize the worst case scenario + for i := 0; i < int(capacity); i++ { + blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee) + blobtx.R = uint256.NewInt(1) + blobtx.S = uint256.NewInt(uint64(100 + i)) + blobtx.V = uint256.NewInt(0) + tx := types.NewTx(blobtx) + addr, err := types.Sender(signer, tx) + if err != nil { + b.Fatal(err) + } + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000)) + pool.add(tx) + } + statedb.Commit(0, true) + defer pool.Close() + + // Benchmark assembling the pending + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + p := pool.Pending(txpool.PendingFilter{ + MinTip: uint256.NewInt(1), + BaseFee: chain.basefee, + BlobFee: chain.blobfee, + }) + if len(p) != int(capacity) { + b.Fatalf("have %d want %d", len(p), capacity) + } + } +} diff --git a/core/txpool/blobpool/config.go b/core/txpool/blobpool/config.go index 99a2002a30..1d180739cd 100644 --- a/core/txpool/blobpool/config.go +++ b/core/txpool/blobpool/config.go @@ -30,8 +30,8 @@ type Config struct { // DefaultConfig contains the default configurations for the transaction pool. var DefaultConfig = Config{ Datadir: "blobpool", - Datacap: 10 * 1024 * 1024 * 1024, - PriceBump: 100, // either have patience or be aggressive, no mushy ground + Datacap: 10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB + PriceBump: 100, // either have patience or be aggressive, no mushy ground } // sanitize checks the provided user configurations and changes anything that's diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index df594099f7..bc4543a352 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -30,7 +30,7 @@ import ( // transaction from each account to determine which account to evict from. // // The heap internally tracks a slice of cheapest transactions from each account -// and a mapping from addresses to indices for direct removals/udates. +// and a mapping from addresses to indices for direct removals/updates. // // The goal of the heap is to decide which account has the worst bottleneck to // evict transactions from. diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index d1fe9c7394..ec754f6894 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -53,7 +53,7 @@ func newLimbo(datadir string) (*limbo, error) { index: make(map[common.Hash]uint64), groups: make(map[uint64]map[uint64]common.Hash), } - // Index all limboed blobs on disk and delete anything inprocessable + // Index all limboed blobs on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, data []byte) { if l.parseBlob(id, data) != nil { @@ -89,7 +89,7 @@ func (l *limbo) parseBlob(id uint64, data []byte) error { item := new(limboBlob) if err := rlp.DecodeBytes(data, item); err != nil { // This path is impossible unless the disk data representation changes - // across restarts. For that ever unprobable case, recover gracefully + // across restarts. For that ever improbable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob limbo entry", "id", id, "err", err) return err @@ -172,7 +172,7 @@ func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) { // update changes the block number under which a blob transaction is tracked. This // method should be used when a reorg changes a transaction's inclusion block. // -// The method may log errors for various unexpcted scenarios but will not return +// The method may log errors for various unexpected scenarios but will not return // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 587804cc61..52419ade09 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -65,8 +65,8 @@ var ( pooltipGauge = metrics.NewRegisteredGauge("blobpool/pooltip", nil) // addwait/time, resetwait/time and getwait/time track the rough health of - // the pool and whether or not it's capable of keeping up with the load from - // the network. + // the pool and whether it's capable of keeping up with the load from the + // network. addwaitHist = metrics.NewRegisteredHistogram("blobpool/addwait", nil, metrics.NewExpDecaySample(1028, 0.015)) addtimeHist = metrics.NewRegisteredHistogram("blobpool/addtime", nil, metrics.NewExpDecaySample(1028, 0.015)) getwaitHist = metrics.NewRegisteredHistogram("blobpool/getwait", nil, metrics.NewExpDecaySample(1028, 0.015)) @@ -75,4 +75,31 @@ var ( pendtimeHist = metrics.NewRegisteredHistogram("blobpool/pendtime", nil, metrics.NewExpDecaySample(1028, 0.015)) resetwaitHist = metrics.NewRegisteredHistogram("blobpool/resetwait", nil, metrics.NewExpDecaySample(1028, 0.015)) resettimeHist = metrics.NewRegisteredHistogram("blobpool/resettime", nil, metrics.NewExpDecaySample(1028, 0.015)) + + // The below metrics track various cases where transactions are dropped out + // of the pool. Most are exceptional, some are chain progression and some + // threshold cappings. + dropInvalidMeter = metrics.NewRegisteredMeter("blobpool/drop/invalid", nil) // Invalid transaction, consensus change or bugfix, neutral-ish + dropDanglingMeter = metrics.NewRegisteredMeter("blobpool/drop/dangling", nil) // First nonce gapped, bad + dropFilledMeter = metrics.NewRegisteredMeter("blobpool/drop/filled", nil) // State full-overlap, chain progress, ok + dropOverlappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overlapped", nil) // State partial-overlap, chain progress, ok + dropRepeatedMeter = metrics.NewRegisteredMeter("blobpool/drop/repeated", nil) // Repeated nonce, bad + dropGappedMeter = metrics.NewRegisteredMeter("blobpool/drop/gapped", nil) // Non-first nonce gapped, bad + dropOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/drop/overdrafted", nil) // Balance exceeded, bad + dropOvercappedMeter = metrics.NewRegisteredMeter("blobpool/drop/overcapped", nil) // Per-account cap exceeded, bad + dropOverflownMeter = metrics.NewRegisteredMeter("blobpool/drop/overflown", nil) // Global disk cap exceeded, neutral-ish + dropUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/drop/underpriced", nil) // Gas tip changed, neutral + dropReplacedMeter = metrics.NewRegisteredMeter("blobpool/drop/replaced", nil) // Transaction replaced, neutral + + // The below metrics track various outcomes of transactions being added to + // the pool. + addInvalidMeter = metrics.NewRegisteredMeter("blobpool/add/invalid", nil) // Invalid transaction, reject, neutral + addUnderpricedMeter = metrics.NewRegisteredMeter("blobpool/add/underpriced", nil) // Gas tip too low, neutral + addStaleMeter = metrics.NewRegisteredMeter("blobpool/add/stale", nil) // Nonce already filled, reject, bad-ish + addGappedMeter = metrics.NewRegisteredMeter("blobpool/add/gapped", nil) // Nonce gapped, reject, bad-ish + addOverdraftedMeter = metrics.NewRegisteredMeter("blobpool/add/overdrafted", nil) // Balance exceeded, reject, neutral + addOvercappedMeter = metrics.NewRegisteredMeter("blobpool/add/overcapped", nil) // Per-account cap exceeded, reject, neutral + addNoreplaceMeter = metrics.NewRegisteredMeter("blobpool/add/noreplace", nil) // Replacement fees or tips too low, neutral + addNonExclusiveMeter = metrics.NewRegisteredMeter("blobpool/add/nonexclusive", nil) // Plain transaction from same account exists, reject, neutral + addValidMeter = metrics.NewRegisteredMeter("blobpool/add/valid", nil) // Valid transaction, add, neutral ) diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 4aad919925..cf0e0454a0 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) { // Benchmarks how many priority recalculations can be done. func BenchmarkPriorityCalculation(b *testing.B) { // The basefee and blob fee is constant for all transactions across a block, - // so we can assume theit absolute jump counts can be pre-computed. + // so we can assume their absolute jump counts can be pre-computed. basefee := uint256.NewInt(17_200_000_000) // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be diff --git a/core/txpool/bundlepool/bundlepool.go b/core/txpool/bundlepool/bundlepool.go index b9b30dbc4a..a1b0e1a838 100644 --- a/core/txpool/bundlepool/bundlepool.go +++ b/core/txpool/bundlepool/bundlepool.go @@ -144,7 +144,7 @@ func (p *BundlePool) SetBundleSimulator(simulator BundleSimulator) { p.simulator = simulator } -func (p *BundlePool) Init(gasTip *big.Int, head *types.Header, reserve txpool.AddressReserver) error { +func (p *BundlePool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { return nil } @@ -298,7 +298,7 @@ func (p *BundlePool) Add(txs []*types.Transaction, local bool, sync bool) []erro // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. -func (p *BundlePool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +func (p *BundlePool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { return nil } diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 61daa999ff..3a6a913976 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -54,4 +54,10 @@ var ( // ErrFutureReplacePending is returned if a future transaction replaces a pending // one. Future transactions should only be able to replace other future transactions. ErrFutureReplacePending = errors.New("future transaction tries to replace pending") + + // ErrAlreadyReserved is returned if the sender address has a pending transaction + // in a different subpool. For example, this error is returned in response to any + // input transaction of non-blob type when a blob transaction from this sender + // remains pending (and vice-versa). + ErrAlreadyReserved = errors.New("address already reserved") ) diff --git a/core/txpool/legacypool/cache_for_miner.go b/core/txpool/legacypool/cache_for_miner.go index b02c06d832..4d1ed2628d 100644 --- a/core/txpool/legacypool/cache_for_miner.go +++ b/core/txpool/legacypool/cache_for_miner.go @@ -1,12 +1,10 @@ package legacypool import ( - "math/big" "sort" "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" ) @@ -69,7 +67,7 @@ func (pc *cacheForMiner) del(txs types.Transactions, signer types.Signer) { } } -func (pc *cacheForMiner) dump(pool txpool.LazyResolver, gasPrice, baseFee *big.Int, enforceTip bool) map[common.Address][]*txpool.LazyTransaction { +func (pc *cacheForMiner) dump() map[common.Address]types.Transactions { pending := make(map[common.Address]types.Transactions) pc.txLock.Lock() for addr, txlist := range pc.pending { @@ -79,37 +77,11 @@ func (pc *cacheForMiner) dump(pool txpool.LazyResolver, gasPrice, baseFee *big.I } } pc.txLock.Unlock() - pendingLazy := make(map[common.Address][]*txpool.LazyTransaction) - for addr, txs := range pending { + for _, txs := range pending { // sorted by nonce sort.Sort(types.TxByNonce(txs)) - // If the miner requests tip enforcement, cap the lists now - if enforceTip && !pc.isLocal(addr) { - for i, tx := range txs { - if tx.EffectiveGasTipIntCmp(gasPrice, baseFee) < 0 { - txs = txs[:i] - break - } - } - } - if len(txs) > 0 { - lazies := make([]*txpool.LazyTransaction, len(txs)) - for i, tx := range txs { - lazies[i] = &txpool.LazyTransaction{ - Pool: pool, - Hash: tx.Hash(), - Tx: tx, - Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), - Gas: tx.Gas(), - BlobGas: tx.BlobGas(), - } - } - pendingLazy[addr] = lazies - } } - return pendingLazy + return pending } func (pc *cacheForMiner) markLocal(addr common.Address) { diff --git a/core/txpool/legacypool/journal.go b/core/txpool/legacypool/journal.go index f04ab8fc14..899ed00bcc 100644 --- a/core/txpool/legacypool/journal.go +++ b/core/txpool/legacypool/journal.go @@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error return err } journal.writer = sink - log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) + + logger := log.Info + if len(all) == 0 { + logger = log.Debug + } + logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) return nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 9e998c4159..a539253fa3 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) const ( @@ -171,6 +172,8 @@ type Config struct { Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + EffectiveGasCeil uint64 // if non-zero, a gas ceiling to enforce independent of the header's gaslimit value + ReannounceTime time.Duration // Duration for announcing local pending transactions again ReannounceRemotes bool // Wether reannounce remote transactions or not } @@ -247,7 +250,7 @@ type LegacyPool struct { config Config chainconfig *params.ChainConfig chain BlockChain - gasTip atomic.Pointer[big.Int] + gasTip atomic.Pointer[uint256.Int] txFeed event.Feed reannoTxFeed event.Feed signer types.Signer @@ -267,8 +270,7 @@ type LegacyPool struct { all *lookup // All transactions to allow lookups priced *pricedList // All transactions sorted by price - pendingCacheDumper func(enforceTip bool) map[common.Address][]*txpool.LazyTransaction - pendingCache *cacheForMiner //pending list cache for miner + pendingCache *cacheForMiner //pending list cache for miner reqResetCh chan *txpoolResetRequest reqPromoteCh chan *accountSet @@ -340,20 +342,15 @@ func (pool *LegacyPool) Filter(tx *types.Transaction) bool { // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. The internal // goroutines will be spun up and the pool deemed operational afterwards. -func (pool *LegacyPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.AddressReserver) error { +func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { // Set the address reserver to request exclusive access to pooled accounts pool.reserve = reserve // Set the basic pool parameters - pool.gasTip.Store(gasTip) - - // set dumper - pool.pendingCacheDumper = func(enforceTip bool) map[common.Address][]*txpool.LazyTransaction { - return pool.pendingCache.dump(pool, gasTip, pool.gasTip.Load(), enforceTip) - } + pool.gasTip.Store(uint256.NewInt(gasTip)) // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). statedb, err := pool.chain.StateAt(head.Root) if err != nil { @@ -535,11 +532,13 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { pool.mu.Lock() defer pool.mu.Unlock() - old := pool.gasTip.Load() - pool.gasTip.Store(new(big.Int).Set(tip)) - + var ( + newTip = uint256.MustFromBig(tip) + old = pool.gasTip.Load() + ) + pool.gasTip.Store(newTip) // If the min miner fee increased, remove transactions below the new threshold - if tip.Cmp(old) > 0 { + if newTip.Cmp(old) > 0 { // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead drop := pool.all.RemotesBelowTip(tip) for _, tx := range drop { @@ -547,7 +546,7 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { } pool.priced.Removed(len(drop)) } - log.Info("Legacy pool tip threshold updated", "tip", tip) + log.Info("Legacy pool tip threshold updated", "tip", newTip) } // Nonce returns the next nonce of an account, with all transactions executable @@ -620,17 +619,62 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, } // Pending retrieves all currently processable transactions, grouped by origin -// account and sorted by nonce. The returned transaction set is a copy and can be -// freely modified by calling code. +// account and sorted by nonce. // -// The enforceTips parameter can be used to do an extra filtering on the pending -// transactions and only return those whose **effective** tip is large enough in -// the next pending execution environment. -func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // TODO need to confirm defer func(t0 time.Time) { getPendingDurationTimer.Update(time.Since(t0)) }(time.Now()) - return pool.pendingCacheDumper(enforceTips) + // If only blob transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyBlobTxs { + return nil + } + + // Convert the new uint256.Int types to the old big.Int ones used by the legacy pool + var ( + minTipBig *big.Int + baseFeeBig *big.Int + ) + if filter.MinTip != nil { + minTipBig = filter.MinTip.ToBig() + } + if filter.BaseFee != nil { + baseFeeBig = filter.BaseFee.ToBig() + } + pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) + for addr, txs := range pool.pendingCache.dump() { + + // If the miner requests tip enforcement, cap the lists now + if minTipBig != nil && !pool.locals.contains(addr) { + for i, tx := range txs { + if tx.EffectiveGasTipIntCmp(minTipBig, baseFeeBig) < 0 { + txs = txs[:i] + break + } + } + } + if len(txs) > 0 { + lazies := make([]*txpool.LazyTransaction, len(txs)) + for i := 0; i < len(txs); i++ { + lazies[i] = &txpool.LazyTransaction{ + Pool: pool, + Hash: txs[i].Hash(), + Tx: txs[i], + Time: txs[i].Time(), + GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()), + GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()), + Gas: txs[i].Gas(), + BlobGas: txs[i].BlobGas(), + } + } + pending[addr] = lazies + } + } + return pending } // Locals retrieves the accounts currently considered local by the pool. @@ -686,8 +730,9 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro 1<= 0 { + // Avoid underflow + balance = uint256.NewInt(0) + } else { + balance = new(uint256.Int).Sub(balance, l1Cost256) + } + } + } + return balance +} + // promoteExecutables moves transactions that have become processable from the // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. @@ -1598,7 +1657,7 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T var promoted []*types.Transaction // Iterate over all accounts and promote any executable transactions - gasLimit := txpool.EffectiveGasLimit(pool.chainconfig, pool.currentHead.Load().GasLimit) + gasLimit := txpool.EffectiveGasLimit(pool.chainconfig, pool.currentHead.Load().GasLimit, pool.config.EffectiveGasCeil) for _, addr := range accounts { list := pool.queue[addr] if list == nil { @@ -1612,13 +1671,7 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T } log.Trace("Removed old queued transactions", "count", len(forwards)) balance := pool.currentState.GetBalance(addr) - if !list.Empty() && pool.l1CostFn != nil { - // Reduce the cost-cap by L1 rollup cost of the first tx if necessary. Other txs will get filtered out afterwards. - el := list.txs.FirstElement() - if l1Cost := pool.l1CostFn(el.RollupCostData()); l1Cost != nil { - balance = new(big.Int).Sub(balance, l1Cost) // negative big int is fine - } - } + balance = pool.reduceBalanceByL1Cost(list, balance) // Drop all transactions that are too costly (low balance or out of gas) drops, _ := list.Filter(balance, gasLimit) for _, tx := range drops { @@ -1822,7 +1875,7 @@ func (pool *LegacyPool) demoteUnexecutables(demoteAddrs []common.Address) { demoteTxMeter.Mark(int64(len(demoteAddrs))) // Iterate over all accounts and demote any non-executable transactions - gasLimit := txpool.EffectiveGasLimit(pool.chainconfig, pool.currentHead.Load().GasLimit) + gasLimit := txpool.EffectiveGasLimit(pool.chainconfig, pool.currentHead.Load().GasLimit, pool.config.EffectiveGasCeil) for _, addr := range demoteAddrs { list := pool.pending[addr] if list == nil { @@ -1839,13 +1892,7 @@ func (pool *LegacyPool) demoteUnexecutables(demoteAddrs []common.Address) { log.Trace("Removed old pending transaction", "hash", hash) } balance := pool.currentState.GetBalance(addr) - if !list.Empty() && pool.l1CostFn != nil { - // Reduce the cost-cap by L1 rollup cost of the first tx if necessary. Other txs will get filtered out afterwards. - el := list.txs.FirstElement() - if l1Cost := pool.l1CostFn(el.RollupCostData()); l1Cost != nil { - balance = new(big.Int).Sub(balance, l1Cost) // negative big int is fine - } - } + balance = pool.reduceBalanceByL1Cost(list, balance) // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later drops, invalids := list.Filter(balance, gasLimit) for _, tx := range drops { diff --git a/core/txpool/legacypool/legacypool2_test.go b/core/txpool/legacypool/legacypool2_test.go index a73c1bb8a7..c8d3a76b83 100644 --- a/core/txpool/legacypool/legacypool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" + "github.com/holiman/uint256" ) func pricedValuedTransaction(nonce uint64, value int64, gaslimit uint64, gasprice *big.Int, key *ecdsa.PrivateKey) *types.Transaction { @@ -49,7 +50,7 @@ func fillPool(t testing.TB, pool *LegacyPool) { nonExecutableTxs := types.Transactions{} for i := 0; i < 384; i++ { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(10000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(10000000000)) // Add executable ones for j := 0; j < int(pool.config.AccountSlots); j++ { executableTxs = append(executableTxs, pricedTransaction(uint64(j), 100000, big.NewInt(300), key)) @@ -84,14 +85,14 @@ func TestTransactionFutureAttack(t *testing.T) { config.GlobalQueue = 100 config.GlobalSlots = 100 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() fillPool(t, pool) pending, _ := pool.Stats() // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for j := 0; j < int(pool.config.GlobalSlots+pool.config.GlobalQueue); j++ { futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 100000, big.NewInt(500), key)) @@ -118,7 +119,7 @@ func TestTransactionFuture1559(t *testing.T) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts, fund them and make transactions @@ -128,7 +129,7 @@ func TestTransactionFuture1559(t *testing.T) { // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for j := 0; j < int(pool.config.GlobalSlots+pool.config.GlobalQueue); j++ { futureTxs = append(futureTxs, dynamicFeeTx(1000+uint64(j), 100000, big.NewInt(200), big.NewInt(101), key)) @@ -151,7 +152,7 @@ func TestTransactionZAttack(t *testing.T) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts, fund them and make transactions fillPool(t, pool) @@ -161,7 +162,7 @@ func TestTransactionZAttack(t *testing.T) { var ivpendingNum int pendingtxs, _ := pool.Content() for account, txs := range pendingtxs { - cur_balance := new(big.Int).Set(pool.currentState.GetBalance(account)) + cur_balance := new(big.Int).Set(pool.currentState.GetBalance(account).ToBig()) for _, tx := range txs { if cur_balance.Cmp(tx.Value()) <= 0 { ivpendingNum++ @@ -182,7 +183,7 @@ func TestTransactionZAttack(t *testing.T) { for j := 0; j < int(pool.config.GlobalQueue); j++ { futureTxs := types.Transactions{} key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 21000, big.NewInt(500), key)) pool.addRemotesSync(futureTxs) } @@ -190,7 +191,7 @@ func TestTransactionZAttack(t *testing.T) { overDraftTxs := types.Transactions{} { key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) for j := 0; j < int(pool.config.GlobalSlots); j++ { overDraftTxs = append(overDraftTxs, pricedValuedTransaction(uint64(j), 600000000000, 21000, big.NewInt(500), key)) } @@ -222,12 +223,12 @@ func BenchmarkFutureAttack(b *testing.B) { config.GlobalQueue = 100 config.GlobalSlots = 100 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() fillPool(b, pool) key, _ := crypto.GenerateKey() - pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), uint256.NewInt(100000000000)) futureTxs := types.Transactions{} for n := 0; n < b.N; n++ { diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index 283bddff14..df25aff5e5 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( @@ -163,7 +164,7 @@ func setupPoolWithConfig(config *params.ChainConfig) (*LegacyPool, *ecdsa.Privat key, _ := crypto.GenerateKey() pool := New(testTxPoolConfig, blockchain) - if err := pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()); err != nil { panic(err) } // wait for the pool to initialize @@ -198,9 +199,6 @@ func validatePoolInternals(pool *LegacyPool) error { if nonce := pool.pendingNonces.get(addr); nonce != last+1 { return fmt.Errorf("pending nonce mismatch: have %v, want %v", nonce, last+1) } - if txs.totalcost.Cmp(common.Big0) < 0 { - return fmt.Errorf("totalcost went negative: %v", txs.totalcost) - } } return nil } @@ -255,7 +253,7 @@ func (c *testChain) State() (*state.StateDB, error) { c.statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) - c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) + c.statedb.SetBalance(c.address, new(uint256.Int).SetUint64(params.Ether)) *c.trigger = false } return stdb, nil @@ -275,14 +273,14 @@ func TestStateChangeDuringReset(t *testing.T) { ) // setup pool with 2 transaction in it - statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) + statedb.SetBalance(address, new(uint256.Int).SetUint64(params.Ether)) blockchain := &testChain{newTestBlockChain(params.TestChainConfig, 1000000000, statedb, new(event.Feed)), address, &trigger} tx0 := transaction(0, 100000, key) tx1 := transaction(1, 100000, key) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() nonce := pool.Nonce(address) @@ -309,7 +307,7 @@ func TestStateChangeDuringReset(t *testing.T) { func testAddBalance(pool *LegacyPool, addr common.Address, amount *big.Int) { pool.mu.Lock() - pool.currentState.AddBalance(addr, amount) + pool.currentState.AddBalance(addr, uint256.MustFromBig(amount)) pool.mu.Unlock() } @@ -348,7 +346,7 @@ func TestInvalidTransactions(t *testing.T) { } tx = transaction(1, 100000, key) - pool.gasTip.Store(big.NewInt(1000)) + pool.gasTip.Store(uint256.NewInt(1000)) if err, want := pool.addRemote(tx), txpool.ErrUnderpriced; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } @@ -470,7 +468,7 @@ func TestChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - statedb.AddBalance(addr, big.NewInt(100000000000000)) + statedb.AddBalance(addr, uint256.NewInt(100000000000000)) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) @@ -499,7 +497,7 @@ func TestDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - statedb.AddBalance(addr, big.NewInt(100000000000000)) + statedb.AddBalance(addr, uint256.NewInt(100000000000000)) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) @@ -702,7 +700,7 @@ func TestPostponing(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create two test accounts to produce different gap profiles with @@ -919,7 +917,7 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { config.GlobalQueue = config.AccountQueue*3 - 1 // reduce the queue limits to shorten test time (-1 to make it non divisible) pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts and fund them (last one will be the local) @@ -1012,7 +1010,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { config.NoLocals = nolocals pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create two test accounts to ensure remotes expire but locals do not @@ -1197,7 +1195,7 @@ func TestPendingGlobalLimiting(t *testing.T) { config.GlobalSlots = config.AccountSlots * 10 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1301,7 +1299,7 @@ func TestCapClearsFromAll(t *testing.T) { config.GlobalSlots = 8 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1334,7 +1332,7 @@ func TestPendingMinimumAllowance(t *testing.T) { config.GlobalSlots = 1 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1380,7 +1378,7 @@ func TestRepricing(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -1502,7 +1500,7 @@ func TestMinGasPriceEnforced(t *testing.T) { txPoolConfig := DefaultConfig txPoolConfig.NoLocals = true pool := New(txPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(txPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(txPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() key, _ := crypto.GenerateKey() @@ -1673,7 +1671,7 @@ func TestRepricingKeepsLocals(t *testing.T) { blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1751,7 +1749,7 @@ func TestUnderpricing(t *testing.T) { config.GlobalQueue = 2 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -1866,7 +1864,7 @@ func TestStableUnderpricing(t *testing.T) { config.GlobalQueue = 0 pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -2095,7 +2093,7 @@ func TestDeduplication(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create a test account to add transactions with @@ -2162,7 +2160,7 @@ func TestReplacement(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -2377,7 +2375,7 @@ func testJournaling(t *testing.T, nolocals bool, journalRemotes bool) { config.Rejournal = time.Second pool := New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) // Create two test accounts to ensure remotes expire but locals do not local, _ := crypto.GenerateKey() @@ -2415,7 +2413,7 @@ func testJournaling(t *testing.T, nolocals bool, journalRemotes bool) { blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool = New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) pending, queued = pool.Stats() if queued != 0 { @@ -2446,7 +2444,7 @@ func testJournaling(t *testing.T, nolocals bool, journalRemotes bool) { statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool = New(config, blockchain) - pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) pending, queued = pool.Stats() if journalRemotes { @@ -2483,7 +2481,7 @@ func TestStatusCheck(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() // Create the test accounts to check various transaction statuses with @@ -2676,7 +2674,7 @@ func BenchmarkMultiAccountBatchInsert(b *testing.B) { for i := 0; i < b.N; i++ { key, _ := crypto.GenerateKey() account := crypto.PubkeyToAddress(key.PublicKey) - pool.currentState.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, uint256.NewInt(1000000)) tx := transaction(uint64(0), 100000, key) batches[i] = tx } diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index 61a83b62fd..5aee83bb37 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) // nonceHeap is a heap.Interface implementation over 64bit unsigned integers for @@ -283,19 +284,19 @@ type list struct { strict bool // Whether nonces are strictly continuous or not txs *sortedMap // Heap indexed sorted hash map of the transactions - costcap *big.Int // Price of the highest costing transaction (reset only if exceeds balance) - gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) - totalcost *big.Int // Total cost of all transactions in the list + costcap *uint256.Int // Price of the highest costing transaction (reset only if exceeds balance) + gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) + totalcost *uint256.Int // Total cost of all transactions in the list } -// newList create a new transaction list for maintaining nonce-indexable fast, +// newList creates a new transaction list for maintaining nonce-indexable fast, // gapped, sortable transaction lists. func newList(strict bool) *list { return &list{ strict: strict, txs: newSortedMap(), - costcap: new(big.Int), - totalcost: new(big.Int), + costcap: new(uint256.Int), + totalcost: new(uint256.Int), } } @@ -337,15 +338,19 @@ func (l *list) Add(tx *types.Transaction, priceBump uint64, l1CostFn txpool.L1Co l.subTotalCost([]*types.Transaction{old}) } // Add new tx cost to totalcost - l.totalcost.Add(l.totalcost, tx.Cost()) + cost, overflow := uint256.FromBig(tx.Cost()) + if overflow { + return false, nil + } + l.totalcost.Add(l.totalcost, cost) if l1CostFn != nil { if l1Cost := l1CostFn(tx.RollupCostData()); l1Cost != nil { // add rollup cost - l.totalcost.Add(l.totalcost, l1Cost) + l.totalcost.Add(l.totalcost, cost) } } // Otherwise overwrite the old transaction with the current one l.txs.Put(tx) - if cost := tx.Cost(); l.costcap.Cmp(cost) < 0 { + if l.costcap.Cmp(cost) < 0 { l.costcap = cost } if gas := tx.Gas(); l.gascap < gas { @@ -372,17 +377,17 @@ func (l *list) Forward(threshold uint64) types.Transactions { // a point in calculating all the costs or if the balance covers all. If the threshold // is lower than the costgas cap, the caps will be reset to a new high after removing // the newly invalidated transactions. -func (l *list) Filter(costLimit *big.Int, gasLimit uint64) (types.Transactions, types.Transactions) { +func (l *list) Filter(costLimit *uint256.Int, gasLimit uint64) (types.Transactions, types.Transactions) { // If all transactions are below the threshold, short circuit if l.costcap.Cmp(costLimit) <= 0 && l.gascap <= gasLimit { return nil, nil } - l.costcap = new(big.Int).Set(costLimit) // Lower the caps to the thresholds + l.costcap = new(uint256.Int).Set(costLimit) // Lower the caps to the thresholds l.gascap = gasLimit // Filter out all the transactions above the account's funds removed := l.txs.Filter(func(tx *types.Transaction) bool { - return tx.Gas() > gasLimit || tx.Cost().Cmp(costLimit) > 0 + return tx.Gas() > gasLimit || tx.Cost().Cmp(costLimit.ToBig()) > 0 }) if len(removed) == 0 { @@ -473,7 +478,10 @@ func (l *list) LastElement() *types.Transaction { // total cost of all transactions. func (l *list) subTotalCost(txs []*types.Transaction) { for _, tx := range txs { - l.totalcost.Sub(l.totalcost, tx.Cost()) + _, underflow := l.totalcost.SubOverflow(l.totalcost, uint256.MustFromBig(tx.Cost())) + if underflow { + panic("totalcost underflow") + } } } diff --git a/core/txpool/legacypool/list_test.go b/core/txpool/legacypool/list_test.go index 011a8581db..9f341a7f20 100644 --- a/core/txpool/legacypool/list_test.go +++ b/core/txpool/legacypool/list_test.go @@ -21,8 +21,10 @@ import ( "math/rand" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) // Tests that transactions can be added to strict lists and list contents and @@ -51,6 +53,21 @@ func TestStrictListAdd(t *testing.T) { } } +// TestListAddVeryExpensive tests adding txs which exceed 256 bits in cost. It is +// expected that the list does not panic. +func TestListAddVeryExpensive(t *testing.T) { + key, _ := crypto.GenerateKey() + list := newList(true) + for i := 0; i < 3; i++ { + value := big.NewInt(100) + gasprice, _ := new(big.Int).SetString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 0) + gaslimit := uint64(i) + tx, _ := types.SignTx(types.NewTransaction(uint64(i), common.Address{}, value, gaslimit, gasprice, nil), types.HomesteadSigner{}, key) + t.Logf("cost: %x bitlen: %d\n", tx.Cost(), tx.Cost().BitLen()) + list.Add(tx, DefaultConfig.PriceBump, nil) + } +} + func BenchmarkListAdd(b *testing.B) { // Generate a list of transactions to insert key, _ := crypto.GenerateKey() @@ -60,7 +77,7 @@ func BenchmarkListAdd(b *testing.B) { txs[i] = transaction(uint64(i), 0, key) } // Insert the transactions in a random order - priceLimit := big.NewInt(int64(DefaultConfig.PriceLimit)) + priceLimit := uint256.NewInt(DefaultConfig.PriceLimit) b.ResetTimer() for i := 0; i < b.N; i++ { list := newList(true) diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index d0a877ab96..e4f89d07f6 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/holiman/uint256" ) // LazyTransaction contains a small subset of the transaction properties that is @@ -34,9 +35,9 @@ type LazyTransaction struct { Hash common.Hash // Transaction hash to pull up if needed Tx *types.Transaction // Transaction if already resolved - Time time.Time // Time when the transaction was first seen - GasFeeCap *big.Int // Maximum fee per gas the transaction may consume - GasTipCap *big.Int // Maximum miner tip per gas the transaction can pay + Time time.Time // Time when the transaction was first seen + GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume + GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay Gas uint64 // Amount of gas required by the transaction BlobGas uint64 // Amount of blob gas required by the transaction @@ -44,11 +45,17 @@ type LazyTransaction struct { // Resolve retrieves the full transaction belonging to a lazy handle if it is still // maintained by the transaction pool. +// +// Note, the method will *not* cache the retrieved transaction if the original +// pool has not cached it. The idea being, that if the tx was too big to insert +// originally, silently saving it will cause more trouble down the line (and +// indeed seems to have caused a memory bloat in the original implementation +// which did just that). func (ltx *LazyTransaction) Resolve() *types.Transaction { - if ltx.Tx == nil { - ltx.Tx = ltx.Pool.Get(ltx.Hash) + if ltx.Tx != nil { + return ltx.Tx } - return ltx.Tx + return ltx.Pool.Get(ltx.Hash) } // LazyResolver is a minimal interface needed for a transaction pool to satisfy @@ -63,13 +70,28 @@ type LazyResolver interface { // may request (and relinquish) exclusive access to certain addresses. type AddressReserver func(addr common.Address, reserve bool) error +// PendingFilter is a collection of filter rules to allow retrieving a subset +// of transactions for announcement or mining. +// +// Note, the entries here are not arbitrary useful filters, rather each one has +// a very specific call site in mind and each one can be evaluated very cheaply +// by the pool implementations. Only add new ones that satisfy those constraints. +type PendingFilter struct { + MinTip *uint256.Int // Minimum miner tip required to include a transaction + BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction + BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction + + OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling) + OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block // production, this interface defines the common methods that allow the primary // transaction pool to manage the subpools. type SubPool interface { - // Filter is a selector used to decide whether a transaction whould be added + // Filter is a selector used to decide whether a transaction would be added // to this particular subpool. Filter(tx *types.Transaction) bool @@ -80,7 +102,7 @@ type SubPool interface { // These should not be passed as a constructor argument - nor should the pools // start by themselves - in order to keep multiple subpools in lockstep with // one another. - Init(gasTip *big.Int, head *types.Header, reserve AddressReserver) error + Init(gasTip uint64, head *types.Header, reserve AddressReserver) error // Close terminates any background processing threads and releases any held // resources. @@ -108,7 +130,10 @@ type SubPool interface { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. - Pending(enforceTips bool) map[common.Address][]*LazyTransaction + // + // The transactions can also be pre-filtered by the dynamic fee components to + // reduce allocations and load on downstream subsystems. + Pending(filter PendingFilter) map[common.Address][]*LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 4eca01f9b4..522a2f74c2 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -74,11 +74,14 @@ type TxPool struct { subs event.SubscriptionScope // Subscription scope to unsubscribe all on shutdown quit chan chan error // Quit channel to tear down the head updater + term chan struct{} // Termination channel to detect a closed pool + + sync chan chan error // Testing / simulator channel to block until internal reset is done } // New creates a new transaction pool to gather, sort and filter inbound // transactions from the network. -func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) { +func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { // Retrieve the current head so that all subpools and this main coordinator // pool will have the same starting state, even if the chain moves forward // during initialization. @@ -88,6 +91,8 @@ func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) subpools: subpools, reservations: make(map[common.Address]SubPool), quit: make(chan chan error), + term: make(chan struct{}), + sync: make(chan chan error), } for i, subpool := range subpools { if err := subpool.Init(gasTip, head, pool.reserver(i, subpool)); err != nil { @@ -119,7 +124,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver { log.Error("pool attempted to reserve already-owned address", "address", addr) return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed } - return errors.New("address already reserved") + return ErrAlreadyReserved } p.reservations[addr] = subpool if metrics.Enabled { @@ -176,6 +181,9 @@ func (p *TxPool) Close() error { // outside blockchain events as well as for various reporting and transaction // eviction events. func (p *TxPool) loop(head *types.Header, chain BlockChain) { + // Close the termination marker when the pool stops + defer close(p.term) + // Subscribe to chain head events to trigger subpool resets var ( newHeadCh = make(chan core.ChainHeadEvent) @@ -192,13 +200,23 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { var ( resetBusy = make(chan struct{}, 1) // Allow 1 reset to run concurrently resetDone = make(chan *types.Header) + + resetForced bool // Whether a forced reset was requested, only used in simulator mode + resetWaiter chan error // Channel waiting on a forced reset, only used in simulator mode ) + // Notify the live reset waiter to not block if the txpool is closed. + defer func() { + if resetWaiter != nil { + resetWaiter <- errors.New("pool already terminated") + resetWaiter = nil + } + }() var errc chan error for errc == nil { // Something interesting might have happened, run a reset if there is // one needed but none is running. The resetter will run on its own // goroutine to allow chain head events to be consumed contiguously. - if newHead != oldHead { + if newHead != oldHead || resetForced { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: @@ -209,8 +227,18 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { } resetDone <- newHead }(oldHead, newHead) + + // If the reset operation was explicitly requested, consider it + // being fulfilled and drop the request marker. If it was not, + // this is a noop. + resetForced = false + default: - // Reset already running, wait until it finishes + // Reset already running, wait until it finishes. + // + // Note, this will not drop any forced reset request. If a forced + // reset was requested, but we were busy, then when the currently + // running reset finishes, a new one will be spun up. } } // Wait for the next chain head event or a previous reset finish @@ -224,8 +252,26 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { oldHead = head <-resetBusy + // If someone is waiting for a reset to finish, notify them, unless + // the forced op is still pending. In that case, wait another round + // of resets. + if resetWaiter != nil && !resetForced { + resetWaiter <- nil + resetWaiter = nil + } + case errc = <-p.quit: // Termination requested, break out on the next loop round + + case syncc := <-p.sync: + // Transaction pool is running inside a simulator, and we are about + // to create a new block. Request a forced sync operation to ensure + // that any running reset operation finishes to make block imports + // deterministic. On top of that, run a new reset operation to make + // transaction insertions deterministic instead of being stuck in a + // queue waiting for a reset. + resetForced = true + resetWaiter = syncc } } // Notify the closer of termination (no error possible for now) @@ -352,10 +398,13 @@ func (p *TxPool) AllBundles() []*types.Bundle { // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. -func (p *TxPool) Pending(enforceTips bool) map[common.Address][]*LazyTransaction { +// +// The transactions can also be pre-filtered by the dynamic fee components to +// reduce allocations and load on downstream subsystems. +func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(enforceTips) { + for addr, set := range subpool.Pending(filter) { txs[addr] = set } } @@ -474,3 +523,20 @@ func (p *TxPool) Status(hash common.Hash) TxStatus { } return TxStatusUnknown } + +// Sync is a helper method for unit tests or simulator runs where the chain events +// are arriving in quick succession, without any time in between them to run the +// internal background reset operations. This method will run an explicit reset +// operation to ensure the pool stabilises, thus avoiding flakey behavior. +// +// Note, do not use this in production / live code. In live code, the pool is +// meant to reset on a separate thread to avoid DoS vectors. +func (p *TxPool) Sync() error { + sync := make(chan error) + select { + case p.sync <- sync: + return <-sync + case <-p.term: + return errors.New("pool already terminated") + } +} diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 30ece38b78..4178a7c7fb 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -35,7 +35,16 @@ import ( // are not able to consume all of the gas in a L2 block as the L1 info deposit is always present. const l1InfoGasOverhead = uint64(70_000) -func EffectiveGasLimit(chainConfig *params.ChainConfig, gasLimit uint64) uint64 { +var ( + // blobTxMinBlobGasPrice is the big.Int version of the configured protocol + // parameter to avoid constucting a new big integer for every transaction. + blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) +) + +func EffectiveGasLimit(chainConfig *params.ChainConfig, gasLimit uint64, effectiveLimit uint64) uint64 { + if effectiveLimit != 0 && effectiveLimit < gasLimit { + gasLimit = effectiveLimit + } if chainConfig.Optimism != nil { if l1InfoGasOverhead < gasLimit { gasLimit -= l1InfoGasOverhead @@ -54,6 +63,8 @@ type ValidationOptions struct { Accept uint8 // Bitmap of transaction types that should be accepted for the calling pool MaxSize uint64 // Maximum size of a transaction that the caller can meaningfully handle MinTip *big.Int // Minimum gas tip needed to allow a transaction into the caller pool + + EffectiveGasCeil uint64 // if non-zero, a gas ceiling to enforce independent of the header's gaslimit value } // ValidateTransaction is a helper method to check whether a transaction is valid @@ -106,7 +117,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return ErrNegativeValue } // Ensure the transaction doesn't exceed the current block limit gas - if EffectiveGasLimit(opts.Config, head.GasLimit) < tx.Gas() { + if EffectiveGasLimit(opts.Config, head.GasLimit, opts.EffectiveGasCeil) < tx.Gas() { return ErrGasLimit } // Sanity check for extremely large numbers (supported by RLP or RPC) @@ -137,16 +148,18 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types } if tx.Gas() < intrGas { Meter(IntrinsicGas).Mark(1) - return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas()) + return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } - // Ensure the gasprice is high enough to cover the requirement of the calling - // pool and/or block producer + // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { Meter(Underpriced).Mark(1) - return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap()) + return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) } - // Ensure blob transactions have valid commitments if tx.Type() == types.BlobTxType { + // Ensure the blob fee cap satisfies the minimum blob gas price + if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + } sidecar := tx.BlobTxSidecar() if sidecar == nil { return fmt.Errorf("missing sidecar in blob transaction") @@ -160,6 +173,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) } + // Ensure commitments, proofs and hashes are valid if err := validateBlobSidecar(hashes, sidecar); err != nil { return err } @@ -180,17 +194,10 @@ func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) err // Blob quantities match up, validate that the provers match with the // transaction hash before getting to the cryptography hasher := sha256.New() - for i, want := range hashes { - hasher.Write(sidecar.Commitments[i][:]) - hash := hasher.Sum(nil) - hasher.Reset() - - var vhash common.Hash - vhash[0] = params.BlobTxHashVersion - copy(vhash[1:], hash[1:]) - - if vhash != want { - return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, vhash, want) + for i, vhash := range hashes { + computed := kzg4844.CalcBlobHashV1(hasher, &sidecar.Commitments[i]) + if vhash != computed { + return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, computed, vhash) } } // Blob commitments match with the hashes in the transaction, verify the @@ -257,7 +264,7 @@ func ValidateTransactionWithState(tx *types.Transaction, signer types.Signer, op } // Ensure the transactor has enough funds to cover the transaction costs var ( - balance = opts.State.GetBalance(from) + balance = opts.State.GetBalance(from).ToBig() cost = tx.Cost() ) if opts.L1CostFn != nil { diff --git a/core/types/account.go b/core/types/account.go new file mode 100644 index 0000000000..bb0f4ca02e --- /dev/null +++ b/core/types/account.go @@ -0,0 +1,87 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" +) + +//go:generate go run github.com/fjl/gencodec -type Account -field-override accountMarshaling -out gen_account.go + +// Account represents an Ethereum account and its attached data. +// This type is used to specify accounts in the genesis block state, and +// is also useful for JSON encoding/decoding of accounts. +type Account struct { + Code []byte `json:"code,omitempty"` + Storage map[common.Hash]common.Hash `json:"storage,omitempty"` + Balance *big.Int `json:"balance" gencodec:"required"` + Nonce uint64 `json:"nonce,omitempty"` + + // used in tests + PrivateKey []byte `json:"secretKey,omitempty"` +} + +type accountMarshaling struct { + Code hexutil.Bytes + Balance *math.HexOrDecimal256 + Nonce math.HexOrDecimal64 + Storage map[storageJSON]storageJSON + PrivateKey hexutil.Bytes +} + +// storageJSON represents a 256 bit byte array, but allows less than 256 bits when +// unmarshaling from hex. +type storageJSON common.Hash + +func (h *storageJSON) UnmarshalText(text []byte) error { + text = bytes.TrimPrefix(text, []byte("0x")) + if len(text) > 64 { + return fmt.Errorf("too many hex characters in storage key/value %q", text) + } + offset := len(h) - len(text)/2 // pad on the left + if _, err := hex.Decode(h[offset:], text); err != nil { + return fmt.Errorf("invalid hex storage key/value %q", text) + } + return nil +} + +func (h storageJSON) MarshalText() ([]byte, error) { + return hexutil.Bytes(h[:]).MarshalText() +} + +// GenesisAlloc specifies the initial state of a genesis block. +type GenesisAlloc map[common.Address]Account + +func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { + m := make(map[common.UnprefixedAddress]Account) + if err := json.Unmarshal(data, &m); err != nil { + return err + } + *ga = make(GenesisAlloc) + for addr, a := range m { + (*ga)[common.Address(addr)] = a + } + return nil +} diff --git a/core/gen_genesis_account.go b/core/types/gen_account.go similarity index 61% rename from core/gen_genesis_account.go rename to core/types/gen_account.go index a9d47e6ba3..4e475896a7 100644 --- a/core/gen_genesis_account.go +++ b/core/types/gen_account.go @@ -1,6 +1,6 @@ // Code generated by github.com/fjl/gencodec. DO NOT EDIT. -package core +package types import ( "encoding/json" @@ -12,62 +12,62 @@ import ( "github.com/ethereum/go-ethereum/common/math" ) -var _ = (*genesisAccountMarshaling)(nil) +var _ = (*accountMarshaling)(nil) // MarshalJSON marshals as JSON. -func (g GenesisAccount) MarshalJSON() ([]byte, error) { - type GenesisAccount struct { +func (a Account) MarshalJSON() ([]byte, error) { + type Account struct { Code hexutil.Bytes `json:"code,omitempty"` Storage map[storageJSON]storageJSON `json:"storage,omitempty"` Balance *math.HexOrDecimal256 `json:"balance" gencodec:"required"` Nonce math.HexOrDecimal64 `json:"nonce,omitempty"` PrivateKey hexutil.Bytes `json:"secretKey,omitempty"` } - var enc GenesisAccount - enc.Code = g.Code - if g.Storage != nil { - enc.Storage = make(map[storageJSON]storageJSON, len(g.Storage)) - for k, v := range g.Storage { + var enc Account + enc.Code = a.Code + if a.Storage != nil { + enc.Storage = make(map[storageJSON]storageJSON, len(a.Storage)) + for k, v := range a.Storage { enc.Storage[storageJSON(k)] = storageJSON(v) } } - enc.Balance = (*math.HexOrDecimal256)(g.Balance) - enc.Nonce = math.HexOrDecimal64(g.Nonce) - enc.PrivateKey = g.PrivateKey + enc.Balance = (*math.HexOrDecimal256)(a.Balance) + enc.Nonce = math.HexOrDecimal64(a.Nonce) + enc.PrivateKey = a.PrivateKey return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. -func (g *GenesisAccount) UnmarshalJSON(input []byte) error { - type GenesisAccount struct { +func (a *Account) UnmarshalJSON(input []byte) error { + type Account struct { Code *hexutil.Bytes `json:"code,omitempty"` Storage map[storageJSON]storageJSON `json:"storage,omitempty"` Balance *math.HexOrDecimal256 `json:"balance" gencodec:"required"` Nonce *math.HexOrDecimal64 `json:"nonce,omitempty"` PrivateKey *hexutil.Bytes `json:"secretKey,omitempty"` } - var dec GenesisAccount + var dec Account if err := json.Unmarshal(input, &dec); err != nil { return err } if dec.Code != nil { - g.Code = *dec.Code + a.Code = *dec.Code } if dec.Storage != nil { - g.Storage = make(map[common.Hash]common.Hash, len(dec.Storage)) + a.Storage = make(map[common.Hash]common.Hash, len(dec.Storage)) for k, v := range dec.Storage { - g.Storage[common.Hash(k)] = common.Hash(v) + a.Storage[common.Hash(k)] = common.Hash(v) } } if dec.Balance == nil { - return errors.New("missing required field 'balance' for GenesisAccount") + return errors.New("missing required field 'balance' for Account") } - g.Balance = (*big.Int)(dec.Balance) + a.Balance = (*big.Int)(dec.Balance) if dec.Nonce != nil { - g.Nonce = uint64(*dec.Nonce) + a.Nonce = uint64(*dec.Nonce) } if dec.PrivateKey != nil { - g.PrivateKey = *dec.PrivateKey + a.PrivateKey = *dec.PrivateKey } return nil } diff --git a/core/types/gen_account_rlp.go b/core/types/gen_account_rlp.go index 3fb36f4038..8b424493af 100644 --- a/core/types/gen_account_rlp.go +++ b/core/types/gen_account_rlp.go @@ -12,10 +12,7 @@ func (obj *StateAccount) EncodeRLP(_w io.Writer) error { if obj.Balance == nil { w.Write(rlp.EmptyString) } else { - if obj.Balance.Sign() == -1 { - return rlp.ErrNegativeBigInt - } - w.WriteBigInt(obj.Balance) + w.WriteUint256(obj.Balance) } w.WriteBytes(obj.Root[:]) w.WriteBytes(obj.CodeHash) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index 14a074b2ba..4e544a0b52 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -34,9 +34,12 @@ func (r Receipt) MarshalJSON() ([]byte, error) { BlockNumber *hexutil.Big `json:"blockNumber,omitempty"` TransactionIndex hexutil.Uint `json:"transactionIndex"` L1GasPrice *hexutil.Big `json:"l1GasPrice,omitempty"` + L1BlobBaseFee *hexutil.Big `json:"l1BlobBaseFee,omitempty"` L1GasUsed *hexutil.Big `json:"l1GasUsed,omitempty"` L1Fee *hexutil.Big `json:"l1Fee,omitempty"` FeeScalar *big.Float `json:"l1FeeScalar,omitempty"` + L1BaseFeeScalar *hexutil.Uint64 `json:"l1BaseFeeScalar,omitempty"` + L1BlobBaseFeeScalar *hexutil.Uint64 `json:"l1BlobBaseFeeScalar,omitempty"` } var enc Receipt enc.Type = hexutil.Uint64(r.Type) @@ -57,9 +60,12 @@ func (r Receipt) MarshalJSON() ([]byte, error) { enc.BlockNumber = (*hexutil.Big)(r.BlockNumber) enc.TransactionIndex = hexutil.Uint(r.TransactionIndex) enc.L1GasPrice = (*hexutil.Big)(r.L1GasPrice) + enc.L1BlobBaseFee = (*hexutil.Big)(r.L1BlobBaseFee) enc.L1GasUsed = (*hexutil.Big)(r.L1GasUsed) enc.L1Fee = (*hexutil.Big)(r.L1Fee) enc.FeeScalar = r.FeeScalar + enc.L1BaseFeeScalar = (*hexutil.Uint64)(r.L1BaseFeeScalar) + enc.L1BlobBaseFeeScalar = (*hexutil.Uint64)(r.L1BlobBaseFeeScalar) return json.Marshal(&enc) } @@ -84,9 +90,12 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { BlockNumber *hexutil.Big `json:"blockNumber,omitempty"` TransactionIndex *hexutil.Uint `json:"transactionIndex"` L1GasPrice *hexutil.Big `json:"l1GasPrice,omitempty"` + L1BlobBaseFee *hexutil.Big `json:"l1BlobBaseFee,omitempty"` L1GasUsed *hexutil.Big `json:"l1GasUsed,omitempty"` L1Fee *hexutil.Big `json:"l1Fee,omitempty"` FeeScalar *big.Float `json:"l1FeeScalar,omitempty"` + L1BaseFeeScalar *hexutil.Uint64 `json:"l1BaseFeeScalar,omitempty"` + L1BlobBaseFeeScalar *hexutil.Uint64 `json:"l1BlobBaseFeeScalar,omitempty"` } var dec Receipt if err := json.Unmarshal(input, &dec); err != nil { @@ -151,6 +160,9 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { if dec.L1GasPrice != nil { r.L1GasPrice = (*big.Int)(dec.L1GasPrice) } + if dec.L1BlobBaseFee != nil { + r.L1BlobBaseFee = (*big.Int)(dec.L1BlobBaseFee) + } if dec.L1GasUsed != nil { r.L1GasUsed = (*big.Int)(dec.L1GasUsed) } @@ -160,5 +172,11 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { if dec.FeeScalar != nil { r.FeeScalar = dec.FeeScalar } + if dec.L1BaseFeeScalar != nil { + r.L1BaseFeeScalar = (*uint64)(dec.L1BaseFeeScalar) + } + if dec.L1BlobBaseFeeScalar != nil { + r.L1BlobBaseFeeScalar = (*uint64)(dec.L1BlobBaseFeeScalar) + } return nil } diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index d2a98ed7bf..a6949414f3 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func TestDeriveSha(t *testing.T) { @@ -39,7 +40,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(txs, trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -86,7 +87,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp = types.DeriveSha(txs, trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) } }) @@ -107,7 +108,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -135,7 +136,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/core/types/receipt.go b/core/types/receipt.go index 12dcf4c2cf..67c1addb3d 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -84,11 +84,14 @@ type Receipt struct { BlockNumber *big.Int `json:"blockNumber,omitempty"` TransactionIndex uint `json:"transactionIndex"` - // OVM legacy: extend receipts with their L1 price (if a rollup tx) - L1GasPrice *big.Int `json:"l1GasPrice,omitempty"` - L1GasUsed *big.Int `json:"l1GasUsed,omitempty"` - L1Fee *big.Int `json:"l1Fee,omitempty"` - FeeScalar *big.Float `json:"l1FeeScalar,omitempty"` // always nil after Ecotone hardfork + // Optimism: extend receipts with L1 fee info + L1GasPrice *big.Int `json:"l1GasPrice,omitempty"` // Present from pre-bedrock. L1 Basefee after Bedrock + L1BlobBaseFee *big.Int `json:"l1BlobBaseFee,omitempty"` // Always nil prior to the Ecotone hardfork + L1GasUsed *big.Int `json:"l1GasUsed,omitempty"` // Present from pre-bedrock, deprecated as of Fjord + L1Fee *big.Int `json:"l1Fee,omitempty"` // Present from pre-bedrock + FeeScalar *big.Float `json:"l1FeeScalar,omitempty"` // Present from pre-bedrock to Ecotone. Nil after Ecotone + L1BaseFeeScalar *uint64 `json:"l1BaseFeeScalar,omitempty"` // Always nil prior to the Ecotone hardfork + L1BlobBaseFeeScalar *uint64 `json:"l1BlobBaseFeeScalar,omitempty"` // Always nil prior to the Ecotone hardfork } type receiptMarshaling struct { @@ -105,9 +108,12 @@ type receiptMarshaling struct { // Optimism L1GasPrice *hexutil.Big + L1BlobBaseFee *hexutil.Big L1GasUsed *hexutil.Big L1Fee *hexutil.Big FeeScalar *big.Float + L1BaseFeeScalar *hexutil.Uint64 + L1BlobBaseFeeScalar *hexutil.Uint64 DepositNonce *hexutil.Uint64 DepositReceiptVersion *hexutil.Uint64 } @@ -570,7 +576,7 @@ func (rs Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, nu } } if config.Optimism != nil && len(txs) >= 2 && config.IsBedrock(new(big.Int).SetUint64(number)) { // need at least an info tx and a non-info tx - l1BaseFee, costFunc, feeScalar, err := extractL1GasParams(config, time, txs[0].Data()) + gasParams, err := extractL1GasParams(config, time, txs[0].Data()) if err != nil { return err } @@ -578,13 +584,24 @@ func (rs Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, nu if txs[i].IsDepositTx() { continue } - rs[i].L1GasPrice = l1BaseFee - rs[i].L1Fee, rs[i].L1GasUsed = costFunc(txs[i].RollupCostData()) + rs[i].L1GasPrice = gasParams.l1BaseFee + rs[i].L1BlobBaseFee = gasParams.l1BlobBaseFee + rs[i].L1Fee, rs[i].L1GasUsed = gasParams.costFunc(txs[i].RollupCostData()) if txs[i].GasPrice().Cmp(big.NewInt(0)) == 0 && config.IsWright(time) { rs[i].L1Fee = big.NewInt(0) } - rs[i].FeeScalar = feeScalar + rs[i].FeeScalar = gasParams.feeScalar + rs[i].L1BaseFeeScalar = u32ptrTou64ptr(gasParams.l1BaseFeeScalar) + rs[i].L1BlobBaseFeeScalar = u32ptrTou64ptr(gasParams.l1BlobBaseFeeScalar) } } return nil } + +func u32ptrTou64ptr(a *uint32) *uint64 { + if a == nil { + return nil + } + b := uint64(*a) + return &b +} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 8eaae91057..1156f3e94f 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -698,10 +698,77 @@ func clearComputedFieldsOnLogs(logs []*Log) []*Log { return l } -func getOptimismTxReceipts( - t *testing.T, l1AttributesPayload []byte, - l1GasPrice, l1GasUsed *big.Int, feeScalar *big.Float, l1Fee *big.Int) ([]*Transaction, []*Receipt) { - //to4 := common.HexToAddress("0x4") +func getOptimismEcotoneTxReceipts(l1AttributesPayload []byte, l1GasPrice, l1BlobGasPrice, l1GasUsed, l1Fee *big.Int, baseFeeScalar, blobBaseFeeScalar *uint64) ([]*Transaction, []*Receipt) { + // Create a few transactions to have receipts for + txs := Transactions{ + NewTx(&DepositTx{ + To: nil, // contract creation + Value: big.NewInt(6), + Gas: 50, + Data: l1AttributesPayload, + }), + emptyTx, + } + + // Create the corresponding receipts + receipts := Receipts{ + &Receipt{ + Type: DepositTxType, + PostState: common.Hash{5}.Bytes(), + CumulativeGasUsed: 50 + 15, + Logs: []*Log{ + { + Address: common.BytesToAddress([]byte{0x33}), + // derived fields: + BlockNumber: blockNumber.Uint64(), + TxHash: txs[0].Hash(), + TxIndex: 0, + BlockHash: blockHash, + Index: 0, + }, + { + Address: common.BytesToAddress([]byte{0x03, 0x33}), + // derived fields: + BlockNumber: blockNumber.Uint64(), + TxHash: txs[0].Hash(), + TxIndex: 0, + BlockHash: blockHash, + Index: 1, + }, + }, + TxHash: txs[0].Hash(), + ContractAddress: common.HexToAddress("0x3bb898b4bbe24f68a4e9be46cfe72d1787fd74f4"), + GasUsed: 65, + EffectiveGasPrice: big.NewInt(0), + BlockHash: blockHash, + BlockNumber: blockNumber, + TransactionIndex: 0, + DepositNonce: &depNonce1, + }, + &Receipt{ + Type: LegacyTxType, + EffectiveGasPrice: big.NewInt(0), + PostState: common.Hash{4}.Bytes(), + CumulativeGasUsed: 10, + Logs: []*Log{}, + // derived fields: + TxHash: txs[1].Hash(), + GasUsed: 18446744073709551561, + BlockHash: blockHash, + BlockNumber: blockNumber, + TransactionIndex: 1, + L1GasPrice: l1GasPrice, + L1BlobBaseFee: l1BlobGasPrice, + L1GasUsed: l1GasUsed, + L1Fee: l1Fee, + L1BaseFeeScalar: baseFeeScalar, + L1BlobBaseFeeScalar: blobBaseFeeScalar, + }, + } + return txs, receipts +} + +func getOptimismTxReceipts(l1AttributesPayload []byte, l1GasPrice, l1GasUsed, l1Fee *big.Int, feeScalar *big.Float) ([]*Transaction, []*Receipt) { // Create a few transactions to have receipts for txs := Transactions{ NewTx(&DepositTx{ @@ -777,7 +844,7 @@ func TestDeriveOptimismBedrockTxReceipts(t *testing.T) { l1GasUsed := bedrockGas feeScalar := big.NewFloat(float64(scalar.Uint64() / 1e6)) l1Fee := bedrockFee - txs, receipts := getOptimismTxReceipts(t, payload, l1GasPrice, l1GasUsed, feeScalar, l1Fee) + txs, receipts := getOptimismTxReceipts(payload, l1GasPrice, l1GasUsed, l1Fee, feeScalar) // Re-derive receipts. baseFee := big.NewInt(1000) @@ -801,10 +868,9 @@ func TestDeriveOptimismEcotoneTxReceipts(t *testing.T) { // Ecotone style l1 attributes with baseFeeScalar=2, blobBaseFeeScalar=3, baseFee=1000*1e6, blobBaseFee=10*1e6 payload := common.Hex2Bytes("440a5e20000000020000000300000000000004d200000000000004d200000000000004d2000000000000000000000000000000000000000000000000000000003b9aca00000000000000000000000000000000000000000000000000000000000098968000000000000000000000000000000000000000000000000000000000000004d200000000000000000000000000000000000000000000000000000000000004d2") // the parameters we use below are defined in rollup_test.go - l1GasPrice := baseFee - l1GasUsed := ecotoneGas - l1Fee := ecotoneFee - txs, receipts := getOptimismTxReceipts(t, payload, l1GasPrice, l1GasUsed, nil /*feeScalar*/, l1Fee) + baseFeeScalarUint64 := baseFeeScalar.Uint64() + blobBaseFeeScalarUint64 := blobBaseFeeScalar.Uint64() + txs, receipts := getOptimismEcotoneTxReceipts(payload, baseFee, blobBaseFee, ecotoneGas, ecotoneFee, &baseFeeScalarUint64, &blobBaseFeeScalarUint64) // Re-derive receipts. baseFee := big.NewInt(1000) diff --git a/core/types/rollup_cost.go b/core/types/rollup_cost.go index c40b24db94..e0bc26d92f 100644 --- a/core/types/rollup_cost.go +++ b/core/types/rollup_cost.go @@ -18,6 +18,7 @@ package types import ( "bytes" + "encoding/binary" "fmt" "math/big" @@ -31,9 +32,9 @@ const ( // sequence number and have the following Solidity offsets within the slot. Note that Solidity // offsets correspond to the last byte of the value in the slot, counting backwards from the // end of the slot. For example, The 8-byte sequence number has offset 0, and is therefore - // stored as big-endian format in bytes [24:32] of the slot. - BaseFeeScalarSlotOffset = 12 // bytes [16:20] of the slot - BlobBaseFeeScalarSlotOffset = 8 // bytes [20:24] of the slot + // stored as big-endian format in bytes [24:32) of the slot. + BaseFeeScalarSlotOffset = 12 // bytes [16:20) of the slot + BlobBaseFeeScalarSlotOffset = 8 // bytes [20:24) of the slot // scalarSectionStart is the beginning of the scalar values segment in the slot // array. baseFeeScalar is in the first four bytes of the segment, blobBaseFeeScalar the next @@ -61,7 +62,7 @@ var ( OverheadSlot = common.BigToHash(big.NewInt(5)) ScalarSlot = common.BigToHash(big.NewInt(6)) - // L2BlobBaseFeeSlot was added with the Ecotone upgrade and stores the blobBaseFee L1 gas + // L1BlobBaseFeeSlot was added with the Ecotone upgrade and stores the blobBaseFee L1 gas // attribute. L1BlobBaseFeeSlot = common.BigToHash(big.NewInt(7)) // L1FeeScalarsSlot as of the Ecotone upgrade stores the 32-bit basefeeScalar and @@ -71,15 +72,23 @@ var ( oneMillion = big.NewInt(1_000_000) ecotoneDivisor = big.NewInt(1_000_000 * 16) + fjordDivisor = big.NewInt(1_000_000_000_000) sixteen = big.NewInt(16) + L1CostIntercept = big.NewInt(-42_585_600) + L1CostFastlzCoef = big.NewInt(836_500) + + MinTransactionSize = big.NewInt(100) + MinTransactionSizeScaled = new(big.Int).Mul(MinTransactionSize, big.NewInt(1e6)) + emptyScalars = make([]byte, 8) ) // RollupCostData is a transaction structure that caches data for quickly computing the data -// availablility costs for the transaction. +// availability costs for the transaction. type RollupCostData struct { - zeroes, ones uint64 + Zeroes, Ones uint64 + FastLzSize uint64 } type StateGetter interface { @@ -97,11 +106,12 @@ type l1CostFunc func(rcd RollupCostData) (fee, gasUsed *big.Int) func NewRollupCostData(data []byte) (out RollupCostData) { for _, b := range data { if b == 0 { - out.zeroes++ + out.Zeroes++ } else { - out.ones++ + out.Ones++ } } + out.FastLzSize = uint64(FlzCompressLen(data)) return out } @@ -113,6 +123,44 @@ func NewL1CostFunc(config *params.ChainConfig, statedb StateGetter) L1CostFunc { } forBlock := ^uint64(0) var cachedFunc l1CostFunc + selectFunc := func(blockTime uint64) l1CostFunc { + if !config.IsOptimismEcotone(blockTime) { + return newL1CostFuncBedrock(config, statedb, blockTime) + } + + // Note: the various state variables below are not initialized from the DB until this + // point to allow deposit transactions from the block to be processed first by state + // transition. This behavior is consensus critical! + l1FeeScalars := statedb.GetState(L1BlockAddr, L1FeeScalarsSlot).Bytes() + l1BlobBaseFee := statedb.GetState(L1BlockAddr, L1BlobBaseFeeSlot).Big() + l1BaseFee := statedb.GetState(L1BlockAddr, L1BaseFeeSlot).Big() + + // Edge case: the very first Ecotone block requires we use the Bedrock cost + // function. We detect this scenario by checking if the Ecotone parameters are + // unset. Note here we rely on assumption that the scalar parameters are adjacent + // in the buffer and l1BaseFeeScalar comes first. We need to check this prior to + // other forks, as the first block of Fjord and Ecotone could be the same block. + firstEcotoneBlock := l1BlobBaseFee.BitLen() == 0 && + bytes.Equal(emptyScalars, l1FeeScalars[scalarSectionStart:scalarSectionStart+8]) + if firstEcotoneBlock { + log.Info("using bedrock l1 cost func for first Ecotone block", "time", blockTime) + return newL1CostFuncBedrock(config, statedb, blockTime) + } + + l1BaseFeeScalar, l1BlobBaseFeeScalar := extractEcotoneFeeParams(l1FeeScalars) + + if config.IsOptimismFjord(blockTime) { + return NewL1CostFuncFjord( + l1BaseFee, + l1BlobBaseFee, + l1BaseFeeScalar, + l1BlobBaseFeeScalar, + ) + } else { + return newL1CostFuncEcotone(l1BaseFee, l1BlobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar) + } + } + return func(rollupCostData RollupCostData, blockTime uint64) *big.Int { if rollupCostData == (RollupCostData{}) { return nil // Do not charge if there is no rollup cost-data (e.g. RPC call or deposit). @@ -124,31 +172,7 @@ func NewL1CostFunc(config *params.ChainConfig, statedb StateGetter) L1CostFunc { log.Info("l1 cost func re-used for different L1 block", "oldTime", forBlock, "newTime", blockTime) } forBlock = blockTime - // Note: the various state variables below are not initialized from the DB until this - // point to allow deposit transactions from the block to be processed first by state - // transition. This behavior is consensus critical! - if !config.IsOptimismEcotone(blockTime) { - cachedFunc = newL1CostFuncBedrock(config, statedb, blockTime) - } else { - l1BlobBaseFee := statedb.GetState(L1BlockAddr, L1BlobBaseFeeSlot).Big() - l1FeeScalars := statedb.GetState(L1BlockAddr, L1FeeScalarsSlot).Bytes() - - // Edge case: the very first Ecotone block requires we use the Bedrock cost - // function. We detect this scenario by checking if the Ecotone parameters are - // unset. Not here we rely on assumption that the scalar parameters are adjacent - // in the buffer and basefeeScalar comes first. - if l1BlobBaseFee.BitLen() == 0 && - bytes.Equal(emptyScalars, l1FeeScalars[scalarSectionStart:scalarSectionStart+8]) { - log.Info("using bedrock l1 cost func for first Ecotone block", "time", blockTime) - cachedFunc = newL1CostFuncBedrock(config, statedb, blockTime) - } else { - l1BaseFee := statedb.GetState(L1BlockAddr, L1BaseFeeSlot).Big() - offset := scalarSectionStart - l1BaseFeeScalar := new(big.Int).SetBytes(l1FeeScalars[offset : offset+4]) - l1BlobBaseFeeScalar := new(big.Int).SetBytes(l1FeeScalars[offset+4 : offset+8]) - cachedFunc = newL1CostFuncEcotone(l1BaseFee, l1BlobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar) - } - } + cachedFunc = selectFunc(blockTime) } fee, _ := cachedFunc(rollupCostData) return fee @@ -172,11 +196,11 @@ func newL1CostFuncBedrockHelper(l1BaseFee, overhead, scalar *big.Int, isRegolith if rollupCostData == (RollupCostData{}) { return nil, nil // Do not charge if there is no rollup cost-data (e.g. RPC call or deposit) } - gas := rollupCostData.zeroes * params.TxDataZeroGas + gas := rollupCostData.Zeroes * params.TxDataZeroGas if isRegolith { - gas += rollupCostData.ones * params.TxDataNonZeroGasEIP2028 + gas += rollupCostData.Ones * params.TxDataNonZeroGasEIP2028 } else { - gas += (rollupCostData.ones + 68) * params.TxDataNonZeroGasEIP2028 + gas += (rollupCostData.Ones + 68) * params.TxDataNonZeroGasEIP2028 } gasWithOverhead := new(big.Int).SetUint64(gas) gasWithOverhead.Add(gasWithOverhead, overhead) @@ -189,8 +213,7 @@ func newL1CostFuncBedrockHelper(l1BaseFee, overhead, scalar *big.Int, isRegolith // very first block of the upgrade. func newL1CostFuncEcotone(l1BaseFee, l1BlobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar *big.Int) l1CostFunc { return func(costData RollupCostData) (fee, calldataGasUsed *big.Int) { - calldataGas := (costData.zeroes * params.TxDataZeroGas) + (costData.ones * params.TxDataNonZeroGasEIP2028) - calldataGasUsed = new(big.Int).SetUint64(calldataGas) + calldataGasUsed = bedrockCalldataGasUsed(costData) // Ecotone L1 cost function: // @@ -218,41 +241,82 @@ func newL1CostFuncEcotone(l1BaseFee, l1BlobBaseFee, l1BaseFeeScalar, l1BlobBaseF } } +type gasParams struct { + l1BaseFee *big.Int + l1BlobBaseFee *big.Int + costFunc l1CostFunc + feeScalar *big.Float // pre-ecotone + l1BaseFeeScalar *uint32 // post-ecotone + l1BlobBaseFeeScalar *uint32 // post-ecotone +} + +// intToScaledFloat returns scalar/10e6 as a float +func intToScaledFloat(scalar *big.Int) *big.Float { + fscalar := new(big.Float).SetInt(scalar) + fdivisor := new(big.Float).SetUint64(1_000_000) // 10**6, i.e. 6 decimals + return new(big.Float).Quo(fscalar, fdivisor) +} + // extractL1GasParams extracts the gas parameters necessary to compute gas costs from L1 block info -func extractL1GasParams(config *params.ChainConfig, time uint64, data []byte) (l1BaseFee *big.Int, costFunc l1CostFunc, feeScalar *big.Float, err error) { - if config.IsEcotone(time) { - // edge case: for the very first Ecotone block we still need to use the Bedrock - // function. We detect this edge case by seeing if the function selector is the old one - if len(data) >= 4 && !bytes.Equal(data[0:4], BedrockL1AttributesSelector) { - l1BaseFee, costFunc, err = extractL1GasParamsEcotone(data) - return +func extractL1GasParams(config *params.ChainConfig, time uint64, data []byte) (gasParams, error) { + // edge case: for the very first Ecotone block we still need to use the Bedrock + // function. We detect this edge case by seeing if the function selector is the old one + // If so, fall through to the pre-ecotone format + // Both Ecotone and Fjord use the same function selector + if config.IsEcotone(time) && len(data) >= 4 && !bytes.Equal(data[0:4], BedrockL1AttributesSelector) { + p, err := extractL1GasParamsPostEcotone(data) + if err != nil { + return gasParams{}, err + } + + if config.IsFjord(time) { + p.costFunc = NewL1CostFuncFjord( + p.l1BaseFee, + p.l1BlobBaseFee, + big.NewInt(int64(*p.l1BaseFeeScalar)), + big.NewInt(int64(*p.l1BlobBaseFeeScalar)), + ) + } else { + p.costFunc = newL1CostFuncEcotone( + p.l1BaseFee, + p.l1BlobBaseFee, + big.NewInt(int64(*p.l1BaseFeeScalar)), + big.NewInt(int64(*p.l1BlobBaseFeeScalar)), + ) } + + return p, nil } + return extractL1GasParamsPreEcotone(config, time, data) +} +func extractL1GasParamsPreEcotone(config *params.ChainConfig, time uint64, data []byte) (gasParams, error) { // data consists of func selector followed by 7 ABI-encoded parameters (32 bytes each) if len(data) < 4+32*8 { - return nil, nil, nil, fmt.Errorf("expected at least %d L1 info bytes, got %d", 4+32*8, len(data)) + return gasParams{}, fmt.Errorf("expected at least %d L1 info bytes, got %d", 4+32*8, len(data)) } - data = data[4:] // trim function selector - l1BaseFee = new(big.Int).SetBytes(data[32*2 : 32*3]) // arg index 2 - overhead := new(big.Int).SetBytes(data[32*6 : 32*7]) // arg index 6 - scalar := new(big.Int).SetBytes(data[32*7 : 32*8]) // arg index 7 - fscalar := new(big.Float).SetInt(scalar) // legacy: format fee scalar as big Float - fdivisor := new(big.Float).SetUint64(1_000_000) // 10**6, i.e. 6 decimals - feeScalar = new(big.Float).Quo(fscalar, fdivisor) - costFunc = newL1CostFuncBedrockHelper(l1BaseFee, overhead, scalar, config.IsRegolith(time)) - return + data = data[4:] // trim function selector + l1BaseFee := new(big.Int).SetBytes(data[32*2 : 32*3]) // arg index 2 + overhead := new(big.Int).SetBytes(data[32*6 : 32*7]) // arg index 6 + scalar := new(big.Int).SetBytes(data[32*7 : 32*8]) // arg index 7 + feeScalar := intToScaledFloat(scalar) // legacy: format fee scalar as big Float + costFunc := newL1CostFuncBedrockHelper(l1BaseFee, overhead, scalar, config.IsRegolith(time)) + return gasParams{ + l1BaseFee: l1BaseFee, + costFunc: costFunc, + feeScalar: feeScalar, + }, nil } -// extractEcotoneL1GasParams extracts the gas parameters necessary to compute gas from L1 attribute +// extractL1GasParamsPostEcotone extracts the gas parameters necessary to compute gas from L1 attribute // info calldata after the Ecotone upgrade, but not for the very first Ecotone block. -func extractL1GasParamsEcotone(data []byte) (l1BaseFee *big.Int, costFunc l1CostFunc, err error) { +func extractL1GasParamsPostEcotone(data []byte) (gasParams, error) { if len(data) != 164 { - return nil, nil, fmt.Errorf("expected 164 L1 info bytes, got %d", len(data)) + return gasParams{}, fmt.Errorf("expected 164 L1 info bytes, got %d", len(data)) } // data layout assumed for Ecotone: // offset type varname - // 0 + // 0 // 4 uint32 _basefeeScalar // 8 uint32 _blobBaseFeeScalar // 12 uint64 _sequenceNumber, @@ -260,14 +324,18 @@ func extractL1GasParamsEcotone(data []byte) (l1BaseFee *big.Int, costFunc l1Cost // 28 uint64 _l1BlockNumber // 36 uint256 _basefee, // 68 uint256 _blobBaseFee, - // 100 bytes32 _hash, + // 100 bytes32 _hash, // 132 bytes32 _batcherHash, - l1BaseFee = new(big.Int).SetBytes(data[36:68]) + l1BaseFee := new(big.Int).SetBytes(data[36:68]) l1BlobBaseFee := new(big.Int).SetBytes(data[68:100]) - l1BaseFeeScalar := new(big.Int).SetBytes(data[4:8]) - l1BlobBaseFeeScalar := new(big.Int).SetBytes(data[8:12]) - costFunc = newL1CostFuncEcotone(l1BaseFee, l1BlobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar) - return + l1BaseFeeScalar := binary.BigEndian.Uint32(data[4:8]) + l1BlobBaseFeeScalar := binary.BigEndian.Uint32(data[8:12]) + return gasParams{ + l1BaseFee: l1BaseFee, + l1BlobBaseFee: l1BlobBaseFee, + l1BaseFeeScalar: &l1BaseFeeScalar, + l1BlobBaseFeeScalar: &l1BlobBaseFeeScalar, + }, nil } // L1Cost computes the the data availability fee for transactions in blocks prior to the Ecotone @@ -283,3 +351,123 @@ func l1CostHelper(gasWithOverhead, l1BaseFee, scalar *big.Int) *big.Int { fee.Mul(fee, l1BaseFee).Mul(fee, scalar).Div(fee, oneMillion) return fee } + +// NewL1CostFuncFjord returns an l1 cost function suitable for the Fjord upgrade +func NewL1CostFuncFjord(l1BaseFee, l1BlobBaseFee, baseFeeScalar, blobFeeScalar *big.Int) l1CostFunc { + return func(costData RollupCostData) (fee, calldataGasUsed *big.Int) { + // Fjord L1 cost function: + //l1FeeScaled = baseFeeScalar*l1BaseFee*16 + blobFeeScalar*l1BlobBaseFee + //estimatedSize = max(minTransactionSize, intercept + fastlzCoef*fastlzSize) + //l1Cost = estimatedSize * l1FeeScaled / 1e12 + + scaledL1BaseFee := new(big.Int).Mul(baseFeeScalar, l1BaseFee) + calldataCostPerByte := new(big.Int).Mul(scaledL1BaseFee, sixteen) + blobCostPerByte := new(big.Int).Mul(blobFeeScalar, l1BlobBaseFee) + l1FeeScaled := new(big.Int).Add(calldataCostPerByte, blobCostPerByte) + + fastLzSize := new(big.Int).SetUint64(costData.FastLzSize) + estimatedSize := new(big.Int).Add(L1CostIntercept, new(big.Int).Mul(L1CostFastlzCoef, fastLzSize)) + + if estimatedSize.Cmp(MinTransactionSizeScaled) < 0 { + estimatedSize.Set(MinTransactionSizeScaled) + } + + l1CostScaled := new(big.Int).Mul(estimatedSize, l1FeeScaled) + l1Cost := new(big.Int).Div(l1CostScaled, fjordDivisor) + + calldataGasUsed = new(big.Int).Mul(estimatedSize, new(big.Int).SetUint64(params.TxDataNonZeroGasEIP2028)) + calldataGasUsed.Div(calldataGasUsed, big.NewInt(1e6)) + + return l1Cost, calldataGasUsed + } +} + +func extractEcotoneFeeParams(l1FeeParams []byte) (l1BaseFeeScalar, l1BlobBaseFeeScalar *big.Int) { + offset := scalarSectionStart + l1BaseFeeScalar = new(big.Int).SetBytes(l1FeeParams[offset : offset+4]) + l1BlobBaseFeeScalar = new(big.Int).SetBytes(l1FeeParams[offset+4 : offset+8]) + return +} + +func bedrockCalldataGasUsed(costData RollupCostData) (calldataGasUsed *big.Int) { + calldataGas := (costData.Zeroes * params.TxDataZeroGas) + (costData.Ones * params.TxDataNonZeroGasEIP2028) + return new(big.Int).SetUint64(calldataGas) +} + +// FlzCompressLen returns the length of the data after compression through FastLZ, based on +// https://github.com/Vectorized/solady/blob/5315d937d79b335c668896d7533ac603adac5315/js/solady.js +func FlzCompressLen(ib []byte) uint32 { + n := uint32(0) + ht := make([]uint32, 8192) + u24 := func(i uint32) uint32 { + return uint32(ib[i]) | (uint32(ib[i+1]) << 8) | (uint32(ib[i+2]) << 16) + } + cmp := func(p uint32, q uint32, e uint32) uint32 { + l := uint32(0) + for e -= q; l < e; l++ { + if ib[p+l] != ib[q+l] { + e = 0 + } + } + return l + } + literals := func(r uint32) { + n += 0x21 * (r / 0x20) + r %= 0x20 + if r != 0 { + n += r + 1 + } + } + match := func(l uint32) { + l-- + n += 3 * (l / 262) + if l%262 >= 6 { + n += 3 + } else { + n += 2 + } + } + hash := func(v uint32) uint32 { + return ((2654435769 * v) >> 19) & 0x1fff + } + setNextHash := func(ip uint32) uint32 { + ht[hash(u24(ip))] = ip + return ip + 1 + } + a := uint32(0) + ipLimit := uint32(len(ib)) - 13 + if len(ib) < 13 { + ipLimit = 0 + } + for ip := a + 2; ip < ipLimit; { + r := uint32(0) + d := uint32(0) + for { + s := u24(ip) + h := hash(s) + r = ht[h] + ht[h] = ip + d = ip - r + if ip >= ipLimit { + break + } + ip++ + if d <= 0x1fff && s == u24(r) { + break + } + } + if ip >= ipLimit { + break + } + ip-- + if ip > a { + literals(ip - a) + } + l := cmp(r+3, ip+3, ipLimit+9) + match(l) + ip = setNextHash(setNextHash(ip + l)) + a = ip + } + literals(uint32(len(ib)) - a) + return n +} diff --git a/core/types/rollup_cost_test.go b/core/types/rollup_cost_test.go index d893f4517b..8d0f1e9218 100644 --- a/core/types/rollup_cost_test.go +++ b/core/types/rollup_cost_test.go @@ -1,13 +1,16 @@ package types import ( + "bytes" "encoding/binary" + "encoding/hex" "math/big" "testing" + "github.com/stretchr/testify/require" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/stretchr/testify/require" ) var ( @@ -24,10 +27,13 @@ var ( bedrockFee = big.NewInt(11326000000000) regolithFee = big.NewInt(3710000000000) ecotoneFee = big.NewInt(960900) // (480/16)*(2*16*1000 + 3*10) == 960900 + // the emptyTx is out of bounds for the linear regression so it uses the minimum size + fjordFee = big.NewInt(3203000) // 100_000_000 * (2 * 1000 * 1e6 * 16 + 3 * 10 * 1e6) / 1e12 - bedrockGas = big.NewInt(1618) - regolithGas = big.NewInt(530) // 530 = 1618 - (16*68) - ecotoneGas = big.NewInt(480) + bedrockGas = big.NewInt(1618) + regolithGas = big.NewInt(530) // 530 = 1618 - (16*68) + ecotoneGas = big.NewInt(480) + minimumFjordGas = big.NewInt(1600) // fastlz size of minimum txn, 100_000_000 * 16 / 1e6 ) func TestBedrockL1CostFunc(t *testing.T) { @@ -46,9 +52,64 @@ func TestBedrockL1CostFunc(t *testing.T) { func TestEcotoneL1CostFunc(t *testing.T) { costFunc := newL1CostFuncEcotone(baseFee, blobBaseFee, baseFeeScalar, blobBaseFeeScalar) - c, g := costFunc(emptyTx.RollupCostData()) - require.Equal(t, ecotoneGas, g) - require.Equal(t, ecotoneFee, c) + + c0, g0 := costFunc(emptyTx.RollupCostData()) + + require.Equal(t, ecotoneGas, g0) + require.Equal(t, ecotoneFee, c0) +} + +func TestFjordL1CostFuncMinimumBounds(t *testing.T) { + costFunc := NewL1CostFuncFjord( + baseFee, + blobBaseFee, + baseFeeScalar, + blobBaseFeeScalar, + ) + + // Minimum size transactions: + // -42.5856 + 0.8365*110 = 49.4294 + // -42.5856 + 0.8365*150 = 82.8894 + // -42.5856 + 0.8365*170 = 99.6194 + for _, fastLzsize := range []uint64{100, 150, 170} { + c, g := costFunc(RollupCostData{ + FastLzSize: fastLzsize, + }) + + require.Equal(t, minimumFjordGas, g) + require.Equal(t, fjordFee, c) + } + + // Larger size transactions: + // -42.5856 + 0.8365*171 = 100.4559 + // -42.5856 + 0.8365*175 = 108.8019 + // -42.5856 + 0.8365*200 = 124.7144 + for _, fastLzsize := range []uint64{171, 175, 200} { + c, g := costFunc(RollupCostData{ + FastLzSize: fastLzsize, + }) + + require.Greater(t, g.Uint64(), minimumFjordGas.Uint64()) + require.Greater(t, c.Uint64(), fjordFee.Uint64()) + } +} + +// TestFjordL1CostSolidityParity tests that the cost function for the fjord upgrade matches a Solidity +// test to ensure the outputs are the same. +func TestFjordL1CostSolidityParity(t *testing.T) { + costFunc := NewL1CostFuncFjord( + big.NewInt(2*1e6), + big.NewInt(3*1e6), + big.NewInt(20), + big.NewInt(15), + ) + + c0, g0 := costFunc(RollupCostData{ + FastLzSize: 235, + }) + + require.Equal(t, big.NewInt(2463), g0) + require.Equal(t, big.NewInt(105484), c0) } func TestExtractBedrockGasParams(t *testing.T) { @@ -60,7 +121,8 @@ func TestExtractBedrockGasParams(t *testing.T) { data := getBedrockL1Attributes(baseFee, overhead, scalar) - _, costFuncPreRegolith, _, err := extractL1GasParams(config, regolithTime-1, data) + gasparams, err := extractL1GasParams(config, regolithTime-1, data) + costFuncPreRegolith := gasparams.costFunc require.NoError(t, err) // Function should continue to succeed even with extra data (that just gets ignored) since we @@ -68,7 +130,8 @@ func TestExtractBedrockGasParams(t *testing.T) { // the expected number of bytes. It's unclear if this flexibility was intentional, but since // it's been in production we shouldn't change this behavior. data = append(data, []byte{0xBE, 0xEE, 0xEE, 0xFF}...) // tack on garbage data - _, costFuncRegolith, _, err := extractL1GasParams(config, regolithTime, data) + gasparams, err = extractL1GasParams(config, regolithTime, data) + costFuncRegolith := gasparams.costFunc require.NoError(t, err) c, _ := costFuncPreRegolith(emptyTx.RollupCostData()) @@ -79,7 +142,7 @@ func TestExtractBedrockGasParams(t *testing.T) { // try to extract from data which has not enough params, should get error. data = data[:len(data)-4-32] - _, _, _, err = extractL1GasParams(config, regolithTime, data) + _, err = extractL1GasParams(config, regolithTime, data) require.Error(t, err) } @@ -91,12 +154,18 @@ func TestExtractEcotoneGasParams(t *testing.T) { RegolithTime: &zeroTime, EcotoneTime: &zeroTime, } - require.True(t, config.IsOptimismEcotone(0)) + require.True(t, config.IsOptimismEcotone(zeroTime)) - data := getEcotoneL1Attributes(baseFee, blobBaseFee, baseFeeScalar, blobBaseFeeScalar) + data := getEcotoneL1Attributes( + baseFee, + blobBaseFee, + baseFeeScalar, + blobBaseFeeScalar, + ) - _, costFunc, _, err := extractL1GasParams(config, 0, data) + gasparams, err := extractL1GasParams(config, zeroTime, data) require.NoError(t, err) + costFunc := gasparams.costFunc c, g := costFunc(emptyTx.RollupCostData()) @@ -105,10 +174,38 @@ func TestExtractEcotoneGasParams(t *testing.T) { // make sure wrong amont of data results in error data = append(data, 0x00) // tack on garbage byte - _, _, err = extractL1GasParamsEcotone(data) + _, err = extractL1GasParamsPostEcotone(data) require.Error(t, err) } +func TestExtractFjordGasParams(t *testing.T) { + zeroTime := uint64(0) + // create a config where fjord is active + config := ¶ms.ChainConfig{ + Optimism: params.OptimismTestConfig.Optimism, + RegolithTime: &zeroTime, + EcotoneTime: &zeroTime, + FjordTime: &zeroTime, + } + require.True(t, config.IsOptimismFjord(zeroTime)) + + data := getEcotoneL1Attributes( + baseFee, + blobBaseFee, + baseFeeScalar, + blobBaseFeeScalar, + ) + + gasparams, err := extractL1GasParams(config, zeroTime, data) + require.NoError(t, err) + costFunc := gasparams.costFunc + + c, g := costFunc(emptyTx.RollupCostData()) + + require.Equal(t, minimumFjordGas, g) + require.Equal(t, fjordFee, c) +} + // make sure the first block of the ecotone upgrade is properly detected, and invokes the bedrock // cost function appropriately func TestFirstBlockEcotoneGasParams(t *testing.T) { @@ -123,9 +220,11 @@ func TestFirstBlockEcotoneGasParams(t *testing.T) { data := getBedrockL1Attributes(baseFee, overhead, scalar) - _, oldCostFunc, _, err := extractL1GasParams(config, 0, data) + gasparams, err := extractL1GasParams(config, zeroTime, data) require.NoError(t, err) - c, _ := oldCostFunc(emptyTx.RollupCostData()) + oldCostFunc := gasparams.costFunc + c, g := oldCostFunc(emptyTx.RollupCostData()) + require.Equal(t, regolithGas, g) require.Equal(t, regolithFee, c) } @@ -148,19 +247,19 @@ func getBedrockL1Attributes(baseFee, overhead, scalar *big.Int) []byte { func getEcotoneL1Attributes(baseFee, blobBaseFee, baseFeeScalar, blobBaseFeeScalar *big.Int) []byte { ignored := big.NewInt(1234) data := []byte{} - uint256 := make([]byte, 32) - uint64 := make([]byte, 8) - uint32 := make([]byte, 4) + uint256Slice := make([]byte, 32) + uint64Slice := make([]byte, 8) + uint32Slice := make([]byte, 4) data = append(data, EcotoneL1AttributesSelector...) - data = append(data, baseFeeScalar.FillBytes(uint32)...) - data = append(data, blobBaseFeeScalar.FillBytes(uint32)...) - data = append(data, ignored.FillBytes(uint64)...) - data = append(data, ignored.FillBytes(uint64)...) - data = append(data, ignored.FillBytes(uint64)...) - data = append(data, baseFee.FillBytes(uint256)...) - data = append(data, blobBaseFee.FillBytes(uint256)...) - data = append(data, ignored.FillBytes(uint256)...) - data = append(data, ignored.FillBytes(uint256)...) + data = append(data, baseFeeScalar.FillBytes(uint32Slice)...) + data = append(data, blobBaseFeeScalar.FillBytes(uint32Slice)...) + data = append(data, ignored.FillBytes(uint64Slice)...) + data = append(data, ignored.FillBytes(uint64Slice)...) + data = append(data, ignored.FillBytes(uint64Slice)...) + data = append(data, baseFee.FillBytes(uint256Slice)...) + data = append(data, blobBaseFee.FillBytes(uint256Slice)...) + data = append(data, ignored.FillBytes(uint256Slice)...) + data = append(data, ignored.FillBytes(uint256Slice)...) return data } @@ -181,6 +280,7 @@ func (sg *testStateGetter) GetState(addr common.Address, slot common.Hash) commo case L1BlobBaseFeeSlot: sg.blobBaseFee.FillBytes(buf[:]) case L1FeeScalarsSlot: + // fetch Ecotone fee sclars offset := scalarSectionStart binary.BigEndian.PutUint32(buf[offset:offset+4], sg.baseFeeScalar) binary.BigEndian.PutUint32(buf[offset+4:offset+8], sg.blobBaseFeeScalar) @@ -193,7 +293,8 @@ func (sg *testStateGetter) GetState(addr common.Address, slot common.Hash) commo // TestNewL1CostFunc tests that the appropriate cost function is selected based on the // configuration and statedb values. func TestNewL1CostFunc(t *testing.T) { - time := uint64(1) + time := uint64(10) + timeInFuture := uint64(20) config := ¶ms.ChainConfig{ Optimism: params.OptimismTestConfig.Optimism, } @@ -233,8 +334,16 @@ func TestNewL1CostFunc(t *testing.T) { require.NotNil(t, fee) require.Equal(t, ecotoneFee, fee) + // emptyTx fee w/ fjord config should be the fjord fee + config.FjordTime = &time + costFunc = NewL1CostFunc(config, statedb) + fee = costFunc(emptyTx.RollupCostData(), time) + require.NotNil(t, fee) + require.Equal(t, fjordFee, fee) + // emptyTx fee w/ ecotone config, but simulate first ecotone block by blowing away the ecotone // params. Should result in regolith fee. + config.FjordTime = &timeInFuture statedb.baseFeeScalar = 0 statedb.blobBaseFeeScalar = 0 statedb.blobBaseFee = new(big.Int) @@ -242,4 +351,57 @@ func TestNewL1CostFunc(t *testing.T) { fee = costFunc(emptyTx.RollupCostData(), time) require.NotNil(t, fee) require.Equal(t, regolithFee, fee) + + // emptyTx fee w/ fjord config, but simulate first ecotone block by blowing away the ecotone + // params. Should result in regolith fee. + config.EcotoneTime = &time + config.FjordTime = &time + statedb.baseFeeScalar = 0 + statedb.blobBaseFeeScalar = 0 + statedb.blobBaseFee = new(big.Int) + costFunc = NewL1CostFunc(config, statedb) + fee = costFunc(emptyTx.RollupCostData(), time) + require.NotNil(t, fee) + require.Equal(t, regolithFee, fee) +} + +func TestFlzCompressLen(t *testing.T) { + var ( + emptyTxBytes, _ = emptyTx.MarshalBinary() + contractCallTxStr = "02f901550a758302df1483be21b88304743f94f8" + + "0e51afb613d764fa61751affd3313c190a86bb870151bd62fd12adb8" + + "e41ef24f3f0000000000000000000000000000000000000000000000" + + "00000000000000006e000000000000000000000000af88d065e77c8c" + + "c2239327c5edb3a432268e5831000000000000000000000000000000" + + "000000000000000000000000000003c1e50000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000a0000000" + + "00000000000000000000000000000000000000000000000000000000" + + "148c89ed219d02f1a5be012c689b4f5b731827bebe00000000000000" + + "0000000000c001a033fd89cb37c31b2cba46b6466e040c61fc9b2a36" + + "75a7f5f493ebd5ad77c497f8a07cdf65680e238392693019b4092f61" + + "0222e71b7cec06449cb922b93b6a12744e" + contractCallTx, _ = hex.DecodeString(contractCallTxStr) + ) + + testCases := []struct { + input []byte + expectedLen uint32 + }{ + // empty input + {[]byte{}, 0}, + // all 1 inputs + {bytes.Repeat([]byte{1}, 1000), 21}, + // all 0 inputs + {make([]byte, 1000), 21}, + // empty tx input + {emptyTxBytes, 31}, + // contract call tx: https://optimistic.etherscan.io/tx/0x8eb9dd4eb6d33f4dc25fb015919e4b1e9f7542f9b0322bf6622e268cd116b594 + {contractCallTx, 202}, + } + + for _, tc := range testCases { + output := FlzCompressLen(tc.input) + require.Equal(t, tc.expectedLen, output) + } } diff --git a/core/types/state_account.go b/core/types/state_account.go index ad07ca3f3a..52ef843b35 100644 --- a/core/types/state_account.go +++ b/core/types/state_account.go @@ -18,10 +18,10 @@ package types import ( "bytes" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) //go:generate go run ../../rlp/rlpgen -type StateAccount -out gen_account_rlp.go @@ -30,7 +30,7 @@ import ( // These objects are stored in the main account trie. type StateAccount struct { Nonce uint64 - Balance *big.Int + Balance *uint256.Int Root common.Hash // merkle root of the storage trie CodeHash []byte } @@ -38,7 +38,7 @@ type StateAccount struct { // NewEmptyStateAccount constructs an empty state account. func NewEmptyStateAccount() *StateAccount { return &StateAccount{ - Balance: new(big.Int), + Balance: new(uint256.Int), Root: EmptyRootHash, CodeHash: EmptyCodeHash.Bytes(), } @@ -46,9 +46,9 @@ func NewEmptyStateAccount() *StateAccount { // Copy returns a deep-copied state account object. func (acct *StateAccount) Copy() *StateAccount { - var balance *big.Int + var balance *uint256.Int if acct.Balance != nil { - balance = new(big.Int).Set(acct.Balance) + balance = new(uint256.Int).Set(acct.Balance) } return &StateAccount{ Nonce: acct.Nonce, @@ -63,7 +63,7 @@ func (acct *StateAccount) Copy() *StateAccount { // or slim format which replaces the empty root and code hash as nil byte slice. type SlimAccount struct { Nonce uint64 - Balance *big.Int + Balance *uint256.Int Root []byte // Nil if root equals to types.EmptyRootHash CodeHash []byte // Nil if hash equals to types.EmptyCodeHash } diff --git a/core/types/transaction.go b/core/types/transaction.go index d41c6a5488..d7860b57bd 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -389,6 +389,7 @@ func (tx *Transaction) RollupCostData() RollupCostData { // RawSignatureValues returns the V, R, S signature values of the transaction. // The return values should not be modified by the caller. +// The return values may be nil or zero, if the transaction is unsigned. func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) { return tx.inner.rawSignatureValues() } @@ -605,6 +606,9 @@ func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, e if err != nil { return nil, err } + if r == nil || s == nil || v == nil { + return nil, fmt.Errorf("%w: r: %s, s: %s, v: %s", ErrInvalidSig, r, s, v) + } cpy := tx.inner.copy() cpy.setSignatureValues(signer.ChainID(), v, r, s) return &Transaction{inner: cpy, time: tx.time}, nil diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index fcb8bc21bc..253e4f03de 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/rlp" "github.com/holiman/uint256" ) @@ -55,6 +56,11 @@ type txJSON struct { Mint *hexutil.Big `json:"mint,omitempty"` IsSystemTx *bool `json:"isSystemTx,omitempty"` + // Blob transaction sidecar encoding: + Blobs []kzg4844.Blob `json:"blobs,omitempty"` + Commitments []kzg4844.Commitment `json:"commitments,omitempty"` + Proofs []kzg4844.Proof `json:"proofs,omitempty"` + // Only used for encoding: Hash common.Hash `json:"hash"` } @@ -150,6 +156,12 @@ func (tx *Transaction) MarshalJSON() ([]byte, error) { enc.S = (*hexutil.Big)(itx.S.ToBig()) yparity := itx.V.Uint64() enc.YParity = (*hexutil.Uint64)(&yparity) + if sidecar := itx.Sidecar; sidecar != nil { + enc.Blobs = itx.Sidecar.Blobs + enc.Commitments = itx.Sidecar.Commitments + enc.Proofs = itx.Sidecar.Proofs + } + case *DepositTx: enc.Gas = (*hexutil.Uint64)(&itx.Gas) enc.Value = (*hexutil.Big)(itx.Value) diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 2a9ceb0952..b66577f7ed 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -18,11 +18,13 @@ package types import ( "errors" + "fmt" "math/big" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -41,7 +43,7 @@ func TestEIP155Signing(t *testing.T) { t.Fatal(err) } if from != addr { - t.Errorf("exected from and address to be equal. Got %x want %x", from, addr) + t.Errorf("expected from and address to be equal. Got %x want %x", from, addr) } } @@ -136,3 +138,53 @@ func TestChainId(t *testing.T) { t.Error("expected no error") } } + +type nilSigner struct { + v, r, s *big.Int + Signer +} + +func (ns *nilSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v *big.Int, err error) { + return ns.v, ns.r, ns.s, nil +} + +// TestNilSigner ensures a faulty Signer implementation does not result in nil signature values or panics. +func TestNilSigner(t *testing.T) { + key, _ := crypto.GenerateKey() + innerSigner := LatestSignerForChainID(big.NewInt(1)) + for i, signer := range []Signer{ + &nilSigner{v: nil, r: nil, s: nil, Signer: innerSigner}, + &nilSigner{v: big.NewInt(1), r: big.NewInt(1), s: nil, Signer: innerSigner}, + &nilSigner{v: big.NewInt(1), r: nil, s: big.NewInt(1), Signer: innerSigner}, + &nilSigner{v: nil, r: big.NewInt(1), s: big.NewInt(1), Signer: innerSigner}, + } { + t.Run(fmt.Sprintf("signer_%d", i), func(t *testing.T) { + t.Run("legacy", func(t *testing.T) { + legacyTx := createTestLegacyTxInner() + _, err := SignNewTx(key, signer, legacyTx) + if !errors.Is(err, ErrInvalidSig) { + t.Fatal("expected signature values error, no nil result or panic") + } + }) + // test Blob tx specifically, since the signature value types changed + t.Run("blobtx", func(t *testing.T) { + blobtx := createEmptyBlobTxInner(false) + _, err := SignNewTx(key, signer, blobtx) + if !errors.Is(err, ErrInvalidSig) { + t.Fatal("expected signature values error, no nil result or panic") + } + }) + }) + } +} + +func createTestLegacyTxInner() *LegacyTx { + return &LegacyTx{ + Nonce: uint64(0), + To: nil, + Value: big.NewInt(0), + Gas: params.TxGas, + GasPrice: big.NewInt(params.GWei), + Data: nil, + } +} diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index 2f135a490e..ceaeedb20d 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -43,7 +43,7 @@ type BlobTx struct { BlobHashes []common.Hash // A blob transaction can optionally contain blobs. This field must be set when BlobTx - // is used to create a transaction for sigining. + // is used to create a transaction for signing. Sidecar *BlobTxSidecar `rlp:"-"` // Signature values @@ -61,9 +61,10 @@ type BlobTxSidecar struct { // BlobHashes computes the blob hashes of the given blobs. func (sc *BlobTxSidecar) BlobHashes() []common.Hash { + hasher := sha256.New() h := make([]common.Hash, len(sc.Commitments)) for i := range sc.Blobs { - h[i] = blobHash(&sc.Commitments[i]) + h[i] = kzg4844.CalcBlobHashV1(hasher, &sc.Commitments[i]) } return h } @@ -236,12 +237,3 @@ func (tx *BlobTx) decode(input []byte) error { } return nil } - -func blobHash(commit *kzg4844.Commitment) common.Hash { - hasher := sha256.New() - hasher.Write(commit[:]) - var vhash common.Hash - hasher.Sum(vhash[:0]) - vhash[0] = params.BlobTxHashVersion - return vhash -} diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go index 44ac48cc6f..25d09e31ce 100644 --- a/core/types/tx_blob_test.go +++ b/core/types/tx_blob_test.go @@ -65,6 +65,12 @@ var ( ) func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { + blobtx := createEmptyBlobTxInner(withSidecar) + signer := NewCancunSigner(blobtx.ChainID.ToBig()) + return MustSignNewTx(key, signer, blobtx) +} + +func createEmptyBlobTxInner(withSidecar bool) *BlobTx { sidecar := &BlobTxSidecar{ Blobs: []kzg4844.Blob{emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, @@ -85,6 +91,5 @@ func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { if withSidecar { blobtx.Sidecar = sidecar } - signer := NewCancunSigner(blobtx.ChainID.ToBig()) - return MustSignNewTx(key, signer, blobtx) + return blobtx } diff --git a/core/vm/contract.go b/core/vm/contract.go index 478a728885..0dbb2ebbc9 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -20,7 +20,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/opcodeCompiler/compiler" "github.com/holiman/uint256" - "math/big" ) // ContractRef is a reference to the contract's backing object @@ -59,12 +58,12 @@ type Contract struct { Input []byte Gas uint64 - value *big.Int + value *uint256.Int optimized bool } // NewContract returns a new contract environment for the execution of EVM. -func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uint64) *Contract { +func NewContract(caller ContractRef, object ContractRef, value *uint256.Int, gas uint64) *Contract { c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object} if parent, ok := caller.(*Contract); ok { @@ -183,7 +182,7 @@ func (c *Contract) Address() common.Address { } // Value returns the contract's value (sent to it from it's caller) -func (c *Contract) Value() *big.Int { +func (c *Contract) Value() *uint256.Int { return c.value } diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 2cd9174efb..7b8757dc26 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -152,6 +152,25 @@ var PrecompiledContractsHaber = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{103}): &cometBFTLightBlockValidate{}, } +// PrecompiledContractsFjord contains the default set of pre-compiled Ethereum +// contracts used in the Fjord release. +var PrecompiledContractsFjord = map[common.Address]PrecompiledContract{ + common.BytesToAddress([]byte{1}): &ecrecover{}, + common.BytesToAddress([]byte{2}): &sha256hash{}, + common.BytesToAddress([]byte{3}): &ripemd160hash{}, + common.BytesToAddress([]byte{4}): &dataCopy{}, + common.BytesToAddress([]byte{5}): &bigModExp{eip2565: true}, + common.BytesToAddress([]byte{6}): &bn256AddIstanbul{}, + common.BytesToAddress([]byte{7}): &bn256ScalarMulIstanbul{}, + common.BytesToAddress([]byte{8}): &bn256PairingIstanbul{}, + common.BytesToAddress([]byte{9}): &blake2F{}, + common.BytesToAddress([]byte{0x0a}): &kzgPointEvaluation{}, + common.BytesToAddress([]byte{0x01, 0x00}): &p256Verify{}, + + common.BytesToAddress([]byte{102}): &blsSignatureVerify{}, + common.BytesToAddress([]byte{103}): &cometBFTLightBlockValidate{}, +} + // PrecompiledContractsBLS contains the set of pre-compiled Ethereum // contracts specified in EIP-2537. These are exported for testing purposes. var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{ @@ -167,6 +186,7 @@ var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{ } var ( + PrecompiledAddressesFjord []common.Address PrecompiledAddressesHaber []common.Address PrecompiledAddressesCancun []common.Address PrecompiledAddressesFermat []common.Address @@ -198,11 +218,16 @@ func init() { for k := range PrecompiledContractsHaber { PrecompiledAddressesHaber = append(PrecompiledAddressesHaber, k) } + for k := range PrecompiledContractsFjord { + PrecompiledAddressesFjord = append(PrecompiledAddressesFjord, k) + } } // ActivePrecompiles returns the precompiles enabled with the current configuration. func ActivePrecompiles(rules params.Rules) []common.Address { switch { + case rules.IsOptimismFjord: + return PrecompiledAddressesFjord case rules.IsHaber: return PrecompiledAddressesHaber case rules.IsCancun: diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index b9b415c431..01220723cc 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -230,7 +230,7 @@ func BenchmarkPrecompiledRipeMD(bench *testing.B) { benchmarkPrecompiled("03", t, bench) } -// Benchmarks the sample inputs from the identiy precompile. +// Benchmarks the sample inputs from the identity precompile. func BenchmarkPrecompiledIdentity(bench *testing.B) { t := precompiledTest{ Input: "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02", diff --git a/core/vm/eips.go b/core/vm/eips.go index 35f0a3f7c2..9f06b2818f 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -85,7 +85,7 @@ func enable1884(jt *JumpTable) { } func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - balance, _ := uint256.FromBig(interpreter.evm.StateDB.GetBalance(scope.Contract.Address())) + balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) scope.Stack.push(balance) return nil, nil } diff --git a/core/vm/evm.go b/core/vm/evm.go index aefcb40187..43ab27308b 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -31,9 +31,9 @@ import ( type ( // CanTransferFunc is the signature of a transfer guard function - CanTransferFunc func(StateDB, common.Address, *big.Int) bool + CanTransferFunc func(StateDB, common.Address, *uint256.Int) bool // TransferFunc is the signature of a transfer function - TransferFunc func(StateDB, common.Address, common.Address, *big.Int) + TransferFunc func(StateDB, common.Address, common.Address, *uint256.Int) // GetHashFunc returns the n'th block hash in the blockchain // and is used by the BLOCKHASH EVM op code. GetHashFunc func(uint64) common.Hash @@ -42,6 +42,8 @@ type ( func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) { var precompiles map[common.Address]PrecompiledContract switch { + case evm.chainRules.IsOptimismFjord: + precompiles = PrecompiledContractsFjord case evm.chainRules.IsHaber: precompiles = PrecompiledContractsHaber case evm.chainRules.IsCancun: @@ -194,13 +196,13 @@ func (evm *EVM) Interpreter() *EVMInterpreter { // parameters. It also handles any necessary value transfer required and takes // the necessary steps to create accounts and reverses the state in case of an // execution error or failed value transfer. -func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { +func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Fail if we're trying to execute above the call depth limit if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } // Fail if we're trying to transfer more than the available balance - if value.Sign() != 0 && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { + if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { return nil, gas, ErrInsufficientBalance } snapshot := evm.StateDB.Snapshot() @@ -208,14 +210,14 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas debug := evm.Config.Tracer != nil if !evm.StateDB.Exist(addr) { - if !isPrecompile && evm.chainRules.IsEIP158 && value.Sign() == 0 { + if !isPrecompile && evm.chainRules.IsEIP158 && value.IsZero() { // Calling a non existing account, don't do anything, but ping the tracer if debug { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value.ToBig()) evm.Config.Tracer.CaptureEnd(ret, 0, nil) } else { - evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value.ToBig()) evm.Config.Tracer.CaptureExit(ret, 0, nil) } } @@ -228,13 +230,13 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // Capture the tracer start/end events in debug mode if debug { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value.ToBig()) defer func(startGas uint64) { // Lazy evaluation of the parameters evm.Config.Tracer.CaptureEnd(ret, startGas-gas, err) }(gas) } else { // Handle tracer events for entering and exiting a call frame - evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -293,7 +295,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // // CallCode differs from Call in the sense that it executes the given address' // code with the caller as context. -func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { +func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { // Fail if we're trying to execute above the call depth limit if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth @@ -309,7 +311,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { - evm.Config.Tracer.CaptureEnter(CALLCODE, caller.Address(), addr, input, gas, value) + evm.Config.Tracer.CaptureEnter(CALLCODE, caller.Address(), addr, input, gas, value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -367,7 +369,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // that caller is something other than a Contract. parent := caller.(*Contract) // DELEGATECALL inherits value from parent call - evm.Config.Tracer.CaptureEnter(DELEGATECALL, caller.Address(), addr, input, gas, parent.value) + evm.Config.Tracer.CaptureEnter(DELEGATECALL, caller.Address(), addr, input, gas, parent.value.ToBig()) defer func(startGas uint64) { evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) }(gas) @@ -425,7 +427,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, // but is the correct thing to do and matters on other networks, in tests, and potential // future scenarios - evm.StateDB.AddBalance(addr, big0) + evm.StateDB.AddBalance(addr, new(uint256.Int)) // Invoke tracer hooks that signal entering/exiting a call frame if evm.Config.Tracer != nil { @@ -445,7 +447,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte addrCopy := addr // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. - contract := NewContract(caller, AccountRef(addrCopy), new(big.Int), gas) + contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas) code := evm.StateDB.GetCode(addrCopy) codeHash := evm.StateDB.GetCodeHash(addrCopy) contract.optimized, code = tryGetOptimizedCode(evm, codeHash, code) @@ -462,7 +464,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte addrCopy := addr // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. - contract := NewContract(caller, AccountRef(addrCopy), new(big.Int), gas) + contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas) contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally @@ -507,7 +509,7 @@ func (c *codeAndHash) Hash() common.Hash { } // create creates a new contract using code as deployment code. -func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) { +func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) { // Depth check execution. Fail if we're trying to execute above the // limit. if evm.depth > int(params.CallCreateDepth) { @@ -546,9 +548,9 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, if evm.Config.Tracer != nil { if evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value) + evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value.ToBig()) } else { - evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value) + evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value.ToBig()) } } // We don't optimize creation code as it run only once. @@ -607,7 +609,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } // Create creates a new contract using code as deployment code. -func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { +func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address())) return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr, CREATE) } @@ -616,7 +618,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // // The different between Create2 with Create is Create2 uses keccak256(0xff ++ msg.sender ++ salt ++ keccak256(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. -func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { +func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *uint256.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { codeAndHash := &codeAndHash{code: code} contractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes()) return evm.create(caller, codeAndHash, gas, endowment, contractAddr, CREATE2) diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index c1f7fd70c8..1ee9d691df 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) func TestMemoryGasCost(t *testing.T) { @@ -91,12 +92,12 @@ func TestEIP2200(t *testing.T) { statedb.Finalise(true) // Push the state into the "original" slot vmctx := BlockContext{ - CanTransfer: func(StateDB, common.Address, *big.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) - _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(big.Int)) + _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) if err != tt.failure { t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure) } @@ -141,8 +142,8 @@ func TestCreateGas(t *testing.T) { statedb.SetCode(address, hexutil.MustDecode(tt.code)) statedb.Finalise(true) vmctx := BlockContext{ - CanTransfer: func(StateDB, common.Address, *big.Int) bool { return true }, - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, BlockNumber: big.NewInt(0), } config := Config{} @@ -152,7 +153,7 @@ func TestCreateGas(t *testing.T) { vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, config) var startGas = uint64(testGas) - ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(big.Int)) + ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) if err != nil { return false } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 8dd82aa1fa..431b287415 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,6 +17,8 @@ package vm import ( + "math" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -259,7 +261,7 @@ func opAddress(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] func opBalance(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { slot := scope.Stack.peek() address := common.Address(slot.Bytes20()) - slot.SetFromBig(interpreter.evm.StateDB.GetBalance(address)) + slot.Set(interpreter.evm.StateDB.GetBalance(address)) return nil, nil } @@ -274,8 +276,7 @@ func opCaller(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b } func opCallValue(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - v, _ := uint256.FromBig(scope.Contract.value) - scope.Stack.push(v) + scope.Stack.push(scope.Contract.value) return nil, nil } @@ -345,9 +346,7 @@ func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } func opCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - l := new(uint256.Int) - l.SetUint64(uint64(len(scope.Contract.Code))) - scope.Stack.push(l) + scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Code)))) return nil, nil } @@ -358,7 +357,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } contractRawCode := scope.Contract.Code if interpreter.evm.Config.EnableOpcodeOptimizations && scope.Contract.optimized { @@ -380,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } addr := common.Address(a.Bytes20()) codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) @@ -591,13 +590,8 @@ func opCreate(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b stackvalue := size scope.Contract.UseGas(gas) - //TODO: use uint256.Int instead of converting with toBig() - var bigVal = big0 - if !value.IsZero() { - bigVal = value.ToBig() - } - res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, bigVal) + res, addr, returnGas, suberr := interpreter.evm.Create(scope.Contract, input, gas, &value) // Push item on the stack based on the returned error. If the ruleset is // homestead we must check for CodeStoreOutOfGasError (homestead only // rule) and treat as an error, if the ruleset is frontier we must @@ -635,13 +629,8 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] scope.Contract.UseGas(gas) // reuse size int for stackvalue stackvalue := size - //TODO: use uint256.Int instead of converting with toBig() - bigEndowment := big0 - if !endowment.IsZero() { - bigEndowment = endowment.ToBig() - } res, addr, returnGas, suberr := interpreter.evm.Create2(scope.Contract, input, gas, - bigEndowment, &salt) + &endowment, &salt) // Push item on the stack based on the returned error. if suberr != nil { stackvalue.Clear() @@ -674,16 +663,10 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt if interpreter.readOnly && !value.IsZero() { return nil, ErrWriteProtection } - var bigVal = big0 - //TODO: use uint256.Int instead of converting with toBig() - // By using big0 here, we save an alloc for the most common case (non-ether-transferring contract calls), - // but it would make more sense to extend the usage of uint256.Int if !value.IsZero() { gas += params.CallStipend - bigVal = value.ToBig() } - - ret, returnGas, err := interpreter.evm.Call(scope.Contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.Call(scope.Contract, toAddr, args, gas, &value) if err != nil { temp.Clear() @@ -712,14 +695,11 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ // Get arguments from the memory. args := scope.Memory.GetPtr(int64(inOffset.Uint64()), int64(inSize.Uint64())) - //TODO: use uint256.Int instead of converting with toBig() - var bigVal = big0 if !value.IsZero() { gas += params.CallStipend - bigVal = value.ToBig() } - ret, returnGas, err := interpreter.evm.CallCode(scope.Contract, toAddr, args, gas, bigVal) + ret, returnGas, err := interpreter.evm.CallCode(scope.Contract, toAddr, args, gas, &value) if err != nil { temp.Clear() } else { @@ -823,7 +803,7 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) interpreter.evm.StateDB.SelfDestruct(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) tracer.CaptureExit([]byte{}, 0, nil) } return nil, errStopToken @@ -839,7 +819,7 @@ func opSelfdestruct6780(pc *uint64, interpreter *EVMInterpreter, scope *ScopeCon interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) interpreter.evm.StateDB.Selfdestruct6780(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { - tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) tracer.CaptureExit([]byte{}, 0, nil) } return nil, errStopToken diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 807073336d..8653864d11 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -590,7 +590,7 @@ func TestOpTstore(t *testing.T) { caller = common.Address{} to = common.Address{1} contractRef = contractRef{caller} - contract = NewContract(contractRef, AccountRef(to), new(big.Int), 0) + contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) scopeContext = ScopeContext{mem, stack, contract} value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") ) diff --git a/core/vm/interface.go b/core/vm/interface.go index 26814d3d2f..25bfa06720 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -22,15 +22,16 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // StateDB is an EVM database for full state querying. type StateDB interface { CreateAccount(common.Address) - SubBalance(common.Address, *big.Int) - AddBalance(common.Address, *big.Int) - GetBalance(common.Address) *big.Int + SubBalance(common.Address, *uint256.Int) + AddBalance(common.Address, *uint256.Int) + GetBalance(common.Address) *uint256.Int GetNonce(common.Address) uint64 SetNonce(common.Address, uint64) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index dc5d5fbecc..80acdcc013 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -34,7 +34,7 @@ type Config struct { EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages ExtraEips []int // Additional EIPS that are to be enabled OptimismPrecompileOverrides PrecompileOverrides // Precompile overrides for Optimism - EnableOpcodeOptimizations bool // Enable opcode optimization + EnableOpcodeOptimizations bool // Enable opcode optimization } // ScopeContext contains the things that are per-call, such as stack and memory, @@ -156,7 +156,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( debug = in.evm.Config.Tracer != nil ) // Don't move this deferred function, it's placed before the capturestate-deferred method, - // so that it get's executed _after_: the capturestate needs the stacks before + // so that it gets executed _after_: the capturestate needs the stacks before // they are returned to the pools defer func() { returnStack(stack) diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index 96e681fccd..ff4977d728 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -17,7 +17,6 @@ package vm import ( - "math/big" "testing" "time" @@ -27,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var loopInterruptTests = []string{ @@ -39,7 +39,7 @@ var loopInterruptTests = []string{ func TestLoopInterrupt(t *testing.T) { address := common.BytesToAddress([]byte("contract")) vmctx := BlockContext{ - Transfer: func(StateDB, common.Address, common.Address, *big.Int) {}, + Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } for i, tt := range loopInterruptTests { @@ -54,7 +54,7 @@ func TestLoopInterrupt(t *testing.T) { timeout := make(chan bool) go func(evm *EVM) { - _, _, err := evm.Call(AccountRef(common.Address{}), address, nil, math.MaxUint64, new(big.Int)) + _, _, err := evm.Call(AccountRef(common.Address{}), address, nil, math.MaxUint64, new(uint256.Int)) errChannel <- err }(evm) diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 92af953dc7..b9cf220218 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -122,7 +122,7 @@ func newLondonInstructionSet() JumpTable { // constantinople, istanbul, petersburg and berlin instructions. func newBerlinInstructionSet() JumpTable { instructionSet := newIstanbulInstructionSet() - enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929 + enable2929(&instructionSet) // Gas cost increases for state access opcodes https://eips.ethereum.org/EIPS/eip-2929 return validate(instructionSet) } diff --git a/core/vm/jump_table_test.go b/core/vm/jump_table_test.go index f67915fff3..02558035c0 100644 --- a/core/vm/jump_table_test.go +++ b/core/vm/jump_table_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestJumpTableCopy tests that deep copy is necessery to prevent modify shared jump table +// TestJumpTableCopy tests that deep copy is necessary to prevent modify shared jump table func TestJumpTableCopy(t *testing.T) { tbl := newMergeInstructionSet() require.Equal(t, uint64(0), tbl[SLOAD].constantGas) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index 04c6409ebd..f420a24105 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -187,7 +187,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { // outside of this function, as part of the dynamic gas, and that will make it // also become correctly reported to tracers. contract.Gas += coldCost - return gas + coldCost, nil + + var overflow bool + if gas, overflow = math.SafeAdd(gas, coldCost); overflow { + return 0, ErrGasUintOverflow + } + return gas, nil } } @@ -197,7 +202,7 @@ var ( gasStaticCallEIP2929 = makeCallVariantGasCallEIP2929(gasStaticCall) gasCallCodeEIP2929 = makeCallVariantGasCallEIP2929(gasCallCode) gasSelfdestructEIP2929 = makeSelfdestructGasFn(true) - // gasSelfdestructEIP3529 implements the changes in EIP-2539 (no refunds) + // gasSelfdestructEIP3529 implements the changes in EIP-3529 (no refunds) gasSelfdestructEIP3529 = makeSelfdestructGasFn(false) // gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929 @@ -214,12 +219,12 @@ var ( // see gasSStoreEIP2200(...) in core/vm/gas_table.go for more info about how EIP 2200 is specified gasSStoreEIP2929 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP2200) - // gasSStoreEIP2539 implements gas cost for SSTORE according to EIP-2539 + // gasSStoreEIP3529 implements gas cost for SSTORE according to EIP-3529 // Replace `SSTORE_CLEARS_SCHEDULE` with `SSTORE_RESET_GAS + ACCESS_LIST_STORAGE_KEY_COST` (4,800) gasSStoreEIP3529 = makeGasSStoreFunc(params.SstoreClearsScheduleRefundEIP3529) ) -// makeSelfdestructGasFn can create the selfdestruct dynamic gas function for EIP-2929 and EIP-2539 +// makeSelfdestructGasFn can create the selfdestruct dynamic gas function for EIP-2929 and EIP-3529 func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { gasFunc := func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { var ( diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index d10457e7fa..46f2bb5d5f 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) // Config is a basic type specifying certain configuration flags for running @@ -135,7 +136,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { common.BytesToAddress([]byte("contract")), input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return ret, cfg.State, err } @@ -164,7 +165,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { sender, input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return code, address, leftOverGas, err } @@ -179,7 +180,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er var ( vmenv = NewEnv(cfg) - sender = cfg.State.GetOrNewStateObject(cfg.Origin) + sender = vm.AccountRef(cfg.Origin) statedb = cfg.State rules = cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil, vmenv.Context.Time) ) @@ -194,7 +195,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er address, input, cfg.GasLimit, - cfg.Value, + uint256.MustFromBig(cfg.Value), ) return ret, leftOverGas, err } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index c9b03af7bf..52756b4093 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -40,6 +40,7 @@ import ( // force-load js tracers to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" + "github.com/holiman/uint256" ) func TestDefaults(t *testing.T) { @@ -364,12 +365,12 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode //cfg.State.CreateAccount(cfg.Origin) // set the receiver's (the executing contract) code for execution. cfg.State.SetCode(destination, code) - vmenv.Call(sender, destination, nil, gas, cfg.Value) + vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) b.Run(name, func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - vmenv.Call(sender, destination, nil, gas, cfg.Value) + vmenv.Call(sender, destination, nil, gas, uint256.MustFromBig(cfg.Value)) } }) } diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index e5fe75af20..b942bf94fd 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -27,7 +27,7 @@ import ( // If z is equal to one the point is considered as in affine form. type PointG2 [3]fe2 -// Set copies valeus of one point to another. +// Set copies values of one point to another. func (p *PointG2) Set(p2 *PointG2) *PointG2 { p[0].set(&p2[0]) p[1].set(&p2[1]) diff --git a/crypto/bn256/google/bn256.go b/crypto/bn256/google/bn256.go index 0a9d5cd35d..93953e23a9 100644 --- a/crypto/bn256/google/bn256.go +++ b/crypto/bn256/google/bn256.go @@ -166,7 +166,7 @@ type G2 struct { p *twistPoint } -// RandomG1 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. +// RandomG2 returns x and gâ‚‚Ë£ where x is a random, non-zero number read from r. func RandomG2(r io.Reader) (*big.Int, *G2, error) { var k *big.Int var err error diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 5969d1c2ce..52124df674 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -20,21 +20,61 @@ package kzg4844 import ( "embed" "errors" + "hash" + "reflect" "sync/atomic" + + "github.com/ethereum/go-ethereum/common/hexutil" ) //go:embed trusted_setup.json var content embed.FS +var ( + blobT = reflect.TypeOf(Blob{}) + commitmentT = reflect.TypeOf(Commitment{}) + proofT = reflect.TypeOf(Proof{}) +) + // Blob represents a 4844 data blob. type Blob [131072]byte +// UnmarshalJSON parses a blob in hex syntax. +func (b *Blob) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(blobT, input, b[:]) +} + +// MarshalText returns the hex representation of b. +func (b Blob) MarshalText() ([]byte, error) { + return hexutil.Bytes(b[:]).MarshalText() +} + // Commitment is a serialized commitment to a polynomial. type Commitment [48]byte +// UnmarshalJSON parses a commitment in hex syntax. +func (c *Commitment) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(commitmentT, input, c[:]) +} + +// MarshalText returns the hex representation of c. +func (c Commitment) MarshalText() ([]byte, error) { + return hexutil.Bytes(c[:]).MarshalText() +} + // Proof is a serialized commitment to the quotient polynomial. type Proof [48]byte +// UnmarshalJSON parses a proof in hex syntax. +func (p *Proof) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(proofT, input, p[:]) +} + +// MarshalText returns the hex representation of p. +func (p Proof) MarshalText() ([]byte, error) { + return hexutil.Bytes(p[:]).MarshalText() +} + // Point is a BLS field element. type Point [32]byte @@ -108,3 +148,21 @@ func VerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { } return gokzgVerifyBlobProof(blob, commitment, proof) } + +// CalcBlobHashV1 calculates the 'versioned blob hash' of a commitment. +// The given hasher must be a sha256 hash instance, otherwise the result will be invalid! +func CalcBlobHashV1(hasher hash.Hash, commit *Commitment) (vh [32]byte) { + if hasher.Size() != 32 { + panic("wrong hash size") + } + hasher.Reset() + hasher.Write(commit[:]) + hasher.Sum(vh[:0]) + vh[0] = 0x01 // version + return vh +} + +// IsValidVersionedHash checks that h is a structurally-valid versioned blob hash. +func IsValidVersionedHash(h []byte) bool { + return len(h) == 32 && h[0] == 0x01 +} diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md index 962aa51f64..0986f00b65 100644 --- a/docs/postmortems/2021-08-22-split-postmortem.md +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -87,7 +87,7 @@ The blocks on the 'bad' chain were investigated, and Tim Beiko reached out to th ### Disclosure decision -The geth-team have an official policy regarding [vulnerability disclosure](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities). +The geth-team have an official policy regarding [vulnerability disclosure](https://geth.ethereum.org/docs/developers/geth-developer/disclosures). > The primary goal for the Geth team is the health of the Ethereum network as a whole, and the decision whether or not to publish details about a serious vulnerability boils down to minimizing the risk and/or impact of discovery and exploitation. diff --git a/eth/api_backend.go b/eth/api_backend.go index 7aef569c34..f53561d918 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -270,7 +270,7 @@ func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *st } else { context = core.NewEVMBlockContext(header, b.eth.BlockChain(), nil, b.eth.blockchain.Config(), state) } - return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig) + return vm.NewEVM(context, txContext, state, b.ChainConfig(), *vmConfig) } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { @@ -354,7 +354,7 @@ func (b *EthAPIBackend) BundlePrice() *big.Int { } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(false) + pending := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { @@ -370,9 +370,25 @@ func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction return b.eth.txPool.Get(hash) } -func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - tx, blockHash, blockNumber, index := rawdb.ReadTransaction(b.eth.ChainDb(), txHash) - return tx, blockHash, blockNumber, index, nil +// GetTransaction retrieves the lookup along with the transaction itself associate +// with the given transaction hash. +// +// An error will be returned if the transaction is not found, and background +// indexing for transactions is still in progress. The error is used to indicate the +// scenario explicitly that the transaction might be reachable shortly. +// +// A null will be returned in the transaction is not found and background transaction +// indexing is already finished. The transaction is not existent from the perspective +// of node. +func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { + lookup, tx, err := b.eth.blockchain.GetTransactionLookup(txHash) + if err != nil { + return false, nil, common.Hash{}, 0, 0, err + } + if lookup == nil || tx == nil { + return false, nil, common.Hash{}, 0, 0, nil + } + return true, tx, lookup.BlockHash, lookup.BlockIndex, lookup.Index, nil } func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { @@ -400,7 +416,12 @@ func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.S } func (b *EthAPIBackend) SyncProgress() ethereum.SyncProgress { - return b.eth.Downloader().Progress() + prog := b.eth.Downloader().Progress() + if txProg, err := b.eth.blockchain.TxIndexProgress(); err == nil { + prog.TxIndexFinishedBlocks = txProg.Indexed + prog.TxIndexRemainingBlocks = txProg.Remaining + } + return prog } func (b *EthAPIBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 184b90dd09..671e935beb 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -19,7 +19,6 @@ package eth import ( "bytes" "fmt" - "math/big" "reflect" "strings" "testing" @@ -30,7 +29,8 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/holiman/uint256" "golang.org/x/exp/slices" ) @@ -63,7 +63,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) sdb, _ = state.New(types.EmptyRootHash, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} @@ -73,7 +73,7 @@ func TestAccountRange(t *testing.T) { hash := common.HexToHash(fmt.Sprintf("%x", i)) addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) addrs[i] = addr - sdb.SetBalance(addrs[i], big.NewInt(1)) + sdb.SetBalance(addrs[i], uint256.NewInt(1)) if _, ok := m[addr]; ok { t.Fatalf("bad") } else { @@ -160,7 +160,7 @@ func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + db = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) sdb, _ = state.New(types.EmptyRootHash, db, nil) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage diff --git a/eth/api_miner.go b/eth/api_miner.go index 477531d494..764d0ae5e2 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -29,7 +29,7 @@ type MinerAPI struct { e *Ethereum } -// NewMinerAPI create a new MinerAPI instance. +// NewMinerAPI creates a new MinerAPI instance. func NewMinerAPI(e *Ethereum) *MinerAPI { return &MinerAPI{e} } @@ -64,6 +64,7 @@ func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { api.e.lock.Unlock() api.e.txPool.SetGasTip((*big.Int)(&gasPrice)) + api.e.Miner().SetGasTip((*big.Int)(&gasPrice)) return true } diff --git a/eth/backend.go b/eth/backend.go index f2d02fca67..b690938e87 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -60,7 +60,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) const ( @@ -259,6 +259,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideOptimismEcotone != nil { overrides.OverrideOptimismEcotone = config.OverrideOptimismEcotone } + if config.OverrideOptimismFjord != nil { + overrides.OverrideOptimismFjord = config.OverrideOptimismFjord + } if config.OverrideOptimismInterop != nil { overrides.OverrideOptimismInterop = config.OverrideOptimismInterop } @@ -298,7 +301,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { bundlePool = bundlepool.New(config.BundlePool, config.Miner.Mev, eth.blockchain) txPools = append(txPools, bundlePool) } - eth.txPool, err = txpool.New(new(big.Int).SetUint64(config.TxPool.PriceLimit), eth.blockchain, txPools) + eth.txPool, err = txpool.New(config.TxPool.PriceLimit, eth.blockchain, txPools) if err != nil { return nil, err } @@ -415,7 +418,7 @@ func (s *Ethereum) APIs() []rpc.API { Service: NewMinerAPI(s), }, { Namespace: "eth", - Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.blockchain, s.eventMux), }, { Namespace: "admin", Service: NewAdminAPI(s), diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 056c1c1abc..e8703f7532 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -20,7 +20,6 @@ package catalyst import ( "errors" "fmt" - "math/big" "sync" "time" @@ -31,10 +30,13 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" ) @@ -96,6 +98,7 @@ var caps = []string{ "engine_newPayloadV3", "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByRangeV1", + "engine_getClientVersionV1", } type ConsensusAPI struct { @@ -181,61 +184,65 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // and return its payloadID. func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { - if payloadAttributes.Withdrawals != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) + if payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals and beacon root not supported in V1")) } if api.eth.BlockChain().Config().IsShanghai(api.eth.BlockChain().Config().LondonBlock, payloadAttributes.Timestamp) { return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("forkChoiceUpdateV1 called post-shanghai")) } } - return api.forkchoiceUpdated(update, payloadAttributes) + return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, false) } -// ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload attributes. -func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { - if payloadAttributes != nil { - if err := api.verifyPayloadAttributes(payloadAttributes); err != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(err) +// ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload +// attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. +func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) { + case forks.Paris: + if params.Withdrawals != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai")) + } + case forks.Shanghai: + if params.Withdrawals == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + } + default: + return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads")) } - } - return api.forkchoiceUpdated(update, payloadAttributes) -} - -// ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root in the payload attributes. -func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { - if payloadAttributes != nil { - if err := api.verifyPayloadAttributes(payloadAttributes); err != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(err) + if params.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root")) } } - return api.forkchoiceUpdated(update, payloadAttributes) + return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) } -func (api *ConsensusAPI) verifyPayloadAttributes(attr *engine.PayloadAttributes) error { - c := api.eth.BlockChain().Config() - - // Verify withdrawals attribute for Shanghai. - if err := checkAttribute(c.IsShanghai, attr.Withdrawals != nil, c.LondonBlock, attr.Timestamp); err != nil { - return fmt.Errorf("invalid withdrawals: %w", err) - } - // Verify beacon root attribute for Cancun. - if err := checkAttribute(c.IsCancun, attr.BeaconRoot != nil, c.LondonBlock, attr.Timestamp); err != nil { - return fmt.Errorf("invalid parent beacon block root: %w", err) - } - return nil -} - -func checkAttribute(active func(*big.Int, uint64) bool, exists bool, block *big.Int, time uint64) error { - if active(block, time) && !exists { - return errors.New("fork active, missing expected attribute") - } - if !active(block, time) && exists { - return errors.New("fork inactive, unexpected attribute set") +// ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root +// in the payload attributes. It supports only PayloadAttributesV3. +func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + // TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498, + // payload attributes that are invalid should return error + // engine.InvalidPayloadAttributes. Once hive updates this, we should update + // on our end. + if params.Withdrawals == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + } + if params.BeaconRoot == nil { + return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root")) + } + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { + return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads")) + } } - return nil + // TODO(matt): the spec requires that fcu is applied when called on a valid + // hash, even if params are wrong. To do this we need to split up + // forkchoiceUpdate into a function that only updates the head and then a + // function that kicks off block construction. + return api.forkchoiceUpdated(update, params, engine.PayloadV3, false) } -func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, simulatorMode bool) (engine.ForkChoiceResponse, error) { start := time.Now() api.forkchoiceLock.Lock() @@ -344,7 +351,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if merger := api.eth.Merger(); !merger.PoSFinalized() { merger.FinalizePoS() } - // If the finalized block is not in our canonical tree, somethings wrong + // If the finalized block is not in our canonical tree, something is wrong finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) if finalBlock == nil { log.Warn("Final block not available in database", "hash", update.FinalizedBlockHash) @@ -356,7 +363,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // Set the finalized block api.eth.BlockChain().SetFinalized(finalBlock.Header()) } - // Check if the safe block hash is in our canonical tree, if not somethings wrong + // Check if the safe block hash is in our canonical tree, if not something is wrong if update.SafeBlockHash != (common.Hash{}) { safeBlock := api.eth.BlockChain().GetBlockByHash(update.SafeBlockHash) if safeBlock == nil { @@ -407,6 +414,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl NoTxPool: payloadAttributes.NoTxPool, Transactions: transactions, GasLimit: payloadAttributes.GasLimit, + Version: payloadVersion, } id := args.Id() // If we already are busy generating this work, then we do not need @@ -414,6 +422,19 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if api.localBlocks.has(id) { return valid(&id), nil } + // If the beacon chain is ran by a simulator, then transaction insertion, + // block insertion and block production will happen without any timing + // delay between them. This will cause flaky simulator executions due to + // the transaction pool running its internal reset operation on a back- + // ground thread. To avoid the racey behavior - in simulator mode - the + // pool will be explicitly blocked on its reset before continuing to the + // block production below. + if simulatorMode { + if err := api.eth.TxPool().Sync(); err != nil { + log.Error("Failed to sync transaction pool", "err", err) + return valid(nil), engine.InvalidPayloadAttributes.With(err) + } + } payload, err := api.eth.Miner().BuildPayload(args) if err != nil { log.Error("Failed to build payload", "err", err) @@ -461,6 +482,9 @@ func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.Transit // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { + if !payloadID.Is(engine.PayloadV1) { + return nil, engine.UnsupportedFork + } data, err := api.getPayload(payloadID, false) if err != nil { return nil, err @@ -470,11 +494,17 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.Execu // GetPayloadV2 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + if !payloadID.Is(engine.PayloadV1, engine.PayloadV2) { + return nil, engine.UnsupportedFork + } return api.getPayload(payloadID, false) } // GetPayloadV3 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + if !payloadID.Is(engine.PayloadV3) { + return nil, engine.UnsupportedFork + } return api.getPayload(payloadID, false) } @@ -502,38 +532,49 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { - if api.eth.BlockChain().Config().IsShanghai(new(big.Int).SetUint64(params.Number), params.Timestamp) { + if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun")) + } + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai { if params.Withdrawals == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai")) } - } else if params.Withdrawals != nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai")) + } else { + if params.Withdrawals != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai")) + } } - if api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV2 called post-cancun")) + if params.ExcessBlobGas != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun")) + } + if params.BlobGasUsed != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun")) } return api.newPayload(params, nil, nil) } // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { + if params.Withdrawals == nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai")) + } if params.ExcessBlobGas == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun")) } if params.BlobGasUsed == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun")) } + if versionedHashes == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun")) } if beaconRoot == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun")) } - if !api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV3 called pre-cancun")) + if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV3 must only be called for cancun payloads")) } - return api.newPayload(params, versionedHashes, beaconRoot) } @@ -681,6 +722,9 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS // that the parent state is missing and the syncer rejected extending the // current cycle with the new payload. log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash(), "reason", err) + if errors.Is(err, downloader.ErrForcedNeeded) { + return engine.PayloadStatusV1{Status: engine.SYNCING}, err + } } else { // In non-full sync mode (i.e. snap sync) all payloads are rejected until // snap sync terminates as snap sync relies on direct database injections @@ -834,6 +878,23 @@ func (api *ConsensusAPI) ExchangeCapabilities([]string) []string { return caps } +// GetClientVersionV1 exchanges client version data of this node. +func (api *ConsensusAPI) GetClientVersionV1(info engine.ClientVersionV1) []engine.ClientVersionV1 { + log.Trace("Engine API request received", "method", "GetClientVersionV1", "info", info.String()) + commit := make([]byte, 4) + if vcs, ok := version.VCS(); ok { + commit = common.FromHex(vcs.Commit)[0:4] + } + return []engine.ClientVersionV1{ + { + Code: engine.ClientCode, + Name: engine.ClientName, + Version: params.VersionWithMeta, + Commit: hexutil.Encode(commit), + }, + } +} + // GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which allows for retrieval of a list // of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engine.ExecutionPayloadBodyV1 { @@ -880,8 +941,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { ) for j, tx := range body.Transactions { - data, _ := tx.MarshalBinary() - txs[j] = hexutil.Bytes(data) + txs[j], _ = tx.MarshalBinary() } // Post-shanghai withdrawals MUST be set to empty slice instead of nil diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index cbf744cd6d..64907080af 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" crand "crypto/rand" + "errors" "fmt" "math/big" "math/rand" @@ -30,6 +31,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/mattn/go-colorable" + "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -50,7 +53,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" - "github.com/mattn/go-colorable" ) var ( @@ -73,7 +75,7 @@ func generateMergeChain(n int, merged bool) (*core.Genesis, []*types.Block) { } genesis := &core.Genesis{ Config: &config, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ testAddr: {Balance: testBalance}, params.BeaconRootsStorageAddress: {Balance: common.Big0, Code: common.Hex2Bytes("3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500")}, }, @@ -216,6 +218,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { FeeRecipient: blockParams.SuggestedFeeRecipient, Random: blockParams.Random, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV1, }).Id() require.Equal(t, payloadID, *resp.PayloadID) require.NoError(t, waitForApiPayloadToBuild(api, *resp.PayloadID)) @@ -269,11 +272,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) { {0, true}, {parent.Time, true}, {parent.Time - 1, true}, - - // TODO (MariusVanDerWijden) following tests are currently broken, - // fixed in upcoming merge-kiln-v2 pr - //{parent.Time() + 1, false}, - //{uint64(time.Now().Unix()) + uint64(time.Minute), false}, + {parent.Time + 1, false}, + {uint64(time.Now().Unix()) + uint64(time.Minute), false}, } for i, test := range tests { @@ -717,7 +717,7 @@ func TestEmptyBlocks(t *testing.T) { payload := getNewPayload(t, api, commonAncestor, nil) status, err := api.NewPayloadV1(*payload) - if err != nil { + if err != nil && !errors.Is(err, downloader.ErrForcedNeeded) { t.Fatal(err) } if status.Status != engine.VALID { @@ -733,7 +733,7 @@ func TestEmptyBlocks(t *testing.T) { payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor status, err = api.NewPayloadV1(*payload) - if err != nil { + if err != nil && !errors.Is(err, downloader.ErrForcedNeeded) { t.Fatal(err) } if status.Status != engine.INVALID { @@ -751,7 +751,7 @@ func TestEmptyBlocks(t *testing.T) { payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor status, err = api.NewPayloadV1(*payload) - if err != nil { + if err != nil && !errors.Is(err, downloader.ErrForcedNeeded) { t.Fatal(err) } if status.Status != engine.SYNCING { @@ -863,7 +863,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { // feed the payloads to node B for _, payload := range invalidChain { status, err := apiB.NewPayloadV1(*payload) - if err != nil { + if err != nil && !errors.Is(err, downloader.ErrForcedNeeded) { panic(err) } if status.Status == engine.VALID { @@ -1097,6 +1097,7 @@ func TestWithdrawals(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV2, }).Id() require.Equal(t, payloadID, *resp.PayloadID) require.NoError(t, waitForApiPayloadToBuild(api, payloadID)) @@ -1147,6 +1148,7 @@ func TestWithdrawals(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV2, }).Id() require.Equal(t, payloadID, *resp.PayloadID) require.NoError(t, waitForApiPayloadToBuild(api, payloadID)) @@ -1262,7 +1264,18 @@ func TestNilWithdrawals(t *testing.T) { } for _, test := range tests { - resp, err := api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + var ( + err error + payloadVersion engine.PayloadVersion + shanghai = genesis.Config.IsShanghai(genesis.Config.LondonBlock, test.blockParams.Timestamp) + ) + if !shanghai { + payloadVersion = engine.PayloadV1 + _, err = api.ForkchoiceUpdatedV1(fcState, &test.blockParams) + } else { + payloadVersion = engine.PayloadV2 + _, err = api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + } if test.wantErr { if err == nil { t.Fatal("wanted error on fcuv2 with invalid withdrawals") @@ -1281,15 +1294,20 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, BeaconRoot: test.blockParams.BeaconRoot, Withdrawals: test.blockParams.Withdrawals, + Version: payloadVersion, }).Id() - require.Equal(t, payloadID, *resp.PayloadID) - require.NoError(t, waitForApiPayloadToBuild(api, payloadID)) execData, err := api.GetPayloadV2(payloadID) if err != nil { t.Fatalf("error getting payload, err=%v", err) } - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { - t.Fatalf("error validating payload: %v", err) + var status engine.PayloadStatusV1 + if !shanghai { + status, err = api.NewPayloadV1(*execData.ExecutionPayload) + } else { + status, err = api.NewPayloadV2(*execData.ExecutionPayload) + } + if err != nil { + t.Fatalf("error validating payload: %v", err.(*engine.EngineAPIError).ErrorData()) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") } @@ -1617,7 +1635,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV2(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV3(fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1633,6 +1651,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { Random: blockParams.Random, Withdrawals: blockParams.Withdrawals, BeaconRoot: blockParams.BeaconRoot, + Version: engine.PayloadV3, }).Id() require.Equal(t, payloadID, *resp.PayloadID) require.NoError(t, waitForApiPayloadToBuild(api, *resp.PayloadID)) @@ -1682,3 +1701,26 @@ func waitForPayloadToBuild(payload *miner.Payload) { func waitForApiPayloadToBuild(api *ConsensusAPI, id engine.PayloadID) error { return api.localBlocks.waitFull(id) } + +// TestGetClientVersion verifies the expected version info is returned. +func TestGetClientVersion(t *testing.T) { + genesis, preMergeBlocks := generateMergeChain(10, false) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + api := NewConsensusAPI(ethservice) + info := engine.ClientVersionV1{ + Code: "TT", + Name: "test", + Version: "1.1.1", + Commit: "0x12345678", + } + infos := api.GetClientVersionV1(info) + if len(infos) != 1 { + t.Fatalf("expected only one returned client version, got %d", len(infos)) + } + info = infos[0] + if info.Code != engine.ClientCode || info.Name != engine.ClientName || info.Version != params.VersionWithMeta { + t.Fatalf("client info does match expected, got %s", info.String()) + } +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index b0b13d32cd..0c277ce0d6 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -19,16 +19,18 @@ package catalyst import ( "crypto/rand" "errors" + "math/big" "sync" "time" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -81,6 +83,11 @@ type SimulatedBeacon struct { lastBlockTime uint64 } +// NewSimulatedBeacon constructs a new simulated beacon chain. +// Period sets the period in which blocks should be produced. +// +// - If period is set to 0, a block is produced on every transaction. +// via Commit, Fork and AdjustTime. func NewSimulatedBeacon(period uint64, eth *eth.Ethereum) (*SimulatedBeacon, error) { block := eth.BlockChain().CurrentBlock() current := engine.ForkchoiceStateV1{ @@ -116,7 +123,9 @@ func (c *SimulatedBeacon) setFeeRecipient(feeRecipient common.Address) { // Start invokes the SimulatedBeacon life-cycle function in a goroutine. func (c *SimulatedBeacon) Start() error { if c.period == 0 { - go c.loopOnDemand() + // if period is set to 0, do not mine at all + // this is used in the simulated backend where blocks + // are explicitly mined via Commit, AdjustTime and Fork } else { go c.loop() } @@ -131,10 +140,9 @@ func (c *SimulatedBeacon) Stop() error { // sealBlock initiates payload building for a new block and creates a new block // with the completed payload. -func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { - tstamp := uint64(time.Now().Unix()) - if tstamp <= c.lastBlockTime { - tstamp = c.lastBlockTime + 1 +func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp uint64) error { + if timestamp <= c.lastBlockTime { + timestamp = c.lastBlockTime + 1 } c.feeRecipientLock.Lock() feeRecipient := c.feeRecipient @@ -148,12 +156,12 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { var random [32]byte rand.Read(random[:]) - fcResponse, err := c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, &engine.PayloadAttributes{ - Timestamp: tstamp, + fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{ + Timestamp: timestamp, SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, - }) + }, engine.PayloadV2, true) if err != nil { return err } @@ -186,6 +194,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { return err } c.setCurrentState(payload.BlockHash, finalizedHash) + // Mark the block containing the payload as canonical if _, err = c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, nil); err != nil { return err @@ -194,32 +203,6 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { return nil } -// loopOnDemand runs the block production loop for "on-demand" configuration (period = 0) -func (c *SimulatedBeacon) loopOnDemand() { - var ( - newTxs = make(chan core.NewTxsEvent) - sub = c.eth.TxPool().SubscribeTransactions(newTxs, true) - ) - defer sub.Unsubscribe() - - for { - select { - case <-c.shutdownCh: - return - case w := <-c.withdrawals.pending: - withdrawals := append(c.withdrawals.gatherPending(9), w) - if err := c.sealBlock(withdrawals); err != nil { - log.Warn("Error performing sealing work", "err", err) - } - case <-newTxs: - withdrawals := c.withdrawals.gatherPending(10) - if err := c.sealBlock(withdrawals); err != nil { - log.Warn("Error performing sealing work", "err", err) - } - } - } -} - // loop runs the block production loop for non-zero period configuration func (c *SimulatedBeacon) loop() { timer := time.NewTimer(0) @@ -229,7 +212,7 @@ func (c *SimulatedBeacon) loop() { return case <-timer.C: withdrawals := c.withdrawals.gatherPending(10) - if err := c.sealBlock(withdrawals); err != nil { + if err := c.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil { log.Warn("Error performing sealing work", "err", err) } else { timer.Reset(time.Second * time.Duration(c.period)) @@ -238,8 +221,8 @@ func (c *SimulatedBeacon) loop() { } } -// finalizedBlockHash returns the block hash of the finalized block corresponding to the given number -// or nil if doesn't exist in the chain. +// finalizedBlockHash returns the block hash of the finalized block corresponding +// to the given number or nil if doesn't exist in the chain. func (c *SimulatedBeacon) finalizedBlockHash(number uint64) *common.Hash { var finalizedNumber uint64 if number%devEpochLength == 0 { @@ -247,7 +230,6 @@ func (c *SimulatedBeacon) finalizedBlockHash(number uint64) *common.Hash { } else { finalizedNumber = (number - 1) / devEpochLength * devEpochLength } - if finalizedBlock := c.eth.BlockChain().GetBlockByNumber(finalizedNumber); finalizedBlock != nil { fh := finalizedBlock.Hash() return &fh @@ -264,11 +246,60 @@ func (c *SimulatedBeacon) setCurrentState(headHash, finalizedHash common.Hash) { } } +// Commit seals a block on demand. +func (c *SimulatedBeacon) Commit() common.Hash { + withdrawals := c.withdrawals.gatherPending(10) + if err := c.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil { + log.Warn("Error performing sealing work", "err", err) + } + return c.eth.BlockChain().CurrentBlock().Hash() +} + +// Rollback un-sends previously added transactions. +func (c *SimulatedBeacon) Rollback() { + // Flush all transactions from the transaction pools + maxUint256 := new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 256), common.Big1) + c.eth.TxPool().SetGasTip(maxUint256) + // Set the gas tip back to accept new transactions + // TODO (Marius van der Wijden): set gas tip to parameter passed by config + c.eth.TxPool().SetGasTip(big.NewInt(params.GWei)) +} + +// Fork sets the head to the provided hash. +func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + return errors.New("pending block dirty") + } + parent := c.eth.BlockChain().GetBlockByHash(parentHash) + if parent == nil { + return errors.New("parent not found") + } + return c.eth.BlockChain().SetHead(parent.NumberU64()) +} + +// AdjustTime creates a new block with an adjusted timestamp. +func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + return errors.New("could not adjust time on non-empty block") + } + parent := c.eth.BlockChain().CurrentBlock() + if parent == nil { + return errors.New("parent not found") + } + withdrawals := c.withdrawals.gatherPending(10) + return c.sealBlock(withdrawals, parent.Time+uint64(adjustment)) +} + func RegisterSimulatedBeaconAPIs(stack *node.Node, sim *SimulatedBeacon) { + api := &api{sim} + if sim.period == 0 { + // mine on demand if period is set to 0 + go api.loop() + } stack.RegisterAPIs([]rpc.API{ { Namespace: "dev", - Service: &api{sim}, + Service: api, Version: "1.0", }, }) diff --git a/eth/catalyst/simulated_beacon_api.go b/eth/catalyst/simulated_beacon_api.go index 93670257f6..73d0a5921d 100644 --- a/eth/catalyst/simulated_beacon_api.go +++ b/eth/catalyst/simulated_beacon_api.go @@ -18,19 +18,44 @@ package catalyst import ( "context" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" ) type api struct { - simBeacon *SimulatedBeacon + sim *SimulatedBeacon +} + +func (a *api) loop() { + var ( + newTxs = make(chan core.NewTxsEvent) + sub = a.sim.eth.TxPool().SubscribeTransactions(newTxs, true) + ) + defer sub.Unsubscribe() + + for { + select { + case <-a.sim.shutdownCh: + return + case w := <-a.sim.withdrawals.pending: + withdrawals := append(a.sim.withdrawals.gatherPending(9), w) + if err := a.sim.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil { + log.Warn("Error performing sealing work", "err", err) + } + case <-newTxs: + a.sim.Commit() + } + } } func (a *api) AddWithdrawal(ctx context.Context, withdrawal *types.Withdrawal) error { - return a.simBeacon.withdrawals.add(withdrawal) + return a.sim.withdrawals.add(withdrawal) } func (a *api) SetFeeRecipient(ctx context.Context, feeRecipient common.Address) { - a.simBeacon.setFeeRecipient(feeRecipient) + a.sim.setFeeRecipient(feeRecipient) } diff --git a/eth/downloader/api.go b/eth/downloader/api.go index b3f7113bcd..6b8cb98e23 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -19,50 +19,80 @@ package downloader import ( "context" "sync" + "time" "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -// DownloaderAPI provides an API which gives information about the current synchronisation status. -// It offers only methods that operates on data that can be available to anyone without security risks. +// DownloaderAPI provides an API which gives information about the current +// synchronisation status. It offers only methods that operates on data that +// can be available to anyone without security risks. type DownloaderAPI struct { d *Downloader + chain *core.BlockChain mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI creates a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { +func NewDownloaderAPI(d *Downloader, chain *core.BlockChain, m *event.TypeMux) *DownloaderAPI { api := &DownloaderAPI{ d: d, + chain: chain, mux: m, installSyncSubscription: make(chan chan interface{}), uninstallSyncSubscription: make(chan *uninstallSyncSubscriptionRequest), } - go api.eventLoop() - return api } -// eventLoop runs a loop until the event mux closes. It will install and uninstall new -// sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. +// eventLoop runs a loop until the event mux closes. It will install and uninstall +// new sync subscriptions and broadcasts sync status updates to the installed sync +// subscriptions. +// +// The sync status pushed to subscriptions can be a stream like: +// >>> {Syncing: true, Progress: {...}} +// >>> {false} +// +// If the node is already synced up, then only a single event subscribers will +// receive is {false}. func (api *DownloaderAPI) eventLoop() { var ( - sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) + sub = api.mux.Subscribe(StartEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) + checkInterval = time.Second * 60 + checkTimer = time.NewTimer(checkInterval) + + // status flags + started bool + done bool + + getProgress = func() ethereum.SyncProgress { + prog := api.d.Progress() + if txProg, err := api.chain.TxIndexProgress(); err == nil { + prog.TxIndexFinishedBlocks = txProg.Indexed + prog.TxIndexRemainingBlocks = txProg.Remaining + } + return prog + } ) + defer checkTimer.Stop() for { select { case i := <-api.installSyncSubscription: syncSubscriptions[i] = struct{}{} + if done { + i <- false + } case u := <-api.uninstallSyncSubscription: delete(syncSubscriptions, u.c) close(u.uninstalled) @@ -70,21 +100,31 @@ func (api *DownloaderAPI) eventLoop() { if event == nil { return } - - var notification interface{} switch event.Data.(type) { case StartEvent: - notification = &SyncingResult{ + started = true + } + case <-checkTimer.C: + if !started { + checkTimer.Reset(checkInterval) + continue + } + prog := getProgress() + if !prog.Done() { + notification := &SyncingResult{ Syncing: true, - Status: api.d.Progress(), + Status: prog, + } + for c := range syncSubscriptions { + c <- notification } - case DoneEvent, FailedEvent: - notification = false + checkTimer.Reset(checkInterval) + continue } - // broadcast for c := range syncSubscriptions { - c <- notification + c <- false } + done = true } } } @@ -101,16 +141,15 @@ func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error go func() { statuses := make(chan interface{}) sub := api.SubscribeSyncStatus(statuses) + defer sub.Unsubscribe() for { select { case status := <-statuses: notifier.Notify(rpcSub.ID, status) case <-rpcSub.Err(): - sub.Unsubscribe() return case <-notifier.Closed(): - sub.Unsubscribe() return } } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index df8af68bc7..d3f75c8527 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -50,7 +50,8 @@ func newBeaconBackfiller(dl *Downloader, success func()) backfiller { } // suspend cancels any background downloader threads and returns the last header -// that has been successfully backfilled. +// that has been successfully backfilled (potentially in a previous run), or the +// genesis. func (b *beaconBackfiller) suspend() *types.Header { // If no filling is running, don't waste cycles b.lock.Lock() diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 3f39b19485..185aeed6d0 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -35,7 +35,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -215,7 +215,7 @@ type BlockChain interface { // TrieDB retrieves the low level trie database used for interacting // with trie nodes. - TrieDB() *trie.Database + TrieDB() *triedb.Database } // New creates a new downloader to fetch hashes and blocks from remote peers. @@ -615,6 +615,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * if err := d.lightchain.SetHead(origin); err != nil { return err } + log.Info("Truncated excess ancient chain segment", "oldhead", frozen-1, "newhead", origin) } } // Initiate the sync using a concurrent header and content retrieval algorithm diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 52feaa327b..097888f024 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -69,7 +69,7 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { }) gspec := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } chain, err := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -440,9 +440,6 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH68, FullSync) } func TestCanonicalSynchronisation68Snap(t *testing.T) { testCanonSync(t, eth.ETH68, SnapSync) } func TestCanonicalSynchronisation68Light(t *testing.T) { testCanonSync(t, eth.ETH68, LightSync) } -func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } -func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } -func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -463,8 +460,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) } func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } -func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } -func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -546,9 +541,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync68Full(t *testing.T) { testForkedSync(t, eth.ETH68, FullSync) } func TestForkedSync68Snap(t *testing.T) { testForkedSync(t, eth.ETH68, SnapSync) } func TestForkedSync68Light(t *testing.T) { testForkedSync(t, eth.ETH68, LightSync) } -func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } -func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } -func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -576,9 +568,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestHeavyForkedSync68Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, FullSync) } func TestHeavyForkedSync68Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, SnapSync) } func TestHeavyForkedSync68Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } -func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } -func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -608,9 +597,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync68Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, FullSync) } func TestBoundedForkedSync68Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, SnapSync) } func TestBoundedForkedSync68Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } -func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } -func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -645,15 +631,6 @@ func TestBoundedHeavyForkedSync68Snap(t *testing.T) { func TestBoundedHeavyForkedSync68Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH68, LightSync) } -func TestBoundedHeavyForkedSync67Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) -} -func TestBoundedHeavyForkedSync67Snap(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) -} -func TestBoundedHeavyForkedSync67Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) -} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,9 +658,6 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel68Full(t *testing.T) { testCancel(t, eth.ETH68, FullSync) } func TestCancel68Snap(t *testing.T) { testCancel(t, eth.ETH68, SnapSync) } func TestCancel68Light(t *testing.T) { testCancel(t, eth.ETH68, LightSync) } -func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } -func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } -func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -711,9 +685,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation68Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, FullSync) } func TestMultiSynchronisation68Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, SnapSync) } func TestMultiSynchronisation68Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH68, LightSync) } -func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } -func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } -func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -738,9 +709,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation68Full(t *testing.T) { testMultiProtoSync(t, eth.ETH68, FullSync) } func TestMultiProtoSynchronisation68Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH68, SnapSync) } func TestMultiProtoSynchronisation68Light(t *testing.T) { testMultiProtoSync(t, eth.ETH68, LightSync) } -func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } -func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } -func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -751,7 +719,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -760,7 +727,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{68, 67} { + for _, version := range []int{68} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -773,9 +740,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, FullSync) } func TestEmptyShortCircuit68Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, SnapSync) } func TestEmptyShortCircuit68Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, LightSync) } -func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } -func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } -func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -824,9 +788,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack68Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, FullSync) } func TestMissingHeaderAttack68Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, SnapSync) } func TestMissingHeaderAttack68Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH68, LightSync) } -func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } -func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } -func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -853,9 +814,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack68Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, FullSync) } func TestShiftedHeaderAttack68Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, SnapSync) } func TestShiftedHeaderAttack68Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH68, LightSync) } -func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } -func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } -func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -889,15 +847,6 @@ func TestHighTDStarvationAttack68Snap(t *testing.T) { func TestHighTDStarvationAttack68Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH68, LightSync) } -func TestHighTDStarvationAttack67Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, FullSync) -} -func TestHighTDStarvationAttack67Snap(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, SnapSync) -} -func TestHighTDStarvationAttack67Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH67, LightSync) -} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -912,7 +861,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping68(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH68) } -func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -963,9 +911,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullSync) } func TestSyncProgress68Snap(t *testing.T) { testSyncProgress(t, eth.ETH68, SnapSync) } func TestSyncProgress68Light(t *testing.T) { testSyncProgress(t, eth.ETH68, LightSync) } -func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } -func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } -func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1043,9 +988,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress68Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, FullSync) } func TestForkedSyncProgress68Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, SnapSync) } func TestForkedSyncProgress68Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH68, LightSync) } -func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } -func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } -func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1117,9 +1059,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress68Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, FullSync) } func TestFailedSyncProgress68Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, SnapSync) } func TestFailedSyncProgress68Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH68, LightSync) } -func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } -func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1186,9 +1125,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress68Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, FullSync) } func TestFakedSyncProgress68Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, SnapSync) } func TestFakedSyncProgress68Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH68, LightSync) } -func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } -func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } -func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1332,8 +1268,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // being fast-synced from, avoiding potential cheap eclipse attacks. func TestBeaconSync68Full(t *testing.T) { testBeaconSync(t, eth.ETH68, FullSync) } func TestBeaconSync68Snap(t *testing.T) { testBeaconSync(t, eth.ETH68, SnapSync) } -func TestBeaconSync67Full(t *testing.T) { testBeaconSync(t, eth.ETH67, FullSync) } -func TestBeaconSync67Snap(t *testing.T) { testBeaconSync(t, eth.ETH67, SnapSync) } func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index e557158797..6ff858d755 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" @@ -810,7 +811,7 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH return errInvalidBody } for _, hash := range tx.BlobHashes() { - if hash[0] != params.BlobTxHashVersion { + if !kzg4844.IsValidVersionedHash(hash[:]) { return errInvalidBody } } diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index 0962fd89f0..129ffb7f61 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -82,6 +82,10 @@ var errChainGapped = errors.New("chain gapped") // of the current sync cycle is forked with the one advertised by consensus client. var errChainForked = errors.New("chain forked") +// ErrForcedNeeded is a public error to signal that the header chain +// of the current sync cycle needs a forced flag at startup. +var ErrForcedNeeded = errors.New("forced head needed for startup") + // maxBlockNumGapTolerance is the max gap tolerance by peer var maxBlockNumGapTolerance = uint64(30) @@ -165,7 +169,7 @@ type backfiller interface { // on initial startup. // // The method should return the last block header that has been successfully - // backfilled, or nil if the backfiller was not resumed. + // backfilled (in the current or a previous run), falling back to the genesis. suspend() *types.Header // resume requests the backfiller to start running fill or snap sync based on @@ -269,7 +273,7 @@ func (s *skeleton) startup() { // New head announced, start syncing to it, looping every time a current // cycle is terminated due to a chain event (head reorg, old chain merge). if !event.force { - event.errc <- errors.New("forced head needed for startup") + event.errc <- ErrForcedNeeded continue } event.errc <- nil // forced head accepted for startup @@ -386,14 +390,17 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { done := make(chan struct{}) go func() { defer close(done) - if filled := s.filler.suspend(); filled != nil { - // If something was filled, try to delete stale sync helpers. If - // unsuccessful, warn the user, but not much else we can do (it's - // a programming error, just let users report an issue and don't - // choke in the meantime). - if err := s.cleanStales(filled); err != nil { - log.Error("Failed to clean stale beacon headers", "err", err) - } + filled := s.filler.suspend() + if filled == nil { + log.Error("Latest filled block is not available") + return + } + // If something was filled, try to delete stale sync helpers. If + // unsuccessful, warn the user, but not much else we can do (it's + // a programming error, just let users report an issue and don't + // choke in the meantime). + if err := s.cleanStales(filled); err != nil { + log.Error("Failed to clean stale beacon headers", "err", err) } }() // Wait for the suspend to finish, consuming head events in the meantime @@ -1128,33 +1135,46 @@ func (s *skeleton) cleanStales(filled *types.Header) error { number := filled.Number.Uint64() log.Trace("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) - // If the filled header is below the linked subchain, something's - // corrupted internally. Report and error and refuse to do anything. - if number < s.progress.Subchains[0].Tail { + // If the filled header is below the linked subchain, something's corrupted + // internally. Report and error and refuse to do anything. + if number+1 < s.progress.Subchains[0].Tail { return fmt.Errorf("filled header below beacon header tail: %d < %d", number, s.progress.Subchains[0].Tail) } - // Subchain seems trimmable, push the tail forward up to the last - // filled header and delete everything before it - if available. In - // case we filled past the head, recreate the subchain with a new - // head to keep it consistent with the data on disk. + // If nothing in subchain is filled, don't bother to do cleanup. + if number+1 == s.progress.Subchains[0].Tail { + return nil + } var ( - start = s.progress.Subchains[0].Tail // start deleting from the first known header - end = number // delete until the requested threshold + start uint64 + end uint64 batch = s.db.NewBatch() ) - s.progress.Subchains[0].Tail = number - s.progress.Subchains[0].Next = filled.ParentHash - - if s.progress.Subchains[0].Head < number { - // If more headers were filled than available, push the entire - // subchain forward to keep tracking the node's block imports - end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head - s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) - - // The entire original skeleton chain was deleted and a new one - // defined. Make sure the new single-header chain gets pushed to - // disk to keep internal state consistent. - rawdb.WriteSkeletonHeader(batch, filled) + if number < s.progress.Subchains[0].Head { + // The skeleton chain is partially consumed, set the new tail as filled+1. + tail := rawdb.ReadSkeletonHeader(s.db, number+1) + if tail.ParentHash != filled.Hash() { + return fmt.Errorf("filled header is discontinuous with subchain: %d %s, please file an issue", number, filled.Hash()) + } + start, end = s.progress.Subchains[0].Tail, number+1 // remove headers in [tail, filled] + s.progress.Subchains[0].Tail = tail.Number.Uint64() + s.progress.Subchains[0].Next = tail.ParentHash + } else { + // The skeleton chain is fully consumed, set both head and tail as filled. + start, end = s.progress.Subchains[0].Tail, filled.Number.Uint64() // remove headers in [tail, filled) + s.progress.Subchains[0].Tail = filled.Number.Uint64() + s.progress.Subchains[0].Next = filled.ParentHash + + // If more headers were filled than available, push the entire subchain + // forward to keep tracking the node's block imports. + if number > s.progress.Subchains[0].Head { + end = s.progress.Subchains[0].Head + 1 // delete the entire original range, including the head + s.progress.Subchains[0].Head = number // assign a new head (tail is already assigned to this) + + // The entire original skeleton chain was deleted and a new one + // defined. Make sure the new single-header chain gets pushed to + // disk to keep internal state consistent. + rawdb.WriteSkeletonHeader(batch, filled) + } } // Execute the trimming and the potential rewiring of the progress s.saveSyncStatus(batch) diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index aceadd00d3..2b108dfe93 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -811,7 +811,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Create a peer set to feed headers through peerset := newPeerSet() for _, peer := range tt.peers { - peerset.Register(newPeerConnection(peer.id, eth.ETH67, peer, log.New("id", peer.id))) + peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id))) } // Create a peer dropper to track malicious peers dropped := make(map[string]int) @@ -913,7 +913,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { skeleton.Sync(tt.newHead, nil, true) } if tt.newPeer != nil { - if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH67, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { t.Errorf("test %d: failed to register new peer: %v", i, err) } } diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 1bf03411d1..46f3febd8b 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // Test chain parameters. @@ -41,10 +41,10 @@ var ( testGspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, trie.HashDefaults)) + testGenesis = testGspec.MustCommit(testDB, triedb.NewDatabase(testDB, triedb.HashDefaults)) ) // The common prefix of all test chains: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 0da45c08ed..383641ffc3 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -35,7 +35,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // FullNodeGPO contains default gasprice oracle settings for full node. @@ -204,6 +204,8 @@ type Config struct { OverrideOptimismEcotone *uint64 `toml:",omitempty"` + OverrideOptimismFjord *uint64 `toml:",omitempty"` + OverrideOptimismInterop *uint64 `toml:",omitempty"` // ApplySuperchainUpgrades requests the node to load chain-configuration from the superchain-registry. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 3a1fba387a..e6f4e1528f 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -13,7 +13,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/miner" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // MarshalTOML marshals as TOML. @@ -68,6 +68,7 @@ func (c Config) MarshalTOML() (interface{}, error) { OverrideVerkle *uint64 `toml:",omitempty"` OverrideOptimismCanyon *uint64 `toml:",omitempty"` OverrideOptimismEcotone *uint64 `toml:",omitempty"` + OverrideOptimismFjord *uint64 `toml:",omitempty"` OverrideOptimismInterop *uint64 `toml:",omitempty"` ApplySuperchainUpgrades bool `toml:",omitempty"` RollupSequencerHTTP string @@ -128,6 +129,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.OverrideVerkle = c.OverrideVerkle enc.OverrideOptimismCanyon = c.OverrideOptimismCanyon enc.OverrideOptimismEcotone = c.OverrideOptimismEcotone + enc.OverrideOptimismFjord = c.OverrideOptimismFjord enc.OverrideOptimismInterop = c.OverrideOptimismInterop enc.ApplySuperchainUpgrades = c.ApplySuperchainUpgrades enc.RollupSequencerHTTP = c.RollupSequencerHTTP @@ -192,6 +194,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { OverrideVerkle *uint64 `toml:",omitempty"` OverrideOptimismCanyon *uint64 `toml:",omitempty"` OverrideOptimismEcotone *uint64 `toml:",omitempty"` + OverrideOptimismFjord *uint64 `toml:",omitempty"` OverrideOptimismInterop *uint64 `toml:",omitempty"` ApplySuperchainUpgrades *bool `toml:",omitempty"` RollupSequencerHTTP *string @@ -353,6 +356,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.OverrideOptimismEcotone != nil { c.OverrideOptimismEcotone = dec.OverrideOptimismEcotone } + if dec.OverrideOptimismFjord != nil { + c.OverrideOptimismFjord = dec.OverrideOptimismFjord + } if dec.OverrideOptimismInterop != nil { c.OverrideOptimismInterop = dec.OverrideOptimismInterop } diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index 6927300b1d..cb7cbaf79e 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) var ( @@ -41,10 +42,10 @@ var ( testAddress = crypto.PubkeyToAddress(testKey.PublicKey) gspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults)) + genesis = gspec.MustCommit(testdb, triedb.NewDatabase(testdb, triedb.HashDefaults)) unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/filters/api.go b/eth/filters/api.go index a4eaa9cec8..173e40c972 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -43,6 +43,9 @@ var ( // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 const maxTopics = 4 +// The maximum number of allowed topics within a topic criteria +const maxSubTopics = 1000 + // filter is a helper struct that holds meta information over the filter type // and associated subscription in the event system. type filter struct { @@ -159,6 +162,8 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) go func() { txs := make(chan []*types.Transaction, 128) pendingTxSub := api.events.SubscribePendingTxs(txs) + defer pendingTxSub.Unsubscribe() + chainConfig := api.sys.backend.ChainConfig() for { @@ -176,10 +181,8 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) } } case <-rpcSub.Err(): - pendingTxSub.Unsubscribe() return case <-notifier.Closed(): - pendingTxSub.Unsubscribe() return } } @@ -233,16 +236,15 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { go func() { headers := make(chan *types.Header) headersSub := api.events.SubscribeNewHeads(headers) + defer headersSub.Unsubscribe() for { select { case h := <-headers: notifier.Notify(rpcSub.ID, h) case <-rpcSub.Err(): - headersSub.Unsubscribe() return case <-notifier.Closed(): - headersSub.Unsubscribe() return } } @@ -269,6 +271,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } go func() { + defer logsSub.Unsubscribe() for { select { case logs := <-matchedLogs: @@ -277,10 +280,8 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc notifier.Notify(rpcSub.ID, &log) } case <-rpcSub.Err(): // client send an unsubscribe request - logsSub.Unsubscribe() return case <-notifier.Closed(): // connection dropped - logsSub.Unsubscribe() return } } @@ -547,6 +548,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error { return errors.New("invalid addresses in query") } } + if len(raw.Topics) > maxTopics { + return errExceedMaxTopics + } // topics is an array consisting of strings and/or arrays of strings. // JSON null values are converted to common.Hash{} and ignored by the filter manager. @@ -567,6 +571,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error { case []interface{}: // or case e.g. [null, "topic0", "topic1"] + if len(topic) > maxSubTopics { + return errExceedMaxTopics + } for _, rawTopic := range topic { if rawTopic == nil { // null component, match all diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 27cad8826a..99c012cc84 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -820,7 +820,7 @@ func TestLightFilterLogs(t *testing.T) { key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) genesis = &core.Genesis{Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, }, } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 1db917c960..659ca5ce19 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) func makeReceipt(addr common.Address) *types.Receipt { @@ -57,7 +57,7 @@ func BenchmarkFilters(b *testing.B) { addr4 = common.BytesToAddress([]byte("random addresses please")) gspec = &core.Genesis{ - Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), Config: params.TestChainConfig, } @@ -86,7 +86,7 @@ func BenchmarkFilters(b *testing.B) { // The test txs are not properly signed, can't simply create a chain // and then import blocks. TODO(rjl493456442) try to get rid of the // manual database writes. - gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) + gspec.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)) for i, block := range chain { rawdb.WriteBlock(db, block) @@ -99,6 +99,7 @@ func BenchmarkFilters(b *testing.B) { filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { + filter.begin = 0 logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { b.Fatal("expected 4 logs, got", len(logs)) @@ -164,7 +165,7 @@ func TestFilters(t *testing.T) { gspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(0).Mul(big.NewInt(100), big.NewInt(params.Ether))}, contract: {Balance: big.NewInt(0), Code: bytecode}, contract2: {Balance: big.NewInt(0), Code: bytecode}, @@ -180,7 +181,7 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, trie.NewDatabase(db, nil)) + _, err = gspec.Commit(db, triedb.NewDatabase(db, nil)) if err != nil { t.Fatal(err) } diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index 734b47c73f..a6c9d635d8 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -71,9 +71,9 @@ func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uin } // Recap the highest gas limit with account's available balance. if feeCap.BitLen() != 0 { - balance := opts.State.GetBalance(call.From) + balance := opts.State.GetBalance(call.From).ToBig() - available := new(big.Int).Set(balance) + available := balance if call.Value != nil { if call.Value.Cmp(available) >= 0 { return 0, nil, core.ErrInsufficientFundsForTransfer diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 13f36499d4..354b1bc182 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -227,8 +227,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks uint64, unresolvedL if p < 0 || p > 100 { return common.Big0, nil, nil, nil, fmt.Errorf("%w: %f", errInvalidPercentile, p) } - if i > 0 && p < rewardPercentiles[i-1] { - return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) + if i > 0 && p <= rewardPercentiles[i-1] { + return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f >= #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } var ( diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 4ee5a0d1b2..79217502f7 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -126,7 +126,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config = *params.TestChainConfig // needs copy because it is modified below gspec = &core.Genesis{ Config: &config, - Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, + Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, } signer = types.LatestSigner(gspec.Config) ) diff --git a/eth/handler.go b/eth/handler.go index 2a5580e579..db8f0ed5cd 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -41,7 +41,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) const ( @@ -73,7 +73,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 6dbe03e214..6a11bf3689 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -83,10 +83,7 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { case *eth.NewBlockPacket: return h.handleBlockBroadcast(peer, packet.Block, packet.TD) - case *eth.NewPooledTransactionHashesPacket67: - return h.txFetcher.Notify(peer.ID(), nil, nil, *packet) - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index c9ff2ce263..1eb9a9ea49 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -58,11 +58,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { h.blockBroadcasts.Send(packet.Block) return nil - case *eth.NewPooledTransactionHashesPacket67: - h.txAnnounces.Send(([]common.Hash)(*packet)) - return nil - - case *eth.NewPooledTransactionHashesPacket68: + case *eth.NewPooledTransactionHashesPacket: h.txAnnounces.Send(packet.Hashes) return nil @@ -81,7 +77,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit67(t *testing.T) { testForkIDSplit(t, eth.ETH67) } func TestForkIDSplit68(t *testing.T) { testForkIDSplit(t, eth.ETH68) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +231,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions67(t *testing.T) { testRecvTransactions(t, eth.ETH67) } func TestRecvTransactions68(t *testing.T) { testRecvTransactions(t, eth.ETH68) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +288,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions67(t *testing.T) { testSendTransactions(t, eth.ETH67) } func TestSendTransactions68(t *testing.T) { testSendTransactions(t, eth.ETH68) } func testSendTransactions(t *testing.T, protocol uint) { @@ -353,7 +346,7 @@ func testSendTransactions(t *testing.T, protocol uint) { seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { switch protocol { - case 67, 68: + case 68: select { case hashes := <-anns: for _, hash := range hashes { @@ -379,7 +372,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation67(t *testing.T) { testTransactionPropagation(t, eth.ETH67) } func TestTransactionPropagation68(t *testing.T) { testTransactionPropagation(t, eth.ETH68) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -539,8 +531,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH67, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -592,7 +584,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock67(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH67) } func TestBroadcastMalformedBlock68(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH68) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/handler_test.go b/eth/handler_test.go index 4ad6fc6e36..eacdc52aa6 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) var ( @@ -93,7 +94,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, local bool, sync bool) []erro } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { p.lock.RLock() defer p.lock.RUnlock() @@ -112,8 +113,8 @@ func (p *testTxPool) Pending(enforceTips bool) map[common.Address][]*txpool.Lazy Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -168,7 +169,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { db := rawdb.NewMemoryDatabase() gspec := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, } chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) diff --git a/eth/peerset.go b/eth/peerset.go index 423170516d..01ec64802b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -57,6 +57,7 @@ type peerSet struct { lock sync.RWMutex closed bool + quitCh chan struct{} // Quit channel to signal termination } // newPeerSet creates a new peer set to track the active participants. @@ -65,6 +66,7 @@ func newPeerSet() *peerSet { peers: make(map[string]*ethPeer), snapWait: make(map[string]chan *snap.Peer), snapPend: make(map[string]*snap.Peer), + quitCh: make(chan struct{}), } } @@ -129,7 +131,15 @@ func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { ps.snapWait[id] = wait ps.lock.Unlock() - return <-wait, nil + select { + case p := <-wait: + return p, nil + case <-ps.quitCh: + ps.lock.Lock() + delete(ps.snapWait, id) + ps.lock.Unlock() + return nil, errPeerSetClosed + } } // registerPeer injects a new `eth` peer into the working set, or returns an error @@ -275,5 +285,8 @@ func (ps *peerSet) close() { for _, p := range ps.peers { p.Disconnect(p2p.DiscQuitting) } + if !ps.closed { + close(ps.quitCh) + } ps.closed = true } diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index ee671030e6..002d222e64 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -181,18 +181,11 @@ func (p *Peer) announceTransactions() { if len(pending) > 0 { done = make(chan struct{}) go func() { - if p.version >= ETH68 { - if err := p.sendPooledTransactionHashes68(pending, pendingTypes, pendingSizes); err != nil { - p.Log().Warn("Announce hashes68 failed", "peerId", p.ID(), "peerIP", safeGetPeerIP(p), "lost", len(pending), "hashes", concat(pending), "err", err.Error()) - fail <- err - return - } - } else { - if err := p.sendPooledTransactionHashes66(pending); err != nil { - p.Log().Warn("Announce hashes66 failed", "peerId", p.ID(), "peerIP", safeGetPeerIP(p), "lost", len(pending), "hashes", concat(pending), "err", err.Error()) - fail <- err - return - } + // TODO need check it eth68 condition + if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil { + p.Log().Warn("Announce hashes failed", "peerId", p.ID(), "peerIP", safeGetPeerIP(p), "lost", len(pending), "hashes", concat(pending), "err", err.Error()) + fail <- err + return } close(done) p.Log().Trace("Sent transaction announcements", "count", len(pending), "peer.Id", p.ID(), "peer.IP", p.Node().IP().String(), "hashes", concat(pending)) diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 0fd495fc3f..e86f2dec7a 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -93,10 +93,6 @@ type TxPool interface { func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2p.Protocol { protocols := make([]p2p.Protocol, 0, len(ProtocolVersions)) for _, version := range ProtocolVersions { - // Blob transactions require eth/68 announcements, disable everything else - if version <= ETH67 && backend.Chain().Config().CancunTime != nil { - continue - } version := version // Closure protocols = append(protocols, p2p.Protocol{ @@ -178,26 +174,11 @@ type Decoder interface { Time() time.Time } -var eth67 = map[uint64]msgHandler{ - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes67, - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth68 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes68, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, GetBlockHeadersMsg: handleGetBlockHeaders, BlockHeadersMsg: handleBlockHeaders, GetBlockBodiesMsg: handleGetBlockBodies, @@ -221,10 +202,8 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth67 - if peer.Version() >= ETH68 { - handlers = eth68 - } + var handlers = eth68 + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 41e18bfb3e..fdf551ef21 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -102,7 +102,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, gspec := &core.Genesis{ Config: config, - Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, } chain, _ := core.NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) @@ -117,7 +117,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, txconfig.Journal = "" // Don't litter the disk with test journals pool := legacypool.New(txconfig, chain) - txpool, _ := txpool.New(new(big.Int).SetUint64(txconfig.PriceLimit), chain, []txpool.SubPool{pool}) + txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{pool}) return &testBackend{ db: db, @@ -150,7 +150,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders67(t *testing.T) { testGetBlockHeaders(t, ETH67) } func TestGetBlockHeaders68(t *testing.T) { testGetBlockHeaders(t, ETH68) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -336,7 +335,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies67(t *testing.T) { testGetBlockBodies(t, ETH67) } func TestGetBlockBodies68(t *testing.T) { testGetBlockBodies(t, ETH68) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -431,7 +429,6 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts67(t *testing.T) { testGetBlockReceipts(t, ETH67) } func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } func testGetBlockReceipts(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 069e92dadf..0275708a6c 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -383,30 +383,13 @@ func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { }, metadata) } -func handleNewPooledTransactionHashes67(backend Backend, msg Decoder, peer *Peer) error { +func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) error { // New transaction announcement arrived, make sure we have // a valid and fresh chain to handle them if !backend.AcceptTxs() { return nil } - ann := new(NewPooledTransactionHashesPacket67) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Schedule all the unknown hashes for retrieval - for _, hash := range *ann { - peer.markTransaction(hash) - } - return backend.Handle(peer, ann) -} - -func handleNewPooledTransactionHashes68(backend Backend, msg Decoder, peer *Peer) error { - // New transaction announcement arrived, make sure we have - // a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - ann := new(NewPooledTransactionHashesPacket68) + ann := new(NewPooledTransactionHashesPacket) if err := msg.Decode(ann); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index d96cfc8165..b9fd13d863 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake67(t *testing.T) { testHandshake(t, ETH67) } func TestHandshake68(t *testing.T) { testHandshake(t, ETH68) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98ad22a8cf..ffd78b0594 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -92,7 +92,7 @@ type Peer struct { lock sync.RWMutex // Mutex protecting the internal fields } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { peer := &Peer{ @@ -210,29 +210,17 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { } } -// sendPooledTransactionHashes66 sends transaction hashes to the peer and includes -// them in its transaction hash set for future reference. -// -// This method is a helper used by the async transaction announcer. Don't call it -// directly as the queueing (memory) and transmission (bandwidth) costs should -// not be managed directly. -func (p *Peer) sendPooledTransactionHashes66(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket67(hashes)) -} - -// sendPooledTransactionHashes68 sends transaction hashes (tagged with their type +// sendPooledTransactionHashes sends transaction hashes (tagged with their type // and size) to the peer and includes them in its transaction hash set for future // reference. // // This method is a helper used by the async transaction announcer. Don't call it // directly as the queueing (memory) and transmission (bandwidth) costs should // not be managed directly. -func (p *Peer) sendPooledTransactionHashes68(hashes []common.Hash, types []byte, sizes []uint32) error { +func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error { // Mark all the transactions as known, but ensure we don't overflow our limits p.knownTxs.Add(hashes...) - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket68{Types: types, Sizes: sizes, Hashes: hashes}) + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}) } // AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 0f44f83de1..47e8d97244 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH67 = 67 ETH68 = 68 ) @@ -40,11 +39,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH68, ETH67} +var ProtocolVersions = []uint{ETH68} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH68: 17, ETH67: 17} +var protocolLengths = map[uint]uint64{ETH68: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -283,11 +282,8 @@ type ReceiptsRLPPacket struct { ReceiptsRLPResponse } -// NewPooledTransactionHashesPacket67 represents a transaction announcement packet on eth/67. -type NewPooledTransactionHashesPacket67 []common.Hash - -// NewPooledTransactionHashesPacket68 represents a transaction announcement packet on eth/68 and newer. -type NewPooledTransactionHashesPacket68 struct { +// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. +type NewPooledTransactionHashesPacket struct { Types []byte Sizes []uint32 Hashes []common.Hash @@ -346,10 +342,8 @@ func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } func (*NewBlockPacket) Name() string { return "NewBlock" } func (*NewBlockPacket) Kind() byte { return NewBlockMsg } -func (*NewPooledTransactionHashesPacket67) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket67) Kind() byte { return NewPooledTransactionHashesMsg } -func (*NewPooledTransactionHashesPacket68) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket68) Kind() byte { return NewPooledTransactionHashesMsg } +func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } diff --git a/eth/protocols/snap/gentrie.go b/eth/protocols/snap/gentrie.go new file mode 100644 index 0000000000..8ef1a00753 --- /dev/null +++ b/eth/protocols/snap/gentrie.go @@ -0,0 +1,287 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" +) + +// genTrie interface is used by the snap syncer to generate merkle tree nodes +// based on a received batch of states. +type genTrie interface { + // update inserts the state item into generator trie. + update(key, value []byte) error + + // commit flushes the right boundary nodes if complete flag is true. This + // function must be called before flushing the associated database batch. + commit(complete bool) common.Hash +} + +// pathTrie is a wrapper over the stackTrie, incorporating numerous additional +// logics to handle the semi-completed trie and potential leftover dangling +// nodes in the database. It is utilized for constructing the merkle tree nodes +// in path mode during the snap sync process. +type pathTrie struct { + owner common.Hash // identifier of trie owner, empty for account trie + tr *trie.StackTrie // underlying raw stack trie + first []byte // the path of first committed node by stackTrie + last []byte // the path of last committed node by stackTrie + + // This flag indicates whether nodes on the left boundary are skipped for + // committing. If set, the left boundary nodes are considered incomplete + // due to potentially missing left children. + skipLeftBoundary bool + db ethdb.KeyValueReader + batch ethdb.Batch +} + +// newPathTrie initializes the path trie. +func newPathTrie(owner common.Hash, skipLeftBoundary bool, db ethdb.KeyValueReader, batch ethdb.Batch) *pathTrie { + tr := &pathTrie{ + owner: owner, + skipLeftBoundary: skipLeftBoundary, + db: db, + batch: batch, + } + tr.tr = trie.NewStackTrie(tr.onTrieNode) + return tr +} + +// onTrieNode is invoked whenever a new node is committed by the stackTrie. +// +// As the committed nodes might be incomplete if they are on the boundaries +// (left or right), this function has the ability to detect the incomplete +// ones and filter them out for committing. +// +// Additionally, the assumption is made that there may exist leftover dangling +// nodes in the database. This function has the ability to detect the dangling +// nodes that fall within the path space of committed nodes (specifically on +// the path covered by internal extension nodes) and remove them from the +// database. This property ensures that the entire path space is uniquely +// occupied by committed nodes. +// +// Furthermore, all leftover dangling nodes along the path from committed nodes +// to the trie root (left and right boundaries) should be removed as well; +// otherwise, they might potentially disrupt the state healing process. +func (t *pathTrie) onTrieNode(path []byte, hash common.Hash, blob []byte) { + // Filter out the nodes on the left boundary if skipLeftBoundary is + // configured. Nodes are considered to be on the left boundary if + // it's the first one to be committed, or the parent/ancestor of the + // first committed node. + if t.skipLeftBoundary && (t.first == nil || bytes.HasPrefix(t.first, path)) { + if t.first == nil { + // Memorize the path of first committed node, which is regarded + // as left boundary. Deep-copy is necessary as the path given + // is volatile. + t.first = append([]byte{}, path...) + + // The left boundary can be uniquely determined by the first committed node + // from stackTrie (e.g., N_1), as the shared path prefix between the first + // two inserted state items is deterministic (the path of N_3). The path + // from trie root towards the first committed node is considered the left + // boundary. The potential leftover dangling nodes on left boundary should + // be cleaned out. + // + // +-----+ + // | N_3 | shared path prefix of state_1 and state_2 + // +-----+ + // /- -\ + // +-----+ +-----+ + // First committed node | N_1 | | N_2 | latest inserted node (contain state_2) + // +-----+ +-----+ + // + // The node with the path of the first committed one (e.g, N_1) is not + // removed because it's a sibling of the nodes we want to commit, not + // the parent or ancestor. + for i := 0; i < len(path); i++ { + t.delete(path[:i], false) + } + } + return + } + // If boundary filtering is not configured, or the node is not on the left + // boundary, commit it to database. + // + // Note: If the current committed node is an extension node, then the nodes + // falling within the path between itself and its standalone (not embedded + // in parent) child should be cleaned out for exclusively occupy the inner + // path. + // + // This is essential in snap sync to avoid leaving dangling nodes within + // this range covered by extension node which could potentially break the + // state healing. + // + // The extension node is detected if its path is the prefix of last committed + // one and path gap is larger than one. If the path gap is only one byte, + // the current node could either be a full node, or a extension with single + // byte key. In either case, no gaps will be left in the path. + if t.last != nil && bytes.HasPrefix(t.last, path) && len(t.last)-len(path) > 1 { + for i := len(path) + 1; i < len(t.last); i++ { + t.delete(t.last[:i], true) + } + } + t.write(path, blob) + + // Update the last flag. Deep-copy is necessary as the provided path is volatile. + if t.last == nil { + t.last = append([]byte{}, path...) + } else { + t.last = append(t.last[:0], path...) + } +} + +// write commits the node write to provided database batch in path mode. +func (t *pathTrie) write(path []byte, blob []byte) { + if t.owner == (common.Hash{}) { + rawdb.WriteAccountTrieNode(t.batch, path, blob) + } else { + rawdb.WriteStorageTrieNode(t.batch, t.owner, path, blob) + } +} + +func (t *pathTrie) deleteAccountNode(path []byte, inner bool) { + if inner { + accountInnerLookupGauge.Inc(1) + } else { + accountOuterLookupGauge.Inc(1) + } + if !rawdb.ExistsAccountTrieNode(t.db, path) { + return + } + if inner { + accountInnerDeleteGauge.Inc(1) + } else { + accountOuterDeleteGauge.Inc(1) + } + rawdb.DeleteAccountTrieNode(t.batch, path) +} + +func (t *pathTrie) deleteStorageNode(path []byte, inner bool) { + if inner { + storageInnerLookupGauge.Inc(1) + } else { + storageOuterLookupGauge.Inc(1) + } + if !rawdb.ExistsStorageTrieNode(t.db, t.owner, path) { + return + } + if inner { + storageInnerDeleteGauge.Inc(1) + } else { + storageOuterDeleteGauge.Inc(1) + } + rawdb.DeleteStorageTrieNode(t.batch, t.owner, path) +} + +// delete commits the node deletion to provided database batch in path mode. +func (t *pathTrie) delete(path []byte, inner bool) { + if t.owner == (common.Hash{}) { + t.deleteAccountNode(path, inner) + } else { + t.deleteStorageNode(path, inner) + } +} + +// update implements genTrie interface, inserting a (key, value) pair into the +// stack trie. +func (t *pathTrie) update(key, value []byte) error { + return t.tr.Update(key, value) +} + +// commit implements genTrie interface, flushing the right boundary if it's +// considered as complete. Otherwise, the nodes on the right boundary are +// discarded and cleaned up. +// +// Note, this function must be called before flushing database batch, otherwise, +// dangling nodes might be left in database. +func (t *pathTrie) commit(complete bool) common.Hash { + // If the right boundary is claimed as complete, flush them out. + // The nodes on both left and right boundary will still be filtered + // out if left boundary filtering is configured. + if complete { + // Commit all inserted but not yet committed nodes(on the right + // boundary) in the stackTrie. + hash := t.tr.Hash() + if t.skipLeftBoundary { + return common.Hash{} // hash is meaningless if left side is incomplete + } + return hash + } + // Discard nodes on the right boundary as it's claimed as incomplete. These + // nodes might be incomplete due to missing children on the right side. + // Furthermore, the potential leftover nodes on right boundary should also + // be cleaned out. + // + // The right boundary can be uniquely determined by the last committed node + // from stackTrie (e.g., N_1), as the shared path prefix between the last + // two inserted state items is deterministic (the path of N_3). The path + // from trie root towards the last committed node is considered the right + // boundary (root to N_3). + // + // +-----+ + // | N_3 | shared path prefix of last two states + // +-----+ + // /- -\ + // +-----+ +-----+ + // Last committed node | N_1 | | N_2 | latest inserted node (contain last state) + // +-----+ +-----+ + // + // Another interesting scenario occurs when the trie is committed due to + // too many items being accumulated in the batch. To flush them out to + // the database, the path of the last inserted node (N_2) is temporarily + // treated as an incomplete right boundary, and nodes on this path are + // removed (e.g. from root to N_3). + // However, this path will be reclaimed as an internal path by inserting + // more items after the batch flush. New nodes on this path can be committed + // with no issues as they are actually complete. Also, from a database + // perspective, first deleting and then rewriting is a valid data update. + for i := 0; i < len(t.last); i++ { + t.delete(t.last[:i], false) + } + return common.Hash{} // the hash is meaningless for incomplete commit +} + +// hashTrie is a wrapper over the stackTrie for implementing genTrie interface. +type hashTrie struct { + tr *trie.StackTrie +} + +// newHashTrie initializes the hash trie. +func newHashTrie(batch ethdb.Batch) *hashTrie { + return &hashTrie{tr: trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + rawdb.WriteLegacyTrieNode(batch, hash, blob) + })} +} + +// update implements genTrie interface, inserting a (key, value) pair into +// the stack trie. +func (t *hashTrie) update(key, value []byte) error { + return t.tr.Update(key, value) +} + +// commit implements genTrie interface, committing the nodes on right boundary. +func (t *hashTrie) commit(complete bool) common.Hash { + if !complete { + return common.Hash{} // the hash is meaningless for incomplete commit + } + return t.tr.Hash() // return hash only if it's claimed as complete +} diff --git a/eth/protocols/snap/gentrie_test.go b/eth/protocols/snap/gentrie_test.go new file mode 100644 index 0000000000..1fb2dbce75 --- /dev/null +++ b/eth/protocols/snap/gentrie_test.go @@ -0,0 +1,553 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "bytes" + "math/rand" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/testrand" + "github.com/ethereum/go-ethereum/trie" +) + +type replayer struct { + paths []string // sort in fifo order + hashes []common.Hash // empty for deletion + unknowns int // counter for unknown write +} + +func newBatchReplay() *replayer { + return &replayer{} +} + +func (r *replayer) decode(key []byte, value []byte) { + account := rawdb.IsAccountTrieNode(key) + storage := rawdb.IsStorageTrieNode(key) + if !account && !storage { + r.unknowns += 1 + return + } + var path []byte + if account { + _, path = rawdb.ResolveAccountTrieNodeKey(key) + } else { + _, owner, inner := rawdb.ResolveStorageTrieNode(key) + path = append(owner.Bytes(), inner...) + } + r.paths = append(r.paths, string(path)) + + if len(value) == 0 { + r.hashes = append(r.hashes, common.Hash{}) + } else { + r.hashes = append(r.hashes, crypto.Keccak256Hash(value)) + } +} + +// updates returns a set of effective mutations. Multiple mutations targeting +// the same node path will be merged in FIFO order. +func (r *replayer) modifies() map[string]common.Hash { + set := make(map[string]common.Hash) + for i, path := range r.paths { + set[path] = r.hashes[i] + } + return set +} + +// updates returns the number of updates. +func (r *replayer) updates() int { + var count int + for _, hash := range r.modifies() { + if hash == (common.Hash{}) { + continue + } + count++ + } + return count +} + +// Put inserts the given value into the key-value data store. +func (r *replayer) Put(key []byte, value []byte) error { + r.decode(key, value) + return nil +} + +// Delete removes the key from the key-value data store. +func (r *replayer) Delete(key []byte) error { + r.decode(key, nil) + return nil +} + +func byteToHex(str []byte) []byte { + l := len(str) * 2 + var nibbles = make([]byte, l) + for i, b := range str { + nibbles[i*2] = b / 16 + nibbles[i*2+1] = b % 16 + } + return nibbles +} + +// innerNodes returns the internal nodes narrowed by two boundaries along with +// the leftmost and rightmost sub-trie roots. +func innerNodes(first, last []byte, includeLeft, includeRight bool, nodes map[string]common.Hash, t *testing.T) (map[string]common.Hash, []byte, []byte) { + var ( + leftRoot []byte + rightRoot []byte + firstHex = byteToHex(first) + lastHex = byteToHex(last) + inner = make(map[string]common.Hash) + ) + for path, hash := range nodes { + if hash == (common.Hash{}) { + t.Fatalf("Unexpected deletion, %v", []byte(path)) + } + // Filter out the siblings on the left side or the left boundary nodes. + if !includeLeft && (bytes.Compare(firstHex, []byte(path)) > 0 || bytes.HasPrefix(firstHex, []byte(path))) { + continue + } + // Filter out the siblings on the right side or the right boundary nodes. + if !includeRight && (bytes.Compare(lastHex, []byte(path)) < 0 || bytes.HasPrefix(lastHex, []byte(path))) { + continue + } + inner[path] = hash + + // Track the path of the leftmost sub trie root + if leftRoot == nil || bytes.Compare(leftRoot, []byte(path)) > 0 { + leftRoot = []byte(path) + } + // Track the path of the rightmost sub trie root + if rightRoot == nil || + (bytes.Compare(rightRoot, []byte(path)) < 0) || + (bytes.Compare(rightRoot, []byte(path)) > 0 && bytes.HasPrefix(rightRoot, []byte(path))) { + rightRoot = []byte(path) + } + } + return inner, leftRoot, rightRoot +} + +func buildPartial(owner common.Hash, db ethdb.KeyValueReader, batch ethdb.Batch, entries []*kv, first, last int) *replayer { + tr := newPathTrie(owner, first != 0, db, batch) + for i := first; i <= last; i++ { + tr.update(entries[i].k, entries[i].v) + } + tr.commit(last == len(entries)-1) + + replay := newBatchReplay() + batch.Replay(replay) + + return replay +} + +// TestPartialGentree verifies if the trie constructed with partial states can +// generate consistent trie nodes that match those of the full trie. +func TestPartialGentree(t *testing.T) { + for round := 0; round < 100; round++ { + var ( + n = rand.Intn(1024) + 10 + entries []*kv + ) + for i := 0; i < n; i++ { + var val []byte + if rand.Intn(3) == 0 { + val = testrand.Bytes(3) + } else { + val = testrand.Bytes(32) + } + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: val, + }) + } + slices.SortFunc(entries, (*kv).cmp) + + nodes := make(map[string]common.Hash) + tr := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + nodes[string(path)] = hash + }) + for i := 0; i < len(entries); i++ { + tr.Update(entries[i].k, entries[i].v) + } + tr.Hash() + + check := func(first, last int) { + var ( + db = rawdb.NewMemoryDatabase() + batch = db.NewBatch() + ) + // Build the partial tree with specific boundaries + r := buildPartial(common.Hash{}, db, batch, entries, first, last) + if r.unknowns > 0 { + t.Fatalf("Unknown database write: %d", r.unknowns) + } + + // Ensure all the internal nodes are produced + var ( + set = r.modifies() + inner, _, _ = innerNodes(entries[first].k, entries[last].k, first == 0, last == len(entries)-1, nodes, t) + ) + for path, hash := range inner { + if _, ok := set[path]; !ok { + t.Fatalf("Missing nodes %v", []byte(path)) + } + if hash != set[path] { + t.Fatalf("Inconsistent node, want %x, got: %x", hash, set[path]) + } + } + if r.updates() != len(inner) { + t.Fatalf("Unexpected node write detected, want: %d, got: %d", len(inner), r.updates()) + } + } + for j := 0; j < 100; j++ { + var ( + first int + last int + ) + for { + first = rand.Intn(len(entries)) + last = rand.Intn(len(entries)) + if first <= last { + break + } + } + check(first, last) + } + var cases = []struct { + first int + last int + }{ + {0, len(entries) - 1}, // full + {1, len(entries) - 1}, // no left + {2, len(entries) - 1}, // no left + {2, len(entries) - 2}, // no left and right + {2, len(entries) - 2}, // no left and right + {len(entries) / 2, len(entries) / 2}, // single + {0, 0}, // single first + {len(entries) - 1, len(entries) - 1}, // single last + } + for _, c := range cases { + check(c.first, c.last) + } + } +} + +// TestGentreeDanglingClearing tests if the dangling nodes falling within the +// path space of constructed tree can be correctly removed. +func TestGentreeDanglingClearing(t *testing.T) { + for round := 0; round < 100; round++ { + var ( + n = rand.Intn(1024) + 10 + entries []*kv + ) + for i := 0; i < n; i++ { + var val []byte + if rand.Intn(3) == 0 { + val = testrand.Bytes(3) + } else { + val = testrand.Bytes(32) + } + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: val, + }) + } + slices.SortFunc(entries, (*kv).cmp) + + nodes := make(map[string]common.Hash) + tr := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + nodes[string(path)] = hash + }) + for i := 0; i < len(entries); i++ { + tr.Update(entries[i].k, entries[i].v) + } + tr.Hash() + + check := func(first, last int) { + var ( + db = rawdb.NewMemoryDatabase() + batch = db.NewBatch() + ) + // Write the junk nodes as the dangling + var injects []string + for path := range nodes { + for i := 0; i < len(path); i++ { + _, ok := nodes[path[:i]] + if ok { + continue + } + injects = append(injects, path[:i]) + } + } + if len(injects) == 0 { + return + } + for _, path := range injects { + rawdb.WriteAccountTrieNode(db, []byte(path), testrand.Bytes(32)) + } + + // Build the partial tree with specific range + replay := buildPartial(common.Hash{}, db, batch, entries, first, last) + if replay.unknowns > 0 { + t.Fatalf("Unknown database write: %d", replay.unknowns) + } + set := replay.modifies() + + // Make sure the injected junks falling within the path space of + // committed trie nodes are correctly deleted. + _, leftRoot, rightRoot := innerNodes(entries[first].k, entries[last].k, first == 0, last == len(entries)-1, nodes, t) + for _, path := range injects { + if bytes.Compare([]byte(path), leftRoot) < 0 && !bytes.HasPrefix(leftRoot, []byte(path)) { + continue + } + if bytes.Compare([]byte(path), rightRoot) > 0 { + continue + } + if hash, ok := set[path]; !ok || hash != (common.Hash{}) { + t.Fatalf("Missing delete, %v", []byte(path)) + } + } + } + for j := 0; j < 100; j++ { + var ( + first int + last int + ) + for { + first = rand.Intn(len(entries)) + last = rand.Intn(len(entries)) + if first <= last { + break + } + } + check(first, last) + } + var cases = []struct { + first int + last int + }{ + {0, len(entries) - 1}, // full + {1, len(entries) - 1}, // no left + {2, len(entries) - 1}, // no left + {2, len(entries) - 2}, // no left and right + {2, len(entries) - 2}, // no left and right + {len(entries) / 2, len(entries) / 2}, // single + {0, 0}, // single first + {len(entries) - 1, len(entries) - 1}, // single last + } + for _, c := range cases { + check(c.first, c.last) + } + } +} + +// TestFlushPartialTree tests the gentrie can produce complete inner trie nodes +// even with lots of batch flushes. +func TestFlushPartialTree(t *testing.T) { + var entries []*kv + for i := 0; i < 1024; i++ { + var val []byte + if rand.Intn(3) == 0 { + val = testrand.Bytes(3) + } else { + val = testrand.Bytes(32) + } + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: val, + }) + } + slices.SortFunc(entries, (*kv).cmp) + + nodes := make(map[string]common.Hash) + tr := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + nodes[string(path)] = hash + }) + for i := 0; i < len(entries); i++ { + tr.Update(entries[i].k, entries[i].v) + } + tr.Hash() + + var cases = []struct { + first int + last int + }{ + {0, len(entries) - 1}, // full + {1, len(entries) - 1}, // no left + {10, len(entries) - 1}, // no left + {10, len(entries) - 2}, // no left and right + {10, len(entries) - 10}, // no left and right + {11, 11}, // single + {0, 0}, // single first + {len(entries) - 1, len(entries) - 1}, // single last + } + for _, c := range cases { + var ( + db = rawdb.NewMemoryDatabase() + batch = db.NewBatch() + combined = db.NewBatch() + ) + inner, _, _ := innerNodes(entries[c.first].k, entries[c.last].k, c.first == 0, c.last == len(entries)-1, nodes, t) + + tr := newPathTrie(common.Hash{}, c.first != 0, db, batch) + for i := c.first; i <= c.last; i++ { + tr.update(entries[i].k, entries[i].v) + if rand.Intn(2) == 0 { + tr.commit(false) + + batch.Replay(combined) + batch.Write() + batch.Reset() + } + } + tr.commit(c.last == len(entries)-1) + + batch.Replay(combined) + batch.Write() + batch.Reset() + + r := newBatchReplay() + combined.Replay(r) + + // Ensure all the internal nodes are produced + set := r.modifies() + for path, hash := range inner { + if _, ok := set[path]; !ok { + t.Fatalf("Missing nodes %v", []byte(path)) + } + if hash != set[path] { + t.Fatalf("Inconsistent node, want %x, got: %x", hash, set[path]) + } + } + if r.updates() != len(inner) { + t.Fatalf("Unexpected node write detected, want: %d, got: %d", len(inner), r.updates()) + } + } +} + +// TestBoundSplit ensures two consecutive trie chunks are not overlapped with +// each other. +func TestBoundSplit(t *testing.T) { + var entries []*kv + for i := 0; i < 1024; i++ { + var val []byte + if rand.Intn(3) == 0 { + val = testrand.Bytes(3) + } else { + val = testrand.Bytes(32) + } + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: val, + }) + } + slices.SortFunc(entries, (*kv).cmp) + + for j := 0; j < 100; j++ { + var ( + next int + last int + db = rawdb.NewMemoryDatabase() + + lastRightRoot []byte + ) + for { + if next == len(entries) { + break + } + last = rand.Intn(len(entries)-next) + next + + r := buildPartial(common.Hash{}, db, db.NewBatch(), entries, next, last) + set := r.modifies() + + // Skip if the chunk is zero-size + if r.updates() == 0 { + next = last + 1 + continue + } + + // Ensure the updates in two consecutive chunks are not overlapped. + // The only overlapping part should be deletion. + if lastRightRoot != nil && len(set) > 0 { + // Derive the path of left-most node in this chunk + var leftRoot []byte + for path, hash := range r.modifies() { + if hash == (common.Hash{}) { + t.Fatalf("Unexpected deletion %v", []byte(path)) + } + if leftRoot == nil || bytes.Compare(leftRoot, []byte(path)) > 0 { + leftRoot = []byte(path) + } + } + if bytes.HasPrefix(lastRightRoot, leftRoot) || bytes.HasPrefix(leftRoot, lastRightRoot) { + t.Fatalf("Two chunks are not correctly separated, lastRight: %v, left: %v", lastRightRoot, leftRoot) + } + } + + // Track the updates as the last chunk + var rightRoot []byte + for path := range set { + if rightRoot == nil || + (bytes.Compare(rightRoot, []byte(path)) < 0) || + (bytes.Compare(rightRoot, []byte(path)) > 0 && bytes.HasPrefix(rightRoot, []byte(path))) { + rightRoot = []byte(path) + } + } + lastRightRoot = rightRoot + next = last + 1 + } + } +} + +// TestTinyPartialTree tests if the partial tree is too tiny(has less than two +// states), then nothing should be committed. +func TestTinyPartialTree(t *testing.T) { + var entries []*kv + for i := 0; i < 1024; i++ { + var val []byte + if rand.Intn(3) == 0 { + val = testrand.Bytes(3) + } else { + val = testrand.Bytes(32) + } + entries = append(entries, &kv{ + k: testrand.Bytes(32), + v: val, + }) + } + slices.SortFunc(entries, (*kv).cmp) + + for i := 0; i < len(entries); i++ { + next := i + last := i + 1 + if last >= len(entries) { + last = len(entries) - 1 + } + db := rawdb.NewMemoryDatabase() + r := buildPartial(common.Hash{}, db, db.NewBatch(), entries, next, last) + + if next != 0 && last != len(entries)-1 { + if r.updates() != 0 { + t.Fatalf("Unexpected data writes, got: %d", r.updates()) + } + } + } +} diff --git a/eth/protocols/snap/handler_fuzzing_test.go b/eth/protocols/snap/handler_fuzzing_test.go index daed7ed44a..4e234ad21b 100644 --- a/eth/protocols/snap/handler_fuzzing_test.go +++ b/eth/protocols/snap/handler_fuzzing_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -89,7 +90,7 @@ func doFuzz(input []byte, obj interface{}, code int) { var trieRoot common.Hash func getChain() *core.BlockChain { - ga := make(core.GenesisAlloc, 1000) + ga := make(types.GenesisAlloc, 1000) var a = make([]byte, 20) var mkStorage = func(k, v int) (common.Hash, common.Hash) { var kB = make([]byte, 32) @@ -105,7 +106,7 @@ func getChain() *core.BlockChain { } for i := 0; i < 1000; i++ { binary.LittleEndian.PutUint64(a, uint64(i+0xff)) - acc := core.GenesisAccount{Balance: big.NewInt(int64(i))} + acc := types.Account{Balance: big.NewInt(int64(i))} if i%2 == 1 { acc.Storage = storage } diff --git a/eth/protocols/snap/metrics.go b/eth/protocols/snap/metrics.go index a7d071953f..25dbcc6386 100644 --- a/eth/protocols/snap/metrics.go +++ b/eth/protocols/snap/metrics.go @@ -27,21 +27,28 @@ var ( IngressRegistrationErrorMeter = metrics.NewRegisteredMeter(ingressRegistrationErrorName, nil) EgressRegistrationErrorMeter = metrics.NewRegisteredMeter(egressRegistrationErrorName, nil) - // deletionGauge is the metric to track how many trie node deletions - // are performed in total during the sync process. - deletionGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/delete", nil) + // accountInnerDeleteGauge is the metric to track how many dangling trie nodes + // covered by extension node in account trie are deleted during the sync. + accountInnerDeleteGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/delete/account/inner", nil) - // lookupGauge is the metric to track how many trie node lookups are - // performed to determine if node needs to be deleted. - lookupGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/lookup", nil) + // storageInnerDeleteGauge is the metric to track how many dangling trie nodes + // covered by extension node in storage trie are deleted during the sync. + storageInnerDeleteGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/delete/storage/inner", nil) + + // accountOuterDeleteGauge is the metric to track how many dangling trie nodes + // above the committed nodes in account trie are deleted during the sync. + accountOuterDeleteGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/delete/account/outer", nil) - // boundaryAccountNodesGauge is the metric to track how many boundary trie - // nodes in account trie are met. - boundaryAccountNodesGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/boundary/account", nil) + // storageOuterDeleteGauge is the metric to track how many dangling trie nodes + // above the committed nodes in storage trie are deleted during the sync. + storageOuterDeleteGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/delete/storage/outer", nil) - // boundaryAccountNodesGauge is the metric to track how many boundary trie - // nodes in storage tries are met. - boundaryStorageNodesGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/boundary/storage", nil) + // lookupGauge is the metric to track how many trie node lookups are + // performed to determine if node needs to be deleted. + accountInnerLookupGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/account/lookup/inner", nil) + accountOuterLookupGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/account/lookup/outer", nil) + storageInnerLookupGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/storage/lookup/inner", nil) + storageOuterLookupGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/storage/lookup/outer", nil) // smallStorageGauge is the metric to track how many storages are small enough // to retrieved in one or two request. @@ -54,4 +61,9 @@ var ( // skipStorageHealingGauge is the metric to track how many storages are retrieved // in multiple requests but healing is not necessary. skipStorageHealingGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/storage/noheal", nil) + + // largeStorageDiscardGauge is the metric to track how many chunked storages are + // discarded during the snap sync. + largeStorageDiscardGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/storage/chunk/discard", nil) + largeStorageResumedGauge = metrics.NewRegisteredGauge("eth/protocols/snap/sync/storage/chunk/resume", nil) ) diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 3db6e22cbd..c57931678c 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -33,7 +33,7 @@ type Peer struct { logger log.Logger // Contextual logger with the peer id injected } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { id := p.ID().String() @@ -46,7 +46,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { } } -// NewFakePeer create a fake snap peer without a backing p2p peer, for testing purposes. +// NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { return &Peer{ id: id, diff --git a/eth/protocols/snap/progress_test.go b/eth/protocols/snap/progress_test.go new file mode 100644 index 0000000000..9d923bd2f5 --- /dev/null +++ b/eth/protocols/snap/progress_test.go @@ -0,0 +1,154 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + "encoding/json" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +// Legacy sync progress definitions +type legacyStorageTask struct { + Next common.Hash // Next account to sync in this interval + Last common.Hash // Last account to sync in this interval +} + +type legacyAccountTask struct { + Next common.Hash // Next account to sync in this interval + Last common.Hash // Last account to sync in this interval + SubTasks map[common.Hash][]*legacyStorageTask // Storage intervals needing fetching for large contracts +} + +type legacyProgress struct { + Tasks []*legacyAccountTask // The suspended account tasks (contract tasks within) +} + +func compareProgress(a legacyProgress, b SyncProgress) bool { + if len(a.Tasks) != len(b.Tasks) { + return false + } + for i := 0; i < len(a.Tasks); i++ { + if a.Tasks[i].Next != b.Tasks[i].Next { + return false + } + if a.Tasks[i].Last != b.Tasks[i].Last { + return false + } + // new fields are not checked here + + if len(a.Tasks[i].SubTasks) != len(b.Tasks[i].SubTasks) { + return false + } + for addrHash, subTasksA := range a.Tasks[i].SubTasks { + subTasksB, ok := b.Tasks[i].SubTasks[addrHash] + if !ok || len(subTasksB) != len(subTasksA) { + return false + } + for j := 0; j < len(subTasksA); j++ { + if subTasksA[j].Next != subTasksB[j].Next { + return false + } + if subTasksA[j].Last != subTasksB[j].Last { + return false + } + } + } + } + return true +} + +func makeLegacyProgress() legacyProgress { + return legacyProgress{ + Tasks: []*legacyAccountTask{ + { + Next: common.Hash{}, + Last: common.Hash{0x77}, + SubTasks: map[common.Hash][]*legacyStorageTask{ + common.Hash{0x1}: { + { + Next: common.Hash{}, + Last: common.Hash{0xff}, + }, + }, + }, + }, + { + Next: common.Hash{0x88}, + Last: common.Hash{0xff}, + }, + }, + } +} + +func convertLegacy(legacy legacyProgress) SyncProgress { + var progress SyncProgress + for i, task := range legacy.Tasks { + subTasks := make(map[common.Hash][]*storageTask) + for owner, list := range task.SubTasks { + var cpy []*storageTask + for i := 0; i < len(list); i++ { + cpy = append(cpy, &storageTask{ + Next: list[i].Next, + Last: list[i].Last, + }) + } + subTasks[owner] = cpy + } + accountTask := &accountTask{ + Next: task.Next, + Last: task.Last, + SubTasks: subTasks, + } + if i == 0 { + accountTask.StorageCompleted = []common.Hash{{0xaa}, {0xbb}} // fulfill new fields + } + progress.Tasks = append(progress.Tasks, accountTask) + } + return progress +} + +func TestSyncProgressCompatibility(t *testing.T) { + // Decode serialized bytes of legacy progress, backward compatibility + legacy := makeLegacyProgress() + blob, err := json.Marshal(legacy) + if err != nil { + t.Fatalf("Failed to marshal progress %v", err) + } + var dec SyncProgress + if err := json.Unmarshal(blob, &dec); err != nil { + t.Fatalf("Failed to unmarshal progress %v", err) + } + if !compareProgress(legacy, dec) { + t.Fatal("sync progress is not backward compatible") + } + + // Decode serialized bytes of new format progress + progress := convertLegacy(legacy) + blob, err = json.Marshal(progress) + if err != nil { + t.Fatalf("Failed to marshal progress %v", err) + } + var legacyDec legacyProgress + if err := json.Unmarshal(blob, &legacyDec); err != nil { + t.Fatalf("Failed to unmarshal progress %v", err) + } + if !compareProgress(legacyDec, progress) { + t.Fatal("sync progress is not forward compatible") + } +} diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 887a50775d..208d3ba3bc 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -94,6 +94,9 @@ const ( // trienodeHealThrottleDecrease is the divisor for the throttle when the // rate of arriving data is lower than the rate of processing it. trienodeHealThrottleDecrease = 1.25 + + // batchSizeThreshold is the maximum size allowed for gentrie batch. + batchSizeThreshold = 8 * 1024 * 1024 ) var ( @@ -295,11 +298,19 @@ type bytecodeHealResponse struct { // accountTask represents the sync task for a chunk of the account snapshot. type accountTask struct { - // These fields get serialized to leveldb on shutdown + // These fields get serialized to key-value store on shutdown Next common.Hash // Next account to sync in this interval Last common.Hash // Last account to sync in this interval SubTasks map[common.Hash][]*storageTask // Storage intervals needing fetching for large contracts + // This is a list of account hashes whose storage are already completed + // in this cycle. This field is newly introduced in v1.14 and will be + // empty if the task is resolved from legacy progress data. Furthermore, + // this additional field will be ignored by legacy Geth. The only side + // effect is that these contracts might be resynced in the new cycle, + // retaining the legacy behavior. + StorageCompleted []common.Hash `json:",omitempty"` + // These fields are internals used during runtime req *accountRequest // Pending request to fill this task res *accountResponse // Validate response filling this task @@ -309,15 +320,40 @@ type accountTask struct { needState []bool // Flags whether the filling accounts need storage retrieval needHeal []bool // Flags whether the filling accounts's state was chunked and need healing - codeTasks map[common.Hash]struct{} // Code hashes that need retrieval - stateTasks map[common.Hash]common.Hash // Account hashes->roots that need full state retrieval + codeTasks map[common.Hash]struct{} // Code hashes that need retrieval + stateTasks map[common.Hash]common.Hash // Account hashes->roots that need full state retrieval + stateCompleted map[common.Hash]struct{} // Account hashes whose storage have been completed - genBatch ethdb.Batch // Batch used by the node generator - genTrie *trie.StackTrie // Node generator from storage slots + genBatch ethdb.Batch // Batch used by the node generator + genTrie genTrie // Node generator from storage slots done bool // Flag whether the task can be removed } +// activeSubTasks returns the set of storage tasks covered by the current account +// range. Normally this would be the entire subTask set, but on a sync interrupt +// and later resume it can happen that a shorter account range is retrieved. This +// method ensures that we only start up the subtasks covered by the latest account +// response. +// +// Nil is returned if the account range is empty. +func (task *accountTask) activeSubTasks() map[common.Hash][]*storageTask { + if len(task.res.hashes) == 0 { + return nil + } + var ( + tasks = make(map[common.Hash][]*storageTask) + last = task.res.hashes[len(task.res.hashes)-1] + ) + for hash, subTasks := range task.SubTasks { + subTasks := subTasks // closure + if hash.Cmp(last) <= 0 { + tasks[hash] = subTasks + } + } + return tasks +} + // storageTask represents the sync task for a chunk of the storage snapshot. type storageTask struct { Next common.Hash // Next account to sync in this interval @@ -327,8 +363,8 @@ type storageTask struct { root common.Hash // Storage root hash for this instance req *storageRequest // Pending request to fill this task - genBatch ethdb.Batch // Batch used by the node generator - genTrie *trie.StackTrie // Node generator from storage slots + genBatch ethdb.Batch // Batch used by the node generator + genTrie genTrie // Node generator from storage slots done bool // Flag whether the task can be removed } @@ -716,19 +752,6 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { } } -// cleanPath is used to remove the dangling nodes in the stackTrie. -func (s *Syncer) cleanPath(batch ethdb.Batch, owner common.Hash, path []byte) { - if owner == (common.Hash{}) && rawdb.ExistsAccountTrieNode(s.db, path) { - rawdb.DeleteAccountTrieNode(batch, path) - deletionGauge.Inc(1) - } - if owner != (common.Hash{}) && rawdb.ExistsStorageTrieNode(s.db, owner, path) { - rawdb.DeleteStorageTrieNode(batch, owner, path) - deletionGauge.Inc(1) - } - lookupGauge.Inc(1) -} - // loadSyncStatus retrieves a previously aborted sync status from the database, // or generates a fresh one if none is available. func (s *Syncer) loadSyncStatus() { @@ -745,28 +768,27 @@ func (s *Syncer) loadSyncStatus() { for _, task := range s.tasks { task := task // closure for task.genBatch in the stacktrie writer callback + // Restore the completed storages + task.stateCompleted = make(map[common.Hash]struct{}) + for _, hash := range task.StorageCompleted { + task.stateCompleted[hash] = struct{}{} + } + task.StorageCompleted = nil + + // Allocate batch for account trie generation task.genBatch = ethdb.HookedBatch{ Batch: s.db.NewBatch(), OnPut: func(key []byte, value []byte) { s.accountBytes += common.StorageSize(len(key) + len(value)) }, } - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(task.genBatch, common.Hash{}, path, hash, blob, s.scheme) - }) + if s.scheme == rawdb.HashScheme { + task.genTrie = newHashTrie(task.genBatch) + } if s.scheme == rawdb.PathScheme { - // Configure the dangling node cleaner and also filter out boundary nodes - // only in the context of the path scheme. Deletion is forbidden in the - // hash scheme, as it can disrupt state completeness. - options = options.WithCleaner(func(path []byte) { - s.cleanPath(task.genBatch, common.Hash{}, path) - }) - // Skip the left boundary if it's not the first range. - // Skip the right boundary if it's not the last range. - options = options.WithSkipBoundary(task.Next != (common.Hash{}), task.Last != common.MaxHash, boundaryAccountNodesGauge) + task.genTrie = newPathTrie(common.Hash{}, task.Next != common.Hash{}, s.db, task.genBatch) } - task.genTrie = trie.NewStackTrie(options) + // Restore leftover storage tasks for accountHash, subtasks := range task.SubTasks { for _, subtask := range subtasks { subtask := subtask // closure for subtask.genBatch in the stacktrie writer callback @@ -777,23 +799,12 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - owner := accountHash // local assignment for stacktrie writer closure - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(subtask.genBatch, owner, path, hash, blob, s.scheme) - }) + if s.scheme == rawdb.HashScheme { + subtask.genTrie = newHashTrie(subtask.genBatch) + } if s.scheme == rawdb.PathScheme { - // Configure the dangling node cleaner and also filter out boundary nodes - // only in the context of the path scheme. Deletion is forbidden in the - // hash scheme, as it can disrupt state completeness. - options = options.WithCleaner(func(path []byte) { - s.cleanPath(subtask.genBatch, owner, path) - }) - // Skip the left boundary if it's not the first range. - // Skip the right boundary if it's not the last range. - options = options.WithSkipBoundary(subtask.Next != common.Hash{}, subtask.Last != common.MaxHash, boundaryStorageNodesGauge) + subtask.genTrie = newPathTrie(accountHash, subtask.Next != common.Hash{}, s.db, subtask.genBatch) } - subtask.genTrie = trie.NewStackTrie(options) } } } @@ -845,27 +856,20 @@ func (s *Syncer) loadSyncStatus() { s.accountBytes += common.StorageSize(len(key) + len(value)) }, } - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(batch, common.Hash{}, path, hash, blob, s.scheme) - }) + var tr genTrie + if s.scheme == rawdb.HashScheme { + tr = newHashTrie(batch) + } if s.scheme == rawdb.PathScheme { - // Configure the dangling node cleaner and also filter out boundary nodes - // only in the context of the path scheme. Deletion is forbidden in the - // hash scheme, as it can disrupt state completeness. - options = options.WithCleaner(func(path []byte) { - s.cleanPath(batch, common.Hash{}, path) - }) - // Skip the left boundary if it's not the first range. - // Skip the right boundary if it's not the last range. - options = options.WithSkipBoundary(next != common.Hash{}, last != common.MaxHash, boundaryAccountNodesGauge) + tr = newPathTrie(common.Hash{}, next != common.Hash{}, s.db, batch) } s.tasks = append(s.tasks, &accountTask{ - Next: next, - Last: last, - SubTasks: make(map[common.Hash][]*storageTask), - genBatch: batch, - genTrie: trie.NewStackTrie(options), + Next: next, + Last: last, + SubTasks: make(map[common.Hash][]*storageTask), + genBatch: batch, + stateCompleted: make(map[common.Hash]struct{}), + genTrie: tr, }) log.Debug("Created account sync task", "from", next, "last", last) next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) @@ -876,16 +880,31 @@ func (s *Syncer) loadSyncStatus() { func (s *Syncer) saveSyncStatus() { // Serialize any partial progress to disk before spinning down for _, task := range s.tasks { + // Claim the right boundary as incomplete before flushing the + // accumulated nodes in batch, the nodes on right boundary + // will be discarded and cleaned up by this call. + task.genTrie.commit(false) if err := task.genBatch.Write(); err != nil { log.Error("Failed to persist account slots", "err", err) } for _, subtasks := range task.SubTasks { for _, subtask := range subtasks { + // Same for account trie, discard and cleanup the + // incomplete right boundary. + subtask.genTrie.commit(false) if err := subtask.genBatch.Write(); err != nil { log.Error("Failed to persist storage slots", "err", err) } } } + // Save the account hashes of completed storage. + task.StorageCompleted = make([]common.Hash, 0, len(task.stateCompleted)) + for hash := range task.stateCompleted { + task.StorageCompleted = append(task.StorageCompleted, hash) + } + if len(task.StorageCompleted) > 0 { + log.Debug("Leftover completed storages", "number", len(task.StorageCompleted), "next", task.Next, "last", task.Last) + } } // Store the actual progress markers progress := &SyncProgress{ @@ -970,6 +989,10 @@ func (s *Syncer) cleanStorageTasks() { delete(task.SubTasks, account) task.pend-- + // Mark the state as complete to prevent resyncing, regardless + // if state healing is necessary. + task.stateCompleted[account] = struct{}{} + // If this was the last pending task, forward the account task if task.pend == 0 { s.forwardAccountTask(task) @@ -1209,7 +1232,8 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st continue } // Skip tasks that are already retrieving (or done with) all small states - if len(task.SubTasks) == 0 && len(task.stateTasks) == 0 { + storageTasks := task.activeSubTasks() + if len(storageTasks) == 0 && len(task.stateTasks) == 0 { continue } // Task pending retrieval, try to find an idle peer. If no such peer @@ -1253,7 +1277,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st roots = make([]common.Hash, 0, storageSets) subtask *storageTask ) - for account, subtasks := range task.SubTasks { + for account, subtasks := range storageTasks { for _, st := range subtasks { // Skip any subtasks already filling if st.req != nil { @@ -1850,11 +1874,11 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { res.task.res = res // Ensure that the response doesn't overflow into the subsequent task - last := res.task.Last.Big() + lastBig := res.task.Last.Big() for i, hash := range res.hashes { // Mark the range complete if the last is already included. // Keep iteration to delete the extra states if exists. - cmp := hash.Big().Cmp(last) + cmp := hash.Big().Cmp(lastBig) if cmp == 0 { res.cont = false continue @@ -1890,7 +1914,21 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { } // Check if the account is a contract with an unknown storage trie if account.Root != types.EmptyRootHash { - if !rawdb.HasTrieNode(s.db, res.hashes[i], nil, account.Root, s.scheme) { + // If the storage was already retrieved in the last cycle, there's no need + // to resync it again, regardless of whether the storage root is consistent + // or not. + if _, exist := res.task.stateCompleted[res.hashes[i]]; exist { + // The leftover storage tasks are not expected, unless system is + // very wrong. + if _, ok := res.task.SubTasks[res.hashes[i]]; ok { + panic(fmt.Errorf("unexpected leftover storage tasks, owner: %x", res.hashes[i])) + } + // Mark the healing tag if storage root node is inconsistent, or + // it's non-existent due to storage chunking. + if !rawdb.HasTrieNode(s.db, res.hashes[i], nil, account.Root, s.scheme) { + res.task.needHeal[i] = true + } + } else { // If there was a previous large state retrieval in progress, // don't restart it from scratch. This happens if a sync cycle // is interrupted and resumed later. However, *do* update the @@ -1902,7 +1940,12 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { } res.task.needHeal[i] = true resumed[res.hashes[i]] = struct{}{} + largeStorageResumedGauge.Inc(1) } else { + // It's possible that in the hash scheme, the storage, along + // with the trie nodes of the given root, is already present + // in the database. Schedule the storage task anyway to simplify + // the logic here. res.task.stateTasks[res.hashes[i]] = account.Root } res.task.needState[i] = true @@ -1910,13 +1953,29 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { } } } - // Delete any subtasks that have been aborted but not resumed. This may undo - // some progress if a new peer gives us less accounts than an old one, but for - // now we have to live with that. - for hash := range res.task.SubTasks { - if _, ok := resumed[hash]; !ok { - log.Debug("Aborting suspended storage retrieval", "account", hash) - delete(res.task.SubTasks, hash) + // Delete any subtasks that have been aborted but not resumed. It's essential + // as the corresponding contract might be self-destructed in this cycle(it's + // no longer possible in ethereum as self-destruction is disabled in Cancun + // Fork, but the condition is still necessary for other networks). + // + // Keep the leftover storage tasks if they are not covered by the responded + // account range which should be picked up in next account wave. + if len(res.hashes) > 0 { + // The hash of last delivered account in the response + last := res.hashes[len(res.hashes)-1] + for hash := range res.task.SubTasks { + // TODO(rjl493456442) degrade the log level before merging. + if hash.Cmp(last) > 0 { + log.Info("Keeping suspended storage retrieval", "account", hash) + continue + } + // TODO(rjl493456442) degrade the log level before merging. + // It should never happen in ethereum. + if _, ok := resumed[hash]; !ok { + log.Error("Aborting suspended storage retrieval", "account", hash) + delete(res.task.SubTasks, hash) + largeStorageDiscardGauge.Inc(1) + } } } // If the account range contained no contracts, or all have been fully filled @@ -2014,6 +2073,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { if res.subTask == nil && res.mainTask.needState[j] && (i < len(res.hashes)-1 || !res.cont) { res.mainTask.needState[j] = false res.mainTask.pend-- + res.mainTask.stateCompleted[account] = struct{}{} // mark it as completed smallStorageGauge.Inc(1) } // If the last contract was chunked, mark it as needing healing @@ -2062,25 +2122,20 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - owner := account // local assignment for stacktrie writer closure - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(batch, owner, path, hash, blob, s.scheme) - }) + var tr genTrie + if s.scheme == rawdb.HashScheme { + tr = newHashTrie(batch) + } if s.scheme == rawdb.PathScheme { - options = options.WithCleaner(func(path []byte) { - s.cleanPath(batch, owner, path) - }) // Keep the left boundary as it's the first range. - // Skip the right boundary if it's not the last range. - options = options.WithSkipBoundary(false, r.End() != common.MaxHash, boundaryStorageNodesGauge) + tr = newPathTrie(account, false, s.db, batch) } tasks = append(tasks, &storageTask{ Next: common.Hash{}, Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(options), + genTrie: tr, }) for r.Next() { batch := ethdb.HookedBatch{ @@ -2089,27 +2144,19 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(batch, owner, path, hash, blob, s.scheme) - }) + var tr genTrie + if s.scheme == rawdb.HashScheme { + tr = newHashTrie(batch) + } if s.scheme == rawdb.PathScheme { - // Configure the dangling node cleaner and also filter out boundary nodes - // only in the context of the path scheme. Deletion is forbidden in the - // hash scheme, as it can disrupt state completeness. - options = options.WithCleaner(func(path []byte) { - s.cleanPath(batch, owner, path) - }) - // Skip the left boundary as it's not the first range - // Skip the right boundary if it's not the last range. - options = options.WithSkipBoundary(true, r.End() != common.MaxHash, boundaryStorageNodesGauge) + tr = newPathTrie(account, true, s.db, batch) } tasks = append(tasks, &storageTask{ Next: r.Start(), Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(options), + genTrie: tr, }) } for _, task := range tasks { @@ -2155,26 +2202,18 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { if i < len(res.hashes)-1 || res.subTask == nil { // no need to make local reassignment of account: this closure does not outlive the loop - options := trie.NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(batch, account, path, hash, blob, s.scheme) - }) + var tr genTrie + if s.scheme == rawdb.HashScheme { + tr = newHashTrie(batch) + } if s.scheme == rawdb.PathScheme { - // Configure the dangling node cleaner only in the context of the - // path scheme. Deletion is forbidden in the hash scheme, as it can - // disrupt state completeness. - // - // Notably, boundary nodes can be also kept because the whole storage - // trie is complete. - options = options.WithCleaner(func(path []byte) { - s.cleanPath(batch, account, path) - }) + // Keep the left boundary as it's complete + tr = newPathTrie(account, false, s.db, batch) } - tr := trie.NewStackTrie(options) for j := 0; j < len(res.hashes[i]); j++ { - tr.Update(res.hashes[i][j][:], res.slots[i][j]) + tr.update(res.hashes[i][j][:], res.slots[i][j]) } - tr.Commit() + tr.commit(true) } // Persist the received storage segments. These flat state maybe // outdated during the sync, but it can be fixed later during the @@ -2185,14 +2224,14 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // If we're storing large contracts, generate the trie nodes // on the fly to not trash the gluing points if i == len(res.hashes)-1 && res.subTask != nil { - res.subTask.genTrie.Update(res.hashes[i][j][:], res.slots[i][j]) + res.subTask.genTrie.update(res.hashes[i][j][:], res.slots[i][j]) } } } // Large contracts could have generated new trie nodes, flush them to disk if res.subTask != nil { if res.subTask.done { - root := res.subTask.genTrie.Commit() + root := res.subTask.genTrie.commit(res.subTask.Last == common.MaxHash) if err := res.subTask.genBatch.Write(); err != nil { log.Error("Failed to persist stack slots", "err", err) } @@ -2209,8 +2248,8 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } } } - } - if res.subTask.genBatch.ValueSize() > ethdb.IdealBatchSize { + } else if res.subTask.genBatch.ValueSize() > batchSizeThreshold { + res.subTask.genTrie.commit(false) if err := res.subTask.genBatch.Write(); err != nil { log.Error("Failed to persist stack slots", "err", err) } @@ -2393,7 +2432,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { if err != nil { panic(err) // Really shouldn't ever happen } - task.genTrie.Update(hash[:], full) + task.genTrie.update(hash[:], full) } } // Flush anything written just now and update the stats @@ -2409,17 +2448,30 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { return } task.Next = incHash(hash) + + // Remove the completion flag once the account range is pushed + // forward. The leftover accounts will be skipped in the next + // cycle. + delete(task.stateCompleted, hash) } // All accounts marked as complete, track if the entire task is done task.done = !res.cont + // Error out if there is any leftover completion flag. + if task.done && len(task.stateCompleted) != 0 { + panic(fmt.Errorf("storage completion flags should be emptied, %d left", len(task.stateCompleted))) + } // Stack trie could have generated trie nodes, push them to disk (we need to // flush after finalizing task.done. It's fine even if we crash and lose this // write as it will only cause more data to be downloaded during heal. if task.done { - task.genTrie.Commit() - } - if task.genBatch.ValueSize() > ethdb.IdealBatchSize || task.done { + task.genTrie.commit(task.Last == common.MaxHash) + if err := task.genBatch.Write(); err != nil { + log.Error("Failed to persist stack account", "err", err) + } + task.genBatch.Reset() + } else if task.genBatch.ValueSize() > batchSizeThreshold { + task.genTrie.commit(false) if err := task.genBatch.Write(); err != nil { log.Error("Failed to persist stack account", "err", err) } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 5d4099a814..b780868b4e 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -36,8 +36,10 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/testutil" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" "golang.org/x/exp/slices" ) @@ -1503,14 +1505,14 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: types.EmptyRootHash, CodeHash: getCodeHash(i), }) @@ -1538,7 +1540,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { entries []*kv boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) ) // Initialize boundaries @@ -1561,7 +1563,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { for i := 0; i < len(boundaries); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: uint64(0), - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: getCodeHash(uint64(i)), }) @@ -1573,7 +1575,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: types.EmptyRootHash, CodeHash: getCodeHash(i), }) @@ -1596,7 +1598,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { // has a unique storage set. func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1617,7 +1619,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: stRoot, CodeHash: codehash, }) @@ -1651,7 +1653,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots // makeAccountTrieWithStorage spits out a trie, along with the leafs func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool, uneven bool) (*trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) + db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1683,7 +1685,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, - Balance: big.NewInt(int64(i)), + Balance: uint256.NewInt(i), Root: stRoot, CodeHash: codehash, }) @@ -1724,7 +1726,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { trie, _ := trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) var entries []*kv for i := uint64(1); i <= n; i++ { @@ -1747,7 +1749,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { var ( entries []*kv boundaries []common.Hash @@ -1797,7 +1799,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo // makeUnevenStorageTrie constructs a storage tries will states distributed in // different range unevenly. -func makeUnevenStorageTrie(owner common.Hash, slots int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { +func makeUnevenStorageTrie(owner common.Hash, slots int, db *triedb.Database) (common.Hash, *trienode.NodeSet, []*kv) { var ( entries []*kv tr, _ = trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) @@ -1829,7 +1831,7 @@ func makeUnevenStorageTrie(owner common.Hash, slots int, db *trie.Database) (com func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() - triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) + triedb := triedb.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) @@ -1966,9 +1968,9 @@ func TestSlotEstimation(t *testing.T) { } } -func newDbConfig(scheme string) *trie.Config { +func newDbConfig(scheme string) *triedb.Config { if scheme == rawdb.HashScheme { - return &trie.Config{} + return &triedb.Config{} } - return &trie.Config{PathDB: pathdb.Defaults} + return &triedb.Config{PathDB: pathdb.Defaults} } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 497b38dedb..45df21d629 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) // noopReleaser is returned in case there is no operation expected @@ -41,7 +42,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u var ( current *types.Block database state.Database - triedb *trie.Database + tdb *triedb.Database report = true origin = block.NumberU64() ) @@ -67,14 +68,14 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // the internal junks created by tracing will be persisted into the disk. // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. - database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults) + database = state.NewDatabaseWithConfig(eth.chainDb, triedb.HashDefaults) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block - statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false + statedb, database, tdb, report = base, base.Database(), base.Database().TrieDB(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) } else { // Otherwise, try to reexec blocks until we find a state or reach our limit @@ -84,8 +85,8 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // the internal junks created by tracing will be persisted into the disk. // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. - triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults) - database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb) + tdb = triedb.NewDatabase(eth.chainDb, triedb.HashDefaults) + database = state.NewDatabaseWithNodeDB(eth.chainDb, tdb) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is @@ -161,17 +162,17 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u } // Hold the state reference and also drop the parent state // to prevent accumulating too many nodes in memory. - triedb.Reference(root, common.Hash{}) + tdb.Reference(root, common.Hash{}) if parent != (common.Hash{}) { - triedb.Dereference(parent) + tdb.Dereference(parent) } parent = root } if report { - _, nodes, immutablenodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb + _, nodes, immutablenodes, imgs := tdb.Size() // all memory is contained within the nodes return in hashdb log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "immutablenodes", immutablenodes, "preimages", imgs) } - return statedb, func() { triedb.Dereference(block.Root()) }, nil + return statedb, func() { tdb.Dereference(block.Root()) }, nil } func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) { diff --git a/eth/sync.go b/eth/sync.go index a9732147fa..e8e6ea1fcc 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" @@ -36,7 +37,7 @@ const ( // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(false) { + for _, batch := range h.txpool.Pending(txpool.PendingFilter{OnlyPlainTxs: true}) { for _, tx := range batch { hashes = append(hashes, tx.Hash) } @@ -228,24 +229,6 @@ func (cs *chainSyncer) startSync(op *chainSyncOp) { // doSync synchronizes the local blockchain with a remote peer. func (h *handler) doSync(op *chainSyncOp) error { - if op.mode == downloader.SnapSync { - // Before launch the snap sync, we have to ensure user uses the same - // txlookup limit. - // The main concern here is: during the snap sync Geth won't index the - // block(generate tx indices) before the HEAD-limit. But if user changes - // the limit in the next snap sync(e.g. user kill Geth manually and - // restart) then it will be hard for Geth to figure out the oldest block - // has been indexed. So here for the user-experience wise, it's non-optimal - // that user can't change limit during the snap sync. If changed, Geth - // will just blindly use the original one. - limit := h.chain.TxLookupLimit() - if stored := rawdb.ReadFastTxLookupLimit(h.database); stored == nil { - rawdb.WriteFastTxLookupLimit(h.database, limit) - } else if *stored != limit { - h.chain.SetTxLookupLimit(*stored) - log.Warn("Update txLookup limit", "provided", limit, "updated", *stored) - } - } // Run the sync cycle, and disable snap sync if we're past the pivot block err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) if err != nil { diff --git a/eth/sync_test.go b/eth/sync_test.go index d26cbb66ea..a31986730f 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that snap sync is disabled after a successful sync cycle. -func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully diff --git a/eth/tracers/api.go b/eth/tracers/api.go index f3a3890488..67b8628766 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -79,7 +79,7 @@ type Backend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) - GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) + GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) RPCGasCap() uint64 ChainConfig() *params.ChainConfig Engine() consensus.Engine @@ -857,10 +857,9 @@ func containsTx(block *types.Block, hash common.Hash) bool { // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { - // GetTransaction returns 0 for the blocknumber if the transaction is not found - _, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) + _, _, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) if err != nil { - return nil, err + return nil, ethapi.NewTxIndexingError() } if api.backend.ChainConfig().IsOptimismPreBedrock(new(big.Int).SetUint64(blockNumber)) { @@ -880,6 +879,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * if blockNumber == 0 { return nil, errors.New("genesis is not traceable") } + reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec @@ -966,7 +966,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc config.BlockOverrides.Apply(&vmctx) } // Execute the trace - msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) + msg, err := args.ToMessage(api.backend.RPCGasCap(), vmctx.BaseFee) if err != nil { return nil, err } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index b4725d2417..9c3a423f6f 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -52,9 +52,8 @@ import ( ) var ( - errStateNotFound = errors.New("state not found") - errBlockNotFound = errors.New("block not found") - errTransactionNotFound = errors.New("transaction not found") + errStateNotFound = errors.New("state not found") + errBlockNotFound = errors.New("block not found") ) type mockHistoricalBackend struct { @@ -192,12 +191,9 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) return b.chain.GetBlockByNumber(uint64(number)), nil } -func (b *testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { +func (b *testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { tx, hash, blockNumber, index := rawdb.ReadTransaction(b.chaindb, txHash) - if tx == nil { - return nil, common.Hash{}, 0, 0, errTransactionNotFound - } - return tx, hash, blockNumber, index, nil + return tx != nil, tx, hash, blockNumber, index, nil } func (b *testBackend) RPCGasCap() uint64 { @@ -278,7 +274,7 @@ func TestTraceCall(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -496,7 +492,7 @@ func TestTraceTransaction(t *testing.T) { accounts := newAccounts(2) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, }, @@ -598,7 +594,7 @@ func TestTraceBlock(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -741,7 +737,7 @@ func TestTracingWithOverrides(t *testing.T) { storageAccount := common.Address{0x13, 37} genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -1110,7 +1106,7 @@ func TestTraceChain(t *testing.T) { accounts := newAccounts(3) genesis := &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 5c74baacd1..6216a16ced 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -122,12 +122,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } // Configure a blockchain with the given prestate var ( - signer = types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) - origin, _ = signer.Sender(tx) - txContext = vm.TxContext{ - Origin: origin, - GasPrice: tx.GasPrice(), - } + signer = types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) context = vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -138,19 +133,19 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + state = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) - triedb.Close() + state.Close() tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - msg, err := core.TransactionToMessage(tx, signer, nil) + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer}) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { t.Fatalf("failed to execute transaction: %v", err) @@ -222,10 +217,6 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.Fatalf("failed to parse testcase input: %v", err) } signer := types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) - msg, err := core.TransactionToMessage(tx, signer, nil) - if err != nil { - b.Fatalf("failed to prepare transaction for tracing: %v", err) - } origin, _ := signer.Sender(tx) txContext := vm.TxContext{ Origin: origin, @@ -240,8 +231,12 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) - defer triedb.Close() + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) + if err != nil { + b.Fatalf("failed to prepare transaction for tracing: %v", err) + } + state := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer state.Close() b.ReportAllocs() b.ResetTimer() @@ -250,8 +245,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if err != nil { b.Fatalf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - snap := statedb.Snapshot() + evm := vm.NewEVM(context, txContext, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer}) + snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { b.Fatalf("failed to execute transaction: %v", err) @@ -259,7 +254,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if _, err = tracer.GetResult(); err != nil { b.Fatal(err) } - statedb.RevertToSnapshot(snap) + state.StateDB.RevertToSnapshot(snap) } } @@ -367,18 +362,18 @@ func TestInternals(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), - core.GenesisAlloc{ - to: core.GenesisAccount{ + state := tests.MakePreState(rawdb.NewMemoryDatabase(), + types.GenesisAlloc{ + to: types.Account{ Code: tc.code, }, - origin: core.GenesisAccount{ + origin: types.Account{ Balance: big.NewInt(500000000000000), }, }, false, rawdb.HashScheme) - defer triedb.Close() + defer state.Close() - evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) + evm := vm.NewEVM(context, txContext, state.StateDB, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) msg := &core.Message{ To: &to, From: origin, diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 423167b13c..b4e596ead1 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -16,7 +16,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" // Force-load the native, to trigger registration @@ -82,15 +81,10 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string } // Configure a blockchain with the given prestate tx := new(types.Transaction) - if err := rlp.DecodeBytes(common.FromHex(test.Input), tx); err != nil { + if err := tx.UnmarshalBinary(common.FromHex(test.Input)); err != nil { return fmt.Errorf("failed to parse testcase input: %v", err) } signer := types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) - origin, _ := signer.Sender(tx) - txContext := vm.TxContext{ - Origin: origin, - GasPrice: tx.GasPrice(), - } context := vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -100,20 +94,19 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) - defer triedb.Close() + state := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer state.Close() // Create the tracer, the EVM environment and run it tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { return fmt.Errorf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - - msg, err := core.TransactionToMessage(tx, signer, nil) + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { return fmt.Errorf("failed to prepare transaction for tracing: %v", err) } + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer}) st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index b4fa5b6272..8a60123dc2 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -92,12 +92,7 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { } // Configure a blockchain with the given prestate var ( - signer = types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) - origin, _ = signer.Sender(tx) - txContext = vm.TxContext{ - Origin: origin, - GasPrice: tx.GasPrice(), - } + signer = types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) context = vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -108,19 +103,19 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + state = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) - defer triedb.Close() + defer state.Close() tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - msg, err := core.TransactionToMessage(tx, signer, nil) + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } + evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer}) st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { t.Fatalf("failed to execute transaction: %v", err) diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/failed_deposit.json b/eth/tracers/internal/tracetest/testdata/call_tracer/failed_deposit.json new file mode 100644 index 0000000000..2c43cdae0f --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/failed_deposit.json @@ -0,0 +1,47 @@ +{ + "context": { + "difficulty": "3699098917", + "gasLimit": "5258985", + "miner": "0xd049bfd667cb46aa3ef5df0da3e57db3be39e511", + "number": "2294631", + "timestamp": "1513675366" + }, + "genesis": { + "alloc": {}, + "config": { + "chainId": 3, + "daoForkSupport": true, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "ethash": {}, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0 + }, + "difficulty": "3699098917", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "5263953", + "hash": "0x03a0f62a8106793dafcfae7b75fd2654322062d585a19cea568314d7205790dc", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0x15482cc64b7c00a947f5bf015dfc010db1a6a668c74df61974d6a7848c174408", + "nonce": "0xd1bdb150f6fd170e", + "number": "2294630", + "stateRoot": "0x1ab1a534e84cc787cda1db21e0d5920ab06017948075b759166cfea7274657a1", + "timestamp": "1513675347", + "totalDifficulty": "7160543502214733" + }, + "input": "0x7ef85aa0b4f9f798a5fe956d1b79c3eff355febf9e1039a7440948845536982cb62aa03194bc339e628e6fe32c39e84392d087567b2743ea3594bc339e628e6fe32c39e84392d087567b2743ea3580871aa535d3d0c00083030d408000", + "result": { + "from":"0x0000000000000000000000000000000000000000", + "gas":"0x0", + "gasUsed":"0x30d40", + "input":"0x", + "error": "failed deposit transaction", + "type":"STOP" + } +} diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_flat/failed_deposit.json b/eth/tracers/internal/tracetest/testdata/call_tracer_flat/failed_deposit.json new file mode 100644 index 0000000000..540cd53579 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_flat/failed_deposit.json @@ -0,0 +1,59 @@ +{ + "context": { + "difficulty": "3699098917", + "gasLimit": "5258985", + "miner": "0xd049bfd667cb46aa3ef5df0da3e57db3be39e511", + "number": "2294631", + "timestamp": "1513675366" + }, + "genesis": { + "alloc": {}, + "config": { + "chainId": 3, + "daoForkSupport": true, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "ethash": {}, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0 + }, + "difficulty": "3699098917", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "5263953", + "hash": "0x03a0f62a8106793dafcfae7b75fd2654322062d585a19cea568314d7205790dc", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0x15482cc64b7c00a947f5bf015dfc010db1a6a668c74df61974d6a7848c174408", + "nonce": "0xd1bdb150f6fd170e", + "number": "2294630", + "stateRoot": "0x1ab1a534e84cc787cda1db21e0d5920ab06017948075b759166cfea7274657a1", + "timestamp": "1513675347", + "totalDifficulty": "7160543502214733" + }, + "input": "0x7ef85aa0b4f9f798a5fe956d1b79c3eff355febf9e1039a7440948845536982cb62aa03194bc339e628e6fe32c39e84392d087567b2743ea3594bc339e628e6fe32c39e84392d087567b2743ea3580871aa535d3d0c00083030d408000", + "result": [ + { + "action": { + "callType": "stop", + "from": "0x0000000000000000000000000000000000000000", + "gas": "0x0", + "input": "0x", + "value": "0x0" + }, + "blockNumber": 0, + "error": "failed deposit transaction", + "result": { + "gasUsed": "0x0", + "output": "0x" + }, + "subtraces": 0, + "traceAddress": [], + "type": "call" + } + ] +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json index e80dad5667..561ead05b6 100644 --- a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json @@ -83,7 +83,7 @@ }, "post": { "0x808b4da0be6c9512e948521452227efc619bea52": { - "balance": "0x2cd72a36dd031f089", + "balance": "0x2cd987071ba2346b6", "nonce": 1223933 }, "0x8f03f1a3f10c05e7cccf75c1fd10168e06659be7": { diff --git a/eth/tracers/js/internal/tracers/call_tracer_legacy.js b/eth/tracers/js/internal/tracers/call_tracer_legacy.js index 451a644b91..0760bb1e3f 100644 --- a/eth/tracers/js/internal/tracers/call_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/call_tracer_legacy.js @@ -219,7 +219,7 @@ return this.finalize(result); }, - // finalize recreates a call object using the final desired field oder for json + // finalize recreates a call object using the final desired field order for json // serialization. This is a nicety feature to pass meaningfully ordered results // to users who don't interpret it, just display it. finalize: function(call) { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 20906e07f8..8c8c3ec27e 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) type account struct{} @@ -38,9 +39,9 @@ func (account) SubBalance(amount *big.Int) {} func (account) AddBalance(amount *big.Int) {} func (account) SetAddress(common.Address) {} func (account) Value() *big.Int { return nil } -func (account) SetBalance(*big.Int) {} +func (account) SetBalance(*uint256.Int) {} func (account) SetNonce(uint64) {} -func (account) Balance() *big.Int { return nil } +func (account) Balance() *uint256.Int { return nil } func (account) Address() common.Address { return common.Address{} } func (account) SetCode(common.Hash, []byte) {} func (account) ForEachStorage(cb func(key, value common.Hash) bool) {} @@ -49,8 +50,8 @@ type dummyStatedb struct { state.StateDB } -func (*dummyStatedb) GetRefund() uint64 { return 1337 } -func (*dummyStatedb) GetBalance(addr common.Address) *big.Int { return new(big.Int) } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetBalance(addr common.Address) *uint256.Int { return new(uint256.Int) } type vmContext struct { blockCtx vm.BlockContext @@ -74,7 +75,7 @@ func runTraceWithOption(tracer tracers.Tracer, vmctx *vmContext, chaincfg *param env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer, EnableOpcodeOptimizations: enableOpti}) gasLimit uint64 = 31000 startGas uint64 = 10000 - value = big.NewInt(0) + value = uint256.NewInt(0) contract = vm.NewContract(account{}, account{}, value, startGas) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} @@ -90,7 +91,7 @@ func runTraceWithOption(tracer tracers.Tracer, vmctx *vmContext, chaincfg *param } tracer.CaptureTxStart(gasLimit) - tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value) + tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value.ToBig()) ret, err := env.Interpreter().Run(contract, []byte{}, false) tracer.CaptureEnd(ret, startGas-contract.Gas, err) // Rest gas assumes no refund @@ -273,7 +274,7 @@ func TestHaltBetweenSteps(t *testing.T) { } env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: tracer}) scope := &vm.ScopeContext{ - Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } tracer.CaptureStart(env, common.Address{}, common.Address{}, false, []byte{}, 0, big.NewInt(0)) tracer.CaptureState(0, 0, 0, 0, scope, nil, 0, nil) @@ -364,7 +365,7 @@ func TestEnterExit(t *testing.T) { t.Fatal(err) } scope := &vm.ScopeContext{ - Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } tracer.CaptureEnter(vm.CALL, scope.Contract.Caller(), scope.Contract.Address(), []byte{}, 1000, new(big.Int)) tracer.CaptureExit([]byte{}, 400, nil) diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index 3192a15cba..1d8eb320f6 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) type dummyContractRef struct { @@ -56,7 +57,7 @@ func TestStoreCapture(t *testing.T) { var ( logger = NewStructLogger(nil) env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger}) - contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 100000) + contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(uint256.Int), 100000) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, byte(vm.SSTORE)} var index common.Hash diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index f85cf6206a..eafaf157f6 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -161,7 +161,7 @@ func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, sco return } // Avoid processing nested calls when only caring about top call - if t.config.OnlyTopCall && depth > 0 { + if t.config.OnlyTopCall && depth > 1 { return } // Skip if tracing was interrupted @@ -262,6 +262,10 @@ func (t *callTracer) GetResult() (json.RawMessage, error) { return nil, errors.New("incorrect number of top-level calls") } + if t.callstack[0].Type == vm.STOP { + t.callstack[0].Error = "failed deposit transaction" + } + res, err := json.Marshal(t.callstack[0]) if err != nil { return nil, err diff --git a/eth/tracers/native/call_flat.go b/eth/tracers/native/call_flat.go index 266ab99001..93f36f84a0 100644 --- a/eth/tracers/native/call_flat.go +++ b/eth/tracers/native/call_flat.go @@ -215,6 +215,10 @@ func (t *flatCallTracer) GetResult() (json.RawMessage, error) { return nil, errors.New("invalid number of calls") } + if t.tracer.callstack[0].Type == vm.STOP { + t.tracer.callstack[0].Error = "failed deposit transaction" + } + flat, err := flatFromNested(&t.tracer.callstack[0], []int{}, t.config.ConvertParityErrors, t.ctx) if err != nil { return nil, err @@ -249,7 +253,7 @@ func flatFromNested(input *callFrame, traceAddress []int, convertErrs bool, ctx frame = newFlatCreate(input) case vm.SELFDESTRUCT: frame = newFlatSelfdestruct(input) - case vm.CALL, vm.STATICCALL, vm.CALLCODE, vm.DELEGATECALL: + case vm.CALL, vm.STATICCALL, vm.CALLCODE, vm.DELEGATECALL, vm.STOP: frame = newFlatCall(input) default: return nil, fmt.Errorf("unrecognized call frame type: %s", input.Type) diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 82451c40a6..d7e10173cf 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -195,7 +195,7 @@ func (t *prestateTracer) CaptureTxEnd(restGas uint64) { } modified := false postAccount := &account{Storage: make(map[common.Hash]common.Hash)} - newBalance := t.env.StateDB.GetBalance(addr) + newBalance := t.env.StateDB.GetBalance(addr).ToBig() newNonce := t.env.StateDB.GetNonce(addr) newCode := t.env.StateDB.GetCode(addr) @@ -279,7 +279,7 @@ func (t *prestateTracer) lookupAccount(addr common.Address) { } t.pre[addr] = &account{ - Balance: t.env.StateDB.GetBalance(addr), + Balance: t.env.StateDB.GetBalance(addr).ToBig(), Nonce: t.env.StateDB.GetNonce(addr), Code: t.env.StateDB.GetCode(addr), Storage: make(map[common.Hash]common.Hash), diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index b4989ec984..6ac266e06d 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -61,7 +61,7 @@ func BenchmarkTransactionTrace(b *testing.B) { GasLimit: gas, BaseFee: big.NewInt(8), } - alloc := core.GenesisAlloc{} + alloc := types.GenesisAlloc{} // The code pushes 'deadbeef' into memory, then the other params, and calls CREATE2, then returns // the address loop := []byte{ @@ -69,18 +69,18 @@ func BenchmarkTransactionTrace(b *testing.B) { byte(vm.PUSH1), 0, // jumpdestination byte(vm.JUMP), } - alloc[common.HexToAddress("0x00000000000000000000000000000000deadbeef")] = core.GenesisAccount{ + alloc[common.HexToAddress("0x00000000000000000000000000000000deadbeef")] = types.Account{ Nonce: 1, Code: loop, Balance: big.NewInt(1), } - alloc[from] = core.GenesisAccount{ + alloc[from] = types.Account{ Nonce: 1, Code: []byte{}, Balance: big.NewInt(500000000000000), } - triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) - defer triedb.Close() + state := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer state.Close() // Create the tracer, the EVM environment and run it tracer := logger.NewStructLogger(&logger.Config{ @@ -89,8 +89,8 @@ func BenchmarkTransactionTrace(b *testing.B) { //EnableMemory: false, //EnableReturnData: false, }) - evm := vm.NewEVM(context, txContext, statedb, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer}) - msg, err := core.TransactionToMessage(tx, signer, nil) + evm := vm.NewEVM(context, txContext, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer}) + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) } @@ -98,13 +98,13 @@ func BenchmarkTransactionTrace(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - snap := statedb.Snapshot() + snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) _, err = st.TransitionDb() if err != nil { b.Fatal(err) } - statedb.RevertToSnapshot(snap) + state.StateDB.RevertToSnapshot(snap) if have, want := len(tracer.StructLogs()), 244752; have != want { b.Fatalf("trace wrong, want %d steps, have %d", want, have) } @@ -124,9 +124,9 @@ func TestMemCopying(t *testing.T) { {0, 100, 0, "", 0}, // No need to pad (0 size) {100, 50, 100, "", 100}, // Should pad 100-150 {100, 50, 5, "", 5}, // Wanted range fully within memory - {100, -50, 0, "offset or size must not be negative", 0}, // Errror - {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Errror - {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Errror + {100, -50, 0, "offset or size must not be negative", 0}, // Error + {0, 1, 1024*1024 + 1, "reached limit for padding memory slice: 1048578", 0}, // Error + {10, 0, 1024*1024 + 100, "reached limit for padding memory slice: 1048666", 0}, // Error } { mem := vm.NewMemory() diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 87892b6450..e29d982a29 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -317,10 +317,8 @@ func (ec *Client) TransactionInBlock(ctx context.Context, blockHash common.Hash, func (ec *Client) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { var r *types.Receipt err := ec.c.CallContext(ctx, &r, "eth_getTransactionReceipt", txHash) - if err == nil { - if r == nil { - return nil, ethereum.NotFound - } + if err == nil && r == nil { + return nil, ethereum.NotFound } return r, err } @@ -674,6 +672,15 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.GasTipCap != nil { arg["maxPriorityFeePerGas"] = (*hexutil.Big)(msg.GasTipCap) } + if msg.AccessList != nil { + arg["accessList"] = msg.AccessList + } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } return arg } @@ -716,18 +723,20 @@ type rpcProgress struct { PulledStates hexutil.Uint64 KnownStates hexutil.Uint64 - SyncedAccounts hexutil.Uint64 - SyncedAccountBytes hexutil.Uint64 - SyncedBytecodes hexutil.Uint64 - SyncedBytecodeBytes hexutil.Uint64 - SyncedStorage hexutil.Uint64 - SyncedStorageBytes hexutil.Uint64 - HealedTrienodes hexutil.Uint64 - HealedTrienodeBytes hexutil.Uint64 - HealedBytecodes hexutil.Uint64 - HealedBytecodeBytes hexutil.Uint64 - HealingTrienodes hexutil.Uint64 - HealingBytecode hexutil.Uint64 + SyncedAccounts hexutil.Uint64 + SyncedAccountBytes hexutil.Uint64 + SyncedBytecodes hexutil.Uint64 + SyncedBytecodeBytes hexutil.Uint64 + SyncedStorage hexutil.Uint64 + SyncedStorageBytes hexutil.Uint64 + HealedTrienodes hexutil.Uint64 + HealedTrienodeBytes hexutil.Uint64 + HealedBytecodes hexutil.Uint64 + HealedBytecodeBytes hexutil.Uint64 + HealingTrienodes hexutil.Uint64 + HealingBytecode hexutil.Uint64 + TxIndexFinishedBlocks hexutil.Uint64 + TxIndexRemainingBlocks hexutil.Uint64 } func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { @@ -735,22 +744,24 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { return nil } return ðereum.SyncProgress{ - StartingBlock: uint64(p.StartingBlock), - CurrentBlock: uint64(p.CurrentBlock), - HighestBlock: uint64(p.HighestBlock), - PulledStates: uint64(p.PulledStates), - KnownStates: uint64(p.KnownStates), - SyncedAccounts: uint64(p.SyncedAccounts), - SyncedAccountBytes: uint64(p.SyncedAccountBytes), - SyncedBytecodes: uint64(p.SyncedBytecodes), - SyncedBytecodeBytes: uint64(p.SyncedBytecodeBytes), - SyncedStorage: uint64(p.SyncedStorage), - SyncedStorageBytes: uint64(p.SyncedStorageBytes), - HealedTrienodes: uint64(p.HealedTrienodes), - HealedTrienodeBytes: uint64(p.HealedTrienodeBytes), - HealedBytecodes: uint64(p.HealedBytecodes), - HealedBytecodeBytes: uint64(p.HealedBytecodeBytes), - HealingTrienodes: uint64(p.HealingTrienodes), - HealingBytecode: uint64(p.HealingBytecode), + StartingBlock: uint64(p.StartingBlock), + CurrentBlock: uint64(p.CurrentBlock), + HighestBlock: uint64(p.HighestBlock), + PulledStates: uint64(p.PulledStates), + KnownStates: uint64(p.KnownStates), + SyncedAccounts: uint64(p.SyncedAccounts), + SyncedAccountBytes: uint64(p.SyncedAccountBytes), + SyncedBytecodes: uint64(p.SyncedBytecodes), + SyncedBytecodeBytes: uint64(p.SyncedBytecodeBytes), + SyncedStorage: uint64(p.SyncedStorage), + SyncedStorageBytes: uint64(p.SyncedStorageBytes), + HealedTrienodes: uint64(p.HealedTrienodes), + HealedTrienodeBytes: uint64(p.HealedTrienodeBytes), + HealedBytecodes: uint64(p.HealedBytecodes), + HealedBytecodeBytes: uint64(p.HealedBytecodeBytes), + HealingTrienodes: uint64(p.HealingTrienodes), + HealingBytecode: uint64(p.HealingBytecode), + TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks), + TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks), } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 2b6f19f447..dd643f0c18 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -195,7 +195,7 @@ var ( var genesis = &core.Genesis{ Config: params.AllEthashProtocolChanges, - Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, + Alloc: types.GenesisAlloc{testAddr: {Balance: testBalance}}, ExtraData: []byte("test genesis"), Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), @@ -329,11 +329,21 @@ func newTestBackend(t *testing.T, enableHistoricalState bool) (*node.Node, []*ty if _, err := ethservice.BlockChain().InsertChain(blocks[1:]); err != nil { t.Fatalf("can't import test blocks: %v", err) } + + // Ensure the tx indexing is fully generated + for ; ; time.Sleep(time.Millisecond * 100) { + progress, err := ethservice.BlockChain().TxIndexProgress() + if err == nil && progress.Done() { + break + } + } + if enableHistoricalState { // Now that we have a filled DB, swap the pre-Bedrock consensus to OpLegacy, // which does not support re-processing of pre-bedrock data. ethservice.Engine().(*beacon.Beacon).SwapInner(&beacon.OpLegacy{}) } + return n, blocks } @@ -379,7 +389,7 @@ func TestEthClient(t *testing.T) { func(t *testing.T) { testBalanceAt(t, client) }, }, "TxInBlockInterrupted": { - func(t *testing.T) { testTransactionInBlockInterrupted(t, client) }, + func(t *testing.T) { testTransactionInBlock(t, client) }, }, "ChainID": { func(t *testing.T) { testChainID(t, client) }, @@ -447,7 +457,7 @@ func testHeader(t *testing.T, chain []*types.Block, client *rpc.Client) { got.Number = big.NewInt(0) // hack to make DeepEqual work } if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("HeaderByNumber(%v)\n = %v\nwant %v", tt.block, got, tt.want) + t.Fatalf("HeaderByNumber(%v) got = %v, want %v", tt.block, got, tt.want) } }) } @@ -499,7 +509,7 @@ func testBalanceAt(t *testing.T, client *rpc.Client) { } } -func testTransactionInBlockInterrupted(t *testing.T, client *rpc.Client) { +func testTransactionInBlock(t *testing.T, client *rpc.Client) { ec := NewClient(client) // Get current block by number. @@ -508,22 +518,27 @@ func testTransactionInBlockInterrupted(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected error: %v", err) } - // Test tx in block interrupted. - ctx, cancel := context.WithCancel(context.Background()) - cancel() - <-ctx.Done() // Ensure the close of the Done channel - tx, err := ec.TransactionInBlock(ctx, block.Hash(), 0) - if tx != nil { - t.Fatal("transaction should be nil") - } - if err == nil || err == ethereum.NotFound { - t.Fatal("error should not be nil/notfound") - } - // Test tx in block not found. if _, err := ec.TransactionInBlock(context.Background(), block.Hash(), 20); err != ethereum.NotFound { t.Fatal("error should be ethereum.NotFound") } + + // Test tx in block found. + tx, err := ec.TransactionInBlock(context.Background(), block.Hash(), 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tx.Hash() != testTx1.Hash() { + t.Fatalf("unexpected transaction: %v", tx) + } + + tx, err = ec.TransactionInBlock(context.Background(), block.Hash(), 1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tx.Hash() != testTx2.Hash() { + t.Fatalf("unexpected transaction: %v", tx) + } } func testChainID(t *testing.T, client *rpc.Client) { diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 5cb703ea5b..4ca4219a41 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -218,6 +218,15 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.GasPrice != nil { arg["gasPrice"] = (*hexutil.Big)(msg.GasPrice) } + if msg.GasFeeCap != nil { + arg["maxFeePerGas"] = (*hexutil.Big)(msg.GasFeeCap) + } + if msg.GasTipCap != nil { + arg["maxPriorityFeePerGas"] = (*hexutil.Big)(msg.GasTipCap) + } + if msg.AccessList != nil { + arg["accessList"] = msg.AccessList + } return arg } diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index fdd94a7d73..158886475e 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -81,7 +81,7 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { func generateTestChain() (*core.Genesis, []*types.Block) { genesis := &core.Genesis{ Config: params.AllEthashProtocolChanges, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ testAddr: {Balance: testBalance, Storage: map[common.Hash]common.Hash{testSlot: testValue}}, testContract: {Nonce: 1, Code: []byte{0x13, 0x37}}, testEmpty: {Balance: big.NewInt(1)}, @@ -169,7 +169,7 @@ func testAccessList(t *testing.T, client *rpc.Client) { From: testAddr, To: &common.Address{}, Gas: 21000, - GasPrice: big.NewInt(765625000), + GasPrice: big.NewInt(875000000), Value: big.NewInt(1), } al, gas, vmErr, err := ec.CreateAccessList(context.Background(), msg) diff --git a/ethclient/simulated/backend.go b/ethclient/simulated/backend.go new file mode 100644 index 0000000000..1df0a73150 --- /dev/null +++ b/ethclient/simulated/backend.go @@ -0,0 +1,214 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulated + +import ( + "errors" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// Client exposes the methods provided by the Ethereum RPC client. +type Client interface { + ethereum.BlockNumberReader + ethereum.ChainReader + ethereum.ChainStateReader + ethereum.ContractCaller + ethereum.GasEstimator + ethereum.GasPricer + ethereum.GasPricer1559 + ethereum.FeeHistoryReader + ethereum.LogFilterer + ethereum.PendingStateReader + ethereum.PendingContractCaller + ethereum.TransactionReader + ethereum.TransactionSender + ethereum.ChainIDReader +} + +// simClient wraps ethclient. This exists to prevent extracting ethclient.Client +// from the Client interface returned by Backend. +type simClient struct { + *ethclient.Client +} + +// Backend is a simulated blockchain. You can use it to test your contracts or +// other code that interacts with the Ethereum chain. +type Backend struct { + node *node.Node + beacon *catalyst.SimulatedBeacon + client simClient +} + +// NewBackend creates a new simulated blockchain that can be used as a backend for +// contract bindings in unit tests. +// +// A simulated backend always uses chainID 1337. +func NewBackend(alloc types.GenesisAlloc, options ...func(nodeConf *node.Config, ethConf *ethconfig.Config)) *Backend { + // Create the default configurations for the outer node shell and the Ethereum + // service to mutate with the options afterwards + nodeConf := node.DefaultConfig + nodeConf.DataDir = "" + nodeConf.P2P = p2p.Config{NoDiscovery: true} + + ethConf := ethconfig.Defaults + ethConf.Genesis = &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + GasLimit: ethconfig.Defaults.Miner.GasCeil, + Alloc: alloc, + } + ethConf.SyncMode = downloader.FullSync + ethConf.TxPool.NoLocals = true + + for _, option := range options { + option(&nodeConf, ðConf) + } + // Assemble the Ethereum stack to run the chain with + stack, err := node.New(&nodeConf) + if err != nil { + panic(err) // this should never happen + } + sim, err := newWithNode(stack, ðConf, 0) + if err != nil { + panic(err) // this should never happen + } + return sim +} + +func NewBackendFromConfig(conf ethconfig.Config) *Backend { + // Setup the node object + nodeConf := node.DefaultConfig + nodeConf.DataDir = "" + nodeConf.P2P = p2p.Config{NoDiscovery: true} + stack, err := node.New(&nodeConf) + if err != nil { + // This should never happen, if it does, please open an issue + panic(err) + } + + conf.SyncMode = downloader.FullSync + conf.TxPool.NoLocals = true + sim, err := newWithNode(stack, &conf, 0) + if err != nil { + // This should never happen, if it does, please open an issue + panic(err) + } + return sim +} + +// newWithNode sets up a simulated backend on an existing node. The provided node +// must not be started and will be started by this method. +func newWithNode(stack *node.Node, conf *eth.Config, blockPeriod uint64) (*Backend, error) { + backend, err := eth.New(stack, conf) + if err != nil { + return nil, err + } + // Register the filter system + filterSystem := filters.NewFilterSystem(backend.APIBackend, filters.Config{}) + stack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, false), + }}) + // Start the node + if err := stack.Start(); err != nil { + return nil, err + } + // Set up the simulated beacon + beacon, err := catalyst.NewSimulatedBeacon(blockPeriod, backend) + if err != nil { + return nil, err + } + // Reorg our chain back to genesis + if err := beacon.Fork(backend.BlockChain().GetCanonicalHash(0)); err != nil { + return nil, err + } + return &Backend{ + node: stack, + beacon: beacon, + client: simClient{ethclient.NewClient(stack.Attach())}, + }, nil +} + +// Close shuts down the simBackend. +// The simulated backend can't be used afterwards. +func (n *Backend) Close() error { + if n.client.Client != nil { + n.client.Close() + n.client = simClient{} + } + var err error + if n.beacon != nil { + err = n.beacon.Stop() + n.beacon = nil + } + if n.node != nil { + err = errors.Join(err, n.node.Close()) + n.node = nil + } + return err +} + +// Commit seals a block and moves the chain forward to a new empty block. +func (n *Backend) Commit() common.Hash { + return n.beacon.Commit() +} + +// Rollback removes all pending transactions, reverting to the last committed state. +func (n *Backend) Rollback() { + n.beacon.Rollback() +} + +// Fork creates a side-chain that can be used to simulate reorgs. +// +// This function should be called with the ancestor block where the new side +// chain should be started. Transactions (old and new) can then be applied on +// top and Commit-ed. +// +// Note, the side-chain will only become canonical (and trigger the events) when +// it becomes longer. Until then CallContract will still operate on the current +// canonical chain. +// +// There is a % chance that the side chain becomes canonical at the same length +// to simulate live network behavior. +func (n *Backend) Fork(parentHash common.Hash) error { + return n.beacon.Fork(parentHash) +} + +// AdjustTime changes the block timestamp and creates a new block. +// It can only be called on empty blocks. +func (n *Backend) AdjustTime(adjustment time.Duration) error { + return n.beacon.AdjustTime(adjustment) +} + +// Client returns a client that accesses the simulated chain. +func (n *Backend) Client() Client { + return n.client +} diff --git a/ethclient/simulated/backend_test.go b/ethclient/simulated/backend_test.go new file mode 100644 index 0000000000..a8fd7913c3 --- /dev/null +++ b/ethclient/simulated/backend_test.go @@ -0,0 +1,308 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulated + +import ( + "context" + "crypto/ecdsa" + "math/big" + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +var _ bind.ContractBackend = (Client)(nil) + +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +func simTestBackend(testAddr common.Address) *Backend { + return NewBackend( + types.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000000000)}, + }, + ) +} + +func newTx(sim *Backend, key *ecdsa.PrivateKey) (*types.Transaction, error) { + client := sim.Client() + + // create a signed transaction to send + head, _ := client.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(params.GWei)) + addr := crypto.PubkeyToAddress(key.PublicKey) + chainid, _ := client.ChainID(context.Background()) + nonce, err := client.PendingNonceAt(context.Background(), addr) + if err != nil { + return nil, err + } + tx := types.NewTx(&types.DynamicFeeTx{ + ChainID: chainid, + Nonce: nonce, + GasTipCap: big.NewInt(params.GWei), + GasFeeCap: gasPrice, + Gas: 21000, + To: &addr, + }) + return types.SignTx(tx, types.LatestSignerForChainID(chainid), key) +} + +func TestNewBackend(t *testing.T) { + sim := NewBackend(types.GenesisAlloc{}) + defer sim.Close() + + client := sim.Client() + num, err := client.BlockNumber(context.Background()) + if err != nil { + t.Fatal(err) + } + if num != 0 { + t.Fatalf("expected 0 got %v", num) + } + // Create a block + sim.Commit() + num, err = client.BlockNumber(context.Background()) + if err != nil { + t.Fatal(err) + } + if num != 1 { + t.Fatalf("expected 1 got %v", num) + } +} + +func TestAdjustTime(t *testing.T) { + sim := NewBackend(types.GenesisAlloc{}) + defer sim.Close() + + client := sim.Client() + block1, _ := client.BlockByNumber(context.Background(), nil) + + // Create a block + if err := sim.AdjustTime(time.Minute); err != nil { + t.Fatal(err) + } + block2, _ := client.BlockByNumber(context.Background(), nil) + prevTime := block1.Time() + newTime := block2.Time() + if newTime-prevTime != uint64(time.Minute) { + t.Errorf("adjusted time not equal to 60 seconds. prev: %v, new: %v", prevTime, newTime) + } +} + +func TestSendTransaction(t *testing.T) { + sim := simTestBackend(testAddr) + defer sim.Close() + + client := sim.Client() + ctx := context.Background() + + signedTx, err := newTx(sim, testKey) + if err != nil { + t.Errorf("could not create transaction: %v", err) + } + // send tx to simulated backend + err = client.SendTransaction(ctx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + sim.Commit() + block, err := client.BlockByNumber(ctx, big.NewInt(1)) + if err != nil { + t.Errorf("could not get block at height 1: %v", err) + } + + if signedTx.Hash() != block.Transactions()[0].Hash() { + t.Errorf("did not commit sent transaction. expected hash %v got hash %v", block.Transactions()[0].Hash(), signedTx.Hash()) + } +} + +// TestFork check that the chain length after a reorg is correct. +// Steps: +// 1. Save the current block which will serve as parent for the fork. +// 2. Mine n blocks with n ∈ [0, 20]. +// 3. Assert that the chain length is n. +// 4. Fork by using the parent block as ancestor. +// 5. Mine n+1 blocks which should trigger a reorg. +// 6. Assert that the chain length is n+1. +// Since Commit() was called 2n+1 times in total, +// having a chain length of just n+1 means that a reorg occurred. +func TestFork(t *testing.T) { + t.Parallel() + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + client := sim.Client() + ctx := context.Background() + + // 1. + parent, _ := client.HeaderByNumber(ctx, nil) + + // 2. + n := int(rand.Int31n(21)) + for i := 0; i < n; i++ { + sim.Commit() + } + + // 3. + b, _ := client.BlockNumber(ctx) + if b != uint64(n) { + t.Error("wrong chain length") + } + + // 4. + sim.Fork(parent.Hash()) + + // 5. + for i := 0; i < n+1; i++ { + sim.Commit() + } + + // 6. + b, _ = client.BlockNumber(ctx) + if b != uint64(n+1) { + t.Error("wrong chain length") + } +} + +// TestForkResendTx checks that re-sending a TX after a fork +// is possible and does not cause a "nonce mismatch" panic. +// Steps: +// 1. Save the current block which will serve as parent for the fork. +// 2. Send a transaction. +// 3. Check that the TX is included in block 1. +// 4. Fork by using the parent block as ancestor. +// 5. Mine a block, Re-send the transaction and mine another one. +// 6. Check that the TX is now included in block 2. +func TestForkResendTx(t *testing.T) { + t.Parallel() + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + client := sim.Client() + ctx := context.Background() + + // 1. + parent, _ := client.HeaderByNumber(ctx, nil) + + // 2. + tx, err := newTx(sim, testKey) + if err != nil { + t.Fatalf("could not create transaction: %v", err) + } + client.SendTransaction(ctx, tx) + sim.Commit() + + // 3. + receipt, _ := client.TransactionReceipt(ctx, tx.Hash()) + if h := receipt.BlockNumber.Uint64(); h != 1 { + t.Errorf("TX included in wrong block: %d", h) + } + + // 4. + if err := sim.Fork(parent.Hash()); err != nil { + t.Errorf("forking: %v", err) + } + + // 5. + sim.Commit() + if err := client.SendTransaction(ctx, tx); err != nil { + t.Fatalf("sending transaction: %v", err) + } + sim.Commit() + receipt, _ = client.TransactionReceipt(ctx, tx.Hash()) + if h := receipt.BlockNumber.Uint64(); h != 2 { + t.Errorf("TX included in wrong block: %d", h) + } +} + +func TestCommitReturnValue(t *testing.T) { + t.Parallel() + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + client := sim.Client() + ctx := context.Background() + + // Test if Commit returns the correct block hash + h1 := sim.Commit() + cur, _ := client.HeaderByNumber(ctx, nil) + if h1 != cur.Hash() { + t.Error("Commit did not return the hash of the last block.") + } + + // Create a block in the original chain (containing a transaction to force different block hashes) + head, _ := client.HeaderByNumber(ctx, nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) + tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) + client.SendTransaction(ctx, tx) + + h2 := sim.Commit() + + // Create another block in the original chain + sim.Commit() + + // Fork at the first bock + if err := sim.Fork(h1); err != nil { + t.Errorf("forking: %v", err) + } + + // Test if Commit returns the correct block hash after the reorg + h2fork := sim.Commit() + if h2 == h2fork { + t.Error("The block in the fork and the original block are the same block!") + } + if header, err := client.HeaderByHash(ctx, h2fork); err != nil || header == nil { + t.Error("Could not retrieve the just created block (side-chain)") + } +} + +// TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork +// block's parent rather than the canonical head's parent. +func TestAdjustTimeAfterFork(t *testing.T) { + t.Parallel() + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + client := sim.Client() + ctx := context.Background() + + sim.Commit() // h1 + h1, _ := client.HeaderByNumber(ctx, nil) + + sim.Commit() // h2 + sim.Fork(h1.Hash()) + sim.AdjustTime(1 * time.Second) + sim.Commit() + + head, _ := client.HeaderByNumber(ctx, nil) + if head.Number.Uint64() == 2 && head.ParentHash != h1.Hash() { + t.Errorf("failed to build block on fork") + } +} diff --git a/ethclient/simulated/options.go b/ethclient/simulated/options.go new file mode 100644 index 0000000000..6db995c917 --- /dev/null +++ b/ethclient/simulated/options.go @@ -0,0 +1,55 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulated + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" +) + +// WithBlockGasLimit configures the simulated backend to target a specific gas limit +// when producing blocks. +func WithBlockGasLimit(gaslimit uint64) func(nodeConf *node.Config, ethConf *ethconfig.Config) { + return func(nodeConf *node.Config, ethConf *ethconfig.Config) { + ethConf.Genesis.GasLimit = gaslimit + ethConf.Miner.GasCeil = gaslimit + } +} + +// WithCallGasLimit configures the simulated backend to cap eth_calls to a specific +// gas limit when running client operations. +func WithCallGasLimit(gaslimit uint64) func(nodeConf *node.Config, ethConf *ethconfig.Config) { + return func(nodeConf *node.Config, ethConf *ethconfig.Config) { + ethConf.RPCGasCap = gaslimit + } +} + +// WithMinerMinTip configures the simulated backend to require a specific minimum +// gas tip for a transaction to be included. +// +// 0 is not possible as a live Geth node would reject that due to DoS protection, +// so the simulated backend will replicate that behavior for consistency. +func WithMinerMinTip(tip *big.Int) func(nodeConf *node.Config, ethConf *ethconfig.Config) { + if tip == nil || tip.Cmp(new(big.Int)) <= 0 { + panic("invalid miner minimum tip") + } + return func(nodeConf *node.Config, ethConf *ethconfig.Config) { + ethConf.Miner.GasPrice = tip + } +} diff --git a/ethclient/simulated/options_test.go b/ethclient/simulated/options_test.go new file mode 100644 index 0000000000..9ff2be5ff9 --- /dev/null +++ b/ethclient/simulated/options_test.go @@ -0,0 +1,74 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulated + +import ( + "context" + "math/big" + "strings" + "testing" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +// Tests that the simulator starts with the initial gas limit in the genesis block, +// and that it keeps the same target value. +func TestWithBlockGasLimitOption(t *testing.T) { + // Construct a simulator, targeting a different gas limit + sim := NewBackend(types.GenesisAlloc{}, WithBlockGasLimit(12_345_678)) + defer sim.Close() + + client := sim.Client() + genesis, err := client.BlockByNumber(context.Background(), big.NewInt(0)) + if err != nil { + t.Fatalf("failed to retrieve genesis block: %v", err) + } + if genesis.GasLimit() != 12_345_678 { + t.Errorf("genesis gas limit mismatch: have %v, want %v", genesis.GasLimit(), 12_345_678) + } + // Produce a number of blocks and verify the locked in gas target + sim.Commit() + head, err := client.BlockByNumber(context.Background(), big.NewInt(1)) + if err != nil { + t.Fatalf("failed to retrieve head block: %v", err) + } + if head.GasLimit() != 12_345_678 { + t.Errorf("head gas limit mismatch: have %v, want %v", head.GasLimit(), 12_345_678) + } +} + +// Tests that the simulator honors the RPC call caps set by the options. +func TestWithCallGasLimitOption(t *testing.T) { + // Construct a simulator, targeting a different gas limit + sim := NewBackend(types.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000000000)}, + }, WithCallGasLimit(params.TxGas-1)) + defer sim.Close() + + client := sim.Client() + _, err := client.CallContract(context.Background(), ethereum.CallMsg{ + From: testAddr, + To: &testAddr, + Gas: 21000, + }, nil) + if !strings.Contains(err.Error(), core.ErrIntrinsicGas.Error()) { + t.Fatalf("error mismatch: have %v, want %v", err, core.ErrIntrinsicGas) + } +} diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 75d0faac54..61ceec443e 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -611,6 +611,10 @@ func (s *Service) reportBlock(conn *connWrapper, block *types.Block) error { // Gather the block details from the header or block chain details := s.assembleBlockStats(block) + // Short circuit if the block detail is not available. + if details == nil { + return nil + } // Assemble the block report and send it to the server log.Trace("Sending new block to ethstats", "number", details.Number, "hash", details.Hash) @@ -638,10 +642,16 @@ func (s *Service) assembleBlockStats(block *types.Block) *blockStats { // check if backend is a full node fullBackend, ok := s.backend.(fullNodeBackend) if ok { + // Retrieve current chain head if no block is given. if block == nil { head := fullBackend.CurrentBlock() block, _ = fullBackend.BlockByNumber(context.Background(), rpc.BlockNumber(head.Number.Uint64())) } + // Short circuit if no block is available. It might happen when + // the blockchain is reorging. + if block == nil { + return nil + } header = block.Header() td = fullBackend.GetTd(context.Background(), header.Hash()) @@ -792,7 +802,7 @@ func (s *Service) reportStats(conn *connWrapper) error { } sync := fullBackend.SyncProgress() - syncing = fullBackend.CurrentHeader().Number.Uint64() >= sync.HighestBlock + syncing = !sync.Done() price, _ := fullBackend.SuggestGasTipCap(context.Background()) gasprice = int(price.Uint64()) @@ -801,7 +811,7 @@ func (s *Service) reportStats(conn *connWrapper) error { } } else { sync := s.backend.SyncProgress() - syncing = s.backend.CurrentHeader().Number.Uint64() >= sync.HighestBlock + syncing = !sync.Done() } // Assemble the node stats and send it to the server log.Trace("Sending node details to ethstats") diff --git a/fork.yaml b/fork.yaml index 81b979e303..7244be0e7b 100644 --- a/fork.yaml +++ b/fork.yaml @@ -5,7 +5,7 @@ footer: | base: name: go-ethereum url: https://github.com/ethereum/go-ethereum - hash: b20b4a71598481443d60b261d3e5dcb37f8a0d82 # v1.13.8 + hash: c5ba367eb6232e3eddd7d6226bfd374449c63164 # v1.13.15 fork: name: op-geth url: https://github.com/ethereum-optimism/op-geth @@ -56,7 +56,7 @@ def: globs: - "core/vm/evm.go" - "core/evm.go" - - "core/types/rollup_l1_cost.go" + - "core/types/rollup_cost.go" - "core/state_processor.go" - "core/state_prefetcher.go" - title: Transaction processing @@ -111,7 +111,7 @@ def: description: | Transaction queueing and inclusion needs to account for the L1 cost component. globs: - - "core/txpool/*" + - "core/txpool/**/*" - "core/txpool/legacypool/*" - title: "Chain Configuration" sub: @@ -182,13 +182,16 @@ def: - title: Historical data for Snap-sync description: Snap-sync has access to trusted Deposit Transaction Nonce Data. globs: - - "eth/handler.go" - "eth/downloader/downloader.go" - "eth/downloader/receiptreference.go" - title: Discv5 node discovery description: Fix discv5 option to allow discv5 to be an active source for node-discovery. globs: - "p2p/server.go" + - title: Bootnodes + description: Discovery bootnode addresses. + globs: + - "params/bootnodes.go" - title: Generated TOML config update globs: - "eth/ethconfig/gen_config.go" @@ -223,7 +226,7 @@ def: globs: - "eth/tracers/api.go" - title: "Daisy Chain tests" - globs: + ignore: - "internal/ethapi/transaction_args_test.go" - "ethclient/ethclient_test.go" - "eth/tracers/api_test.go" @@ -248,6 +251,7 @@ def: - title: Simulated Backend globs: - "accounts/abi/bind/backends/simulated.go" + - "ethclient/simulated/backend.go" - title: "Hardware wallet support" description: Extend Ledger wallet support for newer devices on Macos sub: @@ -255,6 +259,10 @@ def: # upstream PR: https://github.com/ethereum/go-ethereum/pull/28863/ globs: - "accounts/usbwallet/hub.go" + - title: "Testing" + description: Additional or modified tests, not already captured by the above diff + ignore: + - "**/*_test.go" # ignored globally, does not count towards line count ignore: @@ -262,4 +270,9 @@ ignore: - "*.sum" - "go.mod" - "fork.yaml" - - ".github/workflows/*" + - "Makefile" + - ".golangci.yml" + - ".github/**" + - "**/*.gob" # data asset, not code + - "core/vm/testdata/precompiles/p256Verify.json" # data asset, not code + - "eth/tracers/internal/tracetest/testdata/**/*.json" diff --git a/go.mod b/go.mod index 3415346fc3..73768e2685 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/credentials v1.13.43 github.com/aws/aws-sdk-go-v2/service/route53 v1.30.2 + github.com/bnb-chain/fastssz v0.1.2 github.com/btcsuite/btcd/btcec/v2 v2.3.2 github.com/cespare/cp v1.1.1 github.com/cloudflare/cloudflare-go v0.79.0 @@ -23,7 +24,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set/v2 v2.1.0 github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 - github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240306093353-c557df8e6f41 + github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240603085035-9c8f6081266e github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.15.0 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e @@ -40,7 +41,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/graph-gophers/graphql-go v1.3.0 github.com/hashicorp/go-bexpr v0.1.10 - github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 + github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.4 github.com/huin/goupnp v1.3.0 @@ -115,7 +116,7 @@ require ( github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index d7ed4975c9..0671fa8e41 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/bnb-chain/fastssz v0.1.2 h1:vTcXw5SwCtRYnl/BEclujiml7GXiVOZ74tub4GHpvlM= +github.com/bnb-chain/fastssz v0.1.2/go.mod h1:KcabV+OEw2QwgyY8Fc88ZG79CKYkFdu0kKWyfA3dI6o= github.com/bnb-chain/greenfield-cometbft v1.0.0 h1:0r6hOJWD/+es0gxP/exKuN/krgXAr3LCn5/XlcgDWr8= github.com/bnb-chain/greenfield-cometbft v1.0.0/go.mod h1:f35mk/r5ab6yvzlqEWZt68LfUje68sYgMpVlt2CUYMk= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -357,8 +359,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240306093353-c557df8e6f41 h1:WKJvsRyW/YNgyT0P2x5U530ITOY8Dv9TrZnbliqSXd8= -github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240306093353-c557df8e6f41/go.mod h1:7xh2awFQqsiZxFrHKTgEd+InVfDRrkKVUIuK8SAFHp0= +github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240603085035-9c8f6081266e h1:PJWaF/45dMhO7xdzSwuZmwIIBwnqnPr84oFNmmnpGNs= +github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240603085035-9c8f6081266e/go.mod h1:7xh2awFQqsiZxFrHKTgEd+InVfDRrkKVUIuK8SAFHp0= github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= @@ -440,8 +442,9 @@ github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= @@ -654,8 +657,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/herumi/bls-eth-go-binary v0.0.0-20210130185500-57372fb27371/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e h1:wCMygKUQhmcQAjlk2Gquzq6dLmyMv2kF+llRspoRgrk= github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= -github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= -github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -1788,6 +1791,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/graphql/graphql.go b/graphql/graphql.go index d8330dffc6..c2807131d8 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -100,7 +100,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { if err != nil { return hexutil.Big{}, err } - balance := state.GetBalance(a.address) + balance := state.GetBalance(a.address).ToBig() if balance == nil { return hexutil.Big{}, fmt.Errorf("failed to load balance %x", a.address) } @@ -230,8 +230,8 @@ func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, *Block) return t.tx, t.block } // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) - if err == nil && tx != nil { + found, tx, blockHash, _, index, _ := t.r.backend.GetTransaction(ctx, t.hash) + if found { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ @@ -1509,6 +1509,12 @@ func (s *SyncState) HealingTrienodes() hexutil.Uint64 { func (s *SyncState) HealingBytecode() hexutil.Uint64 { return hexutil.Uint64(s.progress.HealingBytecode) } +func (s *SyncState) TxIndexFinishedBlocks() hexutil.Uint64 { + return hexutil.Uint64(s.progress.TxIndexFinishedBlocks) +} +func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 { + return hexutil.Uint64(s.progress.TxIndexRemainingBlocks) +} // Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not // yet received the latest block headers from its pears. In case it is synchronizing: @@ -1527,11 +1533,13 @@ func (s *SyncState) HealingBytecode() hexutil.Uint64 { // - healedBytecodeBytes: number of bytecodes persisted to disk // - healingTrienodes: number of state trie nodes pending // - healingBytecode: number of bytecodes pending +// - txIndexFinishedBlocks: number of blocks whose transactions are indexed +// - txIndexRemainingBlocks: number of blocks whose transactions are not indexed yet func (r *Resolver) Syncing() (*SyncState, error) { progress := r.backend.SyncProgress() // Return not syncing if the synchronisation already completed - if progress.CurrentBlock >= progress.HighestBlock { + if progress.Done() { return nil, nil } // Otherwise gather the block sync stats diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index f91229d015..1dda102058 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -189,7 +189,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ address: {Balance: funds}, // The address 0xdad sloads 0x00 and 0x01 dad: { @@ -286,7 +286,7 @@ func TestGraphQLConcurrentResolvers(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, dad: { // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) @@ -379,7 +379,7 @@ func TestWithdrawals(t *testing.T) { Config: params.AllEthashProtocolChanges, GasLimit: 11500000, Difficulty: common.Big1, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ addr: {Balance: big.NewInt(params.Ether)}, }, } diff --git a/interfaces.go b/interfaces.go index c4948191d1..53e2e3ae16 100644 --- a/interfaces.go +++ b/interfaces.go @@ -120,6 +120,18 @@ type SyncProgress struct { HealingTrienodes uint64 // Number of state trie nodes pending HealingBytecode uint64 // Number of bytecodes pending + + // "transaction indexing" fields + TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed + TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet +} + +// Done returns the indicator if the initial sync is finished or not. +func (prog SyncProgress) Done() bool { + if prog.CurrentBlock < prog.HighestBlock { + return false + } + return prog.TxIndexRemainingBlocks == 0 } // ChainSyncReader wraps access to the node's current sync status. If there's no @@ -140,6 +152,10 @@ type CallMsg struct { Data []byte // input data, usually an ABI-encoded contract method invocation AccessList types.AccessList // EIP-2930 access list. + + // For BlobTxType + BlobGasFeeCap *big.Int + BlobHashes []common.Hash } // A ContractCaller provides contract calls, essentially transactions that are executed by @@ -199,6 +215,16 @@ type GasPricer interface { SuggestGasPrice(ctx context.Context) (*big.Int, error) } +// GasPricer1559 provides access to the EIP-1559 gas price oracle. +type GasPricer1559 interface { + SuggestGasTipCap(ctx context.Context) (*big.Int, error) +} + +// FeeHistoryReader provides access to the fee history oracle. +type FeeHistoryReader interface { + FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*FeeHistory, error) +} + // FeeHistory provides recent fee market data that consumers can use to determine // a reasonable maxPriorityFeePerGas value. type FeeHistory struct { @@ -239,3 +265,13 @@ type GasEstimator interface { type PendingStateEventer interface { SubscribePendingTransactions(ctx context.Context, ch chan<- *types.Transaction) (Subscription, error) } + +// BlockNumberReader provides access to the current block number. +type BlockNumberReader interface { + BlockNumber(ctx context.Context) (uint64, error) +} + +// ChainIDReader provides access to the chain ID. +type ChainIDReader interface { + ChainID(ctx context.Context) (*big.Int, error) +} diff --git a/internal/build/download.go b/internal/build/download.go index 903d0308df..fda573df83 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -40,7 +40,7 @@ func MustLoadChecksums(file string) *ChecksumDB { if err != nil { log.Fatal("can't load checksum file: " + err.Error()) } - return &ChecksumDB{strings.Split(string(content), "\n")} + return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")} } // Verify checks whether the given file is valid according to the checksum database. diff --git a/internal/build/env.go b/internal/build/env.go index 0854fba249..6a621495ba 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -102,6 +102,10 @@ func LocalEnv() Environment { env := applyEnvFlags(Environment{Name: "local", Repo: "ethereum/go-ethereum"}) head := readGitFile("HEAD") + if info, err := os.Stat(".git/objects"); err == nil && info.IsDir() && env.Tag == "" { + env.Tag = firstLine(RunGit("tag", "-l", "--points-at", "HEAD")) + } + if fields := strings.Fields(head); len(fields) == 2 { head = fields[1] } else { @@ -111,6 +115,7 @@ func LocalEnv() Environment { commitRe, _ := regexp.Compile("^([0-9a-f]{40})$") if commit := commitRe.FindString(head); commit != "" && env.Commit == "" { env.Commit = commit + env.Date = getDate(env.Commit) } return env } @@ -123,9 +128,6 @@ func LocalEnv() Environment { env.Branch = strings.TrimPrefix(head, "refs/heads/") } } - if info, err := os.Stat(".git/objects"); err == nil && info.IsDir() && env.Tag == "" { - env.Tag = firstLine(RunGit("tag", "-l", "--points-at", "HEAD")) - } return env } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 23e4745e8c..dac878a7b1 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -168,22 +168,12 @@ var Flags = []cli.Flag{ } var ( - glogger *log.GlogHandler - logOutputFile io.WriteCloser - defaultTerminalHandler *log.TerminalHandler + glogger *log.GlogHandler + logOutputFile io.WriteCloser ) func init() { - defaultTerminalHandler = log.NewTerminalHandler(os.Stderr, false) - glogger = log.NewGlogHandler(defaultTerminalHandler) - glogger.Verbosity(log.LvlInfo) - log.SetDefault(log.NewLogger(glogger)) -} - -func ResetLogging() { - if defaultTerminalHandler != nil { - defaultTerminalHandler.ResetFieldPadding() - } + glogger = log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) } // Setup initializes profiling and logging based on the CLI flags. diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go new file mode 100644 index 0000000000..1218350e8d --- /dev/null +++ b/internal/era/accumulator.go @@ -0,0 +1,90 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "fmt" + "math/big" + + ssz "github.com/bnb-chain/fastssz" + "github.com/ethereum/go-ethereum/common" +) + +// ComputeAccumulator calculates the SSZ hash tree root of the Era1 +// accumulator of header records. +func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { + if len(hashes) != len(tds) { + return common.Hash{}, fmt.Errorf("must have equal number hashes as td values") + } + if len(hashes) > MaxEra1Size { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + } + hh := ssz.NewHasher() + for i := range hashes { + rec := headerRecord{hashes[i], tds[i]} + root, err := rec.HashTreeRoot() + if err != nil { + return common.Hash{}, err + } + hh.Append(root[:]) + } + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + return hh.HashRoot() +} + +// headerRecord is an individual record for a historical header. +// +// See https://github.com/ethereum/portal-network-specs/blob/master/history-network.md#the-header-accumulator +// for more information. +type headerRecord struct { + Hash common.Hash + TotalDifficulty *big.Int +} + +// GetTree completes the ssz.HashRoot interface, but is unused. +func (h *headerRecord) GetTree() (*ssz.Node, error) { + return nil, nil +} + +// HashTreeRoot ssz hashes the headerRecord object. +func (h *headerRecord) HashTreeRoot() ([32]byte, error) { + return ssz.HashWithDefaultHasher(h) +} + +// HashTreeRootWith ssz hashes the headerRecord object with a hasher. +func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { + hh.PutBytes(h.Hash[:]) + td := bigToBytes32(h.TotalDifficulty) + hh.PutBytes(td[:]) + hh.Merkleize(0) + return +} + +// bigToBytes32 converts a big.Int into a little-endian 32-byte array. +func bigToBytes32(n *big.Int) (b [32]byte) { + n.FillBytes(b[:]) + reverseOrder(b[:]) + return +} + +// reverseOrder reverses the byte order of a slice. +func reverseOrder(b []byte) []byte { + for i := 0; i < 16; i++ { + b[i], b[32-i-1] = b[32-i-1], b[i] + } + return b +} diff --git a/internal/era/builder.go b/internal/era/builder.go new file mode 100644 index 0000000000..9217c049f3 --- /dev/null +++ b/internal/era/builder.go @@ -0,0 +1,224 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package era + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to create Era1 archives of block data. +// +// Era1 files are themselves e2store files. For more information on this format, +// see https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md. +// +// The overall structure of an Era1 file follows closely the structure of an Era file +// which contains consensus Layer data (and as a byproduct, EL data after the merge). +// +// The structure can be summarized through this definition: +// +// era1 := Version | block-tuple* | other-entries* | Accumulator | BlockIndex +// block-tuple := CompressedHeader | CompressedBody | CompressedReceipts | TotalDifficulty +// +// Each basic element is its own entry: +// +// Version = { type: [0x65, 0x32], data: nil } +// CompressedHeader = { type: [0x03, 0x00], data: snappyFramed(rlp(header)) } +// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) } +// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) } +// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) } +// AccumulatorRoot = { type: [0x07, 0x00], data: accumulator-root } +// BlockIndex = { type: [0x32, 0x66], data: block-index } +// +// Accumulator is computed by constructing an SSZ list of header-records of length at most +// 8192 and then calculating the hash_tree_root of that list. +// +// header-record := { block-hash: Bytes32, total-difficulty: Uint256 } +// accumulator := hash_tree_root([]header-record, 8192) +// +// BlockIndex stores relative offsets to each compressed block entry. The +// format is: +// +// block-index := starting-number | index | index | index ... | count +// +// starting-number is the first block number in the archive. Every index is a +// defined relative to beginning of the record. The total number of block +// entries in the file is recorded with count. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in +// an Era1 batch is also 8192. +type Builder struct { + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + written int + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) *Builder { + buf := bytes.NewBuffer(nil) + return &Builder{ + w: e2store.NewWriter(w), + buf: buf, + snappy: snappy.NewBufferedWriter(buf), + } +} + +// Add writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return err + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return err + } + er, err := rlp.EncodeToBytes(receipts) + if err != nil { + return err + } + return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { + // Write Era1 version entry before first block. + if b.startNum == nil { + n, err := b.w.Write(TypeVersion, nil) + if err != nil { + return err + } + startNum := number + b.startNum = &startNum + b.startTd = new(big.Int).Sub(td, difficulty) + b.written += n + } + if len(b.indexes) >= MaxEra1Size { + return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + } + + b.indexes = append(b.indexes, uint64(b.written)) + b.hashes = append(b.hashes, hash) + b.tds = append(b.tds, td) + + // Write block data. + if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + return err + } + if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + return err + } + + // Also write total difficulty, but don't snappy encode. + btd := bigToBytes32(td) + n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + b.written += n + if err != nil { + return err + } + + return nil +} + +// Finalize computes the accumulator and block index values, then writes the +// corresponding e2store entries. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, fmt.Errorf("finalize called on empty builder") + } + // Compute accumulator root and write entry. + root, err := ComputeAccumulator(b.hashes, b.tds) + if err != nil { + return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) + } + n, err := b.w.Write(TypeAccumulator, root[:]) + b.written += n + if err != nil { + return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) + } + // Get beginning of index entry to calculate block relative offset. + base := int64(b.written) + + // Construct block index. Detailed format described in Builder + // documentation, but it is essentially encoded as: + // "start | index | index | ... | count" + var ( + count = len(b.indexes) + index = make([]byte, 16+count*8) + ) + binary.LittleEndian.PutUint64(index, *b.startNum) + // Each offset is relative from the position it is encoded in the + // index. This means that even if the same block was to be included in + // the index twice (this would be invalid anyways), the relative offset + // would be different. The idea with this is that after reading a + // relative offset, the corresponding block can be quickly read by + // performing a seek relative to the current position. + for i, offset := range b.indexes { + relative := int64(offset) - base + binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative)) + } + binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) + + // Finally, write the block index entry. + if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) + } + + return root, nil +} + +// snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + var ( + buf = b.buf + s = b.snappy + ) + buf.Reset() + s.Reset(buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := s.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += n + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go new file mode 100644 index 0000000000..d85b3e44e9 --- /dev/null +++ b/internal/era/e2store/e2store.go @@ -0,0 +1,220 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "encoding/binary" + "fmt" + "io" +) + +const ( + headerSize = 8 + valueSizeLimit = 1024 * 1024 * 50 +) + +// Entry is a variable-length-data record in an e2store. +type Entry struct { + Type uint16 + Value []byte +} + +// Writer writes entries using e2store encoding. +// For more information on this format, see: +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Writer struct { + w io.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w} +} + +// Write writes a single e2store entry to w. +// An entry is encoded in a type-length-value format. The first 8 bytes of the +// record store the type (2 bytes), the length (4 bytes), and some reserved +// data (2 bytes). The remaining bytes store b. +func (w *Writer) Write(typ uint16, b []byte) (int, error) { + buf := make([]byte, headerSize) + binary.LittleEndian.PutUint16(buf, typ) + binary.LittleEndian.PutUint32(buf[2:], uint32(len(b))) + + // Write header. + if n, err := w.w.Write(buf); err != nil { + return n, err + } + // Write value, return combined write size. + n, err := w.w.Write(b) + return n + headerSize, err +} + +// A Reader reads entries from an e2store-encoded file. +// For more information on this format, see +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Reader struct { + r io.ReaderAt + offset int64 +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.ReaderAt) *Reader { + return &Reader{r, 0} +} + +// Read reads one Entry from r. +func (r *Reader) Read() (*Entry, error) { + var e Entry + n, err := r.ReadAt(&e, r.offset) + if err != nil { + return nil, err + } + r.offset += int64(n) + return &e, nil +} + +// ReadAt reads one Entry from r at the specified offset. +func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) { + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + entry.Type = typ + + // Check length bounds. + if length > valueSizeLimit { + return headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + if length == 0 { + return headerSize, nil + } + + // Read value. + val := make([]byte, length) + if n, err := r.r.ReadAt(val, off+headerSize); err != nil { + n += headerSize + // An entry with a non-zero length should not return EOF when + // reading the value. + if err == io.EOF { + return n, io.ErrUnexpectedEOF + } + return n, err + } + entry.Value = val + return int(headerSize + length), nil +} + +// ReaderAt returns an io.Reader delivering value data for the entry at +// the specified offset. If the entry type does not match the expected type, an +// error is returned. +func (r *Reader) ReaderAt(expectedType uint16, off int64) (io.Reader, int, error) { + // problem = need to return length+headerSize not just value length via section reader + typ, length, err := r.ReadMetadataAt(off) + if err != nil { + return nil, headerSize, err + } + if typ != expectedType { + return nil, headerSize, fmt.Errorf("wrong type, want %d have %d", expectedType, typ) + } + if length > valueSizeLimit { + return nil, headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length) + } + return io.NewSectionReader(r.r, off+headerSize, int64(length)), headerSize + int(length), nil +} + +// LengthAt reads the header at off and returns the total length of the entry, +// including header. +func (r *Reader) LengthAt(off int64) (int64, error) { + _, length, err := r.ReadMetadataAt(off) + if err != nil { + return 0, err + } + return int64(length) + headerSize, nil +} + +// ReadMetadataAt reads the header metadata at the given offset. +func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error) { + b := make([]byte, headerSize) + if n, err := r.r.ReadAt(b, off); err != nil { + if err == io.EOF && n > 0 { + return 0, 0, io.ErrUnexpectedEOF + } + return 0, 0, err + } + typ = binary.LittleEndian.Uint16(b) + length = binary.LittleEndian.Uint32(b[2:]) + + // Check reserved bytes of header. + if b[6] != 0 || b[7] != 0 { + return 0, 0, fmt.Errorf("reserved bytes are non-zero") + } + + return typ, length, nil +} + +// Find returns the first entry with the matching type. +func (r *Reader) Find(want uint16) (*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return nil, io.EOF + } else if err != nil { + return nil, err + } + if typ == want { + var e Entry + if _, err := r.ReadAt(&e, off); err != nil { + return nil, err + } + return &e, nil + } + off += int64(headerSize + length) + } +} + +// FindAll returns all entries with the matching type. +func (r *Reader) FindAll(want uint16) ([]*Entry, error) { + var ( + off int64 + typ uint16 + length uint32 + entries []*Entry + err error + ) + for { + typ, length, err = r.ReadMetadataAt(off) + if err == io.EOF { + return entries, nil + } else if err != nil { + return entries, err + } + if typ == want { + e := new(Entry) + if _, err := r.ReadAt(e, off); err != nil { + return entries, err + } + entries = append(entries, e) + } + off += int64(headerSize + length) + } +} diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go new file mode 100644 index 0000000000..febcffe4cf --- /dev/null +++ b/internal/era/e2store/e2store_test.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "bytes" + "fmt" + "io" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestEncode(t *testing.T) { + for _, test := range []struct { + entries []Entry + want string + name string + }{ + { + name: "emptyEntry", + entries: []Entry{{0xffff, nil}}, + want: "ffff000000000000", + }, + { + name: "beef", + entries: []Entry{{42, common.Hex2Bytes("beef")}}, + want: "2a00020000000000beef", + }, + { + name: "twoEntries", + entries: []Entry{ + {42, common.Hex2Bytes("beef")}, + {9, common.Hex2Bytes("abcdabcd")}, + }, + want: "2a00020000000000beef0900040000000000abcdabcd", + }, + } { + tt := test + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + for _, e := range tt.entries { + if _, err := w.Write(e.Type, e.Value); err != nil { + t.Fatalf("encoding error: %v", err) + } + } + if want, have := common.FromHex(tt.want), b.Bytes(); !bytes.Equal(want, have) { + t.Fatalf("encoding mismatch (want %x, have %x", want, have) + } + r := NewReader(bytes.NewReader(b.Bytes())) + for _, want := range tt.entries { + have, err := r.Read() + if err != nil { + t.Fatalf("decoding error: %v", err) + } + if have.Type != want.Type { + t.Fatalf("decoded entry does type mismatch (want %v, got %v)", want.Type, have.Type) + } + if !bytes.Equal(have.Value, want.Value) { + t.Fatalf("decoded entry does not match (want %#x, got %#x)", want.Value, have.Value) + } + } + }) + } +} + +func TestDecode(t *testing.T) { + for i, tt := range []struct { + have string + err error + }{ + { // basic valid decoding + have: "ffff000000000000", + }, + { // basic invalid decoding + have: "ffff000000000001", + err: fmt.Errorf("reserved bytes are non-zero"), + }, + { // no more entries to read, returns EOF + have: "", + err: io.EOF, + }, + { // malformed type + have: "bad", + err: io.ErrUnexpectedEOF, + }, + { // malformed length + have: "badbeef", + err: io.ErrUnexpectedEOF, + }, + { // specified length longer than actual value + have: "beef010000000000", + err: io.ErrUnexpectedEOF, + }, + } { + r := NewReader(bytes.NewReader(common.FromHex(tt.have))) + if tt.err != nil { + _, err := r.Read() + if err == nil && tt.err != nil { + t.Fatalf("test %d, expected error, got none", i) + } + if err != nil && tt.err == nil { + t.Fatalf("test %d, expected no error, got %v", i, err) + } + if err != nil && tt.err != nil && err.Error() != tt.err.Error() { + t.Fatalf("expected error %v, got %v", tt.err, err) + } + continue + } + } +} + +func FuzzCodec(f *testing.F) { + f.Fuzz(func(t *testing.T, input []byte) { + r := NewReader(bytes.NewReader(input)) + entry, err := r.Read() + if err != nil { + return + } + var ( + b = bytes.NewBuffer(nil) + w = NewWriter(b) + ) + w.Write(entry.Type, entry.Value) + output := b.Bytes() + // Only care about the input that was actually consumed + input = input[:r.offset] + if !bytes.Equal(input, output) { + t.Fatalf("decode-encode mismatch, input %#x output %#x", input, output) + } + }) +} diff --git a/internal/era/era.go b/internal/era/era.go new file mode 100644 index 0000000000..a0e701b7e0 --- /dev/null +++ b/internal/era/era.go @@ -0,0 +1,283 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "path" + "strconv" + "strings" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +var ( + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeBlockIndex uint16 = 0x3266 + + MaxEra1Size = 8192 +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +// ReadDir reads all the era1 files in a directory for a given network. +// Format: --.era1 +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) + } + var ( + next = uint64(0) + eras []string + ) + for _, entry := range entries { + if path.Ext(entry.Name()) != ".era1" { + continue + } + parts := strings.Split(entry.Name(), "-") + if len(parts) != 3 || parts[0] != network { + // invalid era1 filename, skip + continue + } + if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + next += 1 + eras = append(eras, entry.Name()) + } + return eras, nil +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f ReadAtSeekCloser) (*Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (*Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds") + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over next two records. + for i := 0; i < 2; i++ { + length, err := e.s.LengthAt(off) + if err != nil { + return nil, err + } + off += length + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clearBuffer(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// clearBuffer zeroes out the buffer. +func clearBuffer(buf []byte) { + for i := 0; i < len(buf); i++ { + buf[i] = 0 + } +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/internal/era/era_test.go b/internal/era/era_test.go new file mode 100644 index 0000000000..ee5d9e82a0 --- /dev/null +++ b/internal/era/era_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "bytes" + "io" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +type testchain struct { + headers [][]byte + bodies [][]byte + receipts [][]byte + tds []*big.Int +} + +func TestEra1Builder(t *testing.T) { + // Get temp directory. + f, err := os.CreateTemp("", "era1-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + var ( + builder = NewBuilder(f) + chain = testchain{} + ) + for i := 0; i < 128; i++ { + chain.headers = append(chain.headers, []byte{byte('h'), byte(i)}) + chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)}) + chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)}) + chain.tds = append(chain.tds, big.NewInt(int64(i))) + } + + // Write blocks to Era1. + for i := 0; i < len(chain.headers); i++ { + var ( + header = chain.headers[i] + body = chain.bodies[i] + receipts = chain.receipts[i] + hash = common.Hash{byte(i)} + td = chain.tds[i] + ) + if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil { + t.Fatalf("error adding entry: %v", err) + } + } + + // Finalize Era1. + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing era1: %v", err) + } + + // Verify Era1 contents. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %s", err) + } + for i := uint64(0); i < uint64(len(chain.headers)); i++ { + if !it.Next() { + t.Fatalf("expected more entries") + } + if it.Error() != nil { + t.Fatalf("unexpected error %v", it.Error()) + } + // Check headers. + header, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(header, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) + } + // Check bodies. + body, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(body, chain.bodies[i]) { + t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) + } + // Check receipts. + receipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(receipts, chain.receipts[i]) { + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts) + } + + // Check total difficulty. + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading td: %v", err) + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + if td.Cmp(chain.tds[i]) != 0 { + t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) + } + } +} + +func TestEraFilename(t *testing.T) { + for i, tt := range []struct { + network string + epoch int + root common.Hash + expected string + }{ + {"mainnet", 1, common.Hash{1}, "mainnet-00001-01000000.era1"}, + {"goerli", 99999, common.HexToHash("0xdeadbeef00000000000000000000000000000000000000000000000000000000"), "goerli-99999-deadbeef.era1"}, + } { + got := Filename(tt.network, tt.epoch, tt.root) + if tt.expected != got { + t.Errorf("test %d: invalid filename: want %s, got %s", i, tt.expected, got) + } + } +} diff --git a/internal/era/iterator.go b/internal/era/iterator.go new file mode 100644 index 0000000000..e74a8154b1 --- /dev/null +++ b/internal/era/iterator.go @@ -0,0 +1,197 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Iterator wraps RawIterator and returns decoded Era1 entries. +type Iterator struct { + inner *RawIterator +} + +// NewRawIterator returns a new Iterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner}, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Block, Receipts, +// and BlockAndReceipts should no longer be called after false is returned. +func (it *Iterator) Next() bool { + return it.inner.Next() +} + +// Number returns the current number block the iterator will return. +func (it *Iterator) Number() uint64 { + return it.inner.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *Iterator) Error() error { + return it.inner.Error() +} + +// Block returns the block for the iterator's current position. +func (it *Iterator) Block() (*types.Block, error) { + if it.inner.Header == nil || it.inner.Body == nil { + return nil, fmt.Errorf("header and body must be non-nil") + } + var ( + header types.Header + body types.Body + ) + if err := rlp.Decode(it.inner.Header, &header); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Receipts returns the receipts for the iterator's current position. +func (it *Iterator) Receipts() (types.Receipts, error) { + if it.inner.Receipts == nil { + return nil, fmt.Errorf("receipts must be non-nil") + } + var receipts types.Receipts + err := rlp.Decode(it.inner.Receipts, &receipts) + return receipts, err +} + +// BlockAndReceipts returns the block and receipts for the iterator's current +// position. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the total difficulty for the iterator's current +// position. +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + td, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(td)), nil +} + +// RawIterator reads an RLP-encode Era1 entries. +type RawIterator struct { + e *Era // backing Era1 + next uint64 // next block to read + err error // last error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader +} + +// NewRawIterator returns a new RawIterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{ + e: e, + next: e.m.start, + }, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Header, Body, +// Receipts, TotalDifficulty will be set to nil in the case returning false or +// finding an error and should therefore no longer be read from. +func (it *RawIterator) Next() bool { + // Clear old errors. + it.err = nil + if it.e.m.start+it.e.m.count <= it.next { + it.clear() + return false + } + off, err := it.e.readOffset(it.next) + if err != nil { + // Error here means block index is corrupted, so don't + // continue. + it.clear() + it.err = err + return false + } + var n int64 + if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + it.clear() + return true + } + off += n + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + it.clear() + return true + } + it.next += 1 + return true +} + +// Number returns the current number block the iterator will return. +func (it *RawIterator) Number() uint64 { + return it.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +// clear sets all the outputs to nil. +func (it *RawIterator) clear() { + it.Header = nil + it.Body = nil + it.Receipts = nil + it.TotalDifficulty = nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9dffba0323..02fb0fdb7d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -26,11 +26,8 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/tyler-smith/go-bip39" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/common" @@ -51,12 +48,16 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/tyler-smith/go-bip39" ) // estimateGasErrorRatio is the amount of overestimation eth_estimateGas is // allowed to produce in order to speed up calculations. const estimateGasErrorRatio = 0.015 +var errBlobTxNotSupported = errors.New("signing blob transactions not supported") + // EthereumAPI provides an API to access Ethereum related information. type EthereumAPI struct { b Backend @@ -134,26 +135,28 @@ func (s *EthereumAPI) Syncing() (interface{}, error) { progress := s.b.SyncProgress() // Return not syncing if the synchronisation already completed - if progress.CurrentBlock >= progress.HighestBlock { + if progress.Done() { return false, nil } // Otherwise gather the block sync stats return map[string]interface{}{ - "startingBlock": hexutil.Uint64(progress.StartingBlock), - "currentBlock": hexutil.Uint64(progress.CurrentBlock), - "highestBlock": hexutil.Uint64(progress.HighestBlock), - "syncedAccounts": hexutil.Uint64(progress.SyncedAccounts), - "syncedAccountBytes": hexutil.Uint64(progress.SyncedAccountBytes), - "syncedBytecodes": hexutil.Uint64(progress.SyncedBytecodes), - "syncedBytecodeBytes": hexutil.Uint64(progress.SyncedBytecodeBytes), - "syncedStorage": hexutil.Uint64(progress.SyncedStorage), - "syncedStorageBytes": hexutil.Uint64(progress.SyncedStorageBytes), - "healedTrienodes": hexutil.Uint64(progress.HealedTrienodes), - "healedTrienodeBytes": hexutil.Uint64(progress.HealedTrienodeBytes), - "healedBytecodes": hexutil.Uint64(progress.HealedBytecodes), - "healedBytecodeBytes": hexutil.Uint64(progress.HealedBytecodeBytes), - "healingTrienodes": hexutil.Uint64(progress.HealingTrienodes), - "healingBytecode": hexutil.Uint64(progress.HealingBytecode), + "startingBlock": hexutil.Uint64(progress.StartingBlock), + "currentBlock": hexutil.Uint64(progress.CurrentBlock), + "highestBlock": hexutil.Uint64(progress.HighestBlock), + "syncedAccounts": hexutil.Uint64(progress.SyncedAccounts), + "syncedAccountBytes": hexutil.Uint64(progress.SyncedAccountBytes), + "syncedBytecodes": hexutil.Uint64(progress.SyncedBytecodes), + "syncedBytecodeBytes": hexutil.Uint64(progress.SyncedBytecodeBytes), + "syncedStorage": hexutil.Uint64(progress.SyncedStorage), + "syncedStorageBytes": hexutil.Uint64(progress.SyncedStorageBytes), + "healedTrienodes": hexutil.Uint64(progress.HealedTrienodes), + "healedTrienodeBytes": hexutil.Uint64(progress.HealedTrienodeBytes), + "healedBytecodes": hexutil.Uint64(progress.HealedBytecodes), + "healedBytecodeBytes": hexutil.Uint64(progress.HealedBytecodeBytes), + "healingTrienodes": hexutil.Uint64(progress.HealingTrienodes), + "healingBytecode": hexutil.Uint64(progress.HealingBytecode), + "txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks), + "txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks), }, nil } @@ -286,7 +289,7 @@ type PersonalAccountAPI struct { b Backend } -// NewPersonalAccountAPI create a new PersonalAccountAPI. +// NewPersonalAccountAPI creates a new PersonalAccountAPI. func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { return &PersonalAccountAPI{ am: b.AccountManager(), @@ -451,7 +454,7 @@ func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *Transact return nil, err } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and sign with the wallet @@ -470,6 +473,9 @@ func (s *PersonalAccountAPI) SendTransaction(ctx context.Context, args Transacti s.nonceLock.LockAddr(args.from()) defer s.nonceLock.UnlockAddr(args.from()) } + if args.IsEIP4844() { + return common.Hash{}, errBlobTxNotSupported + } signed, err := s.signTransaction(ctx, &args, passwd) if err != nil { log.Warn("Failed transaction send attempt", "from", args.from(), "to", args.To, "value", args.Value.ToInt(), "err", err) @@ -494,6 +500,9 @@ func (s *PersonalAccountAPI) SignTransaction(ctx context.Context, args Transacti if args.GasPrice == nil && (args.MaxFeePerGas == nil || args.MaxPriorityFeePerGas == nil) { return nil, errors.New("missing gasPrice or maxFeePerGas/maxPriorityFeePerGas") } + if args.IsEIP4844() { + return nil, errBlobTxNotSupported + } if args.Nonce == nil { return nil, errors.New("nonce not specified") } @@ -522,7 +531,7 @@ func (s *PersonalAccountAPI) SignTransaction(ctx context.Context, args Transacti // // The key used to calculate the signature is decrypted with the given password. // -// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign +// https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal#personal-sign func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -550,7 +559,7 @@ func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr // Note, the signature must conform to the secp256k1 curve R, S and V values, where // the V value must be 27 or 28 for legacy reasons. // -// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover +// https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal#personal-ecrecover func (s *PersonalAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { if len(sig) != crypto.SignatureLength { return common.Address{}, fmt.Errorf("signature must be %d bytes long", crypto.SignatureLength) @@ -661,10 +670,11 @@ func (s *BlockChainAPI) GetBalance(ctx context.Context, address common.Address, if state == nil || err != nil { return nil, err } - return (*hexutil.Big)(state.GetBalance(address)), state.Error() + b := state.GetBalance(address).ToBig() + return (*hexutil.Big)(b), state.Error() } -// Result structs for GetProof +// AccountResult structs for GetProof type AccountResult struct { Address common.Address `json:"address"` AccountProof []string `json:"accountProof"` @@ -681,6 +691,7 @@ type StorageResult struct { Proof []string `json:"proof"` } +// TODO check proof logic by will // GetProof returns the Merkle-proof for a given account and optionally some storage keys. func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (result *AccountResult, err error) { header, err := headerByNumberOrHash(ctx, s.b, blockNrOrHash) @@ -782,11 +793,12 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st if err = tr.Prove(crypto.Keccak256(address.Bytes()), &accountProof); err != nil { return nil, err } + balance := statedb.GetBalance(address).ToBig() err = statedb.Error() return &AccountResult{ Address: address, AccountProof: accountProof, - Balance: (*hexutil.Big)(statedb.GetBalance(address)), + Balance: (*hexutil.Big)(balance), CodeHash: codeHash, Nonce: hexutil.Uint64(statedb.GetNonce(address)), StorageHash: storageRoot, @@ -1059,7 +1071,8 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { } // Override account balance. if account.Balance != nil { - state.SetBalance(addr, (*big.Int)(*account.Balance)) + u256Balance, _ := uint256.FromBig((*big.Int)(*account.Balance)) + state.SetBalance(addr, u256Balance) } if account.State != nil && account.StateDiff != nil { return fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex()) @@ -1174,14 +1187,14 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S defer cancel() // Get a new instance of the EVM. - msg, err := args.ToMessage(globalGasCap, header.BaseFee) - if err != nil { - return nil, err - } blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil, b.ChainConfig(), state) if blockOverrides != nil { blockOverrides.Apply(&blockCtx) } + msg, err := args.ToMessage(globalGasCap, blockCtx.BaseFee) + if err != nil { + return nil, err + } evm := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) // Wait for the context to be done and cancel the evm. Even if the @@ -1219,37 +1232,6 @@ func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash return doCall(ctx, b, args, state, header, overrides, blockOverrides, timeout, globalGasCap) } -func newRevertError(revert []byte) *revertError { - err := vm.ErrExecutionReverted - - reason, errUnpack := abi.UnpackRevert(revert) - if errUnpack == nil { - err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) - } - return &revertError{ - error: err, - reason: hexutil.Encode(revert), - } -} - -// revertError is an API error that encompasses an EVM revertal with JSON error -// code and a binary data blob. -type revertError struct { - error - reason string // revert reason hex encoded -} - -// ErrorCode returns the JSON error code for a revertal. -// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal -func (e *revertError) ErrorCode() int { - return 3 -} - -// ErrorData returns the hex encoded revert reason. -func (e *revertError) ErrorData() interface{} { - return e.reason -} - // Call executes the given transaction on the state for the given block number. // // Additionally, the caller can specify a batch of contract for fields overriding. @@ -1332,6 +1314,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr // returns error if the transaction would revert or if there are unexpected failures. The returned // value is capped by both `args.Gas` (if non-nil & non-zero) and the backend's RPCGasCap // configuration (if non-zero). +// Note: Required blob gas is not computed in this method. func (s *BlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Uint64, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) if blockNrOrHash != nil { @@ -1656,7 +1639,7 @@ type accessListResult struct { // CreateAccessList creates an EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. func (s *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { - bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) + bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash } @@ -1695,14 +1678,9 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, default to RPC gas cap. - if args.Gas == nil { - tmp := hexutil.Uint64(b.RPCGasCap()) - args.Gas = &tmp - } // Ensure any missing fields are filled, extract the recipient and input data - if err := args.setDefaults(ctx, b); err != nil { + if err := args.setDefaults(ctx, b, true); err != nil { return nil, 0, nil, err } var to common.Address @@ -1855,51 +1833,49 @@ func (s *TransactionAPI) GetTransactionCount(ctx context.Context, address common // GetTransactionByHash returns the transaction for the given hash func (s *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { // Try to return an already finalized transaction - tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) - if err != nil { - return nil, err - } - if tx != nil { - header, err := s.b.HeaderByHash(ctx, blockHash) - if err != nil { - return nil, err + found, tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) + if !found { + // No finalized transaction, try to retrieve it from the pool + if tx := s.b.GetPoolTransaction(hash); tx != nil { + return NewRPCPendingTransaction(tx, s.b.CurrentHeader(), s.b.ChainConfig()), nil } - rcpt := depositTxReceipt(ctx, blockHash, index, s.b, tx) - return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, s.b.ChainConfig(), rcpt), nil + if err == nil { + return nil, nil + } + return nil, NewTxIndexingError() } - // No finalized transaction, try to retrieve it from the pool - if tx := s.b.GetPoolTransaction(hash); tx != nil { - return NewRPCPendingTransaction(tx, s.b.CurrentHeader(), s.b.ChainConfig()), nil + header, err := s.b.HeaderByHash(ctx, blockHash) + if err != nil { + return nil, err } - - // Transaction unknown, return as such - return nil, nil + rcpt := depositTxReceipt(ctx, blockHash, index, s.b, tx) + return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, s.b.ChainConfig(), rcpt), nil } // GetRawTransactionByHash returns the bytes of the transaction for the given hash. func (s *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise - tx, _, _, _, err := s.b.GetTransaction(ctx, hash) - if err != nil { - return nil, err - } - if tx == nil { - if tx = s.b.GetPoolTransaction(hash); tx == nil { - // Transaction not found anywhere, abort + found, tx, _, _, _, err := s.b.GetTransaction(ctx, hash) + if !found { + if tx = s.b.GetPoolTransaction(hash); tx != nil { + return tx.MarshalBinary() + } + if err == nil { return nil, nil } + return nil, NewTxIndexingError() } - // Serialize to RLP and return return tx.MarshalBinary() } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (s *TransactionAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { - tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) - if tx == nil || err != nil { - // When the transaction doesn't exist, the RPC method should return JSON null - // as per specification. - return nil, nil + found, tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) + if err != nil { + return nil, NewTxIndexingError() // transaction is not fully indexed + } + if !found { + return nil, nil // transaction is not existent or reachable } header, err := s.b.HeaderByHash(ctx, blockHash) if err != nil { @@ -1943,9 +1919,20 @@ func marshalReceipt(receipt *types.Receipt, blockHash common.Hash, blockNumber u fields["l1GasPrice"] = (*hexutil.Big)(receipt.L1GasPrice) fields["l1GasUsed"] = (*hexutil.Big)(receipt.L1GasUsed) fields["l1Fee"] = (*hexutil.Big)(receipt.L1Fee) - if receipt.FeeScalar != nil { // removed in Ecotone + // Fields removed with Ecotone + if receipt.FeeScalar != nil { fields["l1FeeScalar"] = receipt.FeeScalar.String() } + // Fields added in Ecotone + if receipt.L1BlobBaseFee != nil { + fields["l1BlobBaseFee"] = (*hexutil.Big)(receipt.L1BlobBaseFee) + } + if receipt.L1BaseFeeScalar != nil { + fields["l1BaseFeeScalar"] = hexutil.Uint64(*receipt.L1BaseFeeScalar) + } + if receipt.L1BlobBaseFeeScalar != nil { + fields["l1BlobBaseFeeScalar"] = hexutil.Uint64(*receipt.L1BlobBaseFeeScalar) + } } if chainConfig.Optimism != nil && tx.IsDepositTx() && receipt.DepositNonce != nil { fields["depositNonce"] = hexutil.Uint64(*receipt.DepositNonce) @@ -2037,9 +2024,12 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr s.nonceLock.LockAddr(args.from()) defer s.nonceLock.UnlockAddr(args.from()) } + if args.IsEIP4844() { + return common.Hash{}, errBlobTxNotSupported + } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } // Assemble the transaction and sign with the wallet @@ -2056,8 +2046,10 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr // on a given unsigned transaction, and returns it to the caller for further // processing (signing + broadcast). func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { + args.blobSidecarAllowed = true + // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and obtain rlp @@ -2120,10 +2112,13 @@ func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionAr if args.GasPrice == nil && (args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil) { return nil, errors.New("missing gasPrice or maxFeePerGas/maxPriorityFeePerGas") } + if args.IsEIP4844() { + return nil, errBlobTxNotSupported + } if args.Nonce == nil { return nil, errors.New("nonce not specified") } - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. @@ -2172,7 +2167,7 @@ func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, g if sendArgs.Nonce == nil { return common.Hash{}, errors.New("missing transaction nonce in transaction spec") } - if err := sendArgs.setDefaults(ctx, s.b); err != nil { + if err := sendArgs.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } matchTx := sendArgs.toTransaction() @@ -2297,15 +2292,15 @@ func (api *DebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.Block // GetRawTransaction returns the bytes of the transaction for the given hash. func (s *DebugAPI) GetRawTransaction(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise - tx, _, _, _, err := s.b.GetTransaction(ctx, hash) - if err != nil { - return nil, err - } - if tx == nil { - if tx = s.b.GetPoolTransaction(hash); tx == nil { - // Transaction not found anywhere, abort + found, tx, _, _, _, err := s.b.GetTransaction(ctx, hash) + if !found { + if tx = s.b.GetPoolTransaction(hash); tx != nil { + return tx.MarshalBinary() + } + if err == nil { return nil, nil } + return nil, NewTxIndexingError() } return tx.MarshalBinary() } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index b1ffd68685..f65c98a50a 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -17,8 +17,10 @@ package ethapi import ( + "bytes" "context" "crypto/ecdsa" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -35,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" @@ -47,6 +50,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/blocktest" @@ -562,10 +566,30 @@ func allBlobTxs(addr common.Address, config *params.ChainConfig) []txData { } } +func newTestAccountManager(t *testing.T) (*accounts.Manager, accounts.Account) { + var ( + dir = t.TempDir() + am = accounts.NewManager(&accounts.Config{InsecureUnlockAllowed: true}) + b = keystore.NewKeyStore(dir, 2, 1) + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + ) + acc, err := b.ImportECDSA(testKey, "") + if err != nil { + t.Fatalf("failed to create test account: %v", err) + } + if err := b.Unlock(acc, ""); err != nil { + t.Fatalf("failed to unlock account: %v\n", err) + } + am.AddBackend(b) + return am, acc +} + type testBackend struct { db ethdb.Database chain *core.BlockChain pending *types.Block + accman *accounts.Manager + acc accounts.Account } func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.Engine, generator func(i int, b *core.BlockGen)) *testBackend { @@ -578,6 +602,8 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.E TrieDirtyDisabled: true, // Archive mode } ) + accman, acc := newTestAccountManager(t) + gspec.Alloc[acc.Address] = types.Account{Balance: big.NewInt(params.Ether)} // Generate blocks for testing db, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, generator) txlookupLimit := uint64(0) @@ -589,7 +615,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.E t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - backend := &testBackend{db: db, chain: chain} + backend := &testBackend{db: db, chain: chain, accman: accman, acc: acc} return backend } @@ -609,7 +635,7 @@ func (b testBackend) FeeHistory(ctx context.Context, blockCount uint64, lastBloc return nil, nil, nil, nil, nil } func (b testBackend) ChainDb() ethdb.Database { return b.db } -func (b testBackend) AccountManager() *accounts.Manager { return nil } +func (b testBackend) AccountManager() *accounts.Manager { return b.accman } func (b testBackend) ExtRPCEnabled() bool { return false } func (b testBackend) RPCGasCap() uint64 { return 10000000 } func (b testBackend) RPCEVMTimeout() time.Duration { return time.Second } @@ -731,14 +757,14 @@ func (b testBackend) SendBundle(ctx context.Context, bundle *types.Bundle, origi func (b testBackend) BundlePrice() *big.Int { panic("implement me") } -func (b testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { +func (b testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { tx, blockHash, blockNumber, index := rawdb.ReadTransaction(b.db, txHash) - return tx, blockHash, blockNumber, index, nil + return true, tx, blockHash, blockNumber, index, nil } func (b testBackend) GetPoolTransactions() (types.Transactions, error) { panic("implement me") } func (b testBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction { panic("implement me") } func (b testBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { - panic("implement me") + return 0, nil } func (b testBackend) Stats() (pending int, queued int) { panic("implement me") } func (b testBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { @@ -781,8 +807,8 @@ func TestEstimateGas(t *testing.T) { var ( accounts = newAccounts(2) genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, }, @@ -791,12 +817,13 @@ func TestEstimateGas(t *testing.T) { signer = types.HomesteadSigner{} randomAccounts = newAccounts(2) ) - api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, ethash.NewFaker(), func(i int, b *core.BlockGen) { + api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &accounts[1].addr, Value: big.NewInt(1000), Gas: params.TxGas, GasPrice: b.BaseFee(), Data: nil}), signer, accounts[0].key) b.AddTx(tx) + b.SetPoS() })) var testSuite = []struct { blockNumber rpc.BlockNumber @@ -896,6 +923,18 @@ func TestEstimateGas(t *testing.T) { expectErr: nil, want: 67595, }, + // Blobs should have no effect on gas estimate + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{common.Hash{0x01, 0x22}}, + BlobFeeCap: (*hexutil.Big)(big.NewInt(1)), + }, + want: 21000, + }, } for i, tc := range testSuite { result, err := api.EstimateGas(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides) @@ -925,8 +964,8 @@ func TestCall(t *testing.T) { var ( accounts = newAccounts(3) genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{ accounts[0].addr: {Balance: big.NewInt(params.Ether)}, accounts[1].addr: {Balance: big.NewInt(params.Ether)}, accounts[2].addr: {Balance: big.NewInt(params.Ether)}, @@ -935,12 +974,13 @@ func TestCall(t *testing.T) { genBlocks = 10 signer = types.HomesteadSigner{} ) - api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, ethash.NewFaker(), func(i int, b *core.BlockGen) { + api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &accounts[1].addr, Value: big.NewInt(1000), Gas: params.TxGas, GasPrice: b.BaseFee(), Data: nil}), signer, accounts[0].key) b.AddTx(tx) + b.SetPoS() })) randomAccounts := newAccounts(3) var testSuite = []struct { @@ -1062,6 +1102,32 @@ func TestCall(t *testing.T) { blockOverrides: BlockOverrides{Number: (*hexutil.Big)(big.NewInt(11))}, want: "0x000000000000000000000000000000000000000000000000000000000000000b", }, + // Invalid blob tx + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[1].addr, + Input: &hexutil.Bytes{0x00}, + BlobHashes: []common.Hash{}, + }, + expectErr: core.ErrBlobTxCreate, + }, + // BLOBHASH opcode + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[1].addr, + To: &randomAccounts[2].addr, + BlobHashes: []common.Hash{common.Hash{0x01, 0x22}}, + BlobFeeCap: (*hexutil.Big)(big.NewInt(1)), + }, + overrides: StateOverride{ + randomAccounts[2].addr: { + Code: hex2Bytes("60004960005260206000f3"), + }, + }, + want: "0x0122000000000000000000000000000000000000000000000000000000000000", + }, } for i, tc := range testSuite { result, err := api.Call(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides) @@ -1088,6 +1154,323 @@ func TestCall(t *testing.T) { } } +func TestSignTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{}, + } + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + res, err := api.FillTransaction(context.Background(), TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + }) + if err != nil { + t.Fatalf("failed to fill tx defaults: %v\n", err) + } + + res, err = api.SignTransaction(context.Background(), argsFromTransaction(res.Tx, b.acc.Address)) + if err != nil { + t.Fatalf("failed to sign tx: %v\n", err) + } + tx, err := json.Marshal(res.Tx) + if err != nil { + t.Fatal(err) + } + expect := `{"type":"0x2","chainId":"0x1","nonce":"0x0","to":"0x703c4b2bd70c169f5717101caee543299fc946c7","gas":"0x5208","gasPrice":null,"maxPriorityFeePerGas":"0x0","maxFeePerGas":"0x684ee180","value":"0x1","input":"0x","accessList":[],"v":"0x0","r":"0x8fabeb142d585dd9247f459f7e6fe77e2520c88d50ba5d220da1533cea8b34e1","s":"0x582dd68b21aef36ba23f34e49607329c20d981d30404daf749077f5606785ce7","yParity":"0x0","hash":"0x93927839207cfbec395da84b8a2bc38b7b65d2cb2819e9fef1f091f5b1d4cc8f"}` + if !bytes.Equal(tx, []byte(expect)) { + t.Errorf("result mismatch. Have:\n%s\nWant:\n%s\n", tx, expect) + } +} + +func TestSignBlobTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{}, + } + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + res, err := api.FillTransaction(context.Background(), TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{{0x01, 0x22}}, + }) + if err != nil { + t.Fatalf("failed to fill tx defaults: %v\n", err) + } + + _, err = api.SignTransaction(context.Background(), argsFromTransaction(res.Tx, b.acc.Address)) + if err == nil { + t.Fatalf("should fail on blob transaction") + } + if !errors.Is(err, errBlobTxNotSupported) { + t.Errorf("error mismatch. Have: %v, want: %v", err, errBlobTxNotSupported) + } +} + +func TestSendBlobTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{}, + } + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + res, err := api.FillTransaction(context.Background(), TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{common.Hash{0x01, 0x22}}, + }) + if err != nil { + t.Fatalf("failed to fill tx defaults: %v\n", err) + } + + _, err = api.SendTransaction(context.Background(), argsFromTransaction(res.Tx, b.acc.Address)) + if err == nil { + t.Errorf("sending tx should have failed") + } else if !errors.Is(err, errBlobTxNotSupported) { + t.Errorf("unexpected error. Have %v, want %v\n", err, errBlobTxNotSupported) + } +} + +func TestFillBlobTransaction(t *testing.T) { + t.Parallel() + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + to = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{}, + } + emptyBlob = kzg4844.Blob{} + emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) + emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) + emptyBlobHash common.Hash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) + ) + b := newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + }) + api := NewTransactionAPI(b, nil) + type result struct { + Hashes []common.Hash + Sidecar *types.BlobTxSidecar + } + suite := []struct { + name string + args TransactionArgs + err string + want *result + }{ + { + name: "TestInvalidParamsCombination1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `blob proofs provided while commitments were not`, + }, + { + name: "TestInvalidParamsCombination2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}}, + }, + err: `blob commitments provided while proofs were not`, + }, + { + name: "TestInvalidParamsCount1", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `number of blobs and commitments mismatch (have=2, want=1)`, + }, + { + name: "TestInvalidParamsCount2", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}}, + }, + err: `number of blobs and proofs mismatch (have=1, want=2)`, + }, + { + name: "TestInvalidProofVerification", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{{}, {}}, + Commitments: []kzg4844.Commitment{{}, {}}, + Proofs: []kzg4844.Proof{{}, {}}, + }, + err: `failed to verify blob proof: short buffer`, + }, + { + name: "TestGenerateBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestValidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{emptyBlobHash}, + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + { + name: "TestInvalidBlobHashes", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{{0x01, 0x22}}, + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + err: fmt.Sprintf("blob hash verification failed (have=%s, want=%s)", common.Hash{0x01, 0x22}, emptyBlobHash), + }, + { + name: "TestGenerateBlobProofs", + args: TransactionArgs{ + From: &b.acc.Address, + To: &to, + Value: (*hexutil.Big)(big.NewInt(1)), + Blobs: []kzg4844.Blob{emptyBlob}, + }, + want: &result{ + Hashes: []common.Hash{emptyBlobHash}, + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, + }, + }, + } + for _, tc := range suite { + t.Run(tc.name, func(t *testing.T) { + res, err := api.FillTransaction(context.Background(), tc.args) + if len(tc.err) > 0 { + if err == nil { + t.Fatalf("missing error. want: %s", tc.err) + } else if err != nil && err.Error() != tc.err { + t.Fatalf("error mismatch. want: %s, have: %s", tc.err, err.Error()) + } + return + } + if err != nil && len(tc.err) == 0 { + t.Fatalf("expected no error. have: %s", err) + } + if res == nil { + t.Fatal("result missing") + } + want, err := json.Marshal(tc.want) + if err != nil { + t.Fatalf("failed to encode expected: %v", err) + } + have, err := json.Marshal(result{Hashes: res.Tx.BlobHashes(), Sidecar: res.Tx.BlobTxSidecar()}) + if err != nil { + t.Fatalf("failed to encode computed sidecar: %v", err) + } + if !bytes.Equal(have, want) { + t.Errorf("blob sidecar mismatch. Have: %s, want: %s", have, want) + } + }) + } +} + +func argsFromTransaction(tx *types.Transaction, from common.Address) TransactionArgs { + var ( + gas = tx.Gas() + nonce = tx.Nonce() + input = tx.Data() + ) + return TransactionArgs{ + From: &from, + To: tx.To(), + Gas: (*hexutil.Uint64)(&gas), + MaxFeePerGas: (*hexutil.Big)(tx.GasFeeCap()), + MaxPriorityFeePerGas: (*hexutil.Big)(tx.GasTipCap()), + Value: (*hexutil.Big)(tx.Value()), + Nonce: (*hexutil.Uint64)(&nonce), + Input: (*hexutil.Bytes)(&input), + ChainID: (*hexutil.Big)(tx.ChainId()), + // TODO: impl accessList conversion + //AccessList: tx.AccessList(), + BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), + BlobHashes: tx.BlobHashes(), + } +} + type account struct { key *ecdsa.PrivateKey addr common.Address @@ -1337,7 +1720,7 @@ func TestRPCGetBlockOrHeader(t *testing.T) { acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) genesis = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ acc1Addr: {Balance: big.NewInt(params.Ether)}, acc2Addr: {Balance: big.NewInt(params.Ether)}, }, @@ -1581,9 +1964,7 @@ func TestRPCGetBlockOrHeader(t *testing.T) { } func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Hash) { - config := *params.TestChainConfig - config.ShanghaiTime = new(uint64) - config.CancunTime = new(uint64) + config := *params.MergedTestChainConfig var ( acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") @@ -1594,7 +1975,7 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha Config: &config, ExcessBlobGas: new(uint64), BlobGasUsed: new(uint64), - Alloc: core.GenesisAlloc{ + Alloc: types.GenesisAlloc{ acc1Addr: {Balance: big.NewInt(params.Ether)}, acc2Addr: {Balance: big.NewInt(params.Ether)}, // // SPDX-License-Identifier: GPL-3.0 @@ -1614,14 +1995,12 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha txHashes = make([]common.Hash, genBlocks) ) - // Set the terminal total difficulty in the config - genesis.Config.TerminalTotalDifficulty = big.NewInt(0) - genesis.Config.TerminalTotalDifficultyPassed = true backend := newTestBackend(t, genBlocks, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { var ( tx *types.Transaction err error ) + b.SetPoS() switch i { case 0: // transfer 1000wei @@ -1670,7 +2049,6 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha b.AddTx(tx) txHashes[i] = tx.Hash() } - b.SetPoS() }) return backend, txHashes } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index f758251c12..96086aa8e9 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -80,7 +80,7 @@ type Backend interface { SimulateGaslessBundle(bundle *types.Bundle) (*types.SimulateGaslessBundleResp, error) SendBundle(ctx context.Context, bundle *types.Bundle, originBundle *types.SendBundleArgs) error BundlePrice() *big.Int - GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) + GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go new file mode 100644 index 0000000000..b5e668a805 --- /dev/null +++ b/internal/ethapi/errors.go @@ -0,0 +1,78 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" +) + +// revertError is an API error that encompasses an EVM revert with JSON error +// code and a binary data blob. +type revertError struct { + error + reason string // revert reason hex encoded +} + +// ErrorCode returns the JSON error code for a revert. +// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal +func (e *revertError) ErrorCode() int { + return 3 +} + +// ErrorData returns the hex encoded revert reason. +func (e *revertError) ErrorData() interface{} { + return e.reason +} + +// newRevertError creates a revertError instance with the provided revert data. +func newRevertError(revert []byte) *revertError { + err := vm.ErrExecutionReverted + + reason, errUnpack := abi.UnpackRevert(revert) + if errUnpack == nil { + err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) + } + return &revertError{ + error: err, + reason: hexutil.Encode(revert), + } +} + +// TxIndexingError is an API error that indicates the transaction indexing is not +// fully finished yet with JSON error code and a binary data blob. +type TxIndexingError struct{} + +// NewTxIndexingError creates a TxIndexingError instance. +func NewTxIndexingError() *TxIndexingError { return &TxIndexingError{} } + +// Error implement error interface, returning the error message. +func (e *TxIndexingError) Error() string { + return "transaction indexing is in progress" +} + +// ErrorCode returns the JSON error code for a revert. +// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal +func (e *TxIndexingError) ErrorCode() int { + return -32000 // to be decided +} + +// ErrorData returns the hex encoded revert reason. +func (e *TxIndexingError) ErrorData() interface{} { return "transaction indexing is in progress" } diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json index 379636d5f3..73da1b1752 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json @@ -4,17 +4,17 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x0da274b315de8e4d5bf8717218ec43540464ef36378cb896469bb731e1d3f3cb", + "hash": "0xeeb5c1852740ca4bbe65b0f57baf80634ed12a2b44affe30eec3fb54437c3926", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x1", - "parentHash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "parentHash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0x92c5c55a698963f5b06e3aee415630f5c48b0760e537af94917ce9c4f42a2e22", + "stateRoot": "0x4acfcd1a6ab9f5e62411021ecd8a749976ae50b0590e967471264b372d7ac55b", "timestamp": "0xa", "totalDifficulty": "0x1", "transactions": [ diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json index 759dbf69e9..d2bdbacd73 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json @@ -4,7 +4,7 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x0", - "hash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "hash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -14,7 +14,7 @@ "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x200", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "stateRoot": "0xd883f48b83cc9c1e8389453beb4ad4e572462eec049ca4fffbe16ecefb3fe937", "timestamp": "0x0", "totalDifficulty": "0x1", "transactions": [], diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json index 3526da1219..8e0748def9 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json @@ -4,22 +4,22 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "hash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x9", - "parentHash": "0x5abd19c39d9f1c6e52998e135ea14e1fbc5db3fa2a108f4538e238ca5c2e68d7", + "parentHash": "0xcd7d78eaa8b0ddbd2956fc37e1883c30df27b43e8cc9a982020310656736637c", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0xbd4aa2c2873df709151075250a8c01c9a14d2b0e2f715dbdd16e0ef8030c2cf0", + "stateRoot": "0x78b2b19ef1a0276dbbc23a875dbf60ae5d10dafa0017098473c4871abd3e7b5c", "timestamp": "0x5a", "totalDifficulty": "0x1", "transactions": [ { - "blockHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json index 32fee83268..6e914e37d0 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json @@ -4,17 +4,17 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x97f540a3577c0f645c5dada5da86f38350e8f847e71f21124f917835003e2607", + "hash": "0xa063415a5020f1569fae73ecb0d37bc5649ebe86d59e764a389eb37814bd42cb", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0xa", - "parentHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "parentHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0xbb62872e4023fa8a8b17b9cc37031f4817d9595779748d01cba408b495707a91", + "stateRoot": "0x118f1433ae23c4d1c12f5bd652baddb72611c55ac1cd6af6620d209db222f9e6", "timestamp": "0x64", "totalDifficulty": "0x1", "transactions": [ diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json index 759dbf69e9..d2bdbacd73 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json @@ -4,7 +4,7 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x0", - "hash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "hash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -14,7 +14,7 @@ "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x200", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "stateRoot": "0xd883f48b83cc9c1e8389453beb4ad4e572462eec049ca4fffbe16ecefb3fe937", "timestamp": "0x0", "totalDifficulty": "0x1", "transactions": [], diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json index 379636d5f3..73da1b1752 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json @@ -4,17 +4,17 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x0da274b315de8e4d5bf8717218ec43540464ef36378cb896469bb731e1d3f3cb", + "hash": "0xeeb5c1852740ca4bbe65b0f57baf80634ed12a2b44affe30eec3fb54437c3926", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x1", - "parentHash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "parentHash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0x92c5c55a698963f5b06e3aee415630f5c48b0760e537af94917ce9c4f42a2e22", + "stateRoot": "0x4acfcd1a6ab9f5e62411021ecd8a749976ae50b0590e967471264b372d7ac55b", "timestamp": "0xa", "totalDifficulty": "0x1", "transactions": [ diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json index 3526da1219..8e0748def9 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json @@ -4,22 +4,22 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "hash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x9", - "parentHash": "0x5abd19c39d9f1c6e52998e135ea14e1fbc5db3fa2a108f4538e238ca5c2e68d7", + "parentHash": "0xcd7d78eaa8b0ddbd2956fc37e1883c30df27b43e8cc9a982020310656736637c", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0xbd4aa2c2873df709151075250a8c01c9a14d2b0e2f715dbdd16e0ef8030c2cf0", + "stateRoot": "0x78b2b19ef1a0276dbbc23a875dbf60ae5d10dafa0017098473c4871abd3e7b5c", "timestamp": "0x5a", "totalDifficulty": "0x1", "transactions": [ { - "blockHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json index 32fee83268..6e914e37d0 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json @@ -4,17 +4,17 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x97f540a3577c0f645c5dada5da86f38350e8f847e71f21124f917835003e2607", + "hash": "0xa063415a5020f1569fae73ecb0d37bc5649ebe86d59e764a389eb37814bd42cb", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0xa", - "parentHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "parentHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", "size": "0x26a", - "stateRoot": "0xbb62872e4023fa8a8b17b9cc37031f4817d9595779748d01cba408b495707a91", + "stateRoot": "0x118f1433ae23c4d1c12f5bd652baddb72611c55ac1cd6af6620d209db222f9e6", "timestamp": "0x64", "totalDifficulty": "0x1", "transactions": [ diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json index 591fab673d..09fb734d39 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json @@ -2,7 +2,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0xe724dfd4349861f4dceef2bc4df086d0a3d88858214f6bee9fcf1bebd1edc2a6", + "blockHash": "0xd1392771155ce83f6403c6af275efd22bed567030c21168fcc9dbad5004eb245", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json index f1e0db22c2..ab14d56394 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0x1e7dcf3abe8bf05d32367a5dc387caa32578b15871bf8b3cbeedf2d8d530f844", + "blockHash": "0x56ea26cf955d7f2e08e194ad212ca4d5f99ee8e0b19dec3c71d8faafa33b1d22", "blockNumber": "0x2", "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", "cumulativeGasUsed": "0xcf50", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json index 520e30e4ea..9e137e241f 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0xffa737e6ce9a9162ffd411dd06169114b3ed5ee9fc1474a2625c92548e4455e0", + "blockHash": "0xf41e7a7a716382f20464cf76c6ae1fa701e9d32f5cc550ebfd2391b9642ae6bc", "blockNumber": "0x4", "contractAddress": null, "cumulativeGasUsed": "0x538d", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json index a71cf4b37f..1db7d02b1c 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0x173dcd9d22ce71929cd17e84ea88702a0f84d6244c6898d2a4f48722e494fe9c", + "blockHash": "0xa1410af902e98b32e0bbe464f8637ff464f1d4344b585127d2ce71f9cb39cb8a", "blockNumber": "0x3", "contractAddress": null, "cumulativeGasUsed": "0x5e28", @@ -19,7 +19,7 @@ "blockNumber": "0x3", "transactionHash": "0xeaf3921cbf03ba45bad4e6ab807b196ce3b2a0b5bacc355b6272fa96b11b4287", "transactionIndex": "0x0", - "blockHash": "0x173dcd9d22ce71929cd17e84ea88702a0f84d6244c6898d2a4f48722e494fe9c", + "blockHash": "0xa1410af902e98b32e0bbe464f8637ff464f1d4344b585127d2ce71f9cb39cb8a", "logIndex": "0x0", "removed": false } diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json index 3e16c3062e..9a55927839 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0xa8a067b3cb3b9ddc6cfb8317bfd08b266fcf9994fc870c1f7ed394acecfadf39", + "blockHash": "0x797d0c5603eccb33cc8ebd1300e977746512ec49e6b89087c7aad28ff760a26f", "blockNumber": "0x1", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json index 591fab673d..09fb734d39 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json @@ -2,7 +2,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0xe724dfd4349861f4dceef2bc4df086d0a3d88858214f6bee9fcf1bebd1edc2a6", + "blockHash": "0xd1392771155ce83f6403c6af275efd22bed567030c21168fcc9dbad5004eb245", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json index dc61aa9a2e..1bd68888b6 100644 --- a/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json @@ -4,7 +4,7 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x0", - "hash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "hash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -13,7 +13,7 @@ "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "stateRoot": "0xd883f48b83cc9c1e8389453beb4ad4e572462eec049ca4fffbe16ecefb3fe937", "timestamp": "0x0", "totalDifficulty": "0x1", "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json index c1dc70f64f..cf662cad75 100644 --- a/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x0da274b315de8e4d5bf8717218ec43540464ef36378cb896469bb731e1d3f3cb", + "hash": "0xeeb5c1852740ca4bbe65b0f57baf80634ed12a2b44affe30eec3fb54437c3926", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x1", - "parentHash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "parentHash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x92c5c55a698963f5b06e3aee415630f5c48b0760e537af94917ce9c4f42a2e22", + "stateRoot": "0x4acfcd1a6ab9f5e62411021ecd8a749976ae50b0590e967471264b372d7ac55b", "timestamp": "0xa", "totalDifficulty": "0x1", "transactionsRoot": "0xca0ebcce920d2cdfbf9e1dbe90ed3441a1a576f344bd80e60508da814916f4e7" diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json index a63ff86700..4721dd1e7a 100644 --- a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "hash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x9", - "parentHash": "0x5abd19c39d9f1c6e52998e135ea14e1fbc5db3fa2a108f4538e238ca5c2e68d7", + "parentHash": "0xcd7d78eaa8b0ddbd2956fc37e1883c30df27b43e8cc9a982020310656736637c", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xbd4aa2c2873df709151075250a8c01c9a14d2b0e2f715dbdd16e0ef8030c2cf0", + "stateRoot": "0x78b2b19ef1a0276dbbc23a875dbf60ae5d10dafa0017098473c4871abd3e7b5c", "timestamp": "0x5a", "totalDifficulty": "0x1", "transactionsRoot": "0x0767ed8359337dc6a8fdc77fe52db611bed1be87aac73c4556b1bf1dd3d190a5" diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json index f2affcc1c9..4dd5909159 100644 --- a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x97f540a3577c0f645c5dada5da86f38350e8f847e71f21124f917835003e2607", + "hash": "0xa063415a5020f1569fae73ecb0d37bc5649ebe86d59e764a389eb37814bd42cb", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0xa", - "parentHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "parentHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xbb62872e4023fa8a8b17b9cc37031f4817d9595779748d01cba408b495707a91", + "stateRoot": "0x118f1433ae23c4d1c12f5bd652baddb72611c55ac1cd6af6620d209db222f9e6", "timestamp": "0x64", "totalDifficulty": "0x1", "transactionsRoot": "0xb0893d21a4a44dc26a962a6e91abae66df87fb61ac9c60e936aee89c76331445" diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json index dc61aa9a2e..1bd68888b6 100644 --- a/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json @@ -4,7 +4,7 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x0", - "hash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "hash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -13,7 +13,7 @@ "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "stateRoot": "0xd883f48b83cc9c1e8389453beb4ad4e572462eec049ca4fffbe16ecefb3fe937", "timestamp": "0x0", "totalDifficulty": "0x1", "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json index c1dc70f64f..cf662cad75 100644 --- a/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x0da274b315de8e4d5bf8717218ec43540464ef36378cb896469bb731e1d3f3cb", + "hash": "0xeeb5c1852740ca4bbe65b0f57baf80634ed12a2b44affe30eec3fb54437c3926", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x1", - "parentHash": "0xbdc7d83b8f876938810462fe8d053263a482e44201e3883d4ae204ff4de7eff5", + "parentHash": "0x98e056de84de969782b238b4509b32814627ba443ea622054a79c2bc7e4d92c7", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x92c5c55a698963f5b06e3aee415630f5c48b0760e537af94917ce9c4f42a2e22", + "stateRoot": "0x4acfcd1a6ab9f5e62411021ecd8a749976ae50b0590e967471264b372d7ac55b", "timestamp": "0xa", "totalDifficulty": "0x1", "transactionsRoot": "0xca0ebcce920d2cdfbf9e1dbe90ed3441a1a576f344bd80e60508da814916f4e7" diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json index a63ff86700..4721dd1e7a 100644 --- a/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "hash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0x9", - "parentHash": "0x5abd19c39d9f1c6e52998e135ea14e1fbc5db3fa2a108f4538e238ca5c2e68d7", + "parentHash": "0xcd7d78eaa8b0ddbd2956fc37e1883c30df27b43e8cc9a982020310656736637c", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xbd4aa2c2873df709151075250a8c01c9a14d2b0e2f715dbdd16e0ef8030c2cf0", + "stateRoot": "0x78b2b19ef1a0276dbbc23a875dbf60ae5d10dafa0017098473c4871abd3e7b5c", "timestamp": "0x5a", "totalDifficulty": "0x1", "transactionsRoot": "0x0767ed8359337dc6a8fdc77fe52db611bed1be87aac73c4556b1bf1dd3d190a5" diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json index f2affcc1c9..4dd5909159 100644 --- a/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json @@ -4,16 +4,16 @@ "extraData": "0x", "gasLimit": "0x47e7c4", "gasUsed": "0x5208", - "hash": "0x97f540a3577c0f645c5dada5da86f38350e8f847e71f21124f917835003e2607", + "hash": "0xa063415a5020f1569fae73ecb0d37bc5649ebe86d59e764a389eb37814bd42cb", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "miner": "0x0000000000000000000000000000000000000000", "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", "nonce": "0x0000000000000000", "number": "0xa", - "parentHash": "0xda97ed946e0d502fb898b0ac881bd44da3c7fee5eaf184431e1ec3d361dad17e", + "parentHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xbb62872e4023fa8a8b17b9cc37031f4817d9595779748d01cba408b495707a91", + "stateRoot": "0x118f1433ae23c4d1c12f5bd652baddb72611c55ac1cd6af6620d209db222f9e6", "timestamp": "0x64", "totalDifficulty": "0x1", "transactionsRoot": "0xb0893d21a4a44dc26a962a6e91abae66df87fb61ac9c60e936aee89c76331445" diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json index c3a4a0deee..58f5657429 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json @@ -1,7 +1,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0xe724dfd4349861f4dceef2bc4df086d0a3d88858214f6bee9fcf1bebd1edc2a6", + "blockHash": "0xd1392771155ce83f6403c6af275efd22bed567030c21168fcc9dbad5004eb245", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json index ad6d6152ec..48aa567f23 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json @@ -1,5 +1,5 @@ { - "blockHash": "0x1e7dcf3abe8bf05d32367a5dc387caa32578b15871bf8b3cbeedf2d8d530f844", + "blockHash": "0x56ea26cf955d7f2e08e194ad212ca4d5f99ee8e0b19dec3c71d8faafa33b1d22", "blockNumber": "0x2", "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", "cumulativeGasUsed": "0xcf50", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json index b3362260a0..a679972b8e 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json @@ -1,5 +1,5 @@ { - "blockHash": "0x3fadc5bc916018a326732be829a2565b3acb960a8406f0f151a5e1fa971ea7dd", + "blockHash": "0x69bf6ba924d95b6c50b0357768e5c892bd1b00cdf2f97e2e81fc06a76dfa57e3", "blockNumber": "0x5", "contractAddress": "0xfdaa97661a584d977b4d3abb5370766ff5b86a18", "cumulativeGasUsed": "0xe01c", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json index cc0be1809e..1cd5656d6f 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json @@ -1,5 +1,5 @@ { - "blockHash": "0xffa737e6ce9a9162ffd411dd06169114b3ed5ee9fc1474a2625c92548e4455e0", + "blockHash": "0xf41e7a7a716382f20464cf76c6ae1fa701e9d32f5cc550ebfd2391b9642ae6bc", "blockNumber": "0x4", "contractAddress": null, "cumulativeGasUsed": "0x538d", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json index d3b6ef1c91..2400bd8252 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json @@ -1,5 +1,5 @@ { - "blockHash": "0xa8a067b3cb3b9ddc6cfb8317bfd08b266fcf9994fc870c1f7ed394acecfadf39", + "blockHash": "0x797d0c5603eccb33cc8ebd1300e977746512ec49e6b89087c7aad28ff760a26f", "blockNumber": "0x1", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json index 45a4f6d670..596bcdaa0d 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json @@ -1,5 +1,5 @@ { - "blockHash": "0x173dcd9d22ce71929cd17e84ea88702a0f84d6244c6898d2a4f48722e494fe9c", + "blockHash": "0xa1410af902e98b32e0bbe464f8637ff464f1d4344b585127d2ce71f9cb39cb8a", "blockNumber": "0x3", "contractAddress": null, "cumulativeGasUsed": "0x5e28", @@ -18,7 +18,7 @@ "blockNumber": "0x3", "transactionHash": "0xeaf3921cbf03ba45bad4e6ab807b196ce3b2a0b5bacc355b6272fa96b11b4287", "transactionIndex": "0x0", - "blockHash": "0x173dcd9d22ce71929cd17e84ea88702a0f84d6244c6898d2a4f48722e494fe9c", + "blockHash": "0xa1410af902e98b32e0bbe464f8637ff464f1d4344b585127d2ce71f9cb39cb8a", "logIndex": "0x0", "removed": false } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 75dbe38a59..bae1c68641 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -19,6 +19,7 @@ package ethapi import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "math/big" @@ -29,11 +30,17 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/holiman/uint256" ) +var ( + maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob +) + // TransactionArgs represents the arguments to construct a new transaction // or a message call. type TransactionArgs struct { @@ -56,9 +63,17 @@ type TransactionArgs struct { AccessList *types.AccessList `json:"accessList,omitempty"` ChainID *hexutil.Big `json:"chainId,omitempty"` - // Introduced by EIP-4844. + // For BlobTxType BlobFeeCap *hexutil.Big `json:"maxFeePerBlobGas"` BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + + // For BlobTxType transactions with blob sidecar + Blobs []kzg4844.Blob `json:"blobs"` + Commitments []kzg4844.Commitment `json:"commitments"` + Proofs []kzg4844.Proof `json:"proofs"` + + // This configures whether blobs are allowed to be passed. + blobSidecarAllowed bool } // from retrieves the transaction sender address. @@ -81,10 +96,14 @@ func (args *TransactionArgs) data() []byte { } // setDefaults fills in default values for unspecified tx fields. -func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGasEstimation bool) error { + if err := args.setBlobTxSidecar(ctx, b); err != nil { + return err + } if err := args.setFeeDefaults(ctx, b); err != nil { return err } + if args.Value == nil { args.Value = new(hexutil.Big) } @@ -98,38 +117,58 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) } - if args.BlobHashes != nil && args.To == nil { - return errors.New(`blob transactions cannot have the form of a create transaction`) - } + + // BlobTx fields if args.BlobHashes != nil && len(args.BlobHashes) == 0 { return errors.New(`need at least 1 blob for a blob transaction`) } - if args.To == nil && len(args.data()) == 0 { - return errors.New(`contract creation without any data provided`) + if args.BlobHashes != nil && len(args.BlobHashes) > maxBlobsPerTransaction { + return fmt.Errorf(`too many blobs in transaction (have=%d, max=%d)`, len(args.BlobHashes), maxBlobsPerTransaction) } - // Estimate the gas usage if necessary. - if args.Gas == nil { - // These fields are immutable during the estimation, safe to - // pass the pointer directly. - data := args.data() - callArgs := TransactionArgs{ - From: args.From, - To: args.To, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: args.Value, - Data: (*hexutil.Bytes)(&data), - AccessList: args.AccessList, + + // create check + if args.To == nil { + if args.BlobHashes != nil { + return errors.New(`missing "to" in blob transaction`) } - latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) - estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) - if err != nil { - return err + if len(args.data()) == 0 { + return errors.New(`contract creation without any data provided`) + } + } + + if args.Gas == nil { + if skipGasEstimation { // Skip gas usage estimation if a precise gas limit is not critical, e.g., in non-transaction calls. + gas := hexutil.Uint64(b.RPCGasCap()) + if gas == 0 { + gas = hexutil.Uint64(math.MaxUint64 / 2) + } + args.Gas = &gas + } else { // Estimate the gas usage otherwise. + // These fields are immutable during the estimation, safe to + // pass the pointer directly. + data := args.data() + callArgs := TransactionArgs{ + From: args.From, + To: args.To, + GasPrice: args.GasPrice, + MaxFeePerGas: args.MaxFeePerGas, + MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, + Value: args.Value, + Data: (*hexutil.Bytes)(&data), + AccessList: args.AccessList, + BlobFeeCap: args.BlobFeeCap, + BlobHashes: args.BlobHashes, + } + latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) + if err != nil { + return err + } + args.Gas = &estimated + log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - args.Gas = &estimated - log.Trace("Estimate gas usage automatically", "gas", args.Gas) } + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local // chain id as the default. want := b.ChainConfig().ChainID @@ -145,6 +184,14 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { // setFeeDefaults fills in default fee values for unspecified tx fields. func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + head := b.CurrentHeader() + // Sanity check the EIP-4844 fee parameters. + if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { + return errors.New("maxFeePerBlobGas, if specified, must be non-zero") + } + if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { + return err + } // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") @@ -154,7 +201,6 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro // other tx values. See https://github.com/ethereum/go-ethereum/pull/23274 // for more information. eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil - // Sanity check the EIP-1559 fee parameters if present. if args.GasPrice == nil && eip1559ParamsSet { if args.MaxFeePerGas.ToInt().Sign() == 0 { @@ -165,12 +211,8 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } - // Sanity check the EIP-4844 fee parameters. - if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { - return errors.New("maxFeePerBlobGas must be non-zero") - } + // Sanity check the non-EIP-1559 fee parameters. - head := b.CurrentHeader() isLondon := b.ChainConfig().IsLondon(head.Number) if args.GasPrice != nil && !eip1559ParamsSet { // Zero gas-price is not allowed after London fork @@ -181,21 +223,14 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } // Now attempt to fill in default value depending on whether London is active or not. - if b.ChainConfig().IsCancun(head.Number, head.Time) { - if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { - return err - } - } else if isLondon { - if args.BlobFeeCap != nil { - return errors.New("maxFeePerBlobGas is not valid before Cancun is active") - } + if isLondon { // London is active, set maxPriorityFeePerGas and maxFeePerGas. if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { return err } } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil || args.BlobFeeCap != nil { - return errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active") + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") } // London not active, set gas price. price, err := b.SuggestGasTipCap(ctx) @@ -211,15 +246,19 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro func (args *TransactionArgs) setCancunFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { // Set maxFeePerBlobGas if it is missing. if args.BlobHashes != nil && args.BlobFeeCap == nil { + var excessBlobGas uint64 + if head.ExcessBlobGas != nil { + excessBlobGas = *head.ExcessBlobGas + } // ExcessBlobGas must be set for a Cancun block. - blobBaseFee := eip4844.CalcBlobFee(*head.ExcessBlobGas) + blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) // Set the max fee to be 2 times larger than the previous block's blob base fee. // The additional slack allows the tx to not become invalidated if the base // fee is rising. val := new(big.Int).Mul(blobBaseFee, big.NewInt(2)) args.BlobFeeCap = (*hexutil.Big)(val) } - return args.setLondonFeeDefaults(ctx, head, b) + return nil } // setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. @@ -250,6 +289,81 @@ func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *typ return nil } +// setBlobTxSidecar adds the blob tx +func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) error { + // No blobs, we're done. + if args.Blobs == nil { + return nil + } + + // Passing blobs is not allowed in all contexts, only in specific methods. + if !args.blobSidecarAllowed { + return errors.New(`"blobs" is not supported for this RPC method`) + } + + n := len(args.Blobs) + // Assume user provides either only blobs (w/o hashes), or + // blobs together with commitments and proofs. + if args.Commitments == nil && args.Proofs != nil { + return errors.New(`blob proofs provided while commitments were not`) + } else if args.Commitments != nil && args.Proofs == nil { + return errors.New(`blob commitments provided while proofs were not`) + } + + // len(blobs) == len(commitments) == len(proofs) == len(hashes) + if args.Commitments != nil && len(args.Commitments) != n { + return fmt.Errorf("number of blobs and commitments mismatch (have=%d, want=%d)", len(args.Commitments), n) + } + if args.Proofs != nil && len(args.Proofs) != n { + return fmt.Errorf("number of blobs and proofs mismatch (have=%d, want=%d)", len(args.Proofs), n) + } + if args.BlobHashes != nil && len(args.BlobHashes) != n { + return fmt.Errorf("number of blobs and hashes mismatch (have=%d, want=%d)", len(args.BlobHashes), n) + } + + if args.Commitments == nil { + // Generate commitment and proof. + commitments := make([]kzg4844.Commitment, n) + proofs := make([]kzg4844.Proof, n) + for i, b := range args.Blobs { + c, err := kzg4844.BlobToCommitment(b) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) + } + commitments[i] = c + p, err := kzg4844.ComputeBlobProof(b, c) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) + } + proofs[i] = p + } + args.Commitments = commitments + args.Proofs = proofs + } else { + for i, b := range args.Blobs { + if err := kzg4844.VerifyBlobProof(b, args.Commitments[i], args.Proofs[i]); err != nil { + return fmt.Errorf("failed to verify blob proof: %v", err) + } + } + } + + hashes := make([]common.Hash, n) + hasher := sha256.New() + for i, c := range args.Commitments { + hashes[i] = kzg4844.CalcBlobHashV1(hasher, &c) + } + if args.BlobHashes != nil { + for i, h := range hashes { + if h != args.BlobHashes[i] { + return fmt.Errorf("blob hash verification failed (have=%s, want=%s)", args.BlobHashes[i], h) + } + } + } else { + args.BlobHashes = hashes + } + return nil +} + // ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. @@ -363,6 +477,14 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { BlobHashes: args.BlobHashes, BlobFeeCap: uint256.MustFromBig((*big.Int)(args.BlobFeeCap)), } + if args.Blobs != nil { + data.(*types.BlobTx).Sidecar = &types.BlobTxSidecar{ + Blobs: args.Blobs, + Commitments: args.Commitments, + Proofs: args.Proofs, + } + } + case args.MaxFeePerGas != nil: al := types.AccessList{} if args.AccessList != nil { @@ -379,6 +501,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: al, } + case args.AccessList != nil: data = &types.AccessListTx{ To: args.To, @@ -390,6 +513,7 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { Data: args.data(), AccessList: *args.AccessList, } + default: data = &types.LegacyTx{ To: args.To, @@ -403,12 +527,6 @@ func (args *TransactionArgs) toTransaction() *types.Transaction { return types.NewTx(data) } -// ToTransaction converts the arguments to a transaction. -// This assumes that setDefaults has been called. -func (args *TransactionArgs) ToTransaction() *types.Transaction { - return args.toTransaction() -} - // IsEIP4844 returns an indicator if the args contains EIP4844 fields. func (args *TransactionArgs) IsEIP4844() bool { return args.BlobHashes != nil || args.BlobFeeCap != nil diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index ad010d9ae8..8608821d24 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -153,14 +153,14 @@ func TestSetFeeDefaults(t *testing.T) { "legacy", &TransactionArgs{MaxFeePerGas: maxFee}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx pre-London, priorityFee set", "legacy", &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx, maxFee < priorityFee", @@ -207,20 +207,6 @@ func TestSetFeeDefaults(t *testing.T) { errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), }, // EIP-4844 - { - "set maxFeePerBlobGas pre cancun", - "london", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerBlobGas is not valid before Cancun is active"), - }, - { - "set maxFeePerBlobGas pre london", - "legacy", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), - }, { "set gas price and maxFee for blob transaction", "cancun", @@ -235,6 +221,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, nil, }, + { + "fill maxFeePerBlobGas when dynamic fees are set", + "cancun", + &TransactionArgs{BlobHashes: []common.Hash{}, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, } ctx := context.Background() @@ -244,11 +237,16 @@ func TestSetFeeDefaults(t *testing.T) { } got := test.in err := got.setFeeDefaults(ctx, b) - if err != nil && err.Error() == test.err.Error() { - // Test threw expected error. + if err != nil { + if test.err == nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if err.Error() != test.err.Error() { + t.Fatalf("test %d (%s): unexpected error: (got: %s, want: %s)", i, test.name, err, test.err) + } + // Matching error. continue - } else if err != nil { - t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if test.err != nil { + t.Fatalf("test %d (%s): expected error: %s", i, test.name, test.err) } if !reflect.DeepEqual(got, test.want) { t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) @@ -390,8 +388,8 @@ func (b *backendMock) SendBundle(ctx context.Context, bundle *types.Bundle, orig return nil } func (b *backendMock) BundlePrice() *big.Int { return nil } -func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - return nil, [32]byte{}, 0, 0, nil +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) { + return false, nil, [32]byte{}, 0, 0, nil } func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 69e9743556..bf62c53adf 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -256,7 +256,8 @@ type BigFlag struct { Hidden bool HasBeenSet bool - Value *big.Int + Value *big.Int + defaultValue *big.Int Aliases []string EnvVars []string @@ -269,6 +270,10 @@ func (f *BigFlag) IsSet() bool { return f.HasBeenSet } func (f *BigFlag) String() string { return cli.FlagStringer(f) } func (f *BigFlag) Apply(set *flag.FlagSet) error { + // Set default value so that environment wont be able to overwrite it + if f.Value != nil { + f.defaultValue = new(big.Int).Set(f.Value) + } for _, envVar := range f.EnvVars { envVar = strings.TrimSpace(envVar) if value, found := syscall.Getenv(envVar); found { @@ -283,7 +288,6 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { f.Value = new(big.Int) set.Var((*bigValue)(f.Value), f.Name, f.Usage) }) - return nil } @@ -310,7 +314,7 @@ func (f *BigFlag) GetDefaultText() string { if f.DefaultText != "" { return f.DefaultText } - return f.GetValue() + return f.defaultValue.String() } // bigValue turns *big.Int into a flag.Value diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index d9d1f79036..0112724fa1 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -41,7 +41,7 @@ func NewApp(usage string) *cli.App { app.EnableBashCompletion = true app.Version = params.VersionWithCommit(git.Commit, git.Date) app.Usage = usage - app.Copyright = "Copyright 2013-2023 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2024 The go-ethereum Authors" app.Before = func(ctx *cli.Context) error { MigrateGlobalFlags(ctx) return nil @@ -115,7 +115,7 @@ func doMigrateFlags(ctx *cli.Context) { for _, parent := range ctx.Lineage()[1:] { if parent.IsSet(name) { // When iterating across the lineage, we will be served both - // the 'canon' and alias formats of all commmands. In most cases, + // the 'canon' and alias formats of all commands. In most cases, // it's fine to set it in the ctx multiple times (one for each // name), however, the Slice-flags are not fine. // The slice-flags accumulate, so if we set it once as diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index f23c65584c..0b360e7415 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -2031,7 +2031,7 @@ var fromAscii = function(str) { * * @method transformToFullName * @param {Object} json-abi - * @return {String} full fnction/event name + * @return {String} full function/event name */ var transformToFullName = function (json) { if (json.name.indexOf('(') !== -1) { @@ -2361,7 +2361,7 @@ var isFunction = function (object) { }; /** - * Returns true if object is Objet, otherwise false + * Returns true if object is Object, otherwise false * * @method isObject * @param {Object} @@ -2757,7 +2757,7 @@ var Batch = function (web3) { * Should be called to add create new request to batch request * * @method add - * @param {Object} jsonrpc requet object + * @param {Object} jsonrpc request object */ Batch.prototype.add = function (request) { this.requests.push(request); @@ -3961,6 +3961,8 @@ var outputSyncingFormatter = function(result) { result.healedBytecodeBytes = utils.toDecimal(result.healedBytecodeBytes); result.healingTrienodes = utils.toDecimal(result.healingTrienodes); result.healingBytecode = utils.toDecimal(result.healingBytecode); + result.txIndexFinishedBlocks = utils.toDecimal(result.txIndexFinishedBlocks); + result.txIndexRemainingBlocks = utils.toDecimal(result.txIndexRemainingBlocks); return result; }; @@ -4557,7 +4559,7 @@ Iban.createIndirect = function (options) { }; /** - * Thos method should be used to check if given string is valid iban object + * This method should be used to check if given string is valid iban object * * @method isValid * @param {String} iban string @@ -6706,7 +6708,7 @@ var exchangeAbi = require('../contracts/SmartExchange.json'); * @method transfer * @param {String} from * @param {String} to iban - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transfer = function (eth, from, to, value, callback) { @@ -6736,7 +6738,7 @@ var transfer = function (eth, from, to, value, callback) { * @method transferToAddress * @param {String} from * @param {String} to - * @param {Value} value to be tranfered + * @param {Value} value to be transferred * @param {Function} callback, callback */ var transferToAddress = function (eth, from, to, value, callback) { @@ -7090,7 +7092,7 @@ module.exports = transfer; /** * Initializes a newly created cipher. * - * @param {number} xformMode Either the encryption or decryption transormation mode constant. + * @param {number} xformMode Either the encryption or decryption transformation mode constant. * @param {WordArray} key The key. * @param {Object} cfg (Optional) The configuration options to use for this operation. * @@ -9444,7 +9446,7 @@ module.exports = transfer; var M_offset_14 = M[offset + 14]; var M_offset_15 = M[offset + 15]; - // Working varialbes + // Working variables var a = H[0]; var b = H[1]; var c = H[2]; diff --git a/internal/testrand/rand.go b/internal/testrand/rand.go new file mode 100644 index 0000000000..690993de05 --- /dev/null +++ b/internal/testrand/rand.go @@ -0,0 +1,53 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testrand + +import ( + crand "crypto/rand" + "encoding/binary" + mrand "math/rand" + + "github.com/ethereum/go-ethereum/common" +) + +// prng is a pseudo random number generator seeded by strong randomness. +// The randomness is printed on startup in order to make failures reproducible. +var prng = initRand() + +func initRand() *mrand.Rand { + var seed [8]byte + crand.Read(seed[:]) + rnd := mrand.New(mrand.NewSource(int64(binary.LittleEndian.Uint64(seed[:])))) + return rnd +} + +// Bytes generates a random byte slice with specified length. +func Bytes(n int) []byte { + r := make([]byte, n) + prng.Read(r) + return r +} + +// Hash generates a random hash. +func Hash() common.Hash { + return common.BytesToHash(Bytes(common.HashLength)) +} + +// Address generates a random address. +func Address() common.Address { + return common.BytesToAddress(Bytes(common.AddressLength)) +} diff --git a/internal/utesting/utesting.go b/internal/utesting/utesting.go index ee99794c64..8260de1d76 100644 --- a/internal/utesting/utesting.go +++ b/internal/utesting/utesting.go @@ -35,6 +35,7 @@ import ( type Test struct { Name string Fn func(*T) + Slow bool } // Result is the result of a test execution. diff --git a/log/handler.go b/log/handler.go index 7459aad891..256b57f2d0 100644 --- a/log/handler.go +++ b/log/handler.go @@ -117,6 +117,7 @@ func (l *leveler) Level() slog.Level { func JSONHandler(wr io.Writer) slog.Handler { return slog.NewJSONHandler(wr, &slog.HandlerOptions{ ReplaceAttr: builtinReplaceJSON, + Level: &leveler{levelMaxVerbosity}, }) } diff --git a/log/handler_glog.go b/log/handler_glog.go index fb1e03c5b5..f51bae2a4a 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -192,7 +192,7 @@ func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error { frame, _ := fs.Next() for _, rule := range h.patterns { - if rule.pattern.MatchString(fmt.Sprintf("%+s", frame.File)) { + if rule.pattern.MatchString(fmt.Sprintf("+%s", frame.File)) { h.siteCache[r.PC], lvl, ok = rule.level, rule.level, true } } diff --git a/log/logger.go b/log/logger.go index 20130146a5..c28bbde568 100644 --- a/log/logger.go +++ b/log/logger.go @@ -83,7 +83,7 @@ func LevelAlignedString(l slog.Level) string { } } -// LevelString returns a 5-character string containing the name of a Lvl. +// LevelString returns a string containing the name of a Lvl. func LevelString(l slog.Level) string { switch l { case LevelTrace: @@ -95,7 +95,7 @@ func LevelString(l slog.Level) string { case slog.LevelWarn: return "warn" case slog.LevelError: - return "eror" + return "error" case LevelCrit: return "crit" default: diff --git a/log/logger_test.go b/log/logger_test.go index a633f5ad7a..ff981fd018 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -2,6 +2,7 @@ package log import ( "bytes" + "errors" "fmt" "io" "math/big" @@ -77,7 +78,7 @@ func benchmarkLogger(b *testing.B, l Logger) { tt = time.Now() bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") ) b.ReportAllocs() b.ResetTimer() @@ -106,7 +107,7 @@ func TestLoggerOutput(t *testing.T) { tt = time.Time{} bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) diff --git a/log/root.go b/log/root.go index 71040fff47..8662d87063 100644 --- a/log/root.go +++ b/log/root.go @@ -10,8 +10,7 @@ import ( var root atomic.Value func init() { - defaultLogger := &logger{slog.New(DiscardHandler())} - SetDefault(defaultLogger) + root.Store(&logger{slog.New(DiscardHandler())}) } // SetDefault sets the default global logger diff --git a/metrics/counter.go b/metrics/counter.go index cb81599c21..dbe8e16a90 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -8,7 +8,7 @@ type CounterSnapshot interface { Count() int64 } -// Counters hold an int64 value that can be incremented and decremented. +// Counter hold an int64 value that can be incremented and decremented. type Counter interface { Clear() Dec(int64) diff --git a/metrics/gauge.go b/metrics/gauge.go index 68f8f11abc..5933df3107 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -2,12 +2,12 @@ package metrics import "sync/atomic" -// gaugeSnapshot contains a readonly int64. +// GaugeSnapshot contains a readonly int64. type GaugeSnapshot interface { Value() int64 } -// Gauges hold an int64 value that can be set arbitrarily. +// Gauge holds an int64 value that can be set arbitrarily. type Gauge interface { Snapshot() GaugeSnapshot Update(int64) @@ -74,7 +74,7 @@ func (g *StandardGauge) Update(v int64) { g.value.Store(v) } -// Update updates the gauge's value if v is larger then the current valie. +// Update updates the gauge's value if v is larger then the current value. func (g *StandardGauge) UpdateIfGt(v int64) { for { exist := g.value.Load() diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 967f2bc60e..c1c3c6b6e6 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -48,7 +48,7 @@ type gaugeFloat64Snapshot float64 // Value returns the value at the time the snapshot was taken. func (g gaugeFloat64Snapshot) Value() float64 { return float64(g) } -// NilGauge is a no-op Gauge. +// NilGaugeFloat64 is a no-op Gauge. type NilGaugeFloat64 struct{} func (NilGaugeFloat64) Snapshot() GaugeFloat64Snapshot { return NilGaugeFloat64{} } diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index c44b2d85f3..0010edc324 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -9,7 +9,7 @@ type GaugeInfoSnapshot interface { Value() GaugeInfoValue } -// GaugeInfos hold a GaugeInfoValue value that can be set arbitrarily. +// GaugeInfo holds a GaugeInfoValue value that can be set arbitrarily. type GaugeInfo interface { Update(GaugeInfoValue) Snapshot() GaugeInfoSnapshot diff --git a/metrics/healthcheck.go b/metrics/healthcheck.go index f1ae31e34a..adcd15ab58 100644 --- a/metrics/healthcheck.go +++ b/metrics/healthcheck.go @@ -1,6 +1,6 @@ package metrics -// Healthchecks hold an error value describing an arbitrary up/down status. +// Healthcheck holds an error value describing an arbitrary up/down status. type Healthcheck interface { Check() Error() error diff --git a/metrics/histogram.go b/metrics/histogram.go index 44de588bc1..10259a2463 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -4,7 +4,7 @@ type HistogramSnapshot interface { SampleSnapshot } -// Histograms calculate distribution statistics from a series of int64 values. +// Histogram calculates distribution statistics from a series of int64 values. type Histogram interface { Clear() Update(int64) diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 0be5137d5e..114d57ae07 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -25,7 +25,7 @@ type v2Reporter struct { write api.WriteAPI } -// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +// InfluxDBV2WithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, token string, bucket string, organization string, namespace string, tags map[string]string) { rep := &v2Reporter{ reg: r, diff --git a/miner/miner.go b/miner/miner.go index 2174d6ca81..b65b226238 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -103,7 +103,8 @@ type Config struct { NewPayloadTimeout time.Duration // The maximum time allowance for creating a new payload - RollupComputePendingBlock bool // Compute the pending block from tx-pool, instead of copying the latest-block + RollupComputePendingBlock bool // Compute the pending block from tx-pool, instead of copying the latest-block + EffectiveGasCeil uint64 // if non-zero, a gas ceiling to apply independent of the header's gaslimit value Mev MevConfig // Mev configuration } @@ -111,7 +112,7 @@ type Config struct { // DefaultConfig contains default settings for miner. var DefaultConfig = Config{ GasCeil: 30000000, - GasPrice: big.NewInt(params.GWei), + GasPrice: big.NewInt(params.Wei), // The default recommit time is chosen as two seconds since // consensus-layer usually will wait a half slot of time(6s) @@ -252,6 +253,11 @@ func (miner *Miner) SetExtra(extra []byte) error { return nil } +func (miner *Miner) SetGasTip(tip *big.Int) error { + miner.worker.setGasTip(tip) + return nil +} + // SetRecommitInterval sets the interval for sealing work resubmitting. func (miner *Miner) SetRecommitInterval(interval time.Duration) { miner.worker.setRecommitInterval(interval) diff --git a/miner/miner_test.go b/miner/miner_test.go index 411d6026ce..5907fb4464 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) type mockBackend struct { @@ -279,7 +280,7 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address GasLimit: gasLimit, BaseFee: big.NewInt(params.InitialBaseFee), Difficulty: big.NewInt(1), - Alloc: map[common.Address]core.GenesisAccount{ + Alloc: map[common.Address]types.Account{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD @@ -300,7 +301,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(chainDB, nil) + triedb := triedb.NewDatabase(chainDB, nil) genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) chainConfig, _, err := core.SetupGenesisBlock(chainDB, triedb, genesis) if err != nil { @@ -317,7 +318,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) - txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain, []txpool.SubPool{pool}) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, blockchain, []txpool.SubPool{pool}) backend := NewMockBackend(bc, txpool) // Create event Mux diff --git a/miner/ordering.go b/miner/ordering.go index 4c3055f0d3..bcf7af46e8 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -21,28 +21,31 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) // txWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap type txWithMinerFee struct { tx *txpool.LazyTransaction from common.Address - fees *big.Int + fees *uint256.Int } // newTxWithMinerFee creates a wrapped transaction, calculating the effective // miner gasTipCap if a base fee is provided. // Returns error in case of a negative effective miner gasTipCap. -func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *big.Int) (*txWithMinerFee, error) { - tip := new(big.Int).Set(tx.GasTipCap) +func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *uint256.Int) (*txWithMinerFee, error) { + tip := new(uint256.Int).Set(tx.GasTipCap) if baseFee != nil { if tx.GasFeeCap.Cmp(baseFee) < 0 { return nil, types.ErrGasFeeCapTooLow } - tip = math.BigMin(tx.GasTipCap, new(big.Int).Sub(tx.GasFeeCap, baseFee)) + tip = new(uint256.Int).Sub(tx.GasFeeCap, baseFee) + if tip.Gt(tx.GasTipCap) { + tip = tx.GasTipCap + } } return &txWithMinerFee{ tx: tx, @@ -87,7 +90,7 @@ type transactionsByPriceAndNonce struct { txs map[common.Address][]*txpool.LazyTransaction // Per account nonce-sorted list of transactions heads txByPriceAndTime // Next transaction for each unique account (price heap) signer types.Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + baseFee *uint256.Int // Current base fee } // newTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -96,10 +99,15 @@ type transactionsByPriceAndNonce struct { // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address][]*txpool.LazyTransaction, baseFee *big.Int) *transactionsByPriceAndNonce { + // Convert the basefee from header format to uint256 format + var baseFeeUint *uint256.Int + if baseFee != nil { + baseFeeUint = uint256.MustFromBig(baseFee) + } // Initialize a price and received time based heap with the head transactions heads := make(txByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFee) + wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFeeUint) if err != nil { delete(txs, from) continue @@ -114,16 +122,16 @@ func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address] txs: txs, heads: heads, signer: signer, - baseFee: baseFee, + baseFee: baseFeeUint, } } // Peek returns the next transaction by price. -func (t *transactionsByPriceAndNonce) Peek() *txpool.LazyTransaction { +func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *uint256.Int) { if len(t.heads) == 0 { - return nil + return nil, nil } - return t.heads[0].tx + return t.heads[0].tx, t.heads[0].fees } // Shift replaces the current best head with the next one from the same account. @@ -145,3 +153,14 @@ func (t *transactionsByPriceAndNonce) Shift() { func (t *transactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } + +// Empty returns if the price heap is empty. It can be used to check it simpler +// than calling peek and checking for nil return. +func (t *transactionsByPriceAndNonce) Empty() bool { + return len(t.heads) == 0 +} + +// Clear removes the entire content of the heap. +func (t *transactionsByPriceAndNonce) Clear() { + t.heads, t.txs = nil, nil +} diff --git a/miner/ordering_test.go b/miner/ordering_test.go index e5868d7a06..3587a835c8 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) func TestTransactionPriceNonceSortLegacy(t *testing.T) { @@ -92,8 +93,8 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -104,7 +105,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { txset := newTransactionsByPriceAndNonce(signer, groups, baseFee) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } @@ -160,8 +161,8 @@ func TestTransactionTimeSort(t *testing.T) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -170,7 +171,7 @@ func TestTransactionTimeSort(t *testing.T) { txset := newTransactionsByPriceAndNonce(signer, groups, nil) txs := types.Transactions{} - for tx := txset.Peek(); tx != nil; tx = txset.Peek() { + for tx, _ := txset.Peek(); tx != nil; tx, _ = txset.Peek() { txs = append(txs, tx.Tx) txset.Shift() } diff --git a/miner/payload_building.go b/miner/payload_building.go index f3b8382783..f65b0f0fe4 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -37,12 +37,13 @@ import ( // Check engine-api specification for more details. // https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#payloadattributesv3 type BuildPayloadArgs struct { - Parent common.Hash // The parent block to build payload on top - Timestamp uint64 // The provided timestamp of generated payload - FeeRecipient common.Address // The provided recipient address for collecting transaction fee - Random common.Hash // The provided randomness value - Withdrawals types.Withdrawals // The provided withdrawals - BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + Parent common.Hash // The parent block to build payload on top + Timestamp uint64 // The provided timestamp of generated payload + FeeRecipient common.Address // The provided recipient address for collecting transaction fee + Random common.Hash // The provided randomness value + Withdrawals types.Withdrawals // The provided withdrawals + BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + Version engine.PayloadVersion // Versioning byte for payload id calculation. NoTxPool bool // Optimism addition: option to disable tx pool contents from being included Transactions []*types.Transaction // Optimism addition: txs forced into the block via engine API @@ -76,6 +77,7 @@ func (args *BuildPayloadArgs) Id() engine.PayloadID { var out engine.PayloadID copy(out[:], hasher.Sum(nil)[:8]) + out[0] = byte(args.Version) return out } @@ -341,7 +343,7 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { log.Info("Stopping work on payload", "id", payload.id, "reason", stopReason, - "elapsed", time.Since(start).Milliseconds()) + "elapsed", common.PrettyDuration(time.Since(start))) }() updatePayload := func() time.Duration { diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 7eea54a25d..cb1220fbfa 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -78,16 +78,16 @@ func testBuildPayload(t *testing.T, noTxPool, interrupt bool) { } payload := outer.ExecutionPayload if payload.ParentHash != b.chain.CurrentBlock().Hash() { - t.Fatal("Unexpect parent hash") + t.Fatal("Unexpected parent hash") } if payload.Random != (common.Hash{}) { - t.Fatal("Unexpect random value") + t.Fatal("Unexpected random value") } if payload.Timestamp != timestamp { - t.Fatal("Unexpect timestamp") + t.Fatal("Unexpected timestamp") } if payload.FeeRecipient != recipient { - t.Fatal("Unexpect fee recipient") + t.Fatal("Unexpected fee recipient") } if !interrupt && len(payload.Transactions) != txs { t.Fatalf("Unexpect transaction set: got %d, expected %d", len(payload.Transactions), txs) diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go index 13336cd83c..6059393845 100644 --- a/miner/stress/clique/main.go +++ b/miner/stress/clique/main.go @@ -154,9 +154,9 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core genesis.Config.ChainID = big.NewInt(18) genesis.Config.Clique.Period = 1 - genesis.Alloc = core.GenesisAlloc{} + genesis.Alloc = types.GenesisAlloc{} for _, faucet := range faucets { - genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), } } diff --git a/miner/worker.go b/miner/worker.go index 292a2e9ad0..c5686f4d5d 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -42,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) const ( @@ -229,6 +230,7 @@ type worker struct { mu sync.RWMutex // The lock used to protect the coinbase and extra fields coinbase common.Address extra []byte + tip *uint256.Int // Minimum tip needed for non-local transaction to include them pendingMu sync.RWMutex pendingTasks map[common.Hash]*task @@ -278,6 +280,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus isLocalBlock: isLocalBlock, coinbase: config.Etherbase, extra: config.ExtraData, + tip: uint256.MustFromBig(config.GasPrice), pendingTasks: make(map[common.Hash]*task), txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), @@ -356,6 +359,13 @@ func (w *worker) setExtra(extra []byte) { w.extra = extra } +// setGasTip sets the minimum miner tip needed to include a non-local transaction. +func (w *worker) setGasTip(tip *big.Int) { + w.mu.Lock() + defer w.mu.Unlock() + w.tip = uint256.MustFromBig(tip) +} + // setRecommitInterval updates the interval for miner sealing work recommitting. func (w *worker) setRecommitInterval(interval time.Duration) { select { @@ -600,15 +610,17 @@ func (w *worker) mainLoop() { Hash: tx.Hash(), Tx: nil, // Do *not* set this! We need to resolve it later to pull blobs in Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) } - txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) + plainTxs := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) // Mixed bag of everrything, yolo + blobTxs := newTransactionsByPriceAndNonce(w.current.signer, nil, w.current.header.BaseFee) // Empty bag, don't bother optimising + tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil) + w.commitTransactions(w.current, plainTxs, blobTxs, nil) // Only update the snapshot if any new transactions were added // to the pending block @@ -886,7 +898,7 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { +func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) @@ -905,8 +917,33 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } + // If we don't have enough blob space for any further blob transactions, + // skip that list altogether + if !blobTxs.Empty() && env.blobs*params.BlobTxBlobGasPerBlob >= params.MaxBlobGasPerBlock { + log.Trace("Not enough blob space for further blob transactions") + blobTxs.Clear() + // Fall though to pick up any plain txs + } // Retrieve the next transaction and abort if all done. - ltx := txs.Peek() + var ( + ltx *txpool.LazyTransaction + txs *transactionsByPriceAndNonce + ) + pltx, ptip := plainTxs.Peek() + bltx, btip := blobTxs.Peek() + + switch { + case pltx == nil: + txs, ltx = blobTxs, bltx + case bltx == nil: + txs, ltx = plainTxs, pltx + default: + if ptip.Lt(btip) { + txs, ltx = blobTxs, bltx + } else { + txs, ltx = plainTxs, pltx + } + } if ltx == nil { break } @@ -990,7 +1027,7 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn // generateParams wraps various of settings for generating sealing task. type generateParams struct { - timestamp uint64 // The timstamp for sealing task + timestamp uint64 // The timestamp for sealing task forceTime bool // Flag whether the given timestamp is immutable or not parentHash common.Hash // Parent block hash, empty means the latest chain head coinbase common.Address // The fee recipient address for including transaction @@ -1137,33 +1174,59 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err } } + w.mu.RLock() + tip := w.tip + w.mu.RUnlock() + start := time.Now() - pending := w.eth.TxPool().Pending(true) + // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees + filter := txpool.PendingFilter{ + MinTip: tip, + } + if env.header.BaseFee != nil { + filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) + } + if env.header.ExcessBlobGas != nil { + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) + } + filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false + pendingPlainTxs := w.eth.TxPool().Pending(filter) + + filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true + pendingBlobTxs := w.eth.TxPool().Pending(filter) + packFromTxpoolTimer.UpdateSince(start) log.Debug("packFromTxpoolTimer", "duration", common.PrettyDuration(time.Since(start)), "hash", env.header.Hash()) // Split the pending transactions into locals and remotes. - localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending - - // TODO will remove after fix txpool perf issue - // for _, account := range w.eth.TxPool().Locals() { - // if txs := remoteTxs[account]; len(txs) > 0 { - // delete(remoteTxs, account) - // localTxs[account] = txs - // } - // } + localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs + for _, account := range w.eth.TxPool().Locals() { + if txs := remotePlainTxs[account]; len(txs) > 0 { + delete(remotePlainTxs, account) + localPlainTxs[account] = txs + } + if txs := remoteBlobTxs[account]; len(txs) > 0 { + delete(remoteBlobTxs, account) + localBlobTxs[account] = txs + } + } // Fill the block with all available pending transactions. start = time.Now() - if len(localTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } - if len(remoteTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } @@ -1186,7 +1249,11 @@ func (w *worker) generateWork(genParams *generateParams) *newPayloadResult { } defer work.discard() if work.gasPool == nil { - work.gasPool = new(core.GasPool).AddGas(work.header.GasLimit) + gasLimit := w.config.EffectiveGasCeil + if gasLimit == 0 || gasLimit > work.header.GasLimit { + gasLimit = work.header.GasLimit + } + work.gasPool = new(core.GasPool).AddGas(gasLimit) } if w.chainConfig.PreContractForkBlock != nil && work.header.Number.Cmp(w.chainConfig.PreContractForkBlock) == 0 { diff --git a/miner/worker_builder.go b/miner/worker_builder.go index 7f59c8659f..7233b3d15d 100644 --- a/miner/worker_builder.go +++ b/miner/worker_builder.go @@ -3,6 +3,8 @@ package miner import ( "errors" mapset "github.com/deckarep/golang-set/v2" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/holiman/uint256" "math/big" "slices" "sync" @@ -54,25 +56,59 @@ func (w *worker) fillTransactionsAndBundles(interrupt *atomic.Int32, env *enviro } log.Info("fill bundles", "bundles_count", len(bundles)) + w.mu.RLock() + tip := w.tip + w.mu.RUnlock() + start := time.Now() - pending := w.eth.TxPool().Pending(true) + // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees + filter := txpool.PendingFilter{ + MinTip: tip, + } + if env.header.BaseFee != nil { + filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) + } + if env.header.ExcessBlobGas != nil { + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) + } + filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false + pendingPlainTxs := w.eth.TxPool().Pending(filter) + + filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true + pendingBlobTxs := w.eth.TxPool().Pending(filter) + packFromTxpoolTimer.UpdateSince(start) log.Debug("packFromTxpoolTimer", "duration", common.PrettyDuration(time.Since(start)), "hash", env.header.Hash()) // Split the pending transactions into locals and remotes. - localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending + localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs + for _, account := range w.eth.TxPool().Locals() { + if txs := remotePlainTxs[account]; len(txs) > 0 { + delete(remotePlainTxs, account) + localPlainTxs[account] = txs + } + if txs := remoteBlobTxs[account]; len(txs) > 0 { + delete(remoteBlobTxs, account) + localBlobTxs[account] = txs + } + } // Fill the block with all available pending transactions. start = time.Now() - if len(localTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } - if len(remoteTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt); err != nil { + if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } diff --git a/miner/worker_test.go b/miner/worker_test.go index aa05565301..7a78b6898f 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" ) const ( @@ -116,7 +117,7 @@ type testWorkerBackend struct { func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { var gspec = &core.Genesis{ Config: chainConfig, - Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, } switch e := engine.(type) { case *clique.Clique: @@ -134,7 +135,7 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine t.Fatalf("core.NewBlockChain failed: %v", err) } pool := legacypool.New(testTxPoolConfig, chain) - txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), chain, []txpool.SubPool{pool}) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) return &testWorkerBackend{ db: db, @@ -230,7 +231,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens taskCh := make(chan struct{}, 2) checkEqual := func(t *testing.T, task *task) { // The work should contain 1 tx - receiptLen, balance := 1, big.NewInt(1000) + receiptLen, balance := 1, uint256.NewInt(1000) if len(task.receipts) != receiptLen { t.Fatalf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } diff --git a/node/defaults.go b/node/defaults.go index e7ec278723..fb53895b95 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,6 +41,7 @@ const ( // needs of all CLs. engineAPIBatchItemLimit = 2000 engineAPIBatchResponseSizeLimit = 250 * 1000 * 1000 + engineAPIBodyLimit = 128 * 1024 * 1024 ) var ( diff --git a/node/node.go b/node/node.go index d10639fe5b..4dc856c345 100644 --- a/node/node.go +++ b/node/node.go @@ -453,14 +453,16 @@ func (n *Node) startRPC() error { jwtSecret: secret, batchItemLimit: engineAPIBatchItemLimit, batchResponseSizeLimit: engineAPIBatchResponseSizeLimit, + httpBodyLimit: engineAPIBodyLimit, } - if err := server.enableRPC(allAPIs, httpConfig{ + err := server.enableRPC(allAPIs, httpConfig{ CorsAllowedOrigins: DefaultAuthCors, Vhosts: n.config.AuthVirtualHosts, Modules: DefaultAuthModules, prefix: DefaultAuthPrefix, rpcEndpointConfig: sharedConfig, - }); err != nil { + }) + if err != nil { return err } servers = append(servers, server) diff --git a/node/rpcstack.go b/node/rpcstack.go index b33c238051..d80d5271a7 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -56,6 +56,7 @@ type rpcEndpointConfig struct { jwtSecret []byte // optional JWT secret batchItemLimit int batchResponseSizeLimit int + httpBodyLimit int } type rpcHandler struct { @@ -304,6 +305,9 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } @@ -336,6 +340,9 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { // Create RPC server and handler. srv := rpc.NewServer() srv.SetBatchLimits(config.batchItemLimit, config.batchResponseSizeLimit) + if config.httpBodyLimit > 0 { + srv.SetHTTPBodyLimit(config.httpBodyLimit) + } if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index da8e9cb817..56aae24285 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -58,7 +58,7 @@ func newMeteredConn(conn UDPConn) UDPConn { return &meteredUdpConn{UDPConn: conn} } -// Read delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. +// ReadFromUDP delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. func (c *meteredUdpConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { n, addr, err = c.UDPConn.ReadFromUDP(b) ingressTrafficMeter.Mark(int64(n)) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 06b7681f18..7d9703a345 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -344,11 +344,11 @@ func parseLink(e string) (*linkEntry, error) { return nil, fmt.Errorf("wrong/missing scheme 'enrtree' in URL") } e = e[len(linkPrefix):] - pos := strings.IndexByte(e, '@') - if pos == -1 { + + keystring, domain, found := strings.Cut(e, "@") + if !found { return nil, entryError{"link", errNoPubkey} } - keystring, domain := e[:pos], e[pos+1:] keybytes, err := b32format.DecodeString(keystring) if err != nil { return nil, entryError{"link", errBadPubkey} diff --git a/p2p/server.go b/p2p/server.go index 7975b787bd..b398c51eb4 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -915,13 +915,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { } // Reject connections that do not match NetRestrict. if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not in netrestrict list") + return errors.New("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { - return fmt.Errorf("too many attempts") + return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) return nil diff --git a/p2p/server_nat.go b/p2p/server_nat.go index 354597cc7a..299d275490 100644 --- a/p2p/server_nat.go +++ b/p2p/server_nat.go @@ -127,7 +127,7 @@ func (srv *Server) portMappingLoop() { } else if !ip.Equal(lastExtIP) { log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) } else { - return + continue } // Here, we either failed to get the external IP, or it has changed. lastExtIP = ip diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index c52917fd0a..349e496b2f 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -172,7 +172,7 @@ type SimNode struct { registerOnce sync.Once } -// Close closes the underlaying node.Node to release +// Close closes the underlying node.Node to release // acquired resources. func (sn *SimNode) Close() error { return sn.node.Close() diff --git a/p2p/transport.go b/p2p/transport.go index 4f6bb569bf..5fc7686feb 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -19,6 +19,7 @@ package p2p import ( "bytes" "crypto/ecdsa" + "errors" "fmt" "io" "net" @@ -157,7 +158,7 @@ func readProtocolHandshake(rw MsgReader) (*protoHandshake, error) { return nil, err } if msg.Size > baseProtocolMaxMsgSize { - return nil, fmt.Errorf("message too big") + return nil, errors.New("message too big") } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the diff --git a/params/bootnodes.go b/params/bootnodes.go index 56a25f1cfc..62cb287da5 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -44,12 +44,6 @@ var MainnetBootnodes = []string{ "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", // bootnode-hetzner-fsn } -var OPMainnetBootnodes = []string{ - "enode://ca2774c3c401325850b2477fd7d0f27911efbf79b1e8b335066516e2bd8c4c9e0ba9696a94b1cb030a88eac582305ff55e905e64fb77fe0edcd70a4e5296d3ec@34.65.175.185:30305", - "enode://dd751a9ef8912be1bfa7a5e34e2c3785cc5253110bd929f385e07ba7ac19929fb0e0c5d93f77827291f4da02b2232240fbc47ea7ce04c46e333e452f8656b667@34.65.107.0:30305", - "enode://c5d289b56a77b6a2342ca29956dfd07aadf45364dde8ab20d1dc4efd4d1bc6b4655d902501daea308f4d8950737a4e93a4dfedd17b49cd5760ffd127837ca965@34.65.202.239:30305", -} - // HoleskyBootnodes are the enode URLs of the P2P bootstrap nodes running on the // Holesky test network. var HoleskyBootnodes = []string{ @@ -69,12 +63,6 @@ var SepoliaBootnodes = []string{ "enode://9e9492e2e8836114cc75f5b929784f4f46c324ad01daf87d956f98b3b6c5fcba95524d6e5cf9861dc96a2c8a171ea7105bb554a197455058de185fa870970c7c@138.68.123.152:30303", // sepolia-bootnode-1-ams3 } -var OPSepoliaBootnodes = []string{ - "enode://2bd2e657bb3c8efffb8ff6db9071d9eb7be70d7c6d7d980ff80fc93b2629675c5f750bc0a5ef27cd788c2e491b8795a7e9a4a6e72178c14acc6753c0e5d77ae4@34.65.205.244:30305", - "enode://db8e1cab24624cc62fc35dbb9e481b88a9ef0116114cd6e41034c55b5b4f18755983819252333509bd8e25f6b12aadd6465710cd2e956558faf17672cce7551f@34.65.173.88:30305", - "enode://bfda2e0110cfd0f4c9f7aa5bf5ec66e6bd18f71a2db028d36b8bf8b0d6fdb03125c1606a6017b31311d96a36f5ef7e1ad11604d7a166745e6075a715dfa67f8a@34.65.229.245:30305", -} - // GoerliBootnodes are the enode URLs of the P2P bootstrap nodes running on the // Görli test network. var GoerliBootnodes = []string{ @@ -115,6 +103,29 @@ var V5Bootnodes = []string{ "enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM", // 3.64.117.223 | aws-eu-central-1-frankfurt} } +var V5OPBootnodes = []string{ + // OP Labs + "enode://ca2774c3c401325850b2477fd7d0f27911efbf79b1e8b335066516e2bd8c4c9e0ba9696a94b1cb030a88eac582305ff55e905e64fb77fe0edcd70a4e5296d3ec@34.65.175.185:30305", + "enode://dd751a9ef8912be1bfa7a5e34e2c3785cc5253110bd929f385e07ba7ac19929fb0e0c5d93f77827291f4da02b2232240fbc47ea7ce04c46e333e452f8656b667@34.65.107.0:30305", + "enode://c5d289b56a77b6a2342ca29956dfd07aadf45364dde8ab20d1dc4efd4d1bc6b4655d902501daea308f4d8950737a4e93a4dfedd17b49cd5760ffd127837ca965@34.65.202.239:30305", + // Base + "enode://87a32fd13bd596b2ffca97020e31aef4ddcc1bbd4b95bb633d16c1329f654f34049ed240a36b449fda5e5225d70fe40bc667f53c304b71f8e68fc9d448690b51@3.231.138.188:30301", + "enode://ca21ea8f176adb2e229ce2d700830c844af0ea941a1d8152a9513b966fe525e809c3a6c73a2c18a12b74ed6ec4380edf91662778fe0b79f6a591236e49e176f9@184.72.129.189:30301", + "enode://acf4507a211ba7c1e52cdf4eef62cdc3c32e7c9c47998954f7ba024026f9a6b2150cd3f0b734d9c78e507ab70d59ba61dfe5c45e1078c7ad0775fb251d7735a2@3.220.145.177:30301", + "enode://8a5a5006159bf079d06a04e5eceab2a1ce6e0f721875b2a9c96905336219dbe14203d38f70f3754686a6324f786c2f9852d8c0dd3adac2d080f4db35efc678c5@3.231.11.52:30301", + "enode://cdadbe835308ad3557f9a1de8db411da1a260a98f8421d62da90e71da66e55e98aaa8e90aa7ce01b408a54e4bd2253d701218081ded3dbe5efbbc7b41d7cef79@54.198.153.150:30301", +} + +var V5OPTestnetBootnodes = []string{ + // OP Labs + "enode://2bd2e657bb3c8efffb8ff6db9071d9eb7be70d7c6d7d980ff80fc93b2629675c5f750bc0a5ef27cd788c2e491b8795a7e9a4a6e72178c14acc6753c0e5d77ae4@34.65.205.244:30305", + "enode://db8e1cab24624cc62fc35dbb9e481b88a9ef0116114cd6e41034c55b5b4f18755983819252333509bd8e25f6b12aadd6465710cd2e956558faf17672cce7551f@34.65.173.88:30305", + "enode://bfda2e0110cfd0f4c9f7aa5bf5ec66e6bd18f71a2db028d36b8bf8b0d6fdb03125c1606a6017b31311d96a36f5ef7e1ad11604d7a166745e6075a715dfa67f8a@34.65.229.245:30305", + // Base + "enode://548f715f3fc388a7c917ba644a2f16270f1ede48a5d88a4d14ea287cc916068363f3092e39936f1a3e7885198bef0e5af951f1d7b1041ce8ba4010917777e71f@18.210.176.114:30301", + "enode://6f10052847a966a725c9f4adf6716f9141155b99a0fb487fea3f51498f4c2a2cb8d534e680ee678f9447db85b93ff7c74562762c3714783a7233ac448603b25f@107.21.251.55:30301", +} + const dnsPrefix = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@" // KnownDNSNetwork returns the address of a public DNS-based node list for the given diff --git a/params/config.go b/params/config.go index b837cceede..baa8586c68 100644 --- a/params/config.go +++ b/params/config.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params/forks" ) // Genesis hashes to enforce below configs on. @@ -32,7 +33,7 @@ var ( OPBNBMainNetGenesisHash = common.HexToHash("0x4dd61178c8b0f01670c231597e7bcb368e84545acd46d940a896d6a791dd6df4") OPBNBTestNetGenesisHash = common.HexToHash("0x51fa57729dfb1c27542c21b06cb72a0459c57440ceb43a465dae1307cd04fe80") - OPBNBQANetGenesisHash = common.HexToHash("0xfd822cb9ed44eac6bb4c5413cc34b247d814ba4d162610f434a19d5ba68749fd") + OPBNBQANetGenesisHash = common.HexToHash("0x73eaf214333f29eed23c4902fdc17889b3e379372e52a42567d0069e1b10cdb0") ) const ( @@ -93,6 +94,7 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 TerminalTotalDifficultyPassed: true, ShanghaiTime: newUint64(1681338455), + CancunTime: newUint64(1710338135), Ethash: new(EthashConfig), } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. @@ -117,6 +119,7 @@ var ( TerminalTotalDifficultyPassed: true, MergeNetsplitBlock: nil, ShanghaiTime: newUint64(1696000704), + CancunTime: newUint64(1707305664), Ethash: new(EthashConfig), } // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. @@ -141,6 +144,7 @@ var ( TerminalTotalDifficultyPassed: true, MergeNetsplitBlock: big.NewInt(1735371), ShanghaiTime: newUint64(1677557088), + CancunTime: newUint64(1706655072), Ethash: new(EthashConfig), } // GoerliChainConfig contains the chain parameters to run a node on the Görli test network. @@ -163,6 +167,7 @@ var ( TerminalTotalDifficulty: big.NewInt(10_790_000), TerminalTotalDifficultyPassed: true, ShanghaiTime: newUint64(1678832736), + CancunTime: newUint64(1705473120), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -202,6 +207,7 @@ var ( EcotoneTime: newUint64(1718871600), // Jun-20-2024 08:20 AM +UTC HaberTime: newUint64(1718872200), // Jun-20-2024 08:30 AM +UTC WrightTime: newUint64(1724738400), // Aug-27-2024 06:00 AM +UTC + FjordTime: newUint64(1727157600), // Sep-24-2024 06:00 AM +UTC } // OPBNBTestNetConfig is the chain parameters to run a node on the opBNB testnet network. OPBNBTestNetConfig = &ChainConfig{ @@ -238,10 +244,11 @@ var ( EcotoneTime: newUint64(1715754600), // May-15-2024 06:30 AM +UTC HaberTime: newUint64(1717048800), // May-30-2024 06:00 AM +UTC WrightTime: newUint64(1723701600), // Aug-15-2024 06:00 AM +UTC + FjordTime: newUint64(1725948000), // Sep-10-2024 06:00 AM +UTC } // OPBNBQANetConfig is the chain parameters to run a node on the opBNB qa network. It is just for internal test. OPBNBQANetConfig = &ChainConfig{ - ChainID: big.NewInt(7180), + ChainID: big.NewInt(1081), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), @@ -266,13 +273,14 @@ var ( EIP1559DenominatorCanyon: 8, }, Fermat: big.NewInt(0), - ShanghaiTime: newUint64(1714993800), // May-06-2024 11:10 AM +UTC - CanyonTime: newUint64(1714993800), // May-06-2024 11:10 AM +UTC + ShanghaiTime: newUint64(0), + CanyonTime: newUint64(0), // Delta: the Delta upgrade does not affect the execution-layer, and is thus not configurable in the chain config. - CancunTime: newUint64(1714995000), // May-06-2024 11:30 AM +UTC - EcotoneTime: newUint64(1714995000), // May-06-2024 11:30 AM +UTC - HaberTime: newUint64(1716361200), // May-22-2024 07:00 AM +UTC - WrightTime: newUint64(1721815200), // July-24-2024 10:00 AM +UTC + CancunTime: newUint64(0), + EcotoneTime: newUint64(0), + HaberTime: newUint64(0), + WrightTime: newUint64(0), + FjordTime: newUint64(1724392800), // Aug-23-2024 06:00 AM +UTC } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced @@ -389,6 +397,36 @@ var ( Clique: nil, } + // MergedTestChainConfig contains every protocol change (EIPs) introduced + // and accepted by the Ethereum core developers for testing purposes. + MergedTestChainConfig = &ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: false, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + ShanghaiTime: newUint64(0), + CancunTime: newUint64(0), + PragueTime: nil, + VerkleTime: nil, + TerminalTotalDifficulty: big.NewInt(0), + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), + Clique: nil, + } + // NonActivatedConfig defines the chain configuration without activating // any protocol change (EIPs). NonActivatedConfig = &ChainConfig{ @@ -486,6 +524,7 @@ type ChainConfig struct { EcotoneTime *uint64 `json:"ecotoneTime,omitempty"` // Ecotone switch time (nil = no fork, 0 = already on optimism ecotone) HaberTime *uint64 `json:"haberTime,omitempty"` // Haber switch time (nil = no fork, 0 = already on haber) WrightTime *uint64 `json:"wrightTime,omitempty"` // Wright switch time (nil = no fork, 0 = already on wright) + FjordTime *uint64 `json:"fjordTime,omitempty"` // Fjord switch time (nil = no fork, 0 = already on Optimism Fjord) InteropTime *uint64 `json:"interopTime,omitempty"` // Interop switch time (nil = no fork, 0 = already on optimism interop) @@ -625,7 +664,7 @@ func (c *ChainConfig) Description() string { banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) } if c.CancunTime != nil { - banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) + banner += fmt.Sprintf(" - Cancun: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/cancun.md)\n", *c.CancunTime) } if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) @@ -642,6 +681,9 @@ func (c *ChainConfig) Description() string { if c.EcotoneTime != nil { banner += fmt.Sprintf(" - Ecotone: @%-10v\n", *c.EcotoneTime) } + if c.FjordTime != nil { + banner += fmt.Sprintf(" - Fjord: @%-10v\n", *c.FjordTime) + } if c.InteropTime != nil { banner += fmt.Sprintf(" - Interop: @%-10v\n", *c.InteropTime) } @@ -780,6 +822,10 @@ func (c *ChainConfig) IsEcotone(time uint64) bool { return isTimestampForked(c.EcotoneTime, time) } +func (c *ChainConfig) IsFjord(time uint64) bool { + return isTimestampForked(c.FjordTime, time) +} + func (c *ChainConfig) IsHaber(time uint64) bool { return isTimestampForked(c.HaberTime, time) } @@ -814,6 +860,10 @@ func (c *ChainConfig) IsOptimismEcotone(time uint64) bool { return c.IsOptimism() && c.IsEcotone(time) } +func (c *ChainConfig) IsOptimismFjord(time uint64) bool { + return c.IsOptimism() && c.IsFjord(time) +} + // IsOptimismPreBedrock returns true iff this is an optimism node & bedrock is not yet active func (c *ChainConfig) IsOptimismPreBedrock(num *big.Int) bool { return c.IsOptimism() && !c.IsBedrock(num) @@ -889,7 +939,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, cur.name, cur.block) } else { return fmt.Errorf("unsupported fork ordering: %v not enabled, but %v enabled at timestamp %v", - lastFork.name, cur.name, cur.timestamp) + lastFork.name, cur.name, *cur.timestamp) } // Fork (whether defined by block or timestamp) must follow the fork definition sequence @@ -899,7 +949,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork.name, lastFork.block, cur.name, cur.block) } else if lastFork.timestamp != nil && *lastFork.timestamp > *cur.timestamp { return fmt.Errorf("unsupported fork ordering: %v enabled at timestamp %v, but %v enabled at timestamp %v", - lastFork.name, lastFork.timestamp, cur.name, cur.timestamp) + lastFork.name, *lastFork.timestamp, cur.name, *cur.timestamp) } // Timestamp based forks can follow block based ones, but not the other way around @@ -1014,6 +1064,23 @@ func (c *ChainConfig) ElasticityMultiplier() uint64 { return DefaultElasticityMultiplier } +// LatestFork returns the latest time-based fork that would be active for the given time. +func (c *ChainConfig) LatestFork(time uint64) forks.Fork { + // Assume last non-time-based fork has passed. + london := c.LondonBlock + + switch { + case c.IsPrague(london, time): + return forks.Prague + case c.IsCancun(london, time): + return forks.Cancun + case c.IsShanghai(london, time): + return forks.Shanghai + default: + return forks.Paris + } +} + // isForkBlockIncompatible returns true if a fork scheduled at block s1 cannot be // rescheduled to block s2 because head is already past the fork. func isForkBlockIncompatible(s1, s2, head *big.Int) bool { @@ -1148,7 +1215,7 @@ type Rules struct { IsMerge, IsShanghai, IsCancun, IsPrague bool IsVerkle bool IsOptimismBedrock, IsOptimismRegolith bool - IsOptimismCanyon bool + IsOptimismCanyon, IsOptimismFjord bool IsFermat bool IsHaber bool IsWright bool @@ -1160,6 +1227,8 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules if chainID == nil { chainID = new(big.Int) } + // disallow setting Merge out of order + isMerge = isMerge && c.IsLondon(num) return Rules{ ChainID: new(big.Int).Set(chainID), IsHomestead: c.IsHomestead(num), @@ -1173,14 +1242,15 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, - IsShanghai: c.IsShanghai(num, timestamp), - IsCancun: c.IsCancun(num, timestamp), - IsPrague: c.IsPrague(num, timestamp), - IsVerkle: c.IsVerkle(num, timestamp), + IsShanghai: isMerge && c.IsShanghai(num, timestamp), + IsCancun: isMerge && c.IsCancun(num, timestamp), + IsPrague: isMerge && c.IsPrague(num, timestamp), + IsVerkle: isMerge && c.IsVerkle(num, timestamp), // Optimism - IsOptimismBedrock: c.IsOptimismBedrock(num), - IsOptimismRegolith: c.IsOptimismRegolith(timestamp), - IsOptimismCanyon: c.IsOptimismCanyon(timestamp), + IsOptimismBedrock: isMerge && c.IsOptimismBedrock(num), + IsOptimismRegolith: isMerge && c.IsOptimismRegolith(timestamp), + IsOptimismCanyon: isMerge && c.IsOptimismCanyon(timestamp), + IsOptimismFjord: isMerge && c.IsOptimismFjord(timestamp), // OPBNB IsFermat: c.IsFermat(num), IsHaber: c.IsHaber(timestamp), diff --git a/params/config_test.go b/params/config_test.go index 14d7f833bb..fc76c604a8 100644 --- a/params/config_test.go +++ b/params/config_test.go @@ -141,6 +141,7 @@ func TestConfigRules(t *testing.T) { func TestConfigRulesRegolith(t *testing.T) { c := &ChainConfig{ RegolithTime: newUint64(500), + LondonBlock: new(big.Int), Optimism: &OptimismConfig{}, } var stamp uint64 diff --git a/params/forks/forks.go b/params/forks/forks.go new file mode 100644 index 0000000000..4f50ff5aed --- /dev/null +++ b/params/forks/forks.go @@ -0,0 +1,42 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package forks + +// Fork is a numerical identifier of specific network upgrades (forks). +type Fork int + +const ( + Frontier = iota + FrontierThawing + Homestead + DAO + TangerineWhistle + SpuriousDragon + Byzantium + Constantinople + Petersburg + Istanbul + MuirGlacier + Berlin + London + ArrowGlacier + GrayGlacier + Paris + Shanghai + Cancun + Prague +) diff --git a/params/protocol_params.go b/params/protocol_params.go index b1dd4ea05c..f9a1e65568 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -181,7 +181,6 @@ const ( BlobTxBytesPerFieldElement = 32 // Size in bytes of a field element BlobTxFieldElementsPerBlob = 4096 // Number of field elements stored in a single data blob - BlobTxHashVersion = 0x01 // Version byte of the commitment hash BlobTxBlobGasPerBlob = 1 << 17 // Gas consumption of a single data blob (== blob byte size) BlobTxMinBlobGasprice = 1 // Minimum gas price for data blobs BlobTxBlobGaspriceUpdateFraction = 3338477 // Controls the maximum rate of change for blob gas price diff --git a/params/superchain.go b/params/superchain.go index 83a5b47f5b..bcce62832f 100644 --- a/params/superchain.go +++ b/params/superchain.go @@ -11,7 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" ) -var OPStackSupport = ProtocolVersionV0{Build: [8]byte{}, Major: 6, Minor: 0, Patch: 0, PreRelease: 0}.Encode() +var OPStackSupport = ProtocolVersionV0{Build: [8]byte{}, Major: 7, Minor: 0, Patch: 0, PreRelease: 0}.Encode() func init() { for id, ch := range superchain.OPChains { @@ -68,6 +68,7 @@ func LoadOPStackChainConfig(chainID uint64) (*ChainConfig, error) { RegolithTime: &genesisActivation, CanyonTime: chConfig.CanyonTime, EcotoneTime: chConfig.EcotoneTime, + FjordTime: chConfig.FjordTime, TerminalTotalDifficulty: common.Big0, TerminalTotalDifficultyPassed: true, Ethash: nil, diff --git a/params/version.go b/params/version.go index 763655c821..86adc36db7 100644 --- a/params/version.go +++ b/params/version.go @@ -26,7 +26,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 13 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release + VersionPatch = 15 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) @@ -35,7 +35,7 @@ var ( OPVersionMajor = 0 // Major version component of the current release OPVersionMinor = 1 // Minor version component of the current release OPVersionPatch = 0 // Patch version component of the current release - OPVersionMeta = "unstable" // Version metadata to append to the version string + OPVersionMeta = "untagged" // Version metadata to append to the version string ) // This is set at build-time by the linker when the build is done by build/ci.go. @@ -60,16 +60,18 @@ var _ = func() (_ string) { // Version holds the textual version string. var Version = func() string { + if OPVersionMeta == "untagged" { + return OPVersionMeta + } return fmt.Sprintf("%d.%d.%d", OPVersionMajor, OPVersionMinor, OPVersionPatch) }() // VersionWithMeta holds the textual version string including the metadata. var VersionWithMeta = func() string { - v := Version - if OPVersionMeta != "" { - v += "-" + OPVersionMeta + if OPVersionMeta != "untagged" { + return Version + "-" + OPVersionMeta } - return v + return Version }() // GethVersion holds the textual geth version string. diff --git a/rpc/http.go b/rpc/http.go index 19e44a2e8c..d79ac462d1 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -34,6 +34,7 @@ import ( const ( maxRequestContentLength = 1024 * 1024 * 32 + defaultBodyLimit = 5 * 1024 * 1024 contentType = "application/json" ) @@ -253,8 +254,8 @@ type httpServerConn struct { r *http.Request } -func newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { - body := io.LimitReader(r.Body, maxRequestContentLength) +func (s *Server) newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec { + body := io.LimitReader(r.Body, int64(s.httpBodyLimit)) conn := &httpServerConn{Reader: body, Writer: w, r: r} encoder := func(v any, isErrorResponse bool) error { @@ -312,7 +313,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) return } - if code, err := validateRequest(r); err != nil { + if code, err := s.validateRequest(r); err != nil { http.Error(w, err.Error(), code) return } @@ -330,19 +331,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // until EOF, writes the response to w, and orders the server to process a // single request. w.Header().Set("content-type", contentType) - codec := newHTTPServerConn(r, w) + codec := s.newHTTPServerConn(r, w) defer codec.close() s.serveSingleRequest(ctx, codec) } // validateRequest returns a non-zero response code and error message if the // request is invalid. -func validateRequest(r *http.Request) (int, error) { +func (s *Server) validateRequest(r *http.Request) (int, error) { if r.Method == http.MethodPut || r.Method == http.MethodDelete { return http.StatusMethodNotAllowed, errors.New("method not allowed") } - if r.ContentLength > maxRequestContentLength { - err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength) + if r.ContentLength > int64(s.httpBodyLimit) { + err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, s.httpBodyLimit) return http.StatusRequestEntityTooLarge, err } // Allow OPTIONS (regardless of content-type) diff --git a/rpc/http_test.go b/rpc/http_test.go index 584842a9aa..ad86ca15ae 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -40,11 +40,13 @@ func confirmStatusCode(t *testing.T, got, want int) { func confirmRequestValidationCode(t *testing.T, method, contentType, body string, expectedStatusCode int) { t.Helper() + + s := NewServer() request := httptest.NewRequest(method, "http://url.com", strings.NewReader(body)) if len(contentType) > 0 { request.Header.Set("Content-Type", contentType) } - code, err := validateRequest(request) + code, err := s.validateRequest(request) if code == 0 { if err != nil { t.Errorf("validation: got error %v, expected nil", err) @@ -64,7 +66,7 @@ func TestHTTPErrorResponseWithPut(t *testing.T) { } func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) { - body := make([]rune, maxRequestContentLength+1) + body := make([]rune, defaultBodyLimit+1) confirmRequestValidationCode(t, http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge) } @@ -104,7 +106,7 @@ func TestHTTPResponseWithEmptyGet(t *testing.T) { // This checks that maxRequestContentLength is not applied to the response of a request. func TestHTTPRespBodyUnlimited(t *testing.T) { - const respLength = maxRequestContentLength * 3 + const respLength = defaultBodyLimit * 3 s := NewServer() defer s.Stop() diff --git a/rpc/server.go b/rpc/server.go index 2742adf07b..e2f9120aa2 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -51,13 +51,15 @@ type Server struct { run atomic.Bool batchItemLimit int batchResponseLimit int + httpBodyLimit int } // NewServer creates a new server instance with no registered handlers. func NewServer() *Server { server := &Server{ - idgen: randomIDGenerator(), - codecs: make(map[ServerCodec]struct{}), + idgen: randomIDGenerator(), + codecs: make(map[ServerCodec]struct{}), + httpBodyLimit: defaultBodyLimit, } server.run.Store(true) // Register the default service providing meta information about the RPC service such @@ -78,6 +80,13 @@ func (s *Server) SetBatchLimits(itemLimit, maxResponseSize int) { s.batchResponseLimit = maxResponseSize } +// SetHTTPBodyLimit sets the size limit for HTTP requests. +// +// This method should be called before processing any requests via ServeHTTP. +func (s *Server) SetHTTPBodyLimit(limit int) { + s.httpBodyLimit = limit +} + // RegisterName creates a service for the given receiver type under the given name. When no // methods on the given receiver match the criteria to be either a RPC method or a // subscription an error is returned. Otherwise a new service is created and added to the diff --git a/rpc/types.go b/rpc/types.go index f88c37c59d..d124081786 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math" "strings" @@ -104,7 +105,7 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("block number larger than int64") + return errors.New("block number larger than int64") } *bn = BlockNumber(blckNum) return nil @@ -154,7 +155,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { err := json.Unmarshal(data, &e) if err == nil { if e.BlockNumber != nil && e.BlockHash != nil { - return fmt.Errorf("cannot specify both BlockHash and BlockNumber, choose one or the other") + return errors.New("cannot specify both BlockHash and BlockNumber, choose one or the other") } bnh.BlockNumber = e.BlockNumber bnh.BlockHash = e.BlockHash @@ -202,7 +203,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("blocknumber too high") + return errors.New("blocknumber too high") } bn := BlockNumber(blckNum) bnh.BlockNumber = &bn diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index d3e15d94c9..8d2bd9d802 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -97,7 +97,7 @@ func TestWebsocketLargeCall(t *testing.T) { // This call sends slightly less than the limit and should work. var result echoResult - arg := strings.Repeat("x", maxRequestContentLength-200) + arg := strings.Repeat("x", defaultBodyLimit-200) if err := client.Call(&result, "test_echo", arg, 1); err != nil { t.Fatalf("valid call didn't work: %v", err) } @@ -106,7 +106,7 @@ func TestWebsocketLargeCall(t *testing.T) { } // This call sends twice the allowed size and shouldn't work. - arg = strings.Repeat("x", maxRequestContentLength*2) + arg = strings.Repeat("x", defaultBodyLimit*2) err = client.Call(&result, "test_echo", arg) if err == nil { t.Fatal("no error for too large call") diff --git a/signer/core/api.go b/signer/core/api.go index 43eb89ee00..a32f24cb18 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -65,7 +65,7 @@ type ExternalAPI interface { EcRecover(ctx context.Context, data hexutil.Bytes, sig hexutil.Bytes) (common.Address, error) // Version info about the APIs Version(ctx context.Context) (string, error) - // SignGnosisSafeTransaction signs/confirms a gnosis-safe multisig transaction + // SignGnosisSafeTx signs/confirms a gnosis-safe multisig transaction SignGnosisSafeTx(ctx context.Context, signerAddress common.MixedcaseAddress, gnosisTx GnosisSafeTx, methodSelector *string) (*GnosisSafeTx, error) } @@ -631,7 +631,7 @@ func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common } } typedData := gnosisTx.ToTypedData() - // might aswell error early. + // might as well error early. // we are expected to sign. If our calculated hash does not match what they want, // The gnosis safetx input contains a 'safeTxHash' which is the expected safeTxHash that sighash, _, err := apitypes.TypedDataAndHash(typedData) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 8218e754d3..6bfcd2a727 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -62,7 +62,7 @@ func (vs *ValidationMessages) Info(msg string) { vs.Messages = append(vs.Messages, ValidationInfo{INFO, msg}) } -// getWarnings returns an error with all messages of type WARN of above, or nil if no warnings were present +// GetWarnings returns an error with all messages of type WARN of above, or nil if no warnings were present func (v *ValidationMessages) GetWarnings() error { var messages []string for _, msg := range v.Messages { diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 3c2b6f5d45..c6ae7b1274 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -302,7 +302,7 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex // Note, the signature must conform to the secp256k1 curve R, S and V values, where // the V value must be 27 or 28 for legacy reasons. // - // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover + // https://geth.ethereum.org/docs/tools/clef/apis#account-ecrecover if len(sig) != 65 { return common.Address{}, errors.New("signature must be 65 bytes long") } diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go index 4a060147a6..b8c3acfb4d 100644 --- a/signer/core/uiapi.go +++ b/signer/core/uiapi.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) -// SignerUIAPI implements methods Clef provides for a UI to query, in the bidirectional communication +// UIServerAPI implements methods Clef provides for a UI to query, in the bidirectional communication // channel. // This API is considered secure, since a request can only // ever arrive from the UI -- and the UI is capable of approving any action, thus we can consider these diff --git a/tests/0001-diff-go-ethereum.patch b/tests/0001-diff-go-ethereum.patch new file mode 100644 index 0000000000..6d8722ab1a --- /dev/null +++ b/tests/0001-diff-go-ethereum.patch @@ -0,0 +1,17 @@ +diff --git a/core/vm/contracts.go b/core/vm/contracts.go +index 7b8757dc2..4c01c1434 100644 +--- a/core/vm/contracts.go ++++ b/core/vm/contracts.go +@@ -128,9 +128,6 @@ var PrecompiledContractsCancun = map[common.Address]PrecompiledContract{ + common.BytesToAddress([]byte{8}): &bn256PairingIstanbul{}, + common.BytesToAddress([]byte{9}): &blake2F{}, + common.BytesToAddress([]byte{0x0a}): &kzgPointEvaluation{}, +- +- common.BytesToAddress([]byte{102}): &blsSignatureVerify{}, +- common.BytesToAddress([]byte{103}): &cometBFTLightBlockValidate{}, + } + + // PrecompiledContractsHaber contains the default set of pre-compiled Ethereum +-- +2.33.0 + diff --git a/tests/block_test.go b/tests/block_test.go index aa6f27b8f3..fb355085fd 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -61,14 +61,14 @@ func TestBlockchain(t *testing.T) { // which run natively, so there's no reason to run them here. } -// TestExecutionSpec runs the test fixtures from execution-spec-tests. -func TestExecutionSpec(t *testing.T) { - if !common.FileExist(executionSpecDir) { - t.Skipf("directory %s does not exist", executionSpecDir) +// TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests. +func TestExecutionSpecBlocktests(t *testing.T) { + if !common.FileExist(executionSpecBlockchainTestDir) { + t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir) } bt := new(testMatcher) - bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) { + bt.walk(t, executionSpecBlockchainTestDir, func(t *testing.T, name string, test *BlockTest) { execBlockTest(t, bt, test) }) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 06b6715181..5f77a1c326 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -36,11 +36,12 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // A BlockTest checks handling of entire blocks. @@ -56,8 +57,8 @@ func (t *BlockTest) UnmarshalJSON(in []byte) error { type btJSON struct { Blocks []btBlock `json:"blocks"` Genesis btHeader `json:"genesisBlockHeader"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"postState"` + Pre types.GenesisAlloc `json:"pre"` + Post types.GenesisAlloc `json:"postState"` BestBlock common.UnprefixedHash `json:"lastblockhash"` Network string `json:"network"` SealEngine string `json:"sealEngine"` @@ -116,7 +117,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po // import pre accounts & construct test genesis block & state root var ( db = rawdb.NewMemoryDatabase() - tconf = &trie.Config{ + tconf = &triedb.Config{ Preimages: true, } ) @@ -127,7 +128,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po } // Commit genesis state gspec := t.genesis(config) - triedb := trie.NewDatabase(db, tconf) + triedb := triedb.NewDatabase(db, tconf) gblock, err := gspec.Commit(db, triedb) if err != nil { return err @@ -225,6 +226,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) cb, err := b.decode() if err != nil { if b.BlockHeader == nil { + log.Info("Block decoding failed", "index", bi, "err", err) continue // OK - block is supposed to be invalid, continue with next block } else { return nil, fmt.Errorf("block RLP decoding failed when expected to succeed: %v", err) @@ -327,7 +329,7 @@ func (t *BlockTest) validatePostState(statedb *state.StateDB) error { for addr, acct := range t.json.Post { // address is indirectly verified by the other fields, as it's the db key code2 := statedb.GetCode(addr) - balance2 := statedb.GetBalance(addr) + balance2 := statedb.GetBalance(addr).ToBig() nonce2 := statedb.GetNonce(addr) if !bytes.Equal(code2, acct.Code) { return fmt.Errorf("account code mismatch for addr: %s want: %v have: %s", addr, acct.Code, hex.EncodeToString(code2)) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 6b5ca90880..dcafebb265 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "golang.org/x/exp/slices" ) @@ -56,7 +57,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := trie.NewEmpty(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff diff --git a/tests/gen_stenv.go b/tests/gen_stenv.go index 71f0063178..a5bd0d5fcb 100644 --- a/tests/gen_stenv.go +++ b/tests/gen_stenv.go @@ -16,13 +16,14 @@ var _ = (*stEnvMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { - Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"optional"` - Random *math.HexOrDecimal256 `json:"currentRandom" gencodec:"optional"` - GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee" gencodec:"optional"` + Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"optional"` + Random *math.HexOrDecimal256 `json:"currentRandom" gencodec:"optional"` + GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee" gencodec:"optional"` + ExcessBlobGas *math.HexOrDecimal64 `json:"currentExcessBlobGas" gencodec:"optional"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) @@ -32,19 +33,21 @@ func (s stEnv) MarshalJSON() ([]byte, error) { enc.Number = math.HexOrDecimal64(s.Number) enc.Timestamp = math.HexOrDecimal64(s.Timestamp) enc.BaseFee = (*math.HexOrDecimal256)(s.BaseFee) + enc.ExcessBlobGas = (*math.HexOrDecimal64)(s.ExcessBlobGas) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { - Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"optional"` - Random *math.HexOrDecimal256 `json:"currentRandom" gencodec:"optional"` - GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee" gencodec:"optional"` + Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"optional"` + Random *math.HexOrDecimal256 `json:"currentRandom" gencodec:"optional"` + GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee" gencodec:"optional"` + ExcessBlobGas *math.HexOrDecimal64 `json:"currentExcessBlobGas" gencodec:"optional"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -75,5 +78,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.BaseFee != nil { s.BaseFee = (*big.Int)(dec.BaseFee) } + if dec.ExcessBlobGas != nil { + s.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) + } return nil } diff --git a/tests/init_test.go b/tests/init_test.go index 3ab15e7658..e9bb99dc7d 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -34,15 +34,16 @@ import ( ) var ( - baseDir = filepath.Join(".", "testdata") - blockTestDir = filepath.Join(baseDir, "BlockchainTests") - stateTestDir = filepath.Join(baseDir, "GeneralStateTests") - legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") - transactionTestDir = filepath.Join(baseDir, "TransactionTests") - rlpTestDir = filepath.Join(baseDir, "RLPTests") - difficultyTestDir = filepath.Join(baseDir, "BasicTests") - executionSpecDir = filepath.Join(".", "spec-tests", "fixtures") - benchmarksDir = filepath.Join(".", "evm-benchmarks", "benchmarks") + baseDir = filepath.Join(".", "testdata") + blockTestDir = filepath.Join(baseDir, "BlockchainTests") + stateTestDir = filepath.Join(baseDir, "GeneralStateTests") + legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") + transactionTestDir = filepath.Join(baseDir, "TransactionTests") + rlpTestDir = filepath.Join(baseDir, "RLPTests") + difficultyTestDir = filepath.Join(baseDir, "BasicTests") + executionSpecBlockchainTestDir = filepath.Join(".", "spec-tests", "fixtures", "blockchain_tests") + executionSpecStateTestDir = filepath.Join(".", "spec-tests", "fixtures", "state_tests") + benchmarksDir = filepath.Join(".", "evm-benchmarks", "benchmarks") ) func readJSON(reader io.Reader, value interface{}) error { diff --git a/tests/state_test.go b/tests/state_test.go index 2b9eb21d6d..fc1c351f07 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,19 +30,16 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/holiman/uint256" ) -func TestState(t *testing.T) { - t.Parallel() - - st := new(testMatcher) +func initMatcher(st *testMatcher) { // Long tests: st.slow(`^stAttackTest/ContractCreationSpam`) st.slow(`^stBadOpcode/badOpcodes`) @@ -61,80 +58,102 @@ func TestState(t *testing.T) { // Broken tests: // EOF is not part of cancun st.skipLoad(`^stEOF/`) +} - // EIP-4844 tests need to be regenerated due to the data-to-blob rename - st.skipLoad(`^stEIP4844-blobtransactions/`) - - // Expected failures: - // These EIP-4844 tests need to be regenerated. - st.fails(`stEIP4844-blobtransactions/opcodeBlobhashOutOfRange.json`, "test has incorrect state root") - st.fails(`stEIP4844-blobtransactions/opcodeBlobhBounds.json`, "test has incorrect state root") +func TestState(t *testing.T) { + t.Parallel() - // For Istanbul, older tests were moved into LegacyTests + st := new(testMatcher) + initMatcher(st) for _, dir := range []string{ filepath.Join(baseDir, "EIPTests", "StateTests"), stateTestDir, - legacyStateTestDir, benchmarksDir, } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - return - } - for _, subtest := range test.Subtests() { - subtest := subtest - key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + execStateTest(t, st, test) + }) + } +} - t.Run(key+"/hash/trie", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - result = st.checkFailure(t, err) - }) - return result - }) +// TestLegacyState tests some older tests, which were moved to the folder +// 'LegacyTests' for the Istanbul fork. +func TestLegacyState(t *testing.T) { + st := new(testMatcher) + initMatcher(st) + st.walk(t, legacyStateTestDir, func(t *testing.T, name string, test *StateTest) { + execStateTest(t, st, test) + }) +} + +// TestExecutionSpecState runs the test fixtures from execution-spec-tests. +func TestExecutionSpecState(t *testing.T) { + if !common.FileExist(executionSpecStateTestDir) { + t.Skipf("directory %s does not exist", executionSpecStateTestDir) + } + st := new(testMatcher) + + st.walk(t, executionSpecStateTestDir, func(t *testing.T, name string, test *StateTest) { + execStateTest(t, st, test) + }) +} + +func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { + if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { + t.Skip("test (randomly) skipped on 32-bit windows") + return + } + for _, subtest := range test.Subtests() { + subtest := subtest + key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + + t.Run(key+"/hash/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) { + result = st.checkFailure(t, err) }) - t.Run(key+"/hash/snap", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - if snaps != nil && state != nil { - if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { - result = err - return - } - } - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/hash/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) { + if state.Snapshots != nil && state.StateDB != nil { + if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) }) - t.Run(key+"/path/trie", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/path/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) { + result = st.checkFailure(t, err) }) - t.Run(key+"/path/snap", func(t *testing.T) { - withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - var result error - test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - if snaps != nil && state != nil { - if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { - result = err - return - } - } - result = st.checkFailure(t, err) - }) - return result - }) + return result + }) + }) + t.Run(key+"/path/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) { + if state.Snapshots != nil && state.StateDB != nil { + if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) }) - } + return result + }) }) } } @@ -229,10 +248,10 @@ func runBenchmark(b *testing.B, t *StateTest) { vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme) + state := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme) defer func() { - triedb.Journal(triedb.Head()) - triedb.Close() + state.TrieDB.Journal(state.TrieDB.Head()) + state.Close() }() var baseFee *big.Int @@ -268,10 +287,10 @@ func runBenchmark(b *testing.B, t *StateTest) { // Prepare the EVM. txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase, config, statedb) + context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase, config, state.StateDB) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) + evm := vm.NewEVM(context, txContext, state.StateDB, config, vmconfig) // Create "contract" for sender to cache code analysis. sender := vm.NewContract(vm.AccountRef(msg.From), vm.AccountRef(msg.From), @@ -284,13 +303,13 @@ func runBenchmark(b *testing.B, t *StateTest) { ) b.ResetTimer() for n := 0; n < b.N; n++ { - snapshot := statedb.Snapshot() - statedb.Prepare(rules, msg.From, context.Coinbase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) + snapshot := state.StateDB.Snapshot() + state.StateDB.Prepare(rules, msg.From, context.Coinbase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) b.StartTimer() start := time.Now() // Execute the message. - _, leftOverGas, err := evm.Call(sender, *msg.To, msg.Data, msg.GasLimit, msg.Value) + _, leftOverGas, err := evm.Call(sender, *msg.To, msg.Data, msg.GasLimit, uint256.MustFromBig(msg.Value)) if err != nil { b.Error(err) return @@ -298,10 +317,10 @@ func runBenchmark(b *testing.B, t *StateTest) { b.StopTimer() elapsed += uint64(time.Since(start)) - refund += statedb.GetRefund() + refund += state.StateDB.GetRefund() gasUsed += msg.GasLimit - leftOverGas - statedb.RevertToSnapshot(snapshot) + state.StateDB.RevertToSnapshot(snapshot) } if elapsed < 1 { elapsed = 1 diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 231bde8def..3c68b870fc 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -25,11 +25,10 @@ import ( "strconv" "strings" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -40,9 +39,11 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/holiman/uint256" + "golang.org/x/crypto/sha3" ) // StateTest checks transaction processing without block context. @@ -63,7 +64,7 @@ func (t *StateTest) UnmarshalJSON(in []byte) error { type stJSON struct { Env stEnv `json:"env"` - Pre core.GenesisAlloc `json:"pre"` + Pre types.GenesisAlloc `json:"pre"` Tx stTransaction `json:"transaction"` Out hexutil.Bytes `json:"out"` Post map[string][]stPostState `json:"post"` @@ -84,23 +85,25 @@ type stPostState struct { //go:generate go run github.com/fjl/gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go type stEnv struct { - Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` - Difficulty *big.Int `json:"currentDifficulty" gencodec:"optional"` - Random *big.Int `json:"currentRandom" gencodec:"optional"` - GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` - Number uint64 `json:"currentNumber" gencodec:"required"` - Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` - BaseFee *big.Int `json:"currentBaseFee" gencodec:"optional"` + Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` + Difficulty *big.Int `json:"currentDifficulty" gencodec:"optional"` + Random *big.Int `json:"currentRandom" gencodec:"optional"` + GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` + Number uint64 `json:"currentNumber" gencodec:"required"` + Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` + BaseFee *big.Int `json:"currentBaseFee" gencodec:"optional"` + ExcessBlobGas *uint64 `json:"currentExcessBlobGas" gencodec:"optional"` } type stEnvMarshaling struct { - Coinbase common.UnprefixedAddress - Difficulty *math.HexOrDecimal256 - Random *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - Number math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + Coinbase common.UnprefixedAddress + Difficulty *math.HexOrDecimal256 + Random *math.HexOrDecimal256 + GasLimit math.HexOrDecimal64 + Number math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + BaseFee *math.HexOrDecimal256 + ExcessBlobGas *math.HexOrDecimal64 } //go:generate go run github.com/fjl/gencodec -type stTransaction -field-override stTransactionMarshaling -out gen_sttransaction.go @@ -191,21 +194,14 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) { - triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) - +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, st *StateTestState)) (result error) { + st, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) // Invoke the callback at the end of function for further analysis. defer func() { - postCheck(result, snaps, statedb) - - if triedb != nil { - triedb.Journal(triedb.Head()) - triedb.Close() - } - if snaps != nil { - snaps.Release() - } + postCheck(result, &st) + st.Close() }() + checkedErr := t.checkError(subtest, err) if checkedErr != nil { return checkedErr @@ -222,23 +218,24 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if root != common.Hash(post.Root) { return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } - if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { + if logs := rlpHash(st.StateDB.Logs()); logs != common.Hash(post.Logs) { return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - statedb, _ = state.New(root, statedb.Database(), snaps) + st.StateDB, _ = state.New(root, st.StateDB.Database(), st.Snapshots) return nil } -// RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) { +// RunNoVerify runs a specific subtest and returns the statedb and post-state root. +// Remember to call state.Close after verifying the test result! +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (state StateTestState, root common.Hash, err error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} + return state, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) + state = MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) var baseFee *big.Int if config.IsLondon(new(big.Int)) { @@ -252,9 +249,18 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - triedb.Journal(triedb.Head()) - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return state, common.Hash{}, err + } + + { // Blob transactions may be present after the Cancun fork. + // In production, + // - the header is verified against the max in eip4844.go:VerifyEIP4844Header + // - the block body is verified against the header in block_validator.go:ValidateBody + // Here, we just do this shortcut smaller fix, since state tests do not + // utilize those codepaths + if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { + return state, common.Hash{}, errors.New("blob gas exceeds maximum") + } } // Try to recover tx with current signer @@ -262,21 +268,16 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - triedb.Journal(triedb.Head()) - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return state, common.Hash{}, err } - if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - triedb.Journal(triedb.Head()) - triedb.Close() - return nil, nil, nil, common.Hash{}, err + return state, common.Hash{}, err } } // Prepare the EVM. txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase, config, statedb) + context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase, config, state.StateDB) context.GetHash = vmTestBlockHash context.BaseFee = baseFee context.Random = nil @@ -288,71 +289,36 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.Random = &rnd context.Difficulty = big.NewInt(0) } - evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) + if config.IsCancun(new(big.Int), block.Time()) && t.json.Env.ExcessBlobGas != nil { + context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) + } + evm := vm.NewEVM(context, txContext, state.StateDB, config, vmconfig) // Execute the message. - snapshot := statedb.Snapshot() + snapshot := state.StateDB.Snapshot() gaspool := new(core.GasPool) gaspool.AddGas(block.GasLimit()) _, err = core.ApplyMessage(evm, msg, gaspool) if err != nil { - statedb.RevertToSnapshot(snapshot) + state.StateDB.RevertToSnapshot(snapshot) } // Add 0-value mining reward. This only makes a difference in the cases // where // - the coinbase self-destructed, or // - there are only 'bad' transactions, which aren't executed. In those cases, // the coinbase gets no txfee, so isn't created, and thus needs to be touched - statedb.AddBalance(block.Coinbase(), new(big.Int)) - - root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) - statedb.SetExpectedStateRoot(root) + state.StateDB.AddBalance(block.Coinbase(), new(uint256.Int)) + root = state.StateDB.IntermediateRoot(config.IsEIP158(block.Number())) + state.StateDB.SetExpectedStateRoot(root) // Commit state mutations into database. - root, _ = statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) - return triedb, snaps, statedb, root, err + root, _ = state.StateDB.Commit(block.NumberU64(), config.IsEIP158(block.Number())) + return state, root, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) { - tconf := &trie.Config{Preimages: true} - if scheme == rawdb.HashScheme { - tconf.HashDB = hashdb.Defaults - } else { - tconf.PathDB = pathdb.Defaults - } - triedb := trie.NewDatabase(db, tconf) - sdb := state.NewDatabaseWithNodeDB(db, triedb) - statedb, _ := state.New(types.EmptyRootHash, sdb, nil) - for addr, a := range accounts { - statedb.SetCode(addr, a.Code) - statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, a.Balance) - for k, v := range a.Storage { - statedb.SetState(addr, k, v) - } - } - // Commit and re-open to start with a clean state. - root := statedb.IntermediateRoot(false) - statedb.SetExpectedStateRoot(root) - root, _ = statedb.Commit(0, false) - - var snaps *snapshot.Tree - if snapshotter { - snapconfig := snapshot.Config{ - CacheSize: 1, - Recovery: false, - NoBuild: false, - AsyncBuild: false, - } - snaps, _ = snapshot.New(snapconfig, db, triedb, root) - } - statedb, _ = state.New(root, sdb, snaps) - return triedb, snaps, statedb -} - func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { genesis := &core.Genesis{ Config: config, @@ -469,3 +435,61 @@ func rlpHash(x interface{}) (h common.Hash) { func vmTestBlockHash(n uint64) common.Hash { return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) } + +// StateTestState groups all the state database objects together for use in tests. +type StateTestState struct { + StateDB *state.StateDB + TrieDB *triedb.Database + Snapshots *snapshot.Tree +} + +// MakePreState creates a state containing the given allocation. +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bool, scheme string) StateTestState { + tconf := &triedb.Config{Preimages: true} + if scheme == rawdb.HashScheme { + tconf.HashDB = hashdb.Defaults + } else { + tconf.PathDB = pathdb.Defaults + } + triedb := triedb.NewDatabase(db, tconf) + sdb := state.NewDatabaseWithNodeDB(db, triedb) + statedb, _ := state.New(types.EmptyRootHash, sdb, nil) + for addr, a := range accounts { + statedb.SetCode(addr, a.Code) + statedb.SetNonce(addr, a.Nonce) + statedb.SetBalance(addr, uint256.MustFromBig(a.Balance)) + for k, v := range a.Storage { + statedb.SetState(addr, k, v) + } + } + // Commit and re-open to start with a clean state. + root, _ := statedb.Commit(0, false) + + // If snapshot is requested, initialize the snapshotter and use it in state. + var snaps *snapshot.Tree + if snapshotter { + snapconfig := snapshot.Config{ + CacheSize: 1, + Recovery: false, + NoBuild: false, + AsyncBuild: false, + } + snaps, _ = snapshot.New(snapconfig, db, triedb, root) + } + statedb, _ = state.New(root, sdb, snaps) + return StateTestState{statedb, triedb, snaps} +} + +// Close should be called when the state is no longer needed, ie. after running the test. +func (st *StateTestState) Close() { + if st.TrieDB != nil { + st.TrieDB.Close() + st.TrieDB = nil + } + if st.Snapshots != nil { + // Need to call Disable here to quit the snapshot generator goroutine. + st.Snapshots.Disable() + st.Snapshots.Release() + st.Snapshots = nil + } +} diff --git a/tests/testdata b/tests/testdata index ee3fa4c86d..fa51c5c164 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit ee3fa4c86d05f99f2717f83a6ad08008490ddf07 +Subproject commit fa51c5c164f79140730ccb8fe26a46c3d3994338 diff --git a/trie/committer.go b/trie/committer.go index 92163cdb3b..4e2f7b8bd6 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -154,12 +154,12 @@ func (c *committer) store(path []byte, n node) node { return hash } -// mptResolver the children resolver in merkle-patricia-tree. -type mptResolver struct{} +// MerkleResolver the children resolver in merkle-patricia-tree. +type MerkleResolver struct{} // ForEach implements childResolver, decodes the provided node and // traverses the children inside. -func (resolver mptResolver) ForEach(node []byte, onChild func(common.Hash)) { +func (resolver MerkleResolver) ForEach(node []byte, onChild func(common.Hash)) { forGatherChildren(mustDecodeNodeUnsafe(nil, node), onChild) } diff --git a/trie/database_test.go b/trie/database_test.go index d508c65533..aed508b368 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -17,24 +17,136 @@ package trie import ( + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) -// newTestDatabase initializes the trie database with specified scheme. -func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { - config := &Config{Preimages: false} - if scheme == rawdb.HashScheme { - config.HashDB = &hashdb.Config{ - CleanCacheSize: 0, - } // disable clean cache - } else { - config.PathDB = &pathdb.Config{ - CleanCacheSize: 0, - DirtyCacheSize: 0, - } // disable clean/dirty cache - } - return NewDatabase(diskdb, config) +// testReader implements database.Reader interface, providing function to +// access trie nodes. +type testReader struct { + db ethdb.Database + scheme string + nodes []*trienode.MergedNodeSet // sorted from new to old +} + +// Node implements database.Reader interface, retrieving trie node with +// all available cached layers. +func (r *testReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + // Check the node presence with the cached layer, from latest to oldest. + for _, nodes := range r.nodes { + if _, ok := nodes.Sets[owner]; !ok { + continue + } + n, ok := nodes.Sets[owner].Nodes[string(path)] + if !ok { + continue + } + if n.IsDeleted() || n.Hash != hash { + return nil, &MissingNodeError{Owner: owner, Path: path, NodeHash: hash} + } + return n.Blob, nil + } + // Check the node presence in database. + return rawdb.ReadTrieNode(r.db, owner, path, hash, r.scheme), nil +} + +// testDb implements database.Database interface, using for testing purpose. +type testDb struct { + disk ethdb.Database + root common.Hash + scheme string + nodes map[common.Hash]*trienode.MergedNodeSet + parents map[common.Hash]common.Hash +} + +func newTestDatabase(diskdb ethdb.Database, scheme string) *testDb { + return &testDb{ + disk: diskdb, + root: types.EmptyRootHash, + scheme: scheme, + nodes: make(map[common.Hash]*trienode.MergedNodeSet), + parents: make(map[common.Hash]common.Hash), + } +} + +func (db *testDb) Reader(stateRoot common.Hash) (database.Reader, error) { + nodes, _ := db.dirties(stateRoot, true) + return &testReader{db: db.disk, scheme: db.scheme, nodes: nodes}, nil +} + +func (db *testDb) Preimage(hash common.Hash) []byte { + return rawdb.ReadPreimage(db.disk, hash) +} + +func (db *testDb) InsertPreimage(preimages map[common.Hash][]byte) { + rawdb.WritePreimages(db.disk, preimages) +} + +func (db *testDb) Scheme() string { return db.scheme } + +func (db *testDb) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error { + if root == parent { + return nil + } + if _, ok := db.nodes[root]; ok { + return nil + } + db.parents[root] = parent + db.nodes[root] = nodes + return nil +} + +func (db *testDb) dirties(root common.Hash, topToBottom bool) ([]*trienode.MergedNodeSet, []common.Hash) { + var ( + pending []*trienode.MergedNodeSet + roots []common.Hash + ) + for { + if root == db.root { + break + } + nodes, ok := db.nodes[root] + if !ok { + break + } + if topToBottom { + pending = append(pending, nodes) + roots = append(roots, root) + } else { + pending = append([]*trienode.MergedNodeSet{nodes}, pending...) + roots = append([]common.Hash{root}, roots...) + } + root = db.parents[root] + } + return pending, roots +} + +func (db *testDb) Commit(root common.Hash) error { + if root == db.root { + return nil + } + pending, roots := db.dirties(root, false) + for i, nodes := range pending { + for owner, set := range nodes.Sets { + if owner == (common.Hash{}) { + continue + } + set.ForEachWithOrder(func(path string, n *trienode.Node) { + rawdb.WriteTrieNode(db.disk, owner, []byte(path), n.Hash, n.Blob, db.scheme) + }) + } + nodes.Sets[common.Hash{}].ForEachWithOrder(func(path string, n *trienode.Node) { + rawdb.WriteTrieNode(db.disk, common.Hash{}, []byte(path), n.Hash, n.Blob, db.scheme) + }) + db.root = roots[i] + } + for _, root := range roots { + delete(db.nodes, root) + delete(db.parents, root) + } + return nil } diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 1c6e842c7f..fe71ffb0fd 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -19,7 +19,7 @@ import ( type Hbss2Pbss struct { trie *Trie // traverse trie - db *Database + db Database blockNum uint64 root node // root of triedb stateRootHash common.Hash @@ -33,7 +33,7 @@ const ( ) // NewHbss2Pbss return a hash2Path obj -func NewHbss2Pbss(tr *Trie, db *Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64) (*Hbss2Pbss, error) { +func NewHbss2Pbss(tr *Trie, db Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64) (*Hbss2Pbss, error) { if tr == nil { return nil, errors.New("trie is nil") } @@ -68,11 +68,11 @@ func (t *Trie) resolveWithoutTrack(n node, prefix []byte) (node, error) { func (h2p *Hbss2Pbss) writeNode(pathKey []byte, n *trienode.Node, owner common.Hash) { if owner == (common.Hash{}) { - rawdb.WriteAccountTrieNode(h2p.db.diskdb, pathKey, n.Blob) + rawdb.WriteAccountTrieNode(h2p.db.DiskDB(), pathKey, n.Blob) log.Debug("Write account trie node", "path", common.Bytes2Hex(pathKey), "hash", n.Hash, "blob hash", crypto.Keccak256Hash(n.Blob)) } else { - rawdb.WriteStorageTrieNode(h2p.db.diskdb, owner, pathKey, n.Blob) + rawdb.WriteStorageTrieNode(h2p.db.DiskDB(), owner, pathKey, n.Blob) log.Debug("Write storage trie node", "path", common.Bytes2Hex(pathKey), "owner", owner.String(), "hash", n.Hash, "blob hash", crypto.Keccak256Hash(n.Blob)) } @@ -88,8 +88,8 @@ func (h2p *Hbss2Pbss) Run() { log.Info("Hbss to pbss statistics", "total complete", h2p.totalNum, "go routines num", runtime.NumGoroutine(), "h2p concurrent queue", len(h2p.concurrentQueue)) - rawdb.WritePersistentStateID(h2p.db.diskdb, h2p.blockNum) - rawdb.WriteStateID(h2p.db.diskdb, h2p.stateRootHash, h2p.blockNum) + rawdb.WritePersistentStateID(h2p.db.DiskDB(), h2p.blockNum) + rawdb.WriteStateID(h2p.db.DiskDB(), h2p.stateRootHash, h2p.blockNum) } func (h2p *Hbss2Pbss) SubConcurrentTraversal(theTrie *Trie, theNode node, path []byte) { diff --git a/trie/inspect_trie.go b/trie/inspect_trie.go index 42b3171058..793e333509 100644 --- a/trie/inspect_trie.go +++ b/trie/inspect_trie.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/triedb/database" "math/big" "os" "runtime" @@ -23,6 +25,13 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +type Database interface { + database.Database + Scheme() string + Cap(limit common.StorageSize) error + DiskDB() ethdb.Database +} + type Account struct { Nonce uint64 Balance *big.Int @@ -32,7 +41,7 @@ type Account struct { type Inspector struct { trie *Trie // traverse trie - db *Database + db Database stateRootHash common.Hash blockNum uint64 root node // root of triedb @@ -112,7 +121,7 @@ func (nodeStat *NodeStat) ValueNodeCount() string { } // NewInspector return a inspector obj -func NewInspector(tr *Trie, db *Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64) (*Inspector, error) { +func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64) (*Inspector, error) { if tr == nil { return nil, errors.New("trie is nil") } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 9679b49ca7..41e83f6cb6 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -30,7 +30,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) @@ -43,7 +43,7 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -60,7 +60,7 @@ func TestIterator(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) found := make(map[string]string) @@ -86,7 +86,7 @@ func (k *kv) cmp(other *kv) int { } func TestIteratorLargeData(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -205,7 +205,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for _, val := range testdata1 { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -246,22 +246,22 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dba := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) @@ -288,22 +288,22 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dba := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) @@ -341,7 +341,8 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + tr := NewEmpty(db) for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -365,9 +366,9 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tr.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := tr.Commit(false) - tdb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - tdb.Commit(root, false) + tdb.Commit(root) } tr, _ = New(TrieID(root), tdb) wantNodeCount := checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) @@ -481,9 +482,9 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin break } } - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, false) + triedb.Commit(root) } var ( barNodeBlob []byte @@ -555,8 +556,8 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - triedb.Commit(root, false) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Commit(root) var found = make(map[common.Hash][]byte) trie, _ = New(TrieID(root), triedb) diff --git a/trie/proof.go b/trie/proof.go index a526a53402..fd892fb4be 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -389,7 +389,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the - // path(it belongs to the range), unset the entrie + // path(it belongs to the range), unset the entries // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil diff --git a/trie/proof_test.go b/trie/proof_test.go index 59ae201cea..5471d0efa6 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -94,7 +94,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -145,7 +145,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -343,7 +343,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + tinyTrie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.MustUpdate(entry.k, entry.v) @@ -414,7 +414,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -520,7 +520,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -592,7 +592,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -934,7 +934,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -953,7 +953,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -978,7 +978,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i, key := range keys { trie.MustUpdate(key, vals[i]) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 423205ec4e..6ac368fe32 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) // SecureTrie is the old name of StateTrie. @@ -29,7 +30,7 @@ type SecureTrie = StateTrie // NewSecure creates a new StateTrie. // Deprecated: use NewStateTrie. -func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { +func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db database.Database) (*SecureTrie, error) { id := &ID{ StateRoot: stateRoot, Owner: owner, @@ -50,7 +51,7 @@ func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *D // StateTrie is not safe for concurrent use. type StateTrie struct { trie Trie - preimages *preimageStore + db database.Database hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch @@ -61,7 +62,7 @@ type StateTrie struct { // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil // and returns MissingNodeError if the root node cannot be found. -func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { +func NewStateTrie(id *ID, db database.Database) (*StateTrie, error) { if db == nil { panic("trie.NewStateTrie called without a database") } @@ -69,12 +70,12 @@ func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { if err != nil { return nil, err } - return &StateTrie{trie: *trie, preimages: db.preimages}, nil + return &StateTrie{trie: *trie, db: db}, nil } // NewStateTrieByInnerReader creates a trie with an existing root node from a backing database and a inner reader. // Which is used by proof keeper to avoid deadlock in pathdb read/write. -func NewStateTrieByInnerReader(id *ID, db *Database, innerReader Reader) (*StateTrie, error) { +func NewStateTrieByInnerReader(id *ID, db database.Database, innerReader database.Reader) (*StateTrie, error) { if db == nil || innerReader == nil { panic("trie.NewStateTrieByInnerReader called without a database or a inner reader") } @@ -91,7 +92,7 @@ func NewStateTrieByInnerReader(id *ID, db *Database, innerReader Reader) (*State } trie.root = rootnode } - return &StateTrie{trie: *trie, preimages: db.preimages}, nil + return &StateTrie{trie: *trie, db: db}, nil } // MustGet returns the value for key stored in the trie. @@ -232,10 +233,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - if t.preimages == nil { - return nil - } - return t.preimages.preimage(common.BytesToHash(shaKey)) + return t.db.Preimage(common.BytesToHash(shaKey)) } // Commit collects all dirty nodes in the trie and replaces them with the @@ -248,13 +246,11 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.preimages != nil { - preimages := make(map[common.Hash][]byte) - for hk, key := range t.secKeyCache { - preimages[common.BytesToHash([]byte(hk))] = key - } - t.preimages.insertPreimage(preimages) + preimages := make(map[common.Hash][]byte) + for hk, key := range t.secKeyCache { + preimages[common.BytesToHash([]byte(hk))] = key } + t.db.InsertPreimage(preimages) t.secKeyCache = make(map[string][]byte) } // Commit the trie and return its modified nodeset. @@ -271,7 +267,7 @@ func (t *StateTrie) Hash() common.Hash { func (t *StateTrie) Copy() *StateTrie { return &StateTrie{ trie: *t.trie.Copy(), - preimages: t.preimages, + db: t.db, secKeyCache: t.secKeyCache, } } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 2087866d38..0a6fd688b7 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -31,14 +31,14 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) return trie } // makeTestStateTrie creates a large enough secure trie for testing. -func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { +func makeTestStateTrie() (*testDb, *StateTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + triedb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data @@ -61,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state diff --git a/trie/stacktrie.go b/trie/stacktrie.go index f2f5355c49..9c574db0bf 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -23,8 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" ) var ( @@ -32,62 +30,32 @@ var ( _ = types.TrieHasher((*StackTrie)(nil)) ) -// StackTrieOptions contains the configured options for manipulating the stackTrie. -type StackTrieOptions struct { - Writer func(path []byte, hash common.Hash, blob []byte) // The function to commit the dirty nodes - Cleaner func(path []byte) // The function to clean up dangling nodes - - SkipLeftBoundary bool // Flag whether the nodes on the left boundary are skipped for committing - SkipRightBoundary bool // Flag whether the nodes on the right boundary are skipped for committing - boundaryGauge metrics.Gauge // Gauge to track how many boundary nodes are met -} - -// NewStackTrieOptions initializes an empty options for stackTrie. -func NewStackTrieOptions() *StackTrieOptions { return &StackTrieOptions{} } - -// WithWriter configures trie node writer within the options. -func (o *StackTrieOptions) WithWriter(writer func(path []byte, hash common.Hash, blob []byte)) *StackTrieOptions { - o.Writer = writer - return o -} - -// WithCleaner configures the cleaner in the option for removing dangling nodes. -func (o *StackTrieOptions) WithCleaner(cleaner func(path []byte)) *StackTrieOptions { - o.Cleaner = cleaner - return o -} - -// WithSkipBoundary configures whether the left and right boundary nodes are -// filtered for committing, along with a gauge metrics to track how many -// boundary nodes are met. -func (o *StackTrieOptions) WithSkipBoundary(skipLeft, skipRight bool, gauge metrics.Gauge) *StackTrieOptions { - o.SkipLeftBoundary = skipLeft - o.SkipRightBoundary = skipRight - o.boundaryGauge = gauge - return o -} +// OnTrieNode is a callback method invoked when a trie node is committed +// by the stack trie. The node is only committed if it's considered complete. +// +// The caller should not modify the contents of the returned path and blob +// slice, and their contents may be changed after the call. It is up to the +// `onTrieNode` receiver function to deep-copy the data if it wants to retain +// it after the call ends. +type OnTrieNode func(path []byte, hash common.Hash, blob []byte) // StackTrie is a trie implementation that expects keys to be inserted // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { - options *StackTrieOptions - root *stNode - h *hasher - - first []byte // The (hex-encoded without terminator) key of first inserted entry, tracked as left boundary. - last []byte // The (hex-encoded without terminator) key of last inserted entry, tracked as right boundary. + root *stNode + h *hasher + last []byte + onTrieNode OnTrieNode } -// NewStackTrie allocates and initializes an empty trie. -func NewStackTrie(options *StackTrieOptions) *StackTrie { - if options == nil { - options = NewStackTrieOptions() - } +// NewStackTrie allocates and initializes an empty trie. The committed nodes +// will be discarded immediately if no callback is configured. +func NewStackTrie(onTrieNode OnTrieNode) *StackTrie { return &StackTrie{ - options: options, - root: stPool.Get().(*stNode), - h: newHasher(false), + root: stPool.Get().(*stNode), + h: newHasher(false), + onTrieNode: onTrieNode, } } @@ -101,10 +69,6 @@ func (t *StackTrie) Update(key, value []byte) error { if bytes.Compare(t.last, k) >= 0 { return errors.New("non-ascending key order") } - // track the first and last inserted entries. - if t.first == nil { - t.first = append([]byte{}, k...) - } if t.last == nil { t.last = append([]byte{}, k...) // allocate key slice } else { @@ -114,19 +78,9 @@ func (t *StackTrie) Update(key, value []byte) error { return nil } -// MustUpdate is a wrapper of Update and will omit any encountered error but -// just print out an error message. -func (t *StackTrie) MustUpdate(key, value []byte) { - if err := t.Update(key, value); err != nil { - log.Error("Unhandled trie error in StackTrie.Update", "err", err) - } -} - // Reset resets the stack trie object to empty state. func (t *StackTrie) Reset() { - t.options = NewStackTrieOptions() t.root = stPool.Get().(*stNode) - t.first = nil t.last = nil } @@ -346,10 +300,7 @@ func (t *StackTrie) insert(st *stNode, key, value []byte, path []byte) { // // This method also sets 'st.type' to hashedNode, and clears 'st.key'. func (t *StackTrie) hash(st *stNode, path []byte) { - var ( - blob []byte // RLP-encoded node blob - internal [][]byte // List of node paths covered by the extension node - ) + var blob []byte // RLP-encoded node blob switch st.typ { case hashedNode: return @@ -384,15 +335,6 @@ func (t *StackTrie) hash(st *stNode, path []byte) { // recursively hash and commit child as the first step t.hash(st.children[0], append(path, st.key...)) - // Collect the path of internal nodes between shortNode and its **in disk** - // child. This is essential in the case of path mode scheme to avoid leaving - // danging nodes within the range of this internal path on disk, which would - // break the guarantee for state healing. - if len(st.children[0].val) >= 32 && t.options.Cleaner != nil { - for i := 1; i < len(st.key); i++ { - internal = append(internal, append(path, st.key[:i]...)) - } - } // encode the extension node n := shortNode{Key: hexToCompactInPlace(st.key)} if len(st.children[0].val) < 32 { @@ -416,11 +358,12 @@ func (t *StackTrie) hash(st *stNode, path []byte) { default: panic("invalid node type") } - + // Convert the node type to hashNode and reset the key slice. st.typ = hashedNode st.key = st.key[:0] - // Skip committing the non-root node if the size is smaller than 32 bytes. + // Skip committing the non-root node if the size is smaller than 32 bytes + // as tiny nodes are always embedded in their parent except root node. if len(blob) < 32 && len(path) > 0 { st.val = common.CopyBytes(blob) return @@ -429,51 +372,20 @@ func (t *StackTrie) hash(st *stNode, path []byte) { // input values. st.val = t.h.hashData(blob) - // Short circuit if the stack trie is not configured for writing. - if t.options.Writer == nil { - return + // Invoke the callback it's provided. Notably, the path and blob slices are + // volatile, please deep-copy the slices in callback if the contents need + // to be retained. + if t.onTrieNode != nil { + t.onTrieNode(path, common.BytesToHash(st.val), blob) } - // Skip committing if the node is on the left boundary and stackTrie is - // configured to filter the boundary. - if t.options.SkipLeftBoundary && bytes.HasPrefix(t.first, path) { - if t.options.boundaryGauge != nil { - t.options.boundaryGauge.Inc(1) - } - return - } - // Skip committing if the node is on the right boundary and stackTrie is - // configured to filter the boundary. - if t.options.SkipRightBoundary && bytes.HasPrefix(t.last, path) { - if t.options.boundaryGauge != nil { - t.options.boundaryGauge.Inc(1) - } - return - } - // Clean up the internal dangling nodes covered by the extension node. - // This should be done before writing the node to adhere to the committing - // order from bottom to top. - for _, path := range internal { - t.options.Cleaner(path) - } - t.options.Writer(path, common.BytesToHash(st.val), blob) } // Hash will firstly hash the entire trie if it's still not hashed and then commit -// all nodes to the associated database. Actually most of the trie nodes have been -// committed already. The main purpose here is to commit the nodes on right boundary. -// -// For stack trie, Hash and Commit are functionally identical. +// all leftover nodes to the associated database. Actually most of the trie nodes +// have been committed already. The main purpose here is to commit the nodes on +// right boundary. func (t *StackTrie) Hash() common.Hash { n := t.root t.hash(n, nil) return common.BytesToHash(n.val) } - -// Commit will firstly hash the entire trie if it's still not hashed and then commit -// all nodes to the associated database. Actually most of the trie nodes have been -// committed already. The main purpose here is to commit the nodes on right boundary. -// -// For stack trie, Hash and Commit are functionally identical. -func (t *StackTrie) Commit() common.Hash { - return t.Hash() -} diff --git a/trie/stacktrie_fuzzer_test.go b/trie/stacktrie_fuzzer_test.go index 1b3f9dbe9c..c8e568355c 100644 --- a/trie/stacktrie_fuzzer_test.go +++ b/trie/stacktrie_fuzzer_test.go @@ -42,15 +42,13 @@ func fuzz(data []byte, debugging bool) { var ( input = bytes.NewReader(data) spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbA = NewDatabase(rawdb.NewDatabase(spongeA), nil) + dbA = newTestDatabase(rawdb.NewDatabase(spongeA), rawdb.HashScheme) trieA = NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbB = NewDatabase(rawdb.NewDatabase(spongeB), nil) - - options = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) { + dbB = newTestDatabase(rawdb.NewDatabase(spongeB), rawdb.HashScheme) + trieB = NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(spongeB, common.Hash{}, path, hash, blob, dbB.Scheme()) }) - trieB = NewStackTrie(options) vals []*kv maxElements = 10000 // operate on unique keys only @@ -87,10 +85,10 @@ func fuzz(data []byte, debugging bool) { panic(err) } if nodes != nil { - dbA.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) } // Flush memdb -> disk (sponge) - dbA.Commit(rootA, false) + dbA.Commit(rootA) // Stacktrie requires sorted insertion slices.SortFunc(vals, (*kv).cmp) @@ -99,10 +97,9 @@ func fuzz(data []byte, debugging bool) { if debugging { fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v) } - trieB.MustUpdate(kv.k, kv.v) + trieB.Update(kv.k, kv.v) } rootB := trieB.Hash() - trieB.Commit() if rootA != rootB { panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootB)) } @@ -114,20 +111,19 @@ func fuzz(data []byte, debugging bool) { // Ensure all the nodes are persisted correctly var ( - nodeset = make(map[string][]byte) // path -> blob - optionsC = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) { + nodeset = make(map[string][]byte) // path -> blob + trieC = NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { if crypto.Keccak256Hash(blob) != hash { panic("invalid node blob") } nodeset[string(path)] = common.CopyBytes(blob) }) - trieC = NewStackTrie(optionsC) checked int ) for _, kv := range vals { - trieC.MustUpdate(kv.k, kv.v) + trieC.Update(kv.k, kv.v) } - rootC := trieC.Commit() + rootC := trieC.Hash() if rootA != rootC { panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootC)) } diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 909a77062a..f053b5112d 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -19,15 +19,12 @@ package trie import ( "bytes" "math/big" - "math/rand" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/testutil" "github.com/stretchr/testify/assert" - "golang.org/x/exp/slices" ) func TestStackTrieInsertAndHash(t *testing.T) { @@ -223,7 +220,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -238,7 +235,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -264,7 +261,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -289,7 +286,7 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) kvs := []struct { K string V string @@ -317,7 +314,7 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + nt := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) kvs := []struct { K string V string @@ -381,90 +378,6 @@ func TestStacktrieNotModifyValues(t *testing.T) { } } -func buildPartialTree(entries []*kv, t *testing.T) map[string]common.Hash { - var ( - options = NewStackTrieOptions() - nodes = make(map[string]common.Hash) - ) - var ( - first int - last = len(entries) - 1 - - noLeft bool - noRight bool - ) - // Enter split mode if there are at least two elements - if rand.Intn(5) != 0 { - for { - first = rand.Intn(len(entries)) - last = rand.Intn(len(entries)) - if first <= last { - break - } - } - if first != 0 { - noLeft = true - } - if last != len(entries)-1 { - noRight = true - } - } - options = options.WithSkipBoundary(noLeft, noRight, nil) - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { - nodes[string(path)] = hash - }) - tr := NewStackTrie(options) - - for i := first; i <= last; i++ { - tr.MustUpdate(entries[i].k, entries[i].v) - } - tr.Commit() - return nodes -} - -func TestPartialStackTrie(t *testing.T) { - for round := 0; round < 100; round++ { - var ( - n = rand.Intn(100) + 1 - entries []*kv - ) - for i := 0; i < n; i++ { - var val []byte - if rand.Intn(3) == 0 { - val = testutil.RandBytes(3) - } else { - val = testutil.RandBytes(32) - } - entries = append(entries, &kv{ - k: testutil.RandBytes(32), - v: val, - }) - } - slices.SortFunc(entries, (*kv).cmp) - - var ( - nodes = make(map[string]common.Hash) - options = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) { - nodes[string(path)] = hash - }) - ) - tr := NewStackTrie(options) - - for i := 0; i < len(entries); i++ { - tr.MustUpdate(entries[i].k, entries[i].v) - } - tr.Commit() - - for j := 0; j < 100; j++ { - for path, hash := range buildPartialTree(entries, t) { - if nodes[path] != hash { - t.Errorf("%v, want %x, got %x", []byte(path), nodes[path], hash) - } - } - } - } -} - func TestStackTrieErrors(t *testing.T) { s := NewStackTrie(nil) // Deletion diff --git a/trie/sync_test.go b/trie/sync_test.go index 585181b48c..7bc68c041f 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -32,7 +32,7 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[string][]byte) { +func makeTestTrie(scheme string) (ethdb.Database, *testDb, *StateTrie, map[string][]byte) { // Create an empty trie db := rawdb.NewMemoryDatabase() triedb := newTestDatabase(db, scheme) @@ -58,10 +58,10 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } - if err := triedb.Commit(root, false); err != nil { + if err := triedb.Commit(root); err != nil { panic(err) } // Re-create the trie based on the new state @@ -143,7 +143,7 @@ func TestEmptySync(t *testing.T) { emptyD, _ := New(TrieID(types.EmptyRootHash), dbD) for i, trie := range []*Trie{emptyA, emptyB, emptyC, emptyD} { - sync := NewSync(trie.Hash(), memorydb.New(), nil, []*Database{dbA, dbB, dbC, dbD}[i].Scheme()) + sync := NewSync(trie.Hash(), memorydb.New(), nil, []*testDb{dbA, dbB, dbC, dbD}[i].Scheme()) if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes) } @@ -684,11 +684,11 @@ func testSyncOrdering(t *testing.T, scheme string) { } } } -func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database) { +func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *testDb) { syncWithHookWriter(t, root, db, srcDb, nil) } -func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database, hookWriter ethdb.KeyValueWriter) { +func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb *testDb, hookWriter ethdb.KeyValueWriter) { // Create a destination trie and sync with the scheduler sched := NewSync(root, db, nil, srcDb.Scheme()) @@ -771,10 +771,10 @@ func testSyncMovingTarget(t *testing.T, scheme string) { diff[string(key)] = val } root, nodes, _ := srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } - if err := srcDb.Commit(root, false); err != nil { + if err := srcDb.Commit(root); err != nil { panic(err) } preRoot = root @@ -796,10 +796,10 @@ func testSyncMovingTarget(t *testing.T, scheme string) { reverted[k] = val } root, nodes, _ = srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } - if err := srcDb.Commit(root, false); err != nil { + if err := srcDb.Commit(root); err != nil { panic(err) } srcTrie, _ = NewStateTrie(TrieID(root), srcDb) @@ -854,10 +854,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x13, 0x44}, nil, srcTrie, stateA) rootA, nodesA, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil); err != nil { + if err := srcTrieDB.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootA, false); err != nil { + if err := srcTrieDB.Commit(rootA); err != nil { panic(err) } // Create a destination trie and sync with the scheduler @@ -873,10 +873,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x01, 0x24}, nil, srcTrie, stateB) rootB, nodesB, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootB, rootA, 0, trienode.NewWithNodeSet(nodesB), nil); err != nil { + if err := srcTrieDB.Update(rootB, rootA, trienode.NewWithNodeSet(nodesB)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootB, false); err != nil { + if err := srcTrieDB.Commit(rootB); err != nil { panic(err) } syncWith(t, rootB, destDisk, srcTrieDB) @@ -891,10 +891,10 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { writeFn([]byte{0x13, 0x44}, nil, srcTrie, stateC) rootC, nodesC, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootC, rootB, 0, trienode.NewWithNodeSet(nodesC), nil); err != nil { + if err := srcTrieDB.Update(rootC, rootB, trienode.NewWithNodeSet(nodesC)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootC, false); err != nil { + if err := srcTrieDB.Commit(rootC); err != nil { panic(err) } syncWith(t, rootC, destDisk, srcTrieDB) @@ -960,10 +960,10 @@ func testSyncAbort(t *testing.T, scheme string) { writeFn(key, val, srcTrie, stateA) rootA, nodesA, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil); err != nil { + if err := srcTrieDB.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootA, false); err != nil { + if err := srcTrieDB.Commit(rootA); err != nil { panic(err) } // Create a destination trie and sync with the scheduler @@ -977,10 +977,10 @@ func testSyncAbort(t *testing.T, scheme string) { deleteFn(key, srcTrie, stateB) rootB, nodesB, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootB, rootA, 0, trienode.NewWithNodeSet(nodesB), nil); err != nil { + if err := srcTrieDB.Update(rootB, rootA, trienode.NewWithNodeSet(nodesB)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootB, false); err != nil { + if err := srcTrieDB.Commit(rootB); err != nil { panic(err) } @@ -1004,10 +1004,10 @@ func testSyncAbort(t *testing.T, scheme string) { writeFn(key, val, srcTrie, stateC) rootC, nodesC, _ := srcTrie.Commit(false) - if err := srcTrieDB.Update(rootC, rootB, 0, trienode.NewWithNodeSet(nodesC), nil); err != nil { + if err := srcTrieDB.Update(rootC, rootB, trienode.NewWithNodeSet(nodesC)); err != nil { panic(err) } - if err := srcTrieDB.Commit(rootC, false); err != nil { + if err := srcTrieDB.Commit(rootC); err != nil { panic(err) } syncWith(t, rootC, destDisk, srcTrieDB) diff --git a/trie/tracer_test.go b/trie/tracer_test.go index acb8c2f6bf..27e42d497a 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) { // Tests if the trie diffs are tracked correctly. Tracer should capture // all non-leaf dirty nodes, no matter the node is embedded or not. func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) // Determine all new nodes are tracked @@ -71,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) seen := setKeys(iterNodes(db, root)) if !compareSet(insertSet, seen) { @@ -104,7 +104,8 @@ func TestTrieTracerNoop(t *testing.T) { } func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie := NewEmpty(db) for _, val := range vals { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -128,7 +129,7 @@ func TestAccessList(t *testing.T) { func testAccessList(t *testing.T, vals []struct{ k, v string }) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) orig = trie.Copy() ) @@ -137,7 +138,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -152,7 +153,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -170,7 +171,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate(key, randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -185,7 +186,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(key), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -200,7 +201,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -211,7 +212,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { // Tests origin values won't be tracked in Iterator or Prover func TestAccessListLeak(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) ) // Create trie from scratch @@ -219,7 +220,7 @@ func TestAccessListLeak(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) var cases = []struct { op func(tr *Trie) @@ -262,14 +263,14 @@ func TestAccessListLeak(t *testing.T) { // in its parent due to the smaller size of the original tree node. func TestTinyTree(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase(), nil) + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie = NewEmpty(db) ) for _, val := range tiny { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, set, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(set), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set)) parent := root trie, _ = New(TrieID(root), db) @@ -278,7 +279,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, set, _ = trie.Commit(false) - db.Update(root, parent, 0, trienode.NewWithNodeSet(set), nil) + db.Update(root, parent, trienode.NewWithNodeSet(set)) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, set); err != nil { @@ -312,7 +313,7 @@ func forNodes(tr *Trie) map[string][]byte { return nodes } -func iterNodes(db *Database, root common.Hash) map[string][]byte { +func iterNodes(db *testDb, root common.Hash) map[string][]byte { tr, _ := New(TrieID(root), db) return forNodes(tr) } diff --git a/trie/trie.go b/trie/trie.go index f4d4d9963c..6f21d3437a 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" ) var ( @@ -87,7 +88,7 @@ func (t *Trie) Copy() *Trie { // zero hash or the sha3 hash of an empty string, then trie is initially // empty, otherwise, the root node must be present in database or returns // a MissingNodeError if not. -func New(id *ID, db *Database) (*Trie, error) { +func New(id *ID, db database.Database) (*Trie, error) { reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { return nil, err @@ -108,7 +109,7 @@ func New(id *ID, db *Database) (*Trie, error) { } // NewEmpty is a shortcut to create empty tree. It's mostly used in tests. -func NewEmpty(db *Database) *Trie { +func NewEmpty(db database.Database) *Trie { tr, _ := New(TrieID(types.EmptyRootHash), db) return tr } diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 4215964559..42bc4316fe 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -21,31 +21,19 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb/database" ) -// Reader wraps the Node method of a backing trie store. -type Reader interface { - // Node retrieves the trie node blob with the provided trie identifier, node path and - // the corresponding node hash. No error will be returned if the node is not found. - // - // When looking up nodes in the account trie, 'owner' is the zero hash. For contract - // storage trie nodes, 'owner' is the hash of the account address that containing the - // storage. - // - // TODO(rjl493456442): remove the 'hash' parameter, it's redundant in PBSS. - Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) -} - // trieReader is a wrapper of the underlying node reader. It's not safe // for concurrent usage. type trieReader struct { owner common.Hash - reader Reader + reader database.Reader banned map[string]struct{} // Marker to prevent node from being accessed, for tests } // newTrieReader initializes the trie reader with the given node reader. -func newTrieReader(stateRoot, owner common.Hash, db *Database) (*trieReader, error) { +func newTrieReader(stateRoot, owner common.Hash, db database.Database) (*trieReader, error) { if stateRoot == (common.Hash{}) || stateRoot == types.EmptyRootHash { if stateRoot == (common.Hash{}) { log.Error("Zero state root hash!") @@ -85,17 +73,22 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { return blob, nil } -// trieLoader implements triestate.TrieLoader for constructing tries. -type trieLoader struct { - db *Database +// MerkleLoader implements triestate.TrieLoader for constructing tries. +type MerkleLoader struct { + db database.Database +} + +// NewMerkleLoader creates the merkle trie loader. +func NewMerkleLoader(db database.Database) *MerkleLoader { + return &MerkleLoader{db: db} } // OpenTrie opens the main account trie. -func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { +func (l *MerkleLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { return New(TrieID(root), l.db) } // OpenStorageTrie opens the storage trie of an account. -func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { +func (l *MerkleLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { return New(StorageTrieID(stateRoot, addrHash, root), l.db) } diff --git a/trie/trie_test.go b/trie/trie_test.go index c5bd3faf53..c141c52078 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -23,9 +23,9 @@ import ( "fmt" "hash" "io" - "math/big" "math/rand" "reflect" + "sort" "testing" "testing/quick" @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -46,7 +47,7 @@ func init() { } func TestEmptyTrie(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) res := trie.Hash() exp := types.EmptyRootHash if res != exp { @@ -55,7 +56,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) key := make([]byte, 32) value := []byte("test") trie.MustUpdate(key, value) @@ -95,10 +96,10 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, false) + triedb.Commit(root) } trie, _ = New(TrieID(root), triedb) @@ -167,7 +168,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { } func TestInsert(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -179,7 +180,7 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie = NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") @@ -190,7 +191,7 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -209,13 +210,14 @@ func TestGet(t *testing.T) { return } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) } } func TestDelete(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -242,7 +244,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -266,7 +268,7 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), nil) + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -281,7 +283,7 @@ func TestReplication(t *testing.T) { updateString(trie, val.k, val.v) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. trie2, err := New(TrieID(root), db) @@ -300,7 +302,7 @@ func TestReplication(t *testing.T) { // recreate the trie after commit if nodes != nil { - db.Update(hash, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) } trie2, err = New(TrieID(hash), db) if err != nil { @@ -327,13 +329,13 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99}) trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() } -// TestRandomCases tests som cases that were found via random fuzzing +// TestRandomCases tests some cases that were found via random fuzzing func TestRandomCases(t *testing.T) { var rt = []randTestStep{ {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 0 @@ -531,7 +533,7 @@ func runRandTest(rt randTest) error { case opCommit: root, nodes, _ := tr.Commit(true) if nodes != nil { - triedb.Update(root, origin, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, origin, trienode.NewWithNodeSet(nodes)) } newtr, err := New(TrieID(root), triedb) if err != nil { @@ -632,7 +634,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B) { - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) + triedb := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { @@ -651,7 +653,7 @@ func benchGet(b *testing.B) { } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -683,7 +685,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) i := 0 for ; i < len(addresses)/2; i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -714,7 +716,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -728,7 +730,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -741,7 +743,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + checktr := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) @@ -754,7 +756,7 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -796,7 +798,7 @@ func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) { numBytes := random.Uint32() % 33 // [0, 32] bytes balanceBytes := make([]byte, numBytes) random.Read(balanceBytes) - balance := new(big.Int).SetBytes(balanceBytes) + balance := new(uint256.Int).SetBytes(balanceBytes) data, _ := rlp.EncodeToBytes(&types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code}) accounts[i] = data } @@ -808,6 +810,8 @@ type spongeDb struct { sponge hash.Hash id string journal []string + keys []string + values map[string]string } func (s *spongeDb) Has(key []byte) (bool, error) { panic("implement me") } @@ -831,12 +835,27 @@ func (s *spongeDb) Put(key []byte, value []byte) error { valbrief = valbrief[:8] } s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief)) - s.sponge.Write(key) - s.sponge.Write(value) + + if s.values == nil { + s.sponge.Write(key) + s.sponge.Write(value) + } else { + s.keys = append(s.keys, string(key)) + s.values[string(key)] = string(value) + } return nil } func (s *spongeDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { panic("implement me") } +func (s *spongeDb) Flush() { + // Bottom-up, the longest path first + sort.Sort(sort.Reverse(sort.StringSlice(s.keys))) + for _, key := range s.keys { + s.sponge.Write([]byte(key)) + s.sponge.Write([]byte(s.values[key])) + } +} + // spongeBatch is a dummy batch which immediately writes to the underlying spongedb type spongeBatch struct { db *spongeDb @@ -861,14 +880,14 @@ func TestCommitSequence(t *testing.T) { count int expWriteSeqHash []byte }{ - {20, common.FromHex("873c78df73d60e59d4a2bcf3716e8bfe14554549fea2fc147cb54129382a8066")}, - {200, common.FromHex("ba03d891bb15408c940eea5ee3d54d419595102648d02774a0268d892add9c8e")}, - {2000, common.FromHex("f7a184f20df01c94f09537401d11e68d97ad0c00115233107f51b9c287ce60c7")}, + {20, common.FromHex("330b0afae2853d96b9f015791fbe0fb7f239bf65f335f16dfc04b76c7536276d")}, + {200, common.FromHex("5162b3735c06b5d606b043a3ee8adbdbbb408543f4966bca9dcc63da82684eeb")}, + {2000, common.FromHex("4574cd8e6b17f3fe8ad89140d1d0bf4f1bd7a87a8ac3fb623b33550544c77635")}, } { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s), nil) + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -876,9 +895,9 @@ func TestCommitSequence(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false) + db.Commit(root) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Errorf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } @@ -892,14 +911,14 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { count int expWriteSeqHash []byte }{ - {20, common.FromHex("8e4a01548551d139fa9e833ebc4e66fc1ba40a4b9b7259d80db32cff7b64ebbc")}, - {200, common.FromHex("6869b4e7b95f3097a19ddb30ff735f922b915314047e041614df06958fc50554")}, - {2000, common.FromHex("444200e6f4e2df49f77752f629a96ccf7445d4698c164f962bbd85a0526ef424")}, + {20, common.FromHex("8016650c7a50cf88485fd06cde52d634a89711051107f00d21fae98234f2f13d")}, + {200, common.FromHex("dde92ca9812e068e6982d04b40846dc65a61a9fd4996fc0f55f2fde172a8e13c")}, + {2000, common.FromHex("ab553a7f9aff82e3929c382908e30ef7dd17a332933e92ba3fe873fc661ef382")}, } { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s), nil) + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -917,9 +936,9 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false) + db.Commit(root) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Fatalf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } @@ -930,17 +949,24 @@ func TestCommitSequenceStackTrie(t *testing.T) { for count := 1; count < 200; count++ { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes - s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s), nil) + s := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "a", + values: make(map[string]string), + } + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) - // Another sponge is used for the stacktrie commits - stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} - options := NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { + // Another sponge is used for the stacktrie commits + stackTrieSponge := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "b", + values: make(map[string]string), + } + stTrie := NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) - stTrie := NewStackTrie(options) + // Fill the trie with elements for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order @@ -960,13 +986,16 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - db.Commit(root, false) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + s.Flush() + // And flush stacktrie -> disk - stRoot := stTrie.Commit() + stRoot := stTrie.Hash() if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } + stackTrieSponge.Flush() if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { // Show the journal t.Logf("Expected:") @@ -989,34 +1018,44 @@ func TestCommitSequenceStackTrie(t *testing.T) { // that even a small trie which contains a leaf will have an extension making it // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { - s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s), nil) + s := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "a", + values: make(map[string]string), + } + db := newTestDatabase(rawdb.NewDatabase(s), rawdb.HashScheme) trie := NewEmpty(db) - // Another sponge is used for the stacktrie commits - stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} - options := NewStackTrieOptions() - options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) { + // Another sponge is used for the stacktrie commits + stackTrieSponge := &spongeDb{ + sponge: sha3.NewLegacyKeccak256(), + id: "b", + values: make(map[string]string), + } + stTrie := NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) - stTrie := NewStackTrie(options) // Add a single small-element to the trie(s) key := make([]byte, 5) key[0] = 1 trie.Update(key, []byte{0x1}) stTrie.Update(key, []byte{0x1}) + // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - db.Commit(root, false) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + // And flush stacktrie -> disk - stRoot := stTrie.Commit() + stRoot := stTrie.Hash() if stRoot != root { t.Fatalf("root wrong, got %x exp %x", stRoot, root) } - t.Logf("root: %x\n", stRoot) + + s.Flush() + stackTrieSponge.Flush() if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) { t.Fatalf("test, disk write sequence wrong:\ngot %x exp %x\n", got, exp) } @@ -1067,7 +1106,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1118,7 +1157,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1129,60 +1168,6 @@ func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accou b.StopTimer() } -func BenchmarkDerefRootFixedSize(b *testing.B) { - b.Run("10", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(20) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("100", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(100) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - - b.Run("1K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(1000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("10K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(10000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) - b.Run("100K", func(b *testing.B) { - b.StopTimer() - acc, add := makeAccounts(100000) - for i := 0; i < b.N; i++ { - benchmarkDerefRootFixedSize(b, acc, add) - } - }) -} - -func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { - b.ReportAllocs() - triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) - trie := NewEmpty(triedb) - for i := 0; i < len(addresses); i++ { - trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) - } - h := trie.Hash() - root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - b.StartTimer() - triedb.Dereference(h) - b.StopTimer() -} - func getString(trie *Trie, k string) []byte { return trie.MustGet([]byte(k)) } diff --git a/trie/verkle.go b/trie/verkle.go index 89e2e53408..01d813d9ec 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -20,13 +20,13 @@ import ( "encoding/binary" "errors" "fmt" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/triedb/database" "github.com/gballet/go-verkle" "github.com/holiman/uint256" ) @@ -40,13 +40,12 @@ var ( // interface so that Verkle trees can be reused verbatim. type VerkleTrie struct { root verkle.VerkleNode - db *Database cache *utils.PointCache reader *trieReader } // NewVerkleTrie constructs a verkle tree based on the specified root hash. -func NewVerkleTrie(root common.Hash, db *Database, cache *utils.PointCache) (*VerkleTrie, error) { +func NewVerkleTrie(root common.Hash, db database.Database, cache *utils.PointCache) (*VerkleTrie, error) { reader, err := newTrieReader(root, common.Hash{}, db) if err != nil { return nil, err @@ -65,7 +64,6 @@ func NewVerkleTrie(root common.Hash, db *Database, cache *utils.PointCache) (*Ve } return &VerkleTrie{ root: node, - db: db, cache: cache, reader: reader, }, nil @@ -108,7 +106,7 @@ func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error for i := 0; i < len(balance)/2; i++ { balance[len(balance)-i-1], balance[i] = balance[i], balance[len(balance)-i-1] } - acc.Balance = new(big.Int).SetBytes(balance[:]) + acc.Balance = new(uint256.Int).SetBytes32(balance[:]) // Decode codehash acc.CodeHash = values[utils.CodeKeccakLeafKey] @@ -262,7 +260,6 @@ func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { func (t *VerkleTrie) Copy() *VerkleTrie { return &VerkleTrie{ root: t.root.Copy(), - db: t.db, cache: t.cache, reader: t.reader, } diff --git a/trie/verkle_test.go b/trie/verkle_test.go index bd31ea3879..0cbe28bf01 100644 --- a/trie/verkle_test.go +++ b/trie/verkle_test.go @@ -18,27 +18,26 @@ package trie import ( "bytes" - "math/big" "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/utils" + "github.com/holiman/uint256" ) var ( accounts = map[common.Address]*types.StateAccount{ {1}: { Nonce: 100, - Balance: big.NewInt(100), + Balance: uint256.NewInt(100), CodeHash: common.Hash{0x1}.Bytes(), }, {2}: { Nonce: 200, - Balance: big.NewInt(200), + Balance: uint256.NewInt(200), CodeHash: common.Hash{0x2}.Bytes(), }, } @@ -57,12 +56,7 @@ var ( ) func TestVerkleTreeReadWrite(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase(), &Config{ - IsVerkle: true, - PathDB: pathdb.Defaults, - }) - defer db.Close() - + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) for addr, acct := range accounts { diff --git a/trie/database.go b/triedb/database.go similarity index 91% rename from trie/database.go rename to triedb/database.go index a49049c61b..e2d5449676 100644 --- a/trie/database.go +++ b/triedb/database.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package triedb import ( "errors" @@ -24,10 +24,12 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/trie/triedb/hashdb" - "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // Config defines all necessary options for database. @@ -128,7 +130,7 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database { if rawdb.ReadStateScheme(diskdb) == rawdb.PathScheme { log.Warn("incompatible state scheme", "old", rawdb.PathScheme, "new", rawdb.HashScheme) } - db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) + db.backend = hashdb.New(diskdb, config.HashDB, trie.MerkleResolver{}) } else if config.PathDB != nil { if rawdb.ReadStateScheme(diskdb) == rawdb.HashScheme { log.Warn("incompatible state scheme", "old", rawdb.HashScheme, "new", rawdb.PathScheme) @@ -140,17 +142,29 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database { } db.backend = pathdb.New(diskdb, config.PathDB) } else { + // TODO need to confirm + var resolver hashdb.ChildResolver + if config.IsVerkle { + // TODO define verkle resolver + log.Crit("Verkle node resolver is not defined") + } else { + resolver = trie.MerkleResolver{} + } if config.HashDB == nil { config.HashDB = hashdb.Defaults } - db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) + db.backend = hashdb.New(diskdb, config.HashDB, resolver) } return db } +func (db *Database) DiskDB() ethdb.Database { + return db.diskdb +} + // Reader returns a reader for accessing all trie nodes with provided state root. // An error will be returned if the requested state is not available. -func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { +func (db *Database) Reader(blockRoot common.Hash) (database.Reader, error) { switch b := db.backend.(type) { case *hashdb.Database: return b.Reader(blockRoot) @@ -225,8 +239,7 @@ func (db *Database) WritePreimages() { } } -// Preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. +// Preimage retrieves a cached trie node pre-image from preimage store. func (db *Database) Preimage(hash common.Hash) []byte { if db.preimages == nil { return nil @@ -234,6 +247,14 @@ func (db *Database) Preimage(hash common.Hash) []byte { return db.preimages.preimage(hash) } +// InsertPreimage writes pre-images of trie node to the preimage store. +func (db *Database) InsertPreimage(preimages map[common.Hash][]byte) { + if db.preimages == nil { + return + } + db.preimages.insertPreimage(preimages) +} + // Cap iteratively flushes old but still referenced trie nodes until the total // memory usage goes below the given threshold. The held pre-images accumulated // up to this point will be flushed in case the size exceeds the threshold. @@ -284,7 +305,14 @@ func (db *Database) Recover(target common.Hash) error { if !ok { return errors.New("not supported") } - return pdb.Recover(target, &trieLoader{db: db}) + var loader triestate.TrieLoader + if db.config.IsVerkle { + // TODO define verkle loader + log.Crit("Verkle loader is not defined") + } else { + loader = trie.NewMerkleLoader(db) + } + return pdb.Recover(target, loader) } // Recoverable returns the indicator if the specified state is enabled to be diff --git a/triedb/database/database.go b/triedb/database/database.go new file mode 100644 index 0000000000..18a8f454e2 --- /dev/null +++ b/triedb/database/database.go @@ -0,0 +1,48 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package database + +import ( + "github.com/ethereum/go-ethereum/common" +) + +// Reader wraps the Node method of a backing trie reader. +type Reader interface { + // Node retrieves the trie node blob with the provided trie identifier, + // node path and the corresponding node hash. No error will be returned + // if the node is not found. + Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) +} + +// PreimageStore wraps the methods of a backing store for reading and writing +// trie node preimages. +type PreimageStore interface { + // Preimage retrieves the preimage of the specified hash. + Preimage(hash common.Hash) []byte + + // InsertPreimage commits a set of preimages along with their hashes. + InsertPreimage(preimages map[common.Hash][]byte) +} + +// Database wraps the methods of a backing trie store. +type Database interface { + PreimageStore + + // Reader returns a node reader associated with the specific state. + // An error will be returned if the specified state is not available. + Reader(stateRoot common.Hash) (Reader, error) +} diff --git a/trie/triedb/hashdb/database.go b/triedb/hashdb/database.go similarity index 100% rename from trie/triedb/hashdb/database.go rename to triedb/hashdb/database.go diff --git a/trie/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go similarity index 100% rename from trie/triedb/pathdb/asyncnodebuffer.go rename to triedb/pathdb/asyncnodebuffer.go diff --git a/trie/triedb/pathdb/database.go b/triedb/pathdb/database.go similarity index 100% rename from trie/triedb/pathdb/database.go rename to triedb/pathdb/database.go diff --git a/trie/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go similarity index 99% rename from trie/triedb/pathdb/database_test.go rename to triedb/pathdb/database_test.go index 52b298bb9f..8006ec0979 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -20,7 +20,6 @@ import ( "bytes" "errors" "fmt" - "math/big" "math/rand" "testing" @@ -32,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/holiman/uint256" ) func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) { @@ -53,7 +53,7 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm func generateAccount(storageRoot common.Hash) types.StateAccount { return types.StateAccount{ Nonce: uint64(rand.Intn(100)), - Balance: big.NewInt(rand.Int63()), + Balance: uint256.NewInt(rand.Uint64()), CodeHash: testutil.RandBytes(32), Root: storageRoot, } diff --git a/trie/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go similarity index 100% rename from trie/triedb/pathdb/difflayer.go rename to triedb/pathdb/difflayer.go diff --git a/trie/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go similarity index 100% rename from trie/triedb/pathdb/difflayer_test.go rename to triedb/pathdb/difflayer_test.go diff --git a/trie/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go similarity index 100% rename from trie/triedb/pathdb/disklayer.go rename to triedb/pathdb/disklayer.go diff --git a/trie/triedb/pathdb/errors.go b/triedb/pathdb/errors.go similarity index 100% rename from trie/triedb/pathdb/errors.go rename to triedb/pathdb/errors.go diff --git a/trie/triedb/pathdb/history.go b/triedb/pathdb/history.go similarity index 100% rename from trie/triedb/pathdb/history.go rename to triedb/pathdb/history.go diff --git a/trie/triedb/pathdb/history_test.go b/triedb/pathdb/history_test.go similarity index 100% rename from trie/triedb/pathdb/history_test.go rename to triedb/pathdb/history_test.go diff --git a/trie/triedb/pathdb/journal.go b/triedb/pathdb/journal.go similarity index 100% rename from trie/triedb/pathdb/journal.go rename to triedb/pathdb/journal.go diff --git a/trie/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go similarity index 100% rename from trie/triedb/pathdb/layertree.go rename to triedb/pathdb/layertree.go diff --git a/trie/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go similarity index 100% rename from trie/triedb/pathdb/metrics.go rename to triedb/pathdb/metrics.go diff --git a/trie/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go similarity index 100% rename from trie/triedb/pathdb/nodebuffer.go rename to triedb/pathdb/nodebuffer.go diff --git a/trie/triedb/pathdb/nodebufferlist.go b/triedb/pathdb/nodebufferlist.go similarity index 100% rename from trie/triedb/pathdb/nodebufferlist.go rename to triedb/pathdb/nodebufferlist.go diff --git a/trie/triedb/pathdb/testutils.go b/triedb/pathdb/testutils.go similarity index 100% rename from trie/triedb/pathdb/testutils.go rename to triedb/pathdb/testutils.go diff --git a/trie/preimages.go b/triedb/preimages.go similarity index 99% rename from trie/preimages.go rename to triedb/preimages.go index 66f34117c1..a5384910f7 100644 --- a/trie/preimages.go +++ b/triedb/preimages.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package triedb import ( "sync"