From d226b0cdb696f4a92f4f87f6e79440bc39fb1095 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 09:37:44 +0800 Subject: [PATCH 01/22] feat(state): implement PreflightStateDB and TouchedAccounts for enhanced state management --- core/state/taiko_statedb.go | 32 ++++ eth/tracers/preflight/state.go | 171 +++++++++++++++++++++ eth/tracers/taiko_api.go | 271 +++++++++++++++++++++++++++++++++ 3 files changed, 474 insertions(+) create mode 100644 core/state/taiko_statedb.go create mode 100644 eth/tracers/preflight/state.go create mode 100644 eth/tracers/taiko_api.go diff --git a/core/state/taiko_statedb.go b/core/state/taiko_statedb.go new file mode 100644 index 000000000000..22cf68eb3063 --- /dev/null +++ b/core/state/taiko_statedb.go @@ -0,0 +1,32 @@ +package state + +import ( + "github.com/ethereum/go-ethereum/common" +) + +func (s Storage) Keys() []common.Hash { + keys := make([]common.Hash, 0, len(s)) + for key := range s { + keys = append(keys, key) + } + return keys +} + +// TouchedAccounts represents the storage of an account at a specific point in time. +type TouchedAccounts map[common.Address][]common.Hash + +// TouchedAccounts returns a map of all touched accounts and their storage. +// Incudes self-destructed accounts, loaded accounts and new accounts exclude empty account. +func (s *StateDB) TouchedAccounts() TouchedAccounts { + touched := make(TouchedAccounts, len(s.stateObjects)) + for addr, obj := range s.stateObjects { + touched[addr] = obj.originStorage.Keys() + } + for addr, obj := range s.stateObjectsDestruct { + // ignore empty account because it won't affect the state + if !obj.empty() { + touched[addr] = obj.originStorage.Keys() + } + } + return touched +} diff --git a/eth/tracers/preflight/state.go b/eth/tracers/preflight/state.go new file mode 100644 index 000000000000..c5106b168c0b --- /dev/null +++ b/eth/tracers/preflight/state.go @@ -0,0 +1,171 @@ +package preflight + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/stateless" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie/utils" + "github.com/holiman/uint256" +) + +type PreflightStateDB struct { + *state.StateDB + touchedAccounts map[common.Address]*types.StateAccount + ancestors []common.Hash +} + +func (s *PreflightStateDB) CreateAccount(addr common.Address) { + s.StateDB.CreateAccount(addr) +} + +func (s *PreflightStateDB) CreateContract(addr common.Address) { + s.StateDB.CreateContract(addr) +} + +func (s *PreflightStateDB) SubBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { + s.StateDB.SubBalance(addr, amount, reason) +} + +func (s *PreflightStateDB) AddBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { + s.StateDB.AddBalance(addr, amount, reason) +} + +func (s *PreflightStateDB) GetBalance(addr common.Address) *uint256.Int { + return s.StateDB.GetBalance(addr) +} + +func (s *PreflightStateDB) GetNonce(addr common.Address) uint64 { + return s.StateDB.GetNonce(addr) +} + +func (s *PreflightStateDB) SetNonce(addr common.Address, nonce uint64) { + s.StateDB.SetNonce(addr, nonce) +} + +func (s *PreflightStateDB) GetCodeHash(addr common.Address) common.Hash { + return s.StateDB.GetCodeHash(addr) +} + +func (s *PreflightStateDB) GetCode(addr common.Address) []byte { + return s.StateDB.GetCode(addr) +} + +func (s *PreflightStateDB) SetCode(addr common.Address, code []byte) { + s.StateDB.SetCode(addr, code) +} + +func (s *PreflightStateDB) GetCodeSize(addr common.Address) int { + return s.StateDB.GetCodeSize(addr) +} + +func (s *PreflightStateDB) AddRefund(gas uint64) { + s.StateDB.AddRefund(gas) +} + +func (s *PreflightStateDB) SubRefund(gas uint64) { + s.StateDB.SubRefund(gas) +} + +func (s *PreflightStateDB) GetRefund() uint64 { + return s.StateDB.GetRefund() +} + +func (s *PreflightStateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { + return s.StateDB.GetCommittedState(addr, hash) +} + +func (s *PreflightStateDB) GetState(addr common.Address, hash common.Hash) common.Hash { + return s.StateDB.GetState(addr, hash) +} + +func (s *PreflightStateDB) SetState(addr common.Address, key common.Hash, value common.Hash) { + s.StateDB.SetState(addr, key, value) +} + +func (s *PreflightStateDB) GetStorageRoot(addr common.Address) common.Hash { + return s.StateDB.GetStorageRoot(addr) +} + +func (s *PreflightStateDB) GetTransientState(addr common.Address, key common.Hash) common.Hash { + return s.StateDB.GetTransientState(addr, key) +} + +func (s *PreflightStateDB) SetTransientState(addr common.Address, key common.Hash, value common.Hash) { + s.StateDB.SetTransientState(addr, key, value) +} + +func (s *PreflightStateDB) SelfDestruct(addr common.Address) { + s.StateDB.SelfDestruct(addr) +} + +func (s *PreflightStateDB) HasSelfDestructed(addr common.Address) bool { + return s.StateDB.HasSelfDestructed(addr) +} + +func (s *PreflightStateDB) Selfdestruct6780(addr common.Address) { + s.StateDB.Selfdestruct6780(addr) +} + +// Exist reports whether the given account exists in state. +// Notably this should also return true for self-destructed accounts. +func (s *PreflightStateDB) Exist(addr common.Address) bool { + return s.StateDB.Exist(addr) +} + +// Empty returns whether the given account is empty. Empty +// is defined according to EIP161 (balance = nonce = code = 0). +func (s *PreflightStateDB) Empty(addr common.Address) bool { + return s.StateDB.Empty(addr) +} + +func (s *PreflightStateDB) AddressInAccessList(addr common.Address) bool { + return s.StateDB.AddressInAccessList(addr) +} + +func (s *PreflightStateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addressOk bool, slotOk bool) { + return s.StateDB.SlotInAccessList(addr, slot) +} + +// AddAddressToAccessList adds the given address to the access list. This operation is safe to perform +// even if the feature/fork is not active yet +func (s *PreflightStateDB) AddAddressToAccessList(addr common.Address) { + s.StateDB.AddAddressToAccessList(addr) +} + +// AddSlotToAccessList adds the given (address,slot) to the access list. This operation is safe to perform +// even if the feature/fork is not active yet +func (s *PreflightStateDB) AddSlotToAccessList(addr common.Address, slot common.Hash) { + s.StateDB.AddSlotToAccessList(addr, slot) +} + +// PointCache returns the point cache used in computations +func (s *PreflightStateDB) PointCache() *utils.PointCache { + return s.StateDB.PointCache() +} + +func (s *PreflightStateDB) Prepare(rules params.Rules, sender common.Address, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) { + s.StateDB.Prepare(rules, sender, coinbase, dest, precompiles, txAccesses) +} + +func (s *PreflightStateDB) RevertToSnapshot(revid int) { + s.StateDB.RevertToSnapshot(0) +} + +func (s *PreflightStateDB) Snapshot() int { + return s.StateDB.Snapshot() +} + +func (s *PreflightStateDB) AddLog(log *types.Log) { + s.StateDB.AddLog(log) +} + +func (s *PreflightStateDB) AddPreimage(hash common.Hash, preimage []byte) { + s.StateDB.AddPreimage(hash, preimage) +} + +func (s *PreflightStateDB) Witness() *stateless.Witness { + return s.StateDB.Witness() +} diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go new file mode 100644 index 000000000000..d55d8f2ba9ce --- /dev/null +++ b/eth/tracers/taiko_api.go @@ -0,0 +1,271 @@ +package tracers + +import ( + "context" + "fmt" + "runtime" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +var noopTracer = "noopTracer" + +// provingPreflightResult is the result of a proving preflight request. +type provingPreflightResult struct { + Block *types.Block `json:"block"` + ParentHeader *types.Header `json:"parentHeader"` + AccountProofs []*ethapi.AccountResult `json:"accountProofs"` + ParentAccountProofs []*ethapi.AccountResult `json:"parentAccountProofs"` + Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` + AncestorHeaders []*types.Header `json:"ancestorHeaders"` +} + +func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { + from, err := api.blockByNumber(ctx, start) + if err != nil { + return nil, err + } + to, err := api.blockByNumber(ctx, end) + if err != nil { + return nil, err + } + if from.Number().Cmp(to.Number()) >= 0 { + return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) + } + // Tracing a chain is a **long** operation, only do with subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported + } + sub := notifier.CreateSubscription() + + resCh := api.provingPreflights(from, to, config, sub.Err()) + go func() { + for result := range resCh { + notifier.Notify(sub.ID, result) + } + }() + return sub, nil +} + +func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *provingPreflightResult { + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec + } + blocks := int(end.NumberU64() - start.NumberU64()) + threads := runtime.NumCPU() + if threads > blocks { + threads = blocks + } + // no need any execution traces when preflighting + if config == nil { + config = &TraceConfig{} + } + config.Tracer = &noopTracer + var ( + pend = new(sync.WaitGroup) + ctx = context.Background() + taskCh = make(chan *blockTraceTask, threads) + resCh = make(chan *blockTraceTask, threads) + tracker = newStateTracker(maximumPendingTraceStates, start.NumberU64()) + ) + for th := 0; th < threads; th++ { + pend.Add(1) + go func() { + defer pend.Done() + + // Fetch and execute the block trace taskCh + for task := range taskCh { + var ( + signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number(), task.block.Time()) + blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) + ) + // Trace all the transactions contained within + for i, tx := range task.block.Transactions() { + if i == 0 && api.backend.ChainConfig().Taiko { + if err := tx.MarkAsAnchor(); err != nil { + log.Warn("Mark anchor transaction error", "error", err) + task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} + break + } + } + msg, _ := core.TransactionToMessage(tx, signer, task.block.BaseFee()) + txctx := &Context{ + BlockHash: task.block.Hash(), + BlockNumber: task.block.Number(), + TxIndex: i, + TxHash: tx.Hash(), + } + res, err := api.traceTx(ctx, tx, msg, txctx, blockCtx, task.statedb, config) + if err != nil { + task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} + log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) + break + } + task.results[i] = &txTraceResult{TxHash: tx.Hash(), Result: res} + } + // Tracing state is used up, queue it for de-referencing. Note the + // state is the parent state of trace block, use block.number-1 as + // the state number. + tracker.releaseState(task.block.NumberU64()-1, task.release) + + // Stream the result back to the result catcher or abort on teardown + select { + case resCh <- task: + case <-closed: + return + } + } + }() + } + // Start a goroutine to feed all the blocks into the tracers + go func() { + var ( + logged time.Time + begin = time.Now() + number uint64 + traced uint64 + failed error + statedb *state.StateDB + release StateReleaseFunc + ) + // Ensure everything is properly cleaned up on any exit path + defer func() { + close(taskCh) + pend.Wait() + + // Clean out any pending release functions of trace states. + tracker.callReleases() + + // Log the chain result + switch { + case failed != nil: + log.Warn("Chain tracing failed", "start", start.NumberU64(), "end", end.NumberU64(), "transactions", traced, "elapsed", time.Since(begin), "err", failed) + case number < end.NumberU64(): + log.Warn("Chain tracing aborted", "start", start.NumberU64(), "end", end.NumberU64(), "abort", number, "transactions", traced, "elapsed", time.Since(begin)) + default: + log.Info("Chain tracing finished", "start", start.NumberU64(), "end", end.NumberU64(), "transactions", traced, "elapsed", time.Since(begin)) + } + close(resCh) + }() + // Feed all the blocks both into the tracer, as well as fast process concurrently + for number = start.NumberU64(); number < end.NumberU64(); number++ { + // Stop tracing if interruption was requested + select { + case <-closed: + return + default: + } + // Print progress logs if long enough time elapsed + if time.Since(logged) > 8*time.Second { + logged = time.Now() + log.Info("Tracing chain segment", "start", start.NumberU64(), "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin)) + } + // Retrieve the parent block and target block for tracing. + block, err := api.blockByNumber(ctx, rpc.BlockNumber(number)) + if err != nil { + failed = err + break + } + next, err := api.blockByNumber(ctx, rpc.BlockNumber(number+1)) + if err != nil { + failed = err + break + } + // Make sure the state creator doesn't go too far. Too many unprocessed + // trace state may cause the oldest state to become stale(e.g. in + // path-based scheme). + if err = tracker.wait(number); err != nil { + failed = err + break + } + // Prepare the statedb for tracing. Don't use the live database for + // tracing to avoid persisting state junks into the database. Switch + // over to `preferDisk` mode only if the memory usage exceeds the + // limit, the trie database will be reconstructed from scratch only + // if the relevant state is available in disk. + var preferDisk bool + if statedb != nil { + s1, s2, s3 := statedb.Database().TrieDB().Size() + preferDisk = s1+s2+s3 > defaultTracechainMemLimit + } + statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, statedb, false, preferDisk) + if err != nil { + failed = err + break + } + // Insert block's parent beacon block root in the state + // as per EIP-4788. + if beaconRoot := next.BeaconRoot(); beaconRoot != nil { + context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) + core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + } + // Insert parent hash in history contract. + if api.backend.ChainConfig().IsPrague(next.Number(), next.Time()) { + context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) + core.ProcessParentBlockHash(next.ParentHash(), vmenv, statedb) + } + // Clean out any pending release functions of trace state. Note this + // step must be done after constructing tracing state, because the + // tracing state of block next depends on the parent state and construction + // may fail if we release too early. + tracker.callReleases() + + // Send the block over to the concurrent tracers (if not in the fast-forward phase) + txs := next.Transactions() + select { + case taskCh <- &blockTraceTask{statedb: statedb.Copy(), block: next, release: release, results: make([]*txTraceResult, len(txs))}: + case <-closed: + tracker.releaseState(number, release) + return + } + traced += uint64(len(txs)) + } + }() + + // Keep reading the trace results and stream them to result channel. + retCh := make(chan *blockTraceResult) + go func() { + defer close(retCh) + var ( + next = start.NumberU64() + 1 + done = make(map[uint64]*blockTraceResult) + ) + for res := range resCh { + // Queue up next received result + result := &blockTraceResult{ + Block: hexutil.Uint64(res.block.NumberU64()), + Hash: res.block.Hash(), + Traces: res.results, + } + done[uint64(result.Block)] = result + + // Stream completed traces to the result channel + for result, ok := done[next]; ok; result, ok = done[next] { + if len(result.Traces) > 0 || next == end.NumberU64() { + // It will be blocked in case the channel consumer doesn't take the + // tracing result in time(e.g. the websocket connect is not stable) + // which will eventually block the entire chain tracer. It's the + // expected behavior to not waste node resources for a non-active user. + retCh <- result + } + delete(done, next) + next++ + } + } + }() + return retCh +} From d40437418398addd2fc7f35225173722b19d7849 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 12:47:52 +0800 Subject: [PATCH 02/22] feat(taiko): add documentation for ProvingPreflights function --- eth/tracers/taiko_api.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index d55d8f2ba9ce..781ebecc2e6d 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -30,6 +30,23 @@ type provingPreflightResult struct { AncestorHeaders []*types.Header `json:"ancestorHeaders"` } +// ProvingPreflights traces the blockchain from the start block to the end block +// and returns a subscription to the results. This function is intended to be +// used with long-running operations, as tracing a chain can be time-consuming. +// +// Parameters: +// - ctx: The context for the operation. +// - start: The starting block number for the trace. +// - end: The ending block number for the trace. +// - config: The configuration for the trace. +// +// Returns: +// - *rpc.Subscription: A subscription to the trace results. +// - error: An error if the operation fails, or if the end block is not after the start block. +// +// Note: +// - The function requires a notifier to be present in the context, otherwise it +// returns an error indicating that notifications are unsupported. func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { from, err := api.blockByNumber(ctx, start) if err != nil { From 5b421e6c07cb32da500070351848fdb32275f546 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 16:32:21 +0800 Subject: [PATCH 03/22] feat(taiko): enhance ProvingPreflights with error handling and account proof retrieval --- eth/tracers/taiko_api.go | 189 +++++++++++++++++++++++++++++++++++---- 1 file changed, 171 insertions(+), 18 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 781ebecc2e6d..82717d277db7 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -2,8 +2,11 @@ package tracers import ( "context" + "encoding/hex" + "errors" "fmt" "runtime" + "strings" "sync" "time" @@ -13,21 +16,32 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) var noopTracer = "noopTracer" // provingPreflightResult is the result of a proving preflight request. type provingPreflightResult struct { - Block *types.Block `json:"block"` - ParentHeader *types.Header `json:"parentHeader"` - AccountProofs []*ethapi.AccountResult `json:"accountProofs"` - ParentAccountProofs []*ethapi.AccountResult `json:"parentAccountProofs"` - Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` - AncestorHeaders []*types.Header `json:"ancestorHeaders"` + Block *types.Block `json:"block"` + InitAccountProofs []*ethapi.AccountResult `json:"initAccountProofs"` + Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` + AncestorHashes []common.Hash `json:"ancestorHashes"` + Error error `json:"error,omitempty"` +} + +// provingPreflightTask represents a single block preflight task. +type provingPreflightTask struct { + statedb *state.StateDB // Intermediate state prepped for preflighting + parent *types.Block // Parent block of the block to preflight + block *types.Block // Block to preflight the transactions from + release StateReleaseFunc // The function to release the held resource for this task + results []*txTraceResult // Trace results produced by the task + preflight *provingPreflightResult // Preflight results produced by the task } // ProvingPreflights traces the blockchain from the start block to the end block @@ -48,6 +62,12 @@ type provingPreflightResult struct { // - The function requires a notifier to be present in the context, otherwise it // returns an error indicating that notifications are unsupported. func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { + if start == 0 { + return nil, fmt.Errorf("start block must be greater than 0") + } + // Adjust the start block to the parent one + start -= 1 + from, err := api.blockByNumber(ctx, start) if err != nil { return nil, err @@ -75,6 +95,21 @@ func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumbe return sub, nil } +// provingPreflights performs preflight checks on a range of blocks to ensure they are ready for proving. +// It traces transactions within the specified block range and returns a channel that streams the results. +// +// Parameters: +// - start: The starting block of the range to be traced. +// - end: The ending block of the range to be traced. +// - config: Configuration for tracing, including re-execution settings and tracer options. +// - closed: A channel to signal when the tracing should be aborted. +// +// Returns: +// - A channel that streams the results of the preflight checks for each block. +// +// The function uses multiple goroutines to trace transactions concurrently, leveraging the available CPU cores. +// It ensures that the state is properly managed and released after tracing each block. Progress and errors are logged +// throughout the process. func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *provingPreflightResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { @@ -93,8 +128,8 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, var ( pend = new(sync.WaitGroup) ctx = context.Background() - taskCh = make(chan *blockTraceTask, threads) - resCh = make(chan *blockTraceTask, threads) + taskCh = make(chan *provingPreflightTask, threads) + resCh = make(chan *provingPreflightTask, threads) tracker = newStateTracker(maximumPendingTraceStates, start.NumberU64()) ) for th := 0; th < threads; th++ { @@ -114,6 +149,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, if err := tx.MarkAsAnchor(); err != nil { log.Warn("Mark anchor transaction error", "error", err) task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} + task.preflight.Error = err break } } @@ -128,6 +164,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, if err != nil { task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) + task.preflight.Error = err break } task.results[i] = &txTraceResult{TxHash: tx.Hash(), Result: res} @@ -137,6 +174,15 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // the state number. tracker.releaseState(task.block.NumberU64()-1, task.release) + // Retrieve the touched accounts from the state + for addr, slots := range task.statedb.TouchedAccounts() { + proof, err := api.getProof(ctx, addr, slots, task.parent, reexec) + if err != nil { + task.preflight.Error = err + break + } + } + // Stream the result back to the result catcher or abort on teardown select { case resCh <- task: @@ -244,7 +290,19 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Send the block over to the concurrent tracers (if not in the fast-forward phase) txs := next.Transactions() select { - case taskCh <- &blockTraceTask{statedb: statedb.Copy(), block: next, release: release, results: make([]*txTraceResult, len(txs))}: + case taskCh <- &provingPreflightTask{ + statedb: statedb.Copy(), + parent: block, + block: next, + release: release, + results: make([]*txTraceResult, len(txs)), + preflight: &provingPreflightResult{ + Block: next, + InitAccountProofs: []*ethapi.AccountResult{}, + Contracts: map[common.Hash]*hexutil.Bytes{}, + AncestorHashes: []common.Hash{}, + }, + }: case <-closed: tracker.releaseState(number, release) return @@ -254,25 +312,21 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, }() // Keep reading the trace results and stream them to result channel. - retCh := make(chan *blockTraceResult) + retCh := make(chan *provingPreflightResult) go func() { defer close(retCh) var ( next = start.NumberU64() + 1 - done = make(map[uint64]*blockTraceResult) + done = make(map[uint64]*provingPreflightResult) ) for res := range resCh { // Queue up next received result - result := &blockTraceResult{ - Block: hexutil.Uint64(res.block.NumberU64()), - Hash: res.block.Hash(), - Traces: res.results, - } - done[uint64(result.Block)] = result + result := res.preflight + done[uint64(res.block.NumberU64())] = result // Stream completed traces to the result channel for result, ok := done[next]; ok; result, ok = done[next] { - if len(result.Traces) > 0 || next == end.NumberU64() { + if next == end.NumberU64() { // It will be blocked in case the channel consumer doesn't take the // tracing result in time(e.g. the websocket connect is not stable) // which will eventually block the entire chain tracer. It's the @@ -286,3 +340,102 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, }() return retCh } + +// getProof returns the Merkle-proof for a given account and optionally some storage keys. +func (api *API) getProof(ctx context.Context, address common.Address, storageKeys []common.Hash, parent *types.Block, reexec uint64) (*ethapi.AccountResult, error) { + var ( + storageProof = make([]ethapi.StorageResult, len(storageKeys)) + ) + + header := parent.Header() + + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + if statedb == nil || err != nil { + return nil, err + } + defer release() + + codeHash := statedb.GetCodeHash(address) + storageRoot := statedb.GetStorageRoot(address) + + if len(storageKeys) > 0 { + var storageTrie state.Trie + if storageRoot != types.EmptyRootHash && storageRoot != (common.Hash{}) { + id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) + st, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) + if err != nil { + return nil, err + } + storageTrie = st + } + // Create the proofs for the storageKeys. + for i, key := range storageKeys { + // Output key encoding is a bit special: if the input was a 32-byte hash, it is + // returned as such. Otherwise, we apply the QUANTITY encoding mandated by the + // JSON-RPC spec for getProof. This behavior exists to preserve backwards + // compatibility with older client versions. + outputKey := hexutil.Encode(key[:]) + if storageTrie == nil { + storageProof[i] = ethapi.StorageResult{Key: outputKey, Value: &hexutil.Big{}, Proof: []string{}} + continue + } + var proof proofList + if err := storageTrie.Prove(crypto.Keccak256(key.Bytes()), &proof); err != nil { + return nil, err + } + value := (*hexutil.Big)(statedb.GetState(address, key).Big()) + storageProof[i] = ethapi.StorageResult{Key: outputKey, Value: value, Proof: proof} + } + } + // Create the accountProof. + tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), statedb.Database().TrieDB()) + if err != nil { + return nil, err + } + var accountProof proofList + if err := tr.Prove(crypto.Keccak256(address.Bytes()), &accountProof); err != nil { + return nil, err + } + balance := statedb.GetBalance(address).ToBig() + return ðapi.AccountResult{ + Address: address, + AccountProof: accountProof, + Balance: (*hexutil.Big)(balance), + CodeHash: codeHash, + Nonce: hexutil.Uint64(statedb.GetNonce(address)), + StorageHash: storageRoot, + StorageProof: storageProof, + }, statedb.Error() +} + +// proofList implements ethdb.KeyValueWriter and collects the proofs as +// hex-strings for delivery to rpc-caller. +type proofList []string + +func (n *proofList) Put(key []byte, value []byte) error { + *n = append(*n, hexutil.Encode(value)) + return nil +} + +func (n *proofList) Delete(key []byte) error { + panic("not supported") +} + +// decodeHash parses a hex-encoded 32-byte hash. The input may optionally +// be prefixed by 0x and can have a byte length up to 32. +func decodeHash(s string) (h common.Hash, inputLength int, err error) { + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + s = s[2:] + } + if (len(s) & 1) > 0 { + s = "0" + s + } + b, err := hex.DecodeString(s) + if err != nil { + return common.Hash{}, 0, errors.New("hex string invalid") + } + if len(b) > 32 { + return common.Hash{}, len(b), errors.New("hex string too long, want at most 32 bytes") + } + return common.BytesToHash(b), len(b), nil +} From 555af287f6bff8b7d91513a3589520451b12718a Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 16:44:32 +0800 Subject: [PATCH 04/22] feat(taiko): enhance provingPreflights to initialize account proofs and streamline result handling --- eth/tracers/taiko_api.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 82717d277db7..8c6e425044d5 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -181,6 +181,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, task.preflight.Error = err break } + task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) } // Stream the result back to the result catcher or abort on teardown @@ -322,17 +323,15 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, for res := range resCh { // Queue up next received result result := res.preflight - done[uint64(res.block.NumberU64())] = result + done[res.block.NumberU64()] = result // Stream completed traces to the result channel for result, ok := done[next]; ok; result, ok = done[next] { - if next == end.NumberU64() { - // It will be blocked in case the channel consumer doesn't take the - // tracing result in time(e.g. the websocket connect is not stable) - // which will eventually block the entire chain tracer. It's the - // expected behavior to not waste node resources for a non-active user. - retCh <- result - } + // It will be blocked in case the channel consumer doesn't take the + // tracing result in time(e.g. the websocket connect is not stable) + // which will eventually block the entire chain tracer. It's the + // expected behavior to not waste node resources for a non-active user. + retCh <- result delete(done, next) next++ } From 74972a1540dbf5f6ebcd89f5745f4eb9f688d7fb Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 16:45:29 +0800 Subject: [PATCH 05/22] feat(state): remove PreflightStateDB implementation to streamline state management --- eth/tracers/preflight/state.go | 171 --------------------------------- 1 file changed, 171 deletions(-) delete mode 100644 eth/tracers/preflight/state.go diff --git a/eth/tracers/preflight/state.go b/eth/tracers/preflight/state.go deleted file mode 100644 index c5106b168c0b..000000000000 --- a/eth/tracers/preflight/state.go +++ /dev/null @@ -1,171 +0,0 @@ -package preflight - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/stateless" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/holiman/uint256" -) - -type PreflightStateDB struct { - *state.StateDB - touchedAccounts map[common.Address]*types.StateAccount - ancestors []common.Hash -} - -func (s *PreflightStateDB) CreateAccount(addr common.Address) { - s.StateDB.CreateAccount(addr) -} - -func (s *PreflightStateDB) CreateContract(addr common.Address) { - s.StateDB.CreateContract(addr) -} - -func (s *PreflightStateDB) SubBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { - s.StateDB.SubBalance(addr, amount, reason) -} - -func (s *PreflightStateDB) AddBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) { - s.StateDB.AddBalance(addr, amount, reason) -} - -func (s *PreflightStateDB) GetBalance(addr common.Address) *uint256.Int { - return s.StateDB.GetBalance(addr) -} - -func (s *PreflightStateDB) GetNonce(addr common.Address) uint64 { - return s.StateDB.GetNonce(addr) -} - -func (s *PreflightStateDB) SetNonce(addr common.Address, nonce uint64) { - s.StateDB.SetNonce(addr, nonce) -} - -func (s *PreflightStateDB) GetCodeHash(addr common.Address) common.Hash { - return s.StateDB.GetCodeHash(addr) -} - -func (s *PreflightStateDB) GetCode(addr common.Address) []byte { - return s.StateDB.GetCode(addr) -} - -func (s *PreflightStateDB) SetCode(addr common.Address, code []byte) { - s.StateDB.SetCode(addr, code) -} - -func (s *PreflightStateDB) GetCodeSize(addr common.Address) int { - return s.StateDB.GetCodeSize(addr) -} - -func (s *PreflightStateDB) AddRefund(gas uint64) { - s.StateDB.AddRefund(gas) -} - -func (s *PreflightStateDB) SubRefund(gas uint64) { - s.StateDB.SubRefund(gas) -} - -func (s *PreflightStateDB) GetRefund() uint64 { - return s.StateDB.GetRefund() -} - -func (s *PreflightStateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { - return s.StateDB.GetCommittedState(addr, hash) -} - -func (s *PreflightStateDB) GetState(addr common.Address, hash common.Hash) common.Hash { - return s.StateDB.GetState(addr, hash) -} - -func (s *PreflightStateDB) SetState(addr common.Address, key common.Hash, value common.Hash) { - s.StateDB.SetState(addr, key, value) -} - -func (s *PreflightStateDB) GetStorageRoot(addr common.Address) common.Hash { - return s.StateDB.GetStorageRoot(addr) -} - -func (s *PreflightStateDB) GetTransientState(addr common.Address, key common.Hash) common.Hash { - return s.StateDB.GetTransientState(addr, key) -} - -func (s *PreflightStateDB) SetTransientState(addr common.Address, key common.Hash, value common.Hash) { - s.StateDB.SetTransientState(addr, key, value) -} - -func (s *PreflightStateDB) SelfDestruct(addr common.Address) { - s.StateDB.SelfDestruct(addr) -} - -func (s *PreflightStateDB) HasSelfDestructed(addr common.Address) bool { - return s.StateDB.HasSelfDestructed(addr) -} - -func (s *PreflightStateDB) Selfdestruct6780(addr common.Address) { - s.StateDB.Selfdestruct6780(addr) -} - -// Exist reports whether the given account exists in state. -// Notably this should also return true for self-destructed accounts. -func (s *PreflightStateDB) Exist(addr common.Address) bool { - return s.StateDB.Exist(addr) -} - -// Empty returns whether the given account is empty. Empty -// is defined according to EIP161 (balance = nonce = code = 0). -func (s *PreflightStateDB) Empty(addr common.Address) bool { - return s.StateDB.Empty(addr) -} - -func (s *PreflightStateDB) AddressInAccessList(addr common.Address) bool { - return s.StateDB.AddressInAccessList(addr) -} - -func (s *PreflightStateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addressOk bool, slotOk bool) { - return s.StateDB.SlotInAccessList(addr, slot) -} - -// AddAddressToAccessList adds the given address to the access list. This operation is safe to perform -// even if the feature/fork is not active yet -func (s *PreflightStateDB) AddAddressToAccessList(addr common.Address) { - s.StateDB.AddAddressToAccessList(addr) -} - -// AddSlotToAccessList adds the given (address,slot) to the access list. This operation is safe to perform -// even if the feature/fork is not active yet -func (s *PreflightStateDB) AddSlotToAccessList(addr common.Address, slot common.Hash) { - s.StateDB.AddSlotToAccessList(addr, slot) -} - -// PointCache returns the point cache used in computations -func (s *PreflightStateDB) PointCache() *utils.PointCache { - return s.StateDB.PointCache() -} - -func (s *PreflightStateDB) Prepare(rules params.Rules, sender common.Address, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) { - s.StateDB.Prepare(rules, sender, coinbase, dest, precompiles, txAccesses) -} - -func (s *PreflightStateDB) RevertToSnapshot(revid int) { - s.StateDB.RevertToSnapshot(0) -} - -func (s *PreflightStateDB) Snapshot() int { - return s.StateDB.Snapshot() -} - -func (s *PreflightStateDB) AddLog(log *types.Log) { - s.StateDB.AddLog(log) -} - -func (s *PreflightStateDB) AddPreimage(hash common.Hash, preimage []byte) { - s.StateDB.AddPreimage(hash, preimage) -} - -func (s *PreflightStateDB) Witness() *stateless.Witness { - return s.StateDB.Witness() -} From c38e3bebeb2d14e84883631481d9482a40d0a144 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 17:00:38 +0800 Subject: [PATCH 06/22] feat(taiko): refactor provingPreflights to enhance account proof retrieval and error handling --- eth/tracers/taiko_api.go | 59 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 8c6e425044d5..acee3a003cba 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -2,11 +2,8 @@ package tracers import ( "context" - "encoding/hex" - "errors" "fmt" "runtime" - "strings" "sync" "time" @@ -30,7 +27,7 @@ type provingPreflightResult struct { Block *types.Block `json:"block"` InitAccountProofs []*ethapi.AccountResult `json:"initAccountProofs"` Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` - AncestorHashes []common.Hash `json:"ancestorHashes"` + AncestorHashes map[uint64]common.Hash `json:"ancestorHashes"` Error error `json:"error,omitempty"` } @@ -143,6 +140,9 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number(), task.block.Time()) blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) ) + + recordHash := newRecordHash(blockCtx.GetHash) + blockCtx.GetHash = recordHash.getHashFunc // Trace all the transactions contained within for i, tx := range task.block.Transactions() { if i == 0 && api.backend.ChainConfig().Taiko { @@ -176,12 +176,14 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Retrieve the touched accounts from the state for addr, slots := range task.statedb.TouchedAccounts() { - proof, err := api.getProof(ctx, addr, slots, task.parent, reexec) + proof, code, err := api.getProof(ctx, addr, slots, task.parent, reexec) if err != nil { task.preflight.Error = err break } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) + task.preflight.Contracts[proof.CodeHash] = (*hexutil.Bytes)(&code) + task.preflight.AncestorHashes = recordHash.hashes } // Stream the result back to the result catcher or abort on teardown @@ -301,7 +303,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, Block: next, InitAccountProofs: []*ethapi.AccountResult{}, Contracts: map[common.Hash]*hexutil.Bytes{}, - AncestorHashes: []common.Hash{}, + AncestorHashes: map[uint64]common.Hash{}, }, }: case <-closed: @@ -341,7 +343,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, } // getProof returns the Merkle-proof for a given account and optionally some storage keys. -func (api *API) getProof(ctx context.Context, address common.Address, storageKeys []common.Hash, parent *types.Block, reexec uint64) (*ethapi.AccountResult, error) { +func (api *API) getProof(ctx context.Context, address common.Address, storageKeys []common.Hash, parent *types.Block, reexec uint64) (*ethapi.AccountResult, []byte, error) { var ( storageProof = make([]ethapi.StorageResult, len(storageKeys)) ) @@ -350,7 +352,7 @@ func (api *API) getProof(ctx context.Context, address common.Address, storageKey statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) if statedb == nil || err != nil { - return nil, err + return nil, nil, err } defer release() @@ -363,7 +365,7 @@ func (api *API) getProof(ctx context.Context, address common.Address, storageKey id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) st, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) if err != nil { - return nil, err + return nil, nil, err } storageTrie = st } @@ -380,7 +382,7 @@ func (api *API) getProof(ctx context.Context, address common.Address, storageKey } var proof proofList if err := storageTrie.Prove(crypto.Keccak256(key.Bytes()), &proof); err != nil { - return nil, err + return nil, nil, err } value := (*hexutil.Big)(statedb.GetState(address, key).Big()) storageProof[i] = ethapi.StorageResult{Key: outputKey, Value: value, Proof: proof} @@ -389,11 +391,11 @@ func (api *API) getProof(ctx context.Context, address common.Address, storageKey // Create the accountProof. tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), statedb.Database().TrieDB()) if err != nil { - return nil, err + return nil, nil, err } var accountProof proofList if err := tr.Prove(crypto.Keccak256(address.Bytes()), &accountProof); err != nil { - return nil, err + return nil, nil, err } balance := statedb.GetBalance(address).ToBig() return ðapi.AccountResult{ @@ -404,7 +406,7 @@ func (api *API) getProof(ctx context.Context, address common.Address, storageKey Nonce: hexutil.Uint64(statedb.GetNonce(address)), StorageHash: storageRoot, StorageProof: storageProof, - }, statedb.Error() + }, statedb.GetCode(address), statedb.Error() } // proofList implements ethdb.KeyValueWriter and collects the proofs as @@ -420,21 +422,20 @@ func (n *proofList) Delete(key []byte) error { panic("not supported") } -// decodeHash parses a hex-encoded 32-byte hash. The input may optionally -// be prefixed by 0x and can have a byte length up to 32. -func decodeHash(s string) (h common.Hash, inputLength int, err error) { - if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { - s = s[2:] - } - if (len(s) & 1) > 0 { - s = "0" + s - } - b, err := hex.DecodeString(s) - if err != nil { - return common.Hash{}, 0, errors.New("hex string invalid") - } - if len(b) > 32 { - return common.Hash{}, len(b), errors.New("hex string too long, want at most 32 bytes") +type recordHash struct { + hashes map[uint64]common.Hash + hashFunc vm.GetHashFunc +} + +func newRecordHash(hashFunc vm.GetHashFunc) *recordHash { + return &recordHash{ + hashes: make(map[uint64]common.Hash), + hashFunc: hashFunc, } - return common.BytesToHash(b), len(b), nil +} + +func (r *recordHash) getHashFunc(n uint64) common.Hash { + hash := r.hashFunc(n) + r.hashes[n] = hash + return hash } From 8a6cfa62d2be2852d9430a77233781c014c02998 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 17:02:35 +0800 Subject: [PATCH 07/22] feat(taiko): refactor hash handling in provingPreflights for improved record management --- eth/tracers/taiko_api.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index acee3a003cba..49d025286591 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -141,8 +141,8 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) ) - recordHash := newRecordHash(blockCtx.GetHash) - blockCtx.GetHash = recordHash.getHashFunc + newHashFunc := newHashFuncWithRecord(blockCtx.GetHash) + blockCtx.GetHash = newHashFunc.getHash // Trace all the transactions contained within for i, tx := range task.block.Transactions() { if i == 0 && api.backend.ChainConfig().Taiko { @@ -183,7 +183,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) task.preflight.Contracts[proof.CodeHash] = (*hexutil.Bytes)(&code) - task.preflight.AncestorHashes = recordHash.hashes + task.preflight.AncestorHashes = newHashFunc.hashes } // Stream the result back to the result catcher or abort on teardown @@ -422,19 +422,19 @@ func (n *proofList) Delete(key []byte) error { panic("not supported") } -type recordHash struct { +type hashFuncWithRecord struct { hashes map[uint64]common.Hash hashFunc vm.GetHashFunc } -func newRecordHash(hashFunc vm.GetHashFunc) *recordHash { - return &recordHash{ +func newHashFuncWithRecord(hashFunc vm.GetHashFunc) *hashFuncWithRecord { + return &hashFuncWithRecord{ hashes: make(map[uint64]common.Hash), hashFunc: hashFunc, } } -func (r *recordHash) getHashFunc(n uint64) common.Hash { +func (r *hashFuncWithRecord) getHash(n uint64) common.Hash { hash := r.hashFunc(n) r.hashes[n] = hash return hash From b4fc8ca0c154195b39b3687cd94cba746643a393 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 17:07:58 +0800 Subject: [PATCH 08/22] fix(taiko): correct assignment of AncestorHashes in provingPreflights for consistency --- eth/tracers/taiko_api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 49d025286591..65505fe06810 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -183,8 +183,8 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) task.preflight.Contracts[proof.CodeHash] = (*hexutil.Bytes)(&code) - task.preflight.AncestorHashes = newHashFunc.hashes } + task.preflight.AncestorHashes = newHashFunc.hashes // Stream the result back to the result catcher or abort on teardown select { From 8705c3a8d084e6ac4af7b35f7fba3865897538b9 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 17:09:51 +0800 Subject: [PATCH 09/22] refactor(state): simplify key retrieval in TouchedAccounts using maps.Keys --- core/state/taiko_statedb.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/core/state/taiko_statedb.go b/core/state/taiko_statedb.go index 22cf68eb3063..68f05b14ceaf 100644 --- a/core/state/taiko_statedb.go +++ b/core/state/taiko_statedb.go @@ -2,16 +2,9 @@ package state import ( "github.com/ethereum/go-ethereum/common" + "golang.org/x/exp/maps" ) -func (s Storage) Keys() []common.Hash { - keys := make([]common.Hash, 0, len(s)) - for key := range s { - keys = append(keys, key) - } - return keys -} - // TouchedAccounts represents the storage of an account at a specific point in time. type TouchedAccounts map[common.Address][]common.Hash @@ -20,12 +13,12 @@ type TouchedAccounts map[common.Address][]common.Hash func (s *StateDB) TouchedAccounts() TouchedAccounts { touched := make(TouchedAccounts, len(s.stateObjects)) for addr, obj := range s.stateObjects { - touched[addr] = obj.originStorage.Keys() + touched[addr] = maps.Keys(obj.originStorage) } for addr, obj := range s.stateObjectsDestruct { // ignore empty account because it won't affect the state if !obj.empty() { - touched[addr] = obj.originStorage.Keys() + touched[addr] = maps.Keys(obj.originStorage) } } return touched From 54e57449ad6d52ff93548e11e503b72cb2c5fdd8 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 17:37:35 +0800 Subject: [PATCH 10/22] docs(taiko): enhance comments in ProvingPreflights and provingPreflights for clarity and detail --- eth/tracers/taiko_api.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 65505fe06810..3df985744b6a 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -41,7 +41,7 @@ type provingPreflightTask struct { preflight *provingPreflightResult // Preflight results produced by the task } -// ProvingPreflights traces the blockchain from the start block to the end block +// ProvingPreflights traces the blockchain and gets preflights from the start block to the end block // and returns a subscription to the results. This function is intended to be // used with long-running operations, as tracing a chain can be time-consuming. // @@ -92,21 +92,29 @@ func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumbe return sub, nil } -// provingPreflights performs preflight checks on a range of blocks to ensure they are ready for proving. -// It traces transactions within the specified block range and returns a channel that streams the results. +// provingPreflights traces the execution of blocks from `start` to `end` and returns a channel +// that streams the results of the preflight checks for each block. The function uses multiple +// goroutines to parallelize the tracing process. // // Parameters: -// - start: The starting block of the range to be traced. -// - end: The ending block of the range to be traced. -// - config: Configuration for tracing, including re-execution settings and tracer options. -// - closed: A channel to signal when the tracing should be aborted. +// - start: The starting block for tracing. +// - end: The ending block for tracing. +// - config: Configuration for tracing, including the tracer to use and reexec settings. +// - closed: A channel to signal when tracing should be aborted. // // Returns: // - A channel that streams the results of the preflight checks for each block. // -// The function uses multiple goroutines to trace transactions concurrently, leveraging the available CPU cores. -// It ensures that the state is properly managed and released after tracing each block. Progress and errors are logged -// throughout the process. +// The function performs the following steps: +// 1. Determines the number of blocks to trace and the number of threads to use. +// 2. Initializes the tracing configuration and state tracker. +// 3. Starts multiple goroutines to trace the blocks concurrently. +// 4. Feeds the blocks into the tracers and processes the results. +// 5. Streams the results back to the caller through the returned channel. +// +// The tracing process involves fetching the blocks, preparing the state, and tracing each +// transaction within the blocks. The results include the transaction trace results, account +// proofs, contract codes, and ancestor hashes. func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *provingPreflightResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { From 72694d2e6b8f70b6a8ecfd3421c1d1ac8ca4e1d8 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 21:15:55 +0800 Subject: [PATCH 11/22] feat(taiko): implement ApplyTransactionWithTimeout for transaction execution with context timeout --- core/taiko_state_processor.go | 55 +++++++++++++++++++++ eth/tracers/taiko_api.go | 67 ++++++++++++++----------- eth/tracers/taiko_api_test.go | 93 +++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 29 deletions(-) create mode 100644 core/taiko_state_processor.go create mode 100644 eth/tracers/taiko_api_test.go diff --git a/core/taiko_state_processor.go b/core/taiko_state_processor.go new file mode 100644 index 000000000000..b8ddd177948e --- /dev/null +++ b/core/taiko_state_processor.go @@ -0,0 +1,55 @@ +package core + +import ( + "context" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/params" +) + +// ApplyTransactionWithTimeout applies a transaction to the state with a timeout context. +// If the context is cancelled or times out, the EVM execution will be stopped. +// +// Parameters: +// - ctx: The context to control the timeout and cancellation. +// - hashFunc: Function to retrieve block hashes. +// - config: The chain configuration parameters. +// - bc: The blockchain context. +// - author: The address of the block author. +// - gp: The gas pool for the transaction. +// - statedb: The state database. +// - header: The block header. +// - tx: The transaction to be applied. +// - usedGas: Pointer to the used gas value. +// - cfg: The EVM configuration. +// +// Returns: +// - *types.Receipt: The receipt of the transaction. +// - error: An error if the transaction application fails. +func ApplyTransactionWithTimeout(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { + msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) + if err != nil { + return nil, err + } + // CHANGE(taiko): decode the basefeeSharingPctg config from the extradata, and + // add it to the Message, if its an ontake block. + if config.IsOntake(header.Number) { + msg.BasefeeSharingPctg = DecodeOntakeExtraData(header.Extra) + } + // Create a new context to be used in the EVM environment + blockContext := NewEVMBlockContext(header, bc, author) + txContext := NewEVMTxContext(msg) + vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) + go func() { + <-ctx.Done() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + // Stop evm execution. Note cancellation is not necessarily immediate. + vmenv.Cancel() + } + }() + return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) +} diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 3df985744b6a..88ffb6a9844b 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -20,7 +20,9 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -var noopTracer = "noopTracer" +type TaikoBackend interface { + BlockChain() *core.BlockChain +} // provingPreflightResult is the result of a proving preflight request. type provingPreflightResult struct { @@ -37,7 +39,6 @@ type provingPreflightTask struct { parent *types.Block // Parent block of the block to preflight block *types.Block // Block to preflight the transactions from release StateReleaseFunc // The function to release the held resource for this task - results []*txTraceResult // Trace results produced by the task preflight *provingPreflightResult // Preflight results produced by the task } @@ -125,11 +126,6 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, if threads > blocks { threads = blocks } - // no need any execution traces when preflighting - if config == nil { - config = &TraceConfig{} - } - config.Tracer = &noopTracer var ( pend = new(sync.WaitGroup) ctx = context.Background() @@ -144,38 +140,22 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Fetch and execute the block trace taskCh for task := range taskCh { - var ( - signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number(), task.block.Time()) - blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) - ) - - newHashFunc := newHashFuncWithRecord(blockCtx.GetHash) - blockCtx.GetHash = newHashFunc.getHash + newHashFunc := newHashFuncWithRecord() // Trace all the transactions contained within for i, tx := range task.block.Transactions() { if i == 0 && api.backend.ChainConfig().Taiko { if err := tx.MarkAsAnchor(); err != nil { log.Warn("Mark anchor transaction error", "error", err) - task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} task.preflight.Error = err break } } - msg, _ := core.TransactionToMessage(tx, signer, task.block.BaseFee()) - txctx := &Context{ - BlockHash: task.block.Hash(), - BlockNumber: task.block.Number(), - TxIndex: i, - TxHash: tx.Hash(), - } - res, err := api.traceTx(ctx, tx, msg, txctx, blockCtx, task.statedb, config) + err := api.applyTx(ctx, newHashFunc.hashFuncWrapper, i, tx, task.block.Header(), task.statedb, config) if err != nil { - task.results[i] = &txTraceResult{TxHash: tx.Hash(), Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) task.preflight.Error = err break } - task.results[i] = &txTraceResult{TxHash: tx.Hash(), Result: res} } // Tracing state is used up, queue it for de-referencing. Note the // state is the parent state of trace block, use block.number-1 as @@ -306,7 +286,6 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, parent: block, block: next, release: release, - results: make([]*txTraceResult, len(txs)), preflight: &provingPreflightResult{ Block: next, InitAccountProofs: []*ethapi.AccountResult{}, @@ -350,6 +329,32 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, return retCh } +// applyTx configures a new tracer according to the provided configuration, and +// executes the given message in the provided environment. The return value will +// be tracer dependent. +func (api *API) applyTx(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, txIdx int, tx *types.Transaction, header *types.Header, statedb *state.StateDB, config *TraceConfig) error { + var ( + err error + timeout = defaultTraceTimeout + usedGas uint64 + ) + if config != nil && config.Timeout != nil { + if timeout, err = time.ParseDuration(*config.Timeout); err != nil { + return err + } + } + + deadlineCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + statedb.SetTxContext(tx.Hash(), txIdx) + _, err = core.ApplyTransactionWithTimeout(deadlineCtx, hashFuncWrapper, api.backend.ChainConfig(), api.backend.(TaikoBackend).BlockChain(), nil, new(core.GasPool).AddGas(tx.Gas()), statedb, header, tx, &usedGas, vm.Config{}) + if err != nil { + return err + } + return nil +} + // getProof returns the Merkle-proof for a given account and optionally some storage keys. func (api *API) getProof(ctx context.Context, address common.Address, storageKeys []common.Hash, parent *types.Block, reexec uint64) (*ethapi.AccountResult, []byte, error) { var ( @@ -435,13 +440,17 @@ type hashFuncWithRecord struct { hashFunc vm.GetHashFunc } -func newHashFuncWithRecord(hashFunc vm.GetHashFunc) *hashFuncWithRecord { +func newHashFuncWithRecord() *hashFuncWithRecord { return &hashFuncWithRecord{ - hashes: make(map[uint64]common.Hash), - hashFunc: hashFunc, + hashes: make(map[uint64]common.Hash), } } +func (r *hashFuncWithRecord) hashFuncWrapper(hashFunc vm.GetHashFunc) vm.GetHashFunc { + r.hashFunc = hashFunc + return r.getHash +} + func (r *hashFuncWithRecord) getHash(n uint64) common.Hash { hash := r.hashFunc(n) r.hashes[n] = hash diff --git a/eth/tracers/taiko_api_test.go b/eth/tracers/taiko_api_test.go new file mode 100644 index 000000000000..b1518b08656e --- /dev/null +++ b/eth/tracers/taiko_api_test.go @@ -0,0 +1,93 @@ +package tracers + +import ( + "context" + "math/big" + "sync/atomic" + "testing" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +func (b *testBackend) BlockChain() *core.BlockChain { + return b.chain +} + +func TestProvingPreflights(t *testing.T) { + // Initialize test accounts + accounts := newAccounts(3) + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }, + } + genBlocks := 50 + signer := types.HomesteadSigner{} + + var ( + ref atomic.Uint32 // total refs has made + rel atomic.Uint32 // total rels has made + nonce uint64 + ) + backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + for j := 0; j < i+1; j++ { + tx, _ := types.SignTx(types.NewTransaction(nonce, accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + b.AddTx(tx) + nonce += 1 + } + }) + backend.refHook = func() { ref.Add(1) } + backend.relHook = func() { rel.Add(1) } + api := NewAPI(backend) + + // single := `{"txHash":"0x0000000000000000000000000000000000000000000000000000000000000000","result":{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}}` + var cases = []struct { + start uint64 + end uint64 + config *TraceConfig + }{ + {0, 50, nil}, // the entire chain range, blocks [1, 50] + {10, 20, nil}, // the middle chain range, blocks [11, 20] + } + for _, c := range cases { + ref.Store(0) + rel.Store(0) + + from, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.start)) + to, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.end)) + resCh := api.provingPreflights(from, to, c.config, nil) + + next := c.start + 1 + for result := range resCh { + if have, want := result.Block.NumberU64(), next; have != want { + t.Fatalf("unexpected tracing block, have %d want %d", have, want) + } + if next == 1 { + if have, want := len(result.InitAccountProofs), 2; have != want { + t.Fatalf("unexpected result length, have %d want %d", have, want) + } + } else { + if have, want := len(result.InitAccountProofs), 3; have != want { + t.Fatalf("unexpected result length, have %d want %d", have, want) + } + } + next += 1 + } + if next != c.end+1 { + t.Error("Missing tracing block") + } + + if nref, nrel := ref.Load(), rel.Load(); nref != nrel { + t.Errorf("Ref and deref actions are not equal, ref %d rel %d", nref, nrel) + } + } +} From 8828c031a13f896cb2c551977b2374f3547158d1 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 21:40:36 +0800 Subject: [PATCH 12/22] feat(taiko): integrate hash function wrapper into block context for transaction execution --- core/taiko_state_processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/taiko_state_processor.go b/core/taiko_state_processor.go index b8ddd177948e..15a75b24c0b2 100644 --- a/core/taiko_state_processor.go +++ b/core/taiko_state_processor.go @@ -42,6 +42,7 @@ func ApplyTransactionWithTimeout(ctx context.Context, hashFuncWrapper func(vm.Ge } // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) + blockContext.GetHash = hashFuncWrapper(blockContext.GetHash) txContext := NewEVMTxContext(msg) vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) go func() { From f976df216b88988275153139ce81fecfbd1603d7 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 21:42:00 +0800 Subject: [PATCH 13/22] refactor(taiko): rename ApplyTransactionWithTimeout to ApplyTransactionWithContext for clarity --- core/taiko_state_processor.go | 4 ++-- eth/tracers/taiko_api.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/taiko_state_processor.go b/core/taiko_state_processor.go index 15a75b24c0b2..ab10bdeb55f3 100644 --- a/core/taiko_state_processor.go +++ b/core/taiko_state_processor.go @@ -11,7 +11,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// ApplyTransactionWithTimeout applies a transaction to the state with a timeout context. +// ApplyTransactionWithContext applies a transaction to the state with a context. // If the context is cancelled or times out, the EVM execution will be stopped. // // Parameters: @@ -30,7 +30,7 @@ import ( // Returns: // - *types.Receipt: The receipt of the transaction. // - error: An error if the transaction application fails. -func ApplyTransactionWithTimeout(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { +func ApplyTransactionWithContext(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) if err != nil { return nil, err diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 88ffb6a9844b..835bfad3ed40 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -348,7 +348,7 @@ func (api *API) applyTx(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc defer cancel() statedb.SetTxContext(tx.Hash(), txIdx) - _, err = core.ApplyTransactionWithTimeout(deadlineCtx, hashFuncWrapper, api.backend.ChainConfig(), api.backend.(TaikoBackend).BlockChain(), nil, new(core.GasPool).AddGas(tx.Gas()), statedb, header, tx, &usedGas, vm.Config{}) + _, err = core.ApplyTransactionWithContext(deadlineCtx, hashFuncWrapper, api.backend.ChainConfig(), api.backend.(TaikoBackend).BlockChain(), nil, new(core.GasPool).AddGas(tx.Gas()), statedb, header, tx, &usedGas, vm.Config{}) if err != nil { return err } From 61962af2e2dbb606d3bc062bb051d23af5d35cb7 Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 21:42:54 +0800 Subject: [PATCH 14/22] docs(taiko): improve comments in ApplyTransactionWithContext for clarity and detail --- core/taiko_state_processor.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/taiko_state_processor.go b/core/taiko_state_processor.go index ab10bdeb55f3..41e9f0c3fe05 100644 --- a/core/taiko_state_processor.go +++ b/core/taiko_state_processor.go @@ -11,25 +11,25 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// ApplyTransactionWithContext applies a transaction to the state with a context. -// If the context is cancelled or times out, the EVM execution will be stopped. +// ApplyTransactionWithContext applies a transaction to the current state with the given context. +// It converts the transaction to a message, creates a new EVM environment, and executes the transaction. // // Parameters: -// - ctx: The context to control the timeout and cancellation. -// - hashFunc: Function to retrieve block hashes. +// - ctx: The context to control the execution timeout and cancellation. +// - hashFuncWrapper: A function to wrap the block's GetHash function. // - config: The chain configuration parameters. // - bc: The blockchain context. // - author: The address of the block author. -// - gp: The gas pool for the transaction. +// - gp: The gas pool for the current block. // - statedb: The state database. // - header: The block header. // - tx: The transaction to be applied. -// - usedGas: Pointer to the used gas value. +// - usedGas: A pointer to the amount of gas used. // - cfg: The EVM configuration. // // Returns: // - *types.Receipt: The receipt of the transaction. -// - error: An error if the transaction application fails. +// - error: An error if the transaction could not be applied. func ApplyTransactionWithContext(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) if err != nil { From 2633d0586f035ff63feb876b5bc1e0844829234b Mon Sep 17 00:00:00 2001 From: john xu Date: Wed, 22 Jan 2025 22:14:21 +0800 Subject: [PATCH 15/22] refactor(taiko): remove ApplyTransactionWithContext and related applyTx function for cleaner codebase --- core/taiko_state_processor.go | 56 ----------------------------------- eth/tracers/taiko_api.go | 45 +++++++++++----------------- 2 files changed, 18 insertions(+), 83 deletions(-) delete mode 100644 core/taiko_state_processor.go diff --git a/core/taiko_state_processor.go b/core/taiko_state_processor.go deleted file mode 100644 index 41e9f0c3fe05..000000000000 --- a/core/taiko_state_processor.go +++ /dev/null @@ -1,56 +0,0 @@ -package core - -import ( - "context" - "errors" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/params" -) - -// ApplyTransactionWithContext applies a transaction to the current state with the given context. -// It converts the transaction to a message, creates a new EVM environment, and executes the transaction. -// -// Parameters: -// - ctx: The context to control the execution timeout and cancellation. -// - hashFuncWrapper: A function to wrap the block's GetHash function. -// - config: The chain configuration parameters. -// - bc: The blockchain context. -// - author: The address of the block author. -// - gp: The gas pool for the current block. -// - statedb: The state database. -// - header: The block header. -// - tx: The transaction to be applied. -// - usedGas: A pointer to the amount of gas used. -// - cfg: The EVM configuration. -// -// Returns: -// - *types.Receipt: The receipt of the transaction. -// - error: An error if the transaction could not be applied. -func ApplyTransactionWithContext(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { - msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) - if err != nil { - return nil, err - } - // CHANGE(taiko): decode the basefeeSharingPctg config from the extradata, and - // add it to the Message, if its an ontake block. - if config.IsOntake(header.Number) { - msg.BasefeeSharingPctg = DecodeOntakeExtraData(header.Extra) - } - // Create a new context to be used in the EVM environment - blockContext := NewEVMBlockContext(header, bc, author) - blockContext.GetHash = hashFuncWrapper(blockContext.GetHash) - txContext := NewEVMTxContext(msg) - vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) - go func() { - <-ctx.Done() - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - // Stop evm execution. Note cancellation is not necessarily immediate. - vmenv.Cancel() - } - }() - return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) -} diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 835bfad3ed40..842793793ffa 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -141,6 +141,11 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Fetch and execute the block trace taskCh for task := range taskCh { newHashFunc := newHashFuncWithRecord() + var ( + signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number(), task.block.Time()) + blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) + ) + blockCtx.GetHash = newHashFunc.hashFuncWrapper(blockCtx.GetHash) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { if i == 0 && api.backend.ChainConfig().Taiko { @@ -150,7 +155,19 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, break } } - err := api.applyTx(ctx, newHashFunc.hashFuncWrapper, i, tx, task.block.Header(), task.statedb, config) + msg, _ := core.TransactionToMessage(tx, signer, task.block.BaseFee()) + // CHANGE(taiko): decode the basefeeSharingPctg config from the extradata, and + // add it to the Message, if its an ontake block. + if api.backend.ChainConfig().IsOntake(task.block.Number()) { + msg.BasefeeSharingPctg = core.DecodeOntakeExtraData(task.block.Header().Extra) + } + txctx := &Context{ + BlockHash: task.block.Hash(), + BlockNumber: task.block.Number(), + TxIndex: i, + TxHash: tx.Hash(), + } + _, err := api.traceTx(ctx, tx, msg, txctx, blockCtx, task.statedb, config) if err != nil { log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) task.preflight.Error = err @@ -329,32 +346,6 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, return retCh } -// applyTx configures a new tracer according to the provided configuration, and -// executes the given message in the provided environment. The return value will -// be tracer dependent. -func (api *API) applyTx(ctx context.Context, hashFuncWrapper func(vm.GetHashFunc) vm.GetHashFunc, txIdx int, tx *types.Transaction, header *types.Header, statedb *state.StateDB, config *TraceConfig) error { - var ( - err error - timeout = defaultTraceTimeout - usedGas uint64 - ) - if config != nil && config.Timeout != nil { - if timeout, err = time.ParseDuration(*config.Timeout); err != nil { - return err - } - } - - deadlineCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - statedb.SetTxContext(tx.Hash(), txIdx) - _, err = core.ApplyTransactionWithContext(deadlineCtx, hashFuncWrapper, api.backend.ChainConfig(), api.backend.(TaikoBackend).BlockChain(), nil, new(core.GasPool).AddGas(tx.Gas()), statedb, header, tx, &usedGas, vm.Config{}) - if err != nil { - return err - } - return nil -} - // getProof returns the Merkle-proof for a given account and optionally some storage keys. func (api *API) getProof(ctx context.Context, address common.Address, storageKeys []common.Hash, parent *types.Block, reexec uint64) (*ethapi.AccountResult, []byte, error) { var ( From fa7801d82f3cfd8e058e5e048d07636632eed179 Mon Sep 17 00:00:00 2001 From: johntaiko Date: Wed, 22 Jan 2025 23:07:26 +0800 Subject: [PATCH 16/22] Update taiko_api.go --- eth/tracers/taiko_api.go | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 842793793ffa..307cd400b00a 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -140,12 +140,19 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Fetch and execute the block trace taskCh for task := range taskCh { - newHashFunc := newHashFuncWithRecord() + touchedHashes := map[uint64]common.Hash{} + hashFuncWrapper := func(hashFunc vm.GetHashFunc) vm.GetHashFunc { + return func(n uint64) common.Hash { + hash := hashFunc(n) + touchedHashes[n] = hash + return hash + } + } var ( signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number(), task.block.Time()) blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) ) - blockCtx.GetHash = newHashFunc.hashFuncWrapper(blockCtx.GetHash) + blockCtx.GetHash = hashFuncWrapper(blockCtx.GetHash) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { if i == 0 && api.backend.ChainConfig().Taiko { @@ -189,7 +196,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) task.preflight.Contracts[proof.CodeHash] = (*hexutil.Bytes)(&code) } - task.preflight.AncestorHashes = newHashFunc.hashes + task.preflight.AncestorHashes = touchedHashes // Stream the result back to the result catcher or abort on teardown select { @@ -425,25 +432,3 @@ func (n *proofList) Put(key []byte, value []byte) error { func (n *proofList) Delete(key []byte) error { panic("not supported") } - -type hashFuncWithRecord struct { - hashes map[uint64]common.Hash - hashFunc vm.GetHashFunc -} - -func newHashFuncWithRecord() *hashFuncWithRecord { - return &hashFuncWithRecord{ - hashes: make(map[uint64]common.Hash), - } -} - -func (r *hashFuncWithRecord) hashFuncWrapper(hashFunc vm.GetHashFunc) vm.GetHashFunc { - r.hashFunc = hashFunc - return r.getHash -} - -func (r *hashFuncWithRecord) getHash(n uint64) common.Hash { - hash := r.hashFunc(n) - r.hashes[n] = hash - return hash -} From dd4ecbf5ca49d616901ea3f994434fe6539e90ad Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 15:21:39 +0800 Subject: [PATCH 17/22] refactor(taiko): update TouchedAccounts logic to include self-destructed accounts and improve error handling in provingPreflights --- core/state/taiko_statedb.go | 2 +- core/state/taiko_statedb_test.go | 58 ++++++++++++++++++++++++++++++++ eth/tracers/taiko_api.go | 8 ++--- 3 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 core/state/taiko_statedb_test.go diff --git a/core/state/taiko_statedb.go b/core/state/taiko_statedb.go index 68f05b14ceaf..0ab82f5642db 100644 --- a/core/state/taiko_statedb.go +++ b/core/state/taiko_statedb.go @@ -17,7 +17,7 @@ func (s *StateDB) TouchedAccounts() TouchedAccounts { } for addr, obj := range s.stateObjectsDestruct { // ignore empty account because it won't affect the state - if !obj.empty() { + if obj.selfDestructed || !obj.empty() { touched[addr] = maps.Keys(obj.originStorage) } } diff --git a/core/state/taiko_statedb_test.go b/core/state/taiko_statedb_test.go new file mode 100644 index 000000000000..8a577f84268c --- /dev/null +++ b/core/state/taiko_statedb_test.go @@ -0,0 +1,58 @@ +package state + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" +) + +// TestCopy tests that copying a StateDB object indeed makes the original and +// the copy independent of each other. This test is a regression test against +// https://github.com/ethereum/go-ethereum/pull/15549. +func TestTouchedAccounts(t *testing.T) { + // Create a random state test to copy and modify "independently" + orig, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) + + accountCounts := 10 + + for i := byte(0); i < byte(accountCounts); i++ { + obj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) + obj.AddBalance(uint256.NewInt(uint64(i)), tracing.BalanceChangeUnspecified) + orig.updateStateObject(obj) + } + orig.Finalise(false) + + touchedAccs := orig.TouchedAccounts() + if have, want := len(touchedAccs), accountCounts; have != want { + t.Fatalf("have %d touched accounts, want %d", have, want) + } + + // destroy accounts + selfDestruct1 := common.BytesToAddress([]byte{1}) + selfDestruct2 := common.BytesToAddress([]byte{2}) + selfDestruct3 := common.BytesToAddress([]byte{3}) + + orig.SelfDestruct(selfDestruct1) + orig.SelfDestruct(selfDestruct2) + orig.SelfDestruct(selfDestruct3) + orig.Finalise(false) + + touchedAccs = orig.TouchedAccounts() + if have, want := len(touchedAccs), accountCounts; have != want { + t.Fatalf("have %d touched accounts, want %d", have, want) + } + + // create empty accounts + for i := byte(10); i < 12; i++ { + orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) + } + orig.Finalise(true) + + touchedAccs = orig.TouchedAccounts() + if have, want := len(touchedAccs), accountCounts; have != want { + t.Fatalf("have %d touched accounts, want %d", have, want) + } +} diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 307cd400b00a..1bd4b4b16cc6 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -30,7 +30,7 @@ type provingPreflightResult struct { InitAccountProofs []*ethapi.AccountResult `json:"initAccountProofs"` Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` AncestorHashes map[uint64]common.Hash `json:"ancestorHashes"` - Error error `json:"error,omitempty"` + Error string `json:"error,omitempty"` } // provingPreflightTask represents a single block preflight task. @@ -158,7 +158,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, if i == 0 && api.backend.ChainConfig().Taiko { if err := tx.MarkAsAnchor(); err != nil { log.Warn("Mark anchor transaction error", "error", err) - task.preflight.Error = err + task.preflight.Error = err.Error() break } } @@ -177,7 +177,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, _, err := api.traceTx(ctx, tx, msg, txctx, blockCtx, task.statedb, config) if err != nil { log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) - task.preflight.Error = err + task.preflight.Error = err.Error() break } } @@ -190,7 +190,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, for addr, slots := range task.statedb.TouchedAccounts() { proof, code, err := api.getProof(ctx, addr, slots, task.parent, reexec) if err != nil { - task.preflight.Error = err + task.preflight.Error = err.Error() break } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) From fb332302bcde646d7c72d978587b3961ed69aaee Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 15:34:37 +0800 Subject: [PATCH 18/22] refactor(taiko): streamline TestTouchedAccounts by consolidating account destruction logic --- core/state/taiko_statedb_test.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/core/state/taiko_statedb_test.go b/core/state/taiko_statedb_test.go index 8a577f84268c..37419ae2c8f7 100644 --- a/core/state/taiko_statedb_test.go +++ b/core/state/taiko_statedb_test.go @@ -30,14 +30,22 @@ func TestTouchedAccounts(t *testing.T) { t.Fatalf("have %d touched accounts, want %d", have, want) } - // destroy accounts - selfDestruct1 := common.BytesToAddress([]byte{1}) - selfDestruct2 := common.BytesToAddress([]byte{2}) - selfDestruct3 := common.BytesToAddress([]byte{3}) + // touch slots + for i := byte(0); i < 3; i++ { + orig.getOrNewStateObject(common.BytesToAddress([]byte{i})).SetState(common.Hash{i}, common.Hash{i}) + } + orig.Finalise(false) + touchedAccs = orig.TouchedAccounts() + for i := byte(0); i < 3; i++ { + if have, want := len(touchedAccs[common.BytesToAddress([]byte{i})]), 1; have != want { + t.Fatalf("have %d touched accounts, want %d", have, want) + } + } - orig.SelfDestruct(selfDestruct1) - orig.SelfDestruct(selfDestruct2) - orig.SelfDestruct(selfDestruct3) + // destroy accounts + for i := byte(0); i < 3; i++ { + orig.SelfDestruct(common.BytesToAddress([]byte{i})) + } orig.Finalise(false) touchedAccs = orig.TouchedAccounts() @@ -55,4 +63,5 @@ func TestTouchedAccounts(t *testing.T) { if have, want := len(touchedAccs), accountCounts; have != want { t.Fatalf("have %d touched accounts, want %d", have, want) } + } From 160e3dc829b4fbe779bf050e337951bc696b9e83 Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 15:47:52 +0800 Subject: [PATCH 19/22] refactor(taiko): remove unnecessary blank line in TestTouchedAccounts for cleaner code --- core/state/taiko_statedb_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/state/taiko_statedb_test.go b/core/state/taiko_statedb_test.go index 37419ae2c8f7..b7de52f8feda 100644 --- a/core/state/taiko_statedb_test.go +++ b/core/state/taiko_statedb_test.go @@ -63,5 +63,4 @@ func TestTouchedAccounts(t *testing.T) { if have, want := len(touchedAccs), accountCounts; have != want { t.Fatalf("have %d touched accounts, want %d", have, want) } - } From ac059aeae0bc8798d82fa4b06e5148c3c22d6b18 Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 16:30:57 +0800 Subject: [PATCH 20/22] refactor(taiko): update provingPreflightResult to use common.Address for contracts --- eth/tracers/taiko_api.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index 1bd4b4b16cc6..f824a9585335 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -26,11 +26,11 @@ type TaikoBackend interface { // provingPreflightResult is the result of a proving preflight request. type provingPreflightResult struct { - Block *types.Block `json:"block"` - InitAccountProofs []*ethapi.AccountResult `json:"initAccountProofs"` - Contracts map[common.Hash]*hexutil.Bytes `json:"contracts"` - AncestorHashes map[uint64]common.Hash `json:"ancestorHashes"` - Error string `json:"error,omitempty"` + Block *types.Block `json:"block"` + InitAccountProofs []*ethapi.AccountResult `json:"initAccountProofs"` + Contracts map[common.Address]*hexutil.Bytes `json:"contracts"` + AncestorHashes map[uint64]common.Hash `json:"ancestorHashes"` + Error string `json:"error,omitempty"` } // provingPreflightTask represents a single block preflight task. @@ -194,7 +194,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, break } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) - task.preflight.Contracts[proof.CodeHash] = (*hexutil.Bytes)(&code) + task.preflight.Contracts[addr] = (*hexutil.Bytes)(&code) } task.preflight.AncestorHashes = touchedHashes From 701116d5b6ef9431a5aede16be80af004763d1ec Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 16:32:17 +0800 Subject: [PATCH 21/22] refactor(taiko): update Contracts map to use common.Address for improved type consistency --- eth/tracers/taiko_api.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index f824a9585335..a911b152b89a 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -194,7 +194,9 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, break } task.preflight.InitAccountProofs = append(task.preflight.InitAccountProofs, proof) - task.preflight.Contracts[addr] = (*hexutil.Bytes)(&code) + if code != nil { + task.preflight.Contracts[addr] = (*hexutil.Bytes)(&code) + } } task.preflight.AncestorHashes = touchedHashes @@ -313,7 +315,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, preflight: &provingPreflightResult{ Block: next, InitAccountProofs: []*ethapi.AccountResult{}, - Contracts: map[common.Hash]*hexutil.Bytes{}, + Contracts: map[common.Address]*hexutil.Bytes{}, AncestorHashes: map[uint64]common.Hash{}, }, }: From 5deb5066ac99fe08d2351ae400a0fbd7f4df0ba9 Mon Sep 17 00:00:00 2001 From: john xu Date: Fri, 24 Jan 2025 22:04:24 +0800 Subject: [PATCH 22/22] refactor(taiko): modify ProvingPreflights to return results directly and use context for cancellation --- eth/tracers/taiko_api.go | 32 ++++++++++++-------------------- eth/tracers/taiko_api_test.go | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/eth/tracers/taiko_api.go b/eth/tracers/taiko_api.go index a911b152b89a..35335bd57eae 100644 --- a/eth/tracers/taiko_api.go +++ b/eth/tracers/taiko_api.go @@ -59,7 +59,7 @@ type provingPreflightTask struct { // Note: // - The function requires a notifier to be present in the context, otherwise it // returns an error indicating that notifications are unsupported. -func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) { +func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) ([]*provingPreflightResult, error) { if start == 0 { return nil, fmt.Errorf("start block must be greater than 0") } @@ -77,20 +77,13 @@ func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumbe if from.Number().Cmp(to.Number()) >= 0 { return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) } - // Tracing a chain is a **long** operation, only do with subscriptions - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported - } - sub := notifier.CreateSubscription() - resCh := api.provingPreflights(from, to, config, sub.Err()) - go func() { - for result := range resCh { - notifier.Notify(sub.ID, result) - } - }() - return sub, nil + resCh := api.provingPreflights(ctx, from, to, config) + var res []*provingPreflightResult + for result := range resCh { + res = append(res, result) + } + return res, nil } // provingPreflights traces the execution of blocks from `start` to `end` and returns a channel @@ -98,10 +91,10 @@ func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumbe // goroutines to parallelize the tracing process. // // Parameters: +// - ctx: The context for the operation. // - start: The starting block for tracing. // - end: The ending block for tracing. // - config: Configuration for tracing, including the tracer to use and reexec settings. -// - closed: A channel to signal when tracing should be aborted. // // Returns: // - A channel that streams the results of the preflight checks for each block. @@ -116,7 +109,7 @@ func (api *API) ProvingPreflights(ctx context.Context, start, end rpc.BlockNumbe // The tracing process involves fetching the blocks, preparing the state, and tracing each // transaction within the blocks. The results include the transaction trace results, account // proofs, contract codes, and ancestor hashes. -func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *provingPreflightResult { +func (api *API) provingPreflights(ctx context.Context, start, end *types.Block, config *TraceConfig) chan *provingPreflightResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec @@ -128,7 +121,6 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, } var ( pend = new(sync.WaitGroup) - ctx = context.Background() taskCh = make(chan *provingPreflightTask, threads) resCh = make(chan *provingPreflightTask, threads) tracker = newStateTracker(maximumPendingTraceStates, start.NumberU64()) @@ -203,7 +195,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, // Stream the result back to the result catcher or abort on teardown select { case resCh <- task: - case <-closed: + case <-ctx.Done(): return } } @@ -243,7 +235,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, for number = start.NumberU64(); number < end.NumberU64(); number++ { // Stop tracing if interruption was requested select { - case <-closed: + case <-ctx.Done(): return default: } @@ -319,7 +311,7 @@ func (api *API) provingPreflights(start, end *types.Block, config *TraceConfig, AncestorHashes: map[uint64]common.Hash{}, }, }: - case <-closed: + case <-ctx.Done(): tracker.releaseState(number, release) return } diff --git a/eth/tracers/taiko_api_test.go b/eth/tracers/taiko_api_test.go index b1518b08656e..e4c07973742f 100644 --- a/eth/tracers/taiko_api_test.go +++ b/eth/tracers/taiko_api_test.go @@ -64,7 +64,7 @@ func TestProvingPreflights(t *testing.T) { from, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.start)) to, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.end)) - resCh := api.provingPreflights(from, to, c.config, nil) + resCh := api.provingPreflights(context.Background(), from, to, c.config) next := c.start + 1 for result := range resCh {