Fixing problems in MPT found by light client (#1610)

### Description

Witness generator fixes:

- When branch is hashed, its children are stored as hashes. This causes
problems when computing the neighbour node which is needed for the case
when a node is deleted from a branch with two nodes (the branch
dissappears, the remaining node moves one level up).
- Placeholder storage leaf RLP - when the leaf value is set to zero, the
RLP of the placeholder leaf RLP needs to be updated to reflect the
shortened value.
- `CreateObject` called when there is no storage trie for an account.
- Setting code hash fixed.

Circuit fix:
When value is set to 0 (which deletes the key) and there are two nodes
in the branch, then the node is deleted and the other node moves up one
level (replaces the branch). The returned node is then this moved node
which doesn't have the value 0 and the lookup into MPT tables fails.
Now the circuit is fixed to ensure the new value in the table is 0.

Besides that, some other problems are fixed which were found by @adria0
when integrating the light client and MPT (these were temporary hacks in
MPT for testing purposes that were forgotten to be removed):

-  `SetState` calls removed in `prepare_witness.go` calls from
-  `oracle.PrefetchStorage` call uncommented in `prepare_witness.go`
- `oracle.PrefetchStorage` call uncommented in `state/state_object.go
updateTrie`
- Account nodes were not appended to all nodes in `prepare_witness.go`.

### Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update

### How Has This Been Tested?

No test until now covered this scenario, now
`TestNeighbourNodeInHashedBranch` has been added.

mpt-witness-generator/README.md

@@ -71,14 +71,12 @@ What is changed compared to geth:
 
 ## Generate witnesses
 
-To generate witnesses for MPT circuit, go into witness folder and execute
+To generate witnesses for MPT circuit, execute
 
 ```
-go test gen_witness_from_infura_blockchain_test.go prepare_witness.go leaf.go extension_node.go modified_extension_node.go nodes.go test_tools.go branch.go util.go
+go test -v ./...
 ```
 
-to generate the tests that use Infura blockchain.
-
 To generate the tests that use a local blockchain you need a local `geth`. You would
 need to run something like:
 ```
@@ -93,14 +91,16 @@ database if you already have some accounts:
 geth removedb
 ```
 
-And to generate the tests:
+The witness files will appear in generated_witnesses folder.
+
+## Format the code
+
+To format the code use:
 
 ```
-go test gen_witness_from_local_blockchain_test.go prepare_witness.go leaf.go extension_node.go modified_extension_node.go nodes.go test_tools.go branch.go util.go
+gofmt -w ./*
 ```
 
-The witness files will appear in generated_witnesses folder.
-
 ## Calling from Rust
 
 Build:

mpt-witness-generator/oracle/preimage.go

@@ -1,27 +1,26 @@
 package oracle
 
 import (
+	"errors"
 	"fmt"
-	"io/ioutil"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
 var preimages = make(map[common.Hash][]byte)
 
-func Preimage(hash common.Hash) []byte {
+func Preimage(hash common.Hash) ([]byte, error) {
 	val, ok := preimages[hash]
-	key := fmt.Sprintf("/tmp/eth/%s", hash)
-	ioutil.WriteFile(key, val, 0644)
 	if !ok {
-		fmt.Println("can't find preimage", hash)
+		return nil, errors.New("can't find preimage")
 	}
 	comphash := crypto.Keccak256Hash(val)
 	if hash != comphash {
-		panic("corruption in hash " + hash.String())
+		fmt.Println("corruption in hash " + hash.String())
+		return nil, errors.New("corruption in hash " + hash.String())
 	}
-	return val
+	return val, nil
 }
 
 // TODO: Maybe we will want to have a seperate preimages for next block's preimages?

mpt-witness-generator/state/database.go

@@ -30,15 +30,14 @@ func NewDatabase(header types.Header) Database {
 // ContractCode retrieves a particular contract's code.
 func (db *Database) ContractCode(addrHash common.Hash, codeHash common.Hash) ([]byte, error) {
 	oracle.PrefetchCode(db.BlockNumber, addrHash)
-	code := oracle.Preimage(codeHash)
-	return code, nil
+	return oracle.Preimage(codeHash)
 }
 
 // ContractCodeSize retrieves a particular contracts code's size.
 func (db *Database) ContractCodeSize(addrHash common.Hash, codeHash common.Hash) (int, error) {
 	oracle.PrefetchCode(db.BlockNumber, addrHash)
-	code := oracle.Preimage(codeHash)
-	return len(code), nil
+	code, err := oracle.Preimage(codeHash)
+	return len(code), err
 }
 
 func (db *Database) CopyTrie(t Trie) Trie {
@@ -96,7 +95,7 @@ type Trie interface {
 	// and external (for account tries) references.
 	Commit(onleaf trie.LeafCallback) (common.Hash, error)
 
-	Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) ([]byte, [][]byte, bool, error)
+	Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) ([]byte, [][]byte, bool, bool, error)
 
 	GetNodeByNibbles(key []byte) ([]byte, error)
 

mpt-witness-generator/state/journal.go

@@ -161,7 +161,7 @@ func (ch resetObjectChange) dirtied() *common.Address {
 }
 
 func (ch suicideChange) revert(s *StateDB) {
-	obj := s.getStateObject(*ch.account)
+	obj := s.GetStateObject(*ch.account)
 	if obj != nil {
 		obj.suicided = ch.prev
 		obj.setBalance(ch.prevbalance)
@@ -182,31 +182,31 @@ func (ch touchChange) dirtied() *common.Address {
 }
 
 func (ch balanceChange) revert(s *StateDB) {
-	s.getStateObject(*ch.account).setBalance(ch.prev)
+	s.GetStateObject(*ch.account).setBalance(ch.prev)
 }
 
 func (ch balanceChange) dirtied() *common.Address {
 	return ch.account
 }
 
 func (ch nonceChange) revert(s *StateDB) {
-	s.getStateObject(*ch.account).setNonce(ch.prev)
+	s.GetStateObject(*ch.account).setNonce(ch.prev)
 }
 
 func (ch nonceChange) dirtied() *common.Address {
 	return ch.account
 }
 
 func (ch codeChange) revert(s *StateDB) {
-	s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
+	s.GetStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
 }
 
 func (ch codeChange) dirtied() *common.Address {
 	return ch.account
 }
 
 func (ch storageChange) revert(s *StateDB) {
-	s.getStateObject(*ch.account).setState(ch.key, ch.prevalue)
+	s.GetStateObject(*ch.account).setState(ch.key, ch.prevalue)
 }
 
 func (ch storageChange) dirtied() *common.Address {

mpt-witness-generator/state/state_object.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/privacy-scaling-explorations/mpt-witness-generator/oracle"
+	"github.com/privacy-scaling-explorations/mpt-witness-generator/trie"
 )
 
 var emptyCodeHash = crypto.Keccak256(nil)
@@ -359,7 +360,7 @@ func (s *stateObject) updateTrie(db Database) Trie {
 			// Get absense proof of key in case the deletion needs the sister node.
 
 			// Note: commented for now because of `ExtNodeDeleted`
-			// oracle.PrefetchStorage(big.NewInt(db.BlockNumber.Int64()+1), s.address, key, trie.GenPossibleShortNodePreimage)
+			oracle.PrefetchStorage(big.NewInt(db.BlockNumber.Int64()+1), s.address, key, trie.GenPossibleShortNodePreimage)
 			s.setError(tr.TryDelete(key[:]))
 		} else {
 			//fmt.Println("update", s.address, key, value)