Skip to content
This repository has been archived by the owner on Jul 5, 2024. It is now read-only.

MPT StorageDoesNotExist #1699

Merged
merged 19 commits into from
Feb 2, 2024
Merged

MPT StorageDoesNotExist #1699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
61b345f
StorageDoesNotExist witness for proof of length 0
miha-stopar Dec 5, 2023
0197485
NonExistingStorageProof parent is nil
miha-stopar Dec 13, 2023
b55f3fd
NonExistingStorageProof for empty trie
miha-stopar Dec 13, 2023
e5e18ed
NonExistingStorageProof: compared to empty trie
miha-stopar Dec 14, 2023
0ba6875
NonExistingAcount constraints for empty or one leaf trie
miha-stopar Dec 15, 2023
cef534e
witness_row.rs docs
miha-stopar Dec 18, 2023
5aa49c7
Resolved an issue with non-existing proof lookups for wrong leaf cases
miha-stopar Dec 20, 2023
3d504b3
gofmt
miha-stopar Dec 21, 2023
1bf654c
Account leaf: marking constraints as in the specs
miha-stopar Jan 4, 2024
3e926b1
empty_trie_word function removed
miha-stopar Jan 8, 2024
a56c0f4
prepare witness: fix when determining whether the node is extension node
miha-stopar Jan 12, 2024
ab72325
gofmt
miha-stopar Jan 24, 2024
0a52120
cargo fmt
miha-stopar Jan 24, 2024
b0b9f1a
Comments, renaming
miha-stopar Jan 25, 2024
c668667
Fix for TestExtensionAddedInFirstStorageLevelTwoKeyBytes
miha-stopar Jan 26, 2024
cc98424
Old parameters branchC16, branchC1 removed - used before refactoring
miha-stopar Jan 31, 2024
7d0c5a8
Merge branch 'main' into mpt-storage-not-exist
miha-stopar Feb 1, 2024
a7dab6f
cargo fmt
miha-stopar Feb 2, 2024
f11b183
Merge branch 'main' into mpt-storage-not-exist
miha-stopar Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 7 additions & 27 deletions mpt-witness-generator/witness/branch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ func prepareBranchWitness(rows [][]byte, branch []byte, branchStart int, branchR
}
}

func prepareBranchNode(branch1, branch2, extNode1, extNode2, extListRlpBytes []byte, extValues [][]byte, key, driftedInd,
branchC16, branchC1 byte, isBranchSPlaceholder, isBranchCPlaceholder, isExtension bool) Node {
func prepareBranchNode(branch1, branch2, extNode1, extNode2, extListRlpBytes []byte, extValues [][]byte, key, driftedInd byte,
isBranchSPlaceholder, isBranchCPlaceholder, isExtension bool) Node {
extensionNode := ExtensionNode{
ListRlpBytes: extListRlpBytes,
}
Expand Down Expand Up @@ -197,7 +197,7 @@ func addBranchAndPlaceholder(proof1, proof2,
leafRow0, key, neighbourNode []byte,
keyIndex, extensionNodeInd int,
additionalBranch, isAccountProof, nonExistingAccountProof,
isShorterProofLastLeaf bool, branchC16, branchC1 byte, toBeHashed *[][]byte) (bool, bool, int, byte, Node) {
isShorterProofLastLeaf bool, toBeHashed *[][]byte) (bool, bool, int, Node) {
len1 := len(proof1)
len2 := len(proof2)

Expand All @@ -211,32 +211,14 @@ func addBranchAndPlaceholder(proof1, proof2,
}

isExtension := (len1 == len2+2) || (len2 == len1+2)
if !isExtension {
if branchC16 == 1 {
branchC16 = 0
branchC1 = 1
} else {
branchC16 = 1
branchC1 = 0
}
} else {
if isExtension {
var numNibbles byte
if len1 > len2 {
numNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesS, extensionNodeInd, proof1[len1-3], proof1[len1-3])
} else {
numNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesC, extensionNodeInd, proof2[len2-3], proof2[len2-3])
}
numberOfNibbles = int(numNibbles)

if numberOfNibbles%2 == 0 {
if branchC16 == 1 {
branchC16 = 0
branchC1 = 1
} else {
branchC16 = 1
branchC1 = 0
}
}
}

/*
Expand Down Expand Up @@ -290,8 +272,7 @@ func addBranchAndPlaceholder(proof1, proof2,
driftedInd := getDriftedPosition(leafRow0, numberOfNibbles)

node = prepareBranchNode(proof1[len1-2], proof1[len1-2], extNode, extNode, extListRlpBytes, extValues,
key[keyIndex+numberOfNibbles], driftedInd,
branchC16, branchC1, false, true, isExtension)
key[keyIndex+numberOfNibbles], driftedInd, false, true, isExtension)

// We now get the first nibble of the leaf that was turned into branch.
// This first nibble presents the position of the leaf once it moved
Expand All @@ -303,9 +284,8 @@ func addBranchAndPlaceholder(proof1, proof2,
driftedInd := getDriftedPosition(leafRow0, numberOfNibbles)

node = prepareBranchNode(proof2[len2-2], proof2[len2-2], extNode, extNode, extListRlpBytes, extValues,
key[keyIndex+numberOfNibbles], driftedInd,
branchC16, branchC1, true, false, isExtension)
key[keyIndex+numberOfNibbles], driftedInd, true, false, isExtension)
}

return isModifiedExtNode, isExtension, numberOfNibbles, branchC16, node
return isModifiedExtNode, isExtension, numberOfNibbles, node
}
31 changes: 31 additions & 0 deletions mpt-witness-generator/witness/gen_witness_from_infura_blockchain_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2302,3 +2302,34 @@ func TestWrongAccount(t *testing.T) {

prepareWitness("WrongAccount", trieModifications, statedb)
}

func TestStorageDoesNotExistOnlySProof(t *testing.T) {
blockNum := 2000003
blockNumberParent := big.NewInt(int64(blockNum))
blockHeaderParent := oracle.PrefetchBlock(blockNumberParent, true, nil)
database := state.NewDatabase(blockHeaderParent)
statedb, _ := state.New(blockHeaderParent.Root, database, nil)
addr := common.HexToAddress("0xcaac46d9bd68bffb533320545a90cd92c6e98e58")

// Implicitly create account:
trieMod1 := TrieModification{
Type: BalanceChanged,
Balance: big.NewInt(98),
Address: addr,
}

key1 := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
// leave the same
val1 := common.Hash{}

trieMod2 := TrieModification{
Type: StorageDoesNotExist,
Key: key1,
Value: val1,
Address: addr,
}

trieModifications := []TrieModification{trieMod1, trieMod2}

prepareWitness("StorageDoesNotExistOnlySProof", trieModifications, statedb)
}
12 changes: 7 additions & 5 deletions mpt-witness-generator/witness/leaf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func prepareEmptyNonExistingStorageRow() []byte {
return nonExistingStorageRow
}

func prepareNonExistingStorageRow(leafC, keyNibbles []byte, noLeaf bool) ([]byte, []byte) {
func prepareNonExistingStorageRow(leafC, keyNibbles []byte) ([]byte, []byte) {
// nonExistingStorageRow is used only for proof that nothing is stored at a particular storage key
nonExistingStorageRow := prepareEmptyNonExistingStorageRow()

Expand Down Expand Up @@ -307,7 +307,7 @@ func prepareAccountLeafNode(addr common.Address, addrh []byte, leafS, leafC, nei

// prepareLeafAndPlaceholderNode prepares a leaf node and its placeholder counterpart
// (used when one of the proofs does not have a leaf).
func prepareLeafAndPlaceholderNode(addr common.Address, addrh []byte, proof1, proof2 [][]byte, storage_key common.Hash, key []byte, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension bool) Node {
func prepareLeafAndPlaceholderNode(addr common.Address, addrh []byte, proof1, proof2 [][]byte, storage_key common.Hash, key []byte, isAccountProof, isSModExtension, isCModExtension bool) Node {
len1 := len(proof1)
len2 := len(proof2)

Expand Down Expand Up @@ -407,7 +407,10 @@ func prepareAccountLeafPlaceholderNode(addr common.Address, addrh, key []byte, k
}

func prepareStorageLeafPlaceholderNode(storage_key common.Hash, key []byte, keyIndex int) Node {
leaf := make([]byte, valueLen)
// valueLen + 1 because the placeholder leaf in the empty trie occupies 35 bytes:
// [227 161 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
// 33 (33 = 161 - 128) bytes for path, as in the example above
leaf := make([]byte, valueLen+1)
setStorageLeafKeyRLP(&leaf, key, keyIndex)
keyLen := getLeafKeyLen(keyIndex)
leaf[0] = 192 + 1 + byte(keyLen) + 1
Expand Down Expand Up @@ -529,8 +532,7 @@ func prepareStorageLeafNode(leafS, leafC, neighbourNode []byte, storage_key comm
var nonExistingStorageRow []byte
var wrongRlpBytes []byte
if nonExistingStorageProof {
noLeaf := false
wrongRlpBytes, nonExistingStorageRow = prepareNonExistingStorageRow(leafC, key, noLeaf)
wrongRlpBytes, nonExistingStorageRow = prepareNonExistingStorageRow(leafC, key)
} else {
nonExistingStorageRow = prepareEmptyNonExistingStorageRow()
}
Expand Down
2 changes: 1 addition & 1 deletion mpt-witness-generator/witness/modified_extension_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func equipLeafWithModExtensionNode(statedb *state.StateDB, leafNode Node, addr c
key, neighbourNode []byte,
keyIndex, extensionNodeInd, numberOfNibbles int,
additionalBranch, isAccountProof, nonExistingAccountProof,
isShorterProofLastLeaf bool, branchC16, branchC1 byte, toBeHashed *[][]byte) Node {
isShorterProofLastLeaf bool, toBeHashed *[][]byte) Node {
len1 := len(proof1)
len2 := len(proof2)

Expand Down
50 changes: 18 additions & 32 deletions mpt-witness-generator/witness/prepare_witness.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,11 +369,18 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []

var nodes []Node

branchC16 := byte(0)
branchC1 := byte(1)
for i := 0; i < upTo; i++ {
if !isBranch(proof1[i]) {
if i != len1-1 { // extension node
isNonExistingProof := (isAccountProof && nonExistingAccountProof) || (!isAccountProof && nonExistingStorageProof)
areThereNibbles := len(extNibblesS) != 0 || len(extNibblesC) != 0
// If i < upTo-1, it means it's not a leaf, so it's an extension node.
// There is no any special relation between isNonExistingProof and isExtension,
// except that in the non-existing proof the extension node can appear in `i == upTo-1`.
// For non-existing proof, the last node in the proof could be an extension node (we have
// nil in the underlying branch). For the non-existing proof with the wrong leaf
// (non-existing proofs can be with a nil leaf or with a wrong leaf),
// we don't need to worry because it appears in i = upTo-1).
if (i != upTo-1) || (areThereNibbles && isNonExistingProof) { // extension node
var numberOfNibbles byte
isExtension = true
Comment on lines +383 to 385
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you confirm if we can set isExtension like the following?

let isExtension = (i != upTo-1) || (areThereNibbles && isNonExistingProof);

If that is the case, we can set isExtension at the earliest possible line. We don't have to mutate it in multiple places.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to know whether in the previous iteration of the loop there was an extension node - see:

} else {
	var extNode1 []byte = nil
	var extNode2 []byte = nil
	if isExtension {
		extNode1 = proof1[i-1]
		extNode2 = proof2[i-1]
	}

	bNode := prepareBranchNode(proof1[i], proof2[i], extNode1, extNode2, extListRlpBytes, extValues,
		key[keyIndex], key[keyIndex], false, false, isExtension)
	nodes = append(nodes, bNode)

	keyIndex += 1

	isExtension = false
}

This is why isExtension is set to false only here (after one iteration). Does it make sense?

However, when inspecting this I realized there were some leftovers from before refactoring, I removed them in the last commit: cc98424

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense to me; thanks for elaborating on this.

numberOfNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesS, extensionNodeInd, proof1[i], proof2[i])
Expand All @@ -393,27 +400,6 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []

nodes = append(nodes, node)
} else {
switchC16 := true // If not extension node, switchC16 = true.
if isExtension {
keyLen := getExtensionNodeKeyLen(proof1[i-1])
if keyLen == 1 {
switchC16 = false
} else {
if proof1[i-1][2] != 0 { // If even, switch16 = true.
switchC16 = false
}
}
}
if switchC16 {
if branchC16 == 1 {
branchC16 = 0
branchC1 = 1
} else {
branchC16 = 1
branchC1 = 0
}
}

var extNode1 []byte = nil
var extNode2 []byte = nil
if isExtension {
Expand All @@ -422,7 +408,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
}

bNode := prepareBranchNode(proof1[i], proof2[i], extNode1, extNode2, extListRlpBytes, extValues,
key[keyIndex], key[keyIndex], branchC16, branchC1, false, false, isExtension)
key[keyIndex], key[keyIndex], false, false, isExtension)
nodes = append(nodes, bNode)

keyIndex += 1
Expand All @@ -438,10 +424,10 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
leafRow0 = proof2[len2-1]
}

isModifiedExtNode, _, numberOfNibbles, branchC16, bNode := addBranchAndPlaceholder(proof1, proof2, extNibblesS, extNibblesC,
isModifiedExtNode, _, numberOfNibbles, bNode := addBranchAndPlaceholder(proof1, proof2, extNibblesS, extNibblesC,
leafRow0, key, neighbourNode,
keyIndex, extensionNodeInd, additionalBranch,
isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, branchC16, branchC1, &toBeHashed)
isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, &toBeHashed)

nodes = append(nodes, bNode)

Expand All @@ -458,7 +444,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
} else {
isCModExtension = true
}
leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension)
leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, isSModExtension, isCModExtension)
}
} else {
// Add storage leaf after branch placeholder
Expand All @@ -472,7 +458,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
} else {
isCModExtension = true
}
leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension)
leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, isSModExtension, isCModExtension)
}
}

Expand All @@ -482,14 +468,14 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
if isModifiedExtNode {
leafNode = equipLeafWithModExtensionNode(statedb, leafNode, addr, proof1, proof2, extNibblesS, extNibblesC, key, neighbourNode,
keyIndex, extensionNodeInd, numberOfNibbles, additionalBranch,
isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, branchC16, branchC1, &toBeHashed)
isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, &toBeHashed)
}
nodes = append(nodes, leafNode)
} else {
node := prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, false, false)
node := prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, false, false)
nodes = append(nodes, node)
}
} else if isBranch(proof2[len(proof2)-1]) {
} else if (len1 == 0 && len2 == 0) || isBranch(proof2[len(proof2)-1]) {
// Account proof has drifted leaf as the last row, storage proof has non-existing-storage row
// as the last row.
// When non existing proof and only the branches are returned, we add a placeholder leaf.
Expand Down
Loading
Loading