From e892f700a38db14d2dea2960acf6a9baddea97a0 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Mon, 21 Nov 2022 23:36:40 +0800 Subject: [PATCH 1/6] feat(core/vm/contracts.go): support sstoragePisa.removeRaw() --- core/vm/contracts.go | 36 +++- core/vm/contracts_test.go | 415 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 447 insertions(+), 4 deletions(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 4d0096f3d886..9b0ef20f3db2 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -23,6 +23,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/ethereum/go-ethereum/sstorage" "math/big" "github.com/ethereum/go-ethereum/common" @@ -298,9 +299,10 @@ var ( // modexpMultComplexity implements bigModexp multComplexity formula, as defined in EIP-198 // // def mult_complexity(x): -// if x <= 64: return x ** 2 -// elif x <= 1024: return x ** 2 // 4 + 96 * x - 3072 -// else: return x ** 2 // 16 + 480 * x - 199680 +// +// if x <= 64: return x ** 2 +// elif x <= 1024: return x ** 2 // 4 + 96 * x - 3072 +// else: return x ** 2 // 16 + 480 * x - 199680 // // where is x is max(length_of_MODULUS, length_of_BASE) func modexpMultComplexity(x *big.Int) *big.Int { @@ -774,8 +776,34 @@ func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([ binary.BigEndian.PutUint64(pb[32-8:32], 32) binary.BigEndian.PutUint64(pb[64-8:64], uint64(len(b))) return append(pb, b...), nil + } else if bytes.Equal(input[0:4], removeRawMethodId) { + if evm.interpreter.readOnly { + return nil, ErrWriteProtection + } + // The execution of remove should not effect the result when validator running without data node. + + // The remove operation consists of two steps. + // First, remove the data corresponding to the kvIdx selected by the operator. + // Second, move the data corresponding to lastKvIdx to the data storage location of kvIdx. + lastKvIdx := new(big.Int).SetBytes(getData(input, 4, 32)) + updateKvIdx := new(big.Int).SetBytes(getData(input, 4+32, 32)) + + if lastKvIdx.Cmp(updateKvIdx) == 0 { + // Delete the data corresponding to lastKvIdx + evm.StateDB.SstorageWrite(caller, lastKvIdx.Uint64(), make([]byte, sstorage.ShardInfos[0].KVSize)) + } else { + // Read the data corresponding to lastKvIdx + replaceData, _, _ := evm.StateDB.SstorageRead(caller, lastKvIdx.Uint64(), int(sstorage.ShardInfos[0].KVSize), common.Hash{}) + + // Delete the data corresponding to lastKvIdx + evm.StateDB.SstorageWrite(caller, lastKvIdx.Uint64(), make([]byte, sstorage.ShardInfos[0].KVSize)) + + // Replace the data corresponding to kvIdx with the data corresponding to lastKvIdx + evm.StateDB.SstorageWrite(caller, updateKvIdx.Uint64(), replaceData) + } + + return nil, nil } - // TODO: remove is not supported yet return nil, errors.New("unsupported method") } diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index e2450fe4644c..fc075f942562 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -20,7 +20,13 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/sstorage" "io/ioutil" + "math/big" + "math/rand" "testing" "time" @@ -312,6 +318,415 @@ func TestPrecompiledBLS12381Pairing(t *testing.T) { testJson("blsPairing", "1 func TestPrecompiledBLS12381MapG1(t *testing.T) { testJson("blsMapG1", "11", t) } func TestPrecompiledBLS12381MapG2(t *testing.T) { testJson("blsMapG2", "12", t) } +func constructPutSDOperation(kvIdx uint64, dataLen uint64, specificData uint64) (putInput []byte, getInput []byte, getOutput []byte) { + + validKvIdx := common.BigToHash(big.NewInt(0).SetUint64(kvIdx)).Bytes() + putDataPtr := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040") + validdataLen := common.BigToHash(big.NewInt(0).SetUint64(dataLen)).Bytes() + validdata := generateSpecificData(dataLen, specificData) + + putInput = make([]byte, 0) + putInput = append(putInput, putRawMethodId...) + putInput = append(putInput, validKvIdx...) + putInput = append(putInput, putDataPtr...) + putInput = append(putInput, validdataLen...) + putInput = append(putInput, validdata...) + + // hash is not enable at this time, thus hash can set as any number + hash := common.BigToHash(big.NewInt(1)).Bytes() + readOffset := common.BigToHash(big.NewInt(0)).Bytes() + outputDataPtr := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020") + getInput = make([]byte, 0) + getInput = append(getInput, getRawMethodId...) + getInput = append(getInput, hash...) + getInput = append(getInput, validKvIdx...) + getInput = append(getInput, readOffset...) + getInput = append(getInput, validdataLen...) + + getOutput = make([]byte, 0) + getOutput = append(getOutput, outputDataPtr...) + getOutput = append(getOutput, validdataLen...) + getOutput = append(getOutput, validdata...) + + return putInput, getInput, getOutput +} + +func constructPutOperation(kvIdx uint64, dataLen uint64) (putInput []byte, getInput []byte, getOutput []byte) { + + validKvIdx := common.BigToHash(big.NewInt(0).SetUint64(kvIdx)).Bytes() + putDataPtr := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040") + validdataLen := common.BigToHash(big.NewInt(0).SetUint64(dataLen)).Bytes() + validdata := generateRandomData(dataLen) + + putInput = make([]byte, 0) + putInput = append(putInput, putRawMethodId...) + putInput = append(putInput, validKvIdx...) + putInput = append(putInput, putDataPtr...) + putInput = append(putInput, validdataLen...) + putInput = append(putInput, validdata...) + + // hash is not enable at this time, thus hash can set as any number + hash := common.BigToHash(big.NewInt(1)).Bytes() + readOffset := common.BigToHash(big.NewInt(0)).Bytes() + outputDataPtr := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020") + getInput = make([]byte, 0) + getInput = append(getInput, getRawMethodId...) + getInput = append(getInput, hash...) + getInput = append(getInput, validKvIdx...) + getInput = append(getInput, readOffset...) + getInput = append(getInput, validdataLen...) + + getOutput = make([]byte, 0) + getOutput = append(getOutput, outputDataPtr...) + getOutput = append(getOutput, validdataLen...) + getOutput = append(getOutput, validdata...) + + return putInput, getInput, getOutput +} + +func constructGetOperation(kvIdx uint64, datalen uint64) (getInput []byte) { + validKvIdx := common.BigToHash(big.NewInt(0).SetUint64(kvIdx)).Bytes() + validDatalen := common.BigToHash(big.NewInt(0).SetUint64(datalen)).Bytes() + // hash is not enable at this time, thus hash can set as any number + hash := common.BigToHash(big.NewInt(1)).Bytes() + readOffset := common.BigToHash(big.NewInt(0)).Bytes() + getInput = make([]byte, 0) + getInput = append(getInput, getRawMethodId...) + getInput = append(getInput, hash...) + getInput = append(getInput, validKvIdx...) + getInput = append(getInput, readOffset...) + getInput = append(getInput, validDatalen...) + + return getInput +} + +func constructRemoveOperation(lastKvIdx uint64, updateKvIdx uint64, kvSize uint64) (removeInput []byte, lastKvIdxGetInput []byte, updateKvIdxGetInput []byte) { + + validLastKvIdx := common.BigToHash(big.NewInt(0).SetUint64(lastKvIdx)).Bytes() + validUpdateKvIdx := common.BigToHash(big.NewInt(0).SetUint64(updateKvIdx)).Bytes() + + removeInput = make([]byte, 0) + removeInput = append(removeInput, removeRawMethodId...) + removeInput = append(removeInput, validLastKvIdx...) + removeInput = append(removeInput, validUpdateKvIdx...) + + lastKvIdxGetInput = constructGetOperation(lastKvIdx, kvSize) + updateKvIdxGetInput = constructGetOperation(updateKvIdx, kvSize) + + return removeInput, lastKvIdxGetInput, updateKvIdxGetInput +} + +func generateRandomData(dataLen uint64) []byte { + buf := make([]byte, 0) + for i := 0; i < int(dataLen); i++ { + rv := byte(rand.Uint32() % 256) + buf = append(buf, rv) + } + return buf +} + +func generateSpecificData(dataLen uint64, expectNum uint64) []byte { + buf := make([]byte, 0) + for i := 0; i < int(dataLen); i++ { + rv := byte(expectNum) + buf = append(buf, rv) + } + return buf +} + +type operationList struct { + Name string + KvIdx uint64 + KvSize uint64 + List []*operationDetail +} + +type operationDetail struct { + Method string + Commit bool + Input []byte + Output []byte + ExpectOutput []byte + OtherOperationIdx int // -1 means that do not need to set other operation expectOutput +} + +func (o *operationDetail) checkOutput(output []byte) error { + o.Output = output + if len(o.ExpectOutput) != 0 { + if !bytes.Equal(output, o.ExpectOutput) { + output = output[64:] + if !bytes.Equal(output, o.ExpectOutput) { + return fmt.Errorf("no match with expectOutput; \nexpect:%s \nactual:%s", common.Bytes2Hex(o.ExpectOutput), common.Bytes2Hex(o.Output)) + } + } + } + return nil +} + +func newOperationDetail(method string, input []byte, expectOutput []byte, setOtherOperationIdx int) *operationDetail { + return &operationDetail{Method: method, Input: input, ExpectOutput: expectOutput, OtherOperationIdx: setOtherOperationIdx} +} + +func newOperationDetailWithCommit(method string, input []byte, expectOutput []byte, setOtherOperationIdx int) *operationDetail { + return &operationDetail{Method: method, Input: input, ExpectOutput: expectOutput, OtherOperationIdx: setOtherOperationIdx, Commit: true} +} + +func (ol *operationList) run(db *state.StateDB, env *PrecompiledContractCallEnv, p PrecompiledContract) error { + for i, o := range ol.List { + output, _, err := RunPrecompiledContract(env, p, o.Input, p.RequiredGas(o.Input)) + if err != nil { + return fmt.Errorf("the %dth operation[%s], %s", i, o.Method, err.Error()) + } + + err = o.checkOutput(output) + if err != nil { + return fmt.Errorf("the %dth operation[%s], %s", i, o.Method, err.Error()) + } + + if o.OtherOperationIdx > 0 { + ol.List[o.OtherOperationIdx].ExpectOutput = output + } + + if o.Commit { + + hash, err := db.Commit(true) + if err != nil { + return err + } + + err = db.Database().TrieDB().Commit(hash, true, nil) + if err != nil { + return err + } + } + } + + return nil +} + +func newCheckPutOperationList(kvIdx uint64, kvSize uint64) *operationList { + putInput, getInput, getOutput := constructPutOperation(kvIdx, kvSize) + putOperate := newOperationDetailWithCommit("put", putInput, nil, -1) + getOperate := newOperationDetail("get", getInput, getOutput, -1) + return &operationList{Name: "CheckPut", KvIdx: kvIdx, KvSize: kvSize, List: []*operationDetail{putOperate, getOperate}} +} + +func newCheckPutSDOperationList(kvIdx uint64, kvSize uint64, specificData uint64) *operationList { + putInput, getInput, getOutput := constructPutSDOperation(kvIdx, kvSize, specificData) + putOperate := newOperationDetailWithCommit("put", putInput, nil, -1) + getOperate := newOperationDetail("get", getInput, getOutput, -1) + return &operationList{Name: "CheckPut", KvIdx: kvIdx, KvSize: kvSize, List: []*operationDetail{putOperate, getOperate}} +} + +func newCheckRemoveOperationList(lastKvIdx, updateKvIdx, maxKvSize uint64) *operationList { + removeInput, lastKvIdxGetInput, updateKvIdxGetInput := constructRemoveOperation(lastKvIdx, updateKvIdx, maxKvSize) + if lastKvIdx == updateKvIdx { + removeOperator := newOperationDetailWithCommit("remove", removeInput, nil, -1) + lastKvIdxGetOperatorAfterRemove := newOperationDetail("get", lastKvIdxGetInput, make([]byte, maxKvSize), -1) + return &operationList{Name: "CheckRemove", KvIdx: updateKvIdx, KvSize: maxKvSize, List: []*operationDetail{removeOperator, lastKvIdxGetOperatorAfterRemove}} + } + lastKvIdxGetOperator := newOperationDetail("get", lastKvIdxGetInput, nil, 2) + removeOperator := newOperationDetailWithCommit("remove", removeInput, nil, -1) + updateKvIdxGetOperator := newOperationDetail("get", updateKvIdxGetInput, nil, -1) + lastKvIdxGetOperatorAfterRemove := newOperationDetail("get", lastKvIdxGetInput, make([]byte, maxKvSize), -1) + return &operationList{Name: "CheckRemove", KvIdx: updateKvIdx, KvSize: maxKvSize, List: []*operationDetail{lastKvIdxGetOperator, removeOperator, updateKvIdxGetOperator, lastKvIdxGetOperatorAfterRemove}} +} + +func prepareShardManager() (*sstorage.ShardManager, common.Address, error) { + var ( + KvSize uint64 = 4 * 1024 + KvEntries uint64 = 10 + ShardManagerContract = common.HexToAddress("0x0000000000000000000000000000000003331111") + ) + sm := sstorage.NewShardManager(ShardManagerContract, KvSize, KvEntries) + // we need to update this ,NewDatabaseWithConfig + sstorage.ContractToShardManager[ShardManagerContract] = sm + err := sm.AddDataShard(0) + if err != nil { + return nil, common.Address{}, err + } + err = sm.AddDataShard(1) + if err != nil { + return nil, common.Address{}, err + } + err = sm.AddDataShard(2) + if err != nil { + return nil, common.Address{}, err + } + + create0, err := sstorage.Create("EthStorage-Data0", 0, 10, sstorage.MASK_KECCAK_256) + if err != nil { + return nil, common.Address{}, err + } + create1, err := sstorage.Create("EthStorage-Data1", 10, 10, sstorage.MASK_KECCAK_256) + if err != nil { + return nil, common.Address{}, err + } + create2, err := sstorage.Create("EthStorage-Data2", 20, 10, sstorage.MASK_KECCAK_256) + if err != nil { + return nil, common.Address{}, err + } + + err = sm.AddDataFile(create0) + if err != nil { + return nil, common.Address{}, err + } + err = sm.AddDataFile(create1) + if err != nil { + return nil, common.Address{}, err + } + err = sm.AddDataFile(create2) + if err != nil { + return nil, common.Address{}, err + } + return sm, ShardManagerContract, nil +} + +func TestPutAndUpdateKvData(t *testing.T) { + // prepare three shards. + // Shard Info: MaxKvEntries = 10; KVSize = 4 * 1024; chunksPerKv = 1; ChunkSize = 4 *1024 + sm, ShardManagerContract, err := prepareShardManager() + if err != nil { + t.Error(err) + } + + ols := make([]*operationList, 0) + var i uint64 + // first put data + for i = 0; i < sm.MaxKvSizeEntries()*3; i++ { + ol := newCheckPutOperationList(i, 4*1024) + ols = append(ols, ol) + } + // update data + for i = 0; i < sm.MaxKvSizeEntries()*3; i++ { + ol := newCheckPutOperationList(i, 4*1024) + ols = append(ols, ol) + } + + p := &sstoragePisa{} + vmctx := BlockContext{ + BlockNumber: big.NewInt(0), + CanTransfer: func(db StateDB, address common.Address, b *big.Int) bool { + return true + }, + Transfer: func(db StateDB, address common.Address, address2 common.Address, b *big.Int) {}, + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + jsonRpcVmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{IsJsonRpc: true}) + jsonRpcEnv := &PrecompiledContractCallEnv{jsonRpcVmenv, AccountRef(ShardManagerContract)} + + for j, ol := range ols { + t.Run(fmt.Sprintf("olIdx:%d KvIdx:%d KvSize:%d Method:%s", j, ol.KvIdx, ol.KvSize, ol.Name), func(t *testing.T) { + err := ol.run(statedb, jsonRpcEnv, p) + if err != nil { + t.Error(err) + } + }) + + } +} + +func TestRemoveKvData(t *testing.T) { + // prepare three shards. + // Shard Info: MaxKvEntries = 10; KVSize = 4 * 1024; chunksPerKv = 1; ChunkSize = 4 *1024 + sm, ShardManagerContract, err := prepareShardManager() + if err != nil { + t.Error(err) + } + + ols := make([]*operationList, 0) + var i uint64 + // first put data + for i = 0; i < sm.MaxKvSizeEntries()*3-1; i++ { + ol := newCheckPutOperationList(i, 4*1024) + ols = append(ols, ol) + } + + lastKvIdx := sm.MaxKvSizeEntries()*3 - 2 + // remove the lastKvIdxData + removeKvIdx := sm.MaxKvSizeEntries()*3 - 2 + olr0 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + + // remove the data of kvIdx which shardId is equal with lastKvIdx + lastKvIdx-- + removeKvIdx = sm.MaxKvSizeEntries()*2 + 1 + olr1 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + + // remove the data of kvIdx which shardId is different with lastKvIdx + lastKvIdx-- + removeKvIdx = 1 + olr2 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + + ols = append(ols, olr0, olr1, olr2) + + p := &sstoragePisa{} + vmctx := BlockContext{ + BlockNumber: big.NewInt(0), + CanTransfer: func(db StateDB, address common.Address, b *big.Int) bool { + return true + }, + Transfer: func(db StateDB, address common.Address, address2 common.Address, b *big.Int) {}, + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + jsonRpcVmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{IsJsonRpc: true}) + jsonRpcEnv := &PrecompiledContractCallEnv{jsonRpcVmenv, AccountRef(ShardManagerContract)} + + for j, ol := range ols { + t.Run(fmt.Sprintf("olIdx:%d KvIdx:%d KvSize:%d Method:%s", j, ol.KvIdx, ol.KvSize, ol.Name), func(t *testing.T) { + err := ol.run(statedb, jsonRpcEnv, p) + if err != nil { + t.Error(err) + } + }) + + } +} + +func TestRemoveSpecificKvData(t *testing.T) { + // prepare three shards. + // Shard Info: MaxKvEntries = 10; KVSize = 4 * 1024; chunksPerKv = 1; ChunkSize = 4 *1024 + _, ShardManagerContract, err := prepareShardManager() + if err != nil { + t.Error(err) + } + + ols := make([]*operationList, 0) + var i uint64 + // first put data + for i = 0; i < 3; i++ { + ol := newCheckPutSDOperationList(i, 4*1024, i) + ols = append(ols, ol) + } + + var lastKvIdx uint64 = 2 + var removeKvIdx uint64 = 1 + olr1 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + + ols = append(ols, olr1) + + p := &sstoragePisa{} + vmctx := BlockContext{ + BlockNumber: big.NewInt(0), + CanTransfer: func(db StateDB, address common.Address, b *big.Int) bool { + return true + }, + Transfer: func(db StateDB, address common.Address, address2 common.Address, b *big.Int) {}, + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + jsonRpcVmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{IsJsonRpc: true}) + jsonRpcEnv := &PrecompiledContractCallEnv{jsonRpcVmenv, AccountRef(ShardManagerContract)} + + for j, ol := range ols { + t.Run(fmt.Sprintf("olIdx:%d KvIdx:%d KvSize:%d Method:%s", j, ol.KvIdx, ol.KvSize, ol.Name), func(t *testing.T) { + err := ol.run(statedb, jsonRpcEnv, p) + if err != nil { + t.Error(err) + } + }) + + } +} + func BenchmarkPrecompiledBLS12381G1Add(b *testing.B) { benchJson("blsG1Add", "0a", b) } func BenchmarkPrecompiledBLS12381G1Mul(b *testing.B) { benchJson("blsG1Mul", "0b", b) } func BenchmarkPrecompiledBLS12381G1MultiExp(b *testing.B) { benchJson("blsG1MultiExp", "0c", b) } From 747a081ae4e5be3eac216fb9562a321b050e7223 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Mon, 21 Nov 2022 23:39:47 +0800 Subject: [PATCH 2/6] refactor: fix word --- sstorage/data_shard.go | 4 ++-- sstorage/shard_manager.go | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sstorage/data_shard.go b/sstorage/data_shard.go index 400f4e1141a5..f3fe6f9de935 100644 --- a/sstorage/data_shard.go +++ b/sstorage/data_shard.go @@ -82,7 +82,7 @@ func (ds *DataShard) Read(kvIdx uint64, readLen int, hash common.Hash, isMasked } readLen = readLen - chunkReadLen - chunkIdx := ds.ChunkIdx() + kvIdx*ds.chunksPerKv + i + chunkIdx := kvIdx*ds.chunksPerKv + i cdata, err := ds.ReadChunk(chunkIdx, chunkReadLen, hash, isMasked) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (ds *DataShard) Write(kvIdx uint64, b []byte, isMasked bool) error { writeLen = int(CHUNK_SIZE) } - chunkIdx := ds.ChunkIdx() + kvIdx*ds.chunksPerKv + i + chunkIdx := kvIdx*ds.chunksPerKv + i err := ds.WriteChunk(chunkIdx, b[off:off+writeLen], isMasked) if err != nil { return nil diff --git a/sstorage/shard_manager.go b/sstorage/shard_manager.go index 77f05da1a30a..f591db365c91 100644 --- a/sstorage/shard_manager.go +++ b/sstorage/shard_manager.go @@ -28,6 +28,10 @@ func (sm *ShardManager) MaxKvSize() uint64 { return sm.kvSize } +func (sm *ShardManager) MaxKvSizeEntries() uint64 { + return sm.kvEntries +} + func (sm *ShardManager) AddDataShard(shardIdx uint64) error { if _, ok := sm.shardMap[shardIdx]; !ok { ds := NewDataShard(shardIdx, sm.kvSize, sm.kvEntries) @@ -53,7 +57,7 @@ func (sm *ShardManager) AddDataFile(df *DataFile) error { func (sm *ShardManager) TryWrite(kvIdx uint64, b []byte) (bool, error) { shardIdx := kvIdx / sm.kvEntries if ds, ok := sm.shardMap[shardIdx]; ok { - return true, ds.Write(kvIdx, b, false) + return true, ds.Write(kvIdx, b, true) } else { return false, nil } @@ -85,7 +89,7 @@ func (sm *ShardManager) UnmaskKV(kvIdx uint64, b []byte, hash common.Hash) ([]by } datalen = datalen - chunkReadLen - chunkIdx := ds.ChunkIdx() + kvIdx*ds.chunksPerKv + i + chunkIdx := kvIdx*ds.chunksPerKv + i df := ds.GetStorageFile(chunkIdx) if df == nil { return nil, false, fmt.Errorf("Cannot find storage file for chunkIdx", "chunkIdx", chunkIdx) From f79694bea0c8ddcf0d0365b717ae8ae8cfebe836 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Mon, 21 Nov 2022 23:40:41 +0800 Subject: [PATCH 3/6] refactor: update --- core/vm/contracts.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 9b0ef20f3db2..ef62a29220e1 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -780,10 +780,11 @@ func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([ if evm.interpreter.readOnly { return nil, ErrWriteProtection } - // The execution of remove should not effect the result when validator running without data node. + + // The execution of remove should not affect the result when validator running without data node. // The remove operation consists of two steps. - // First, remove the data corresponding to the kvIdx selected by the operator. + // First, remove the data corresponding to kvIdx selected by the operator. // Second, move the data corresponding to lastKvIdx to the data storage location of kvIdx. lastKvIdx := new(big.Int).SetBytes(getData(input, 4, 32)) updateKvIdx := new(big.Int).SetBytes(getData(input, 4+32, 32)) From 6a27daa99eb25706ea8938de0e802cdcf2690854 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Tue, 22 Nov 2022 00:23:23 +0800 Subject: [PATCH 4/6] refactor: update comment --- core/vm/contracts.go | 9 +++++---- core/vm/contracts_test.go | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index ef62a29220e1..4ce04dfad5cc 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -23,7 +23,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/ethereum/go-ethereum/sstorage" "math/big" "github.com/ethereum/go-ethereum/common" @@ -33,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/bls12381" "github.com/ethereum/go-ethereum/crypto/bn256" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/sstorage" //lint:ignore SA1019 Needed for precompile "golang.org/x/crypto/ripemd160" @@ -717,8 +717,9 @@ func (l *sstoragePisa) RequiredGas(input []byte) uint64 { return params.SstoreResetGasEIP2200 } else if bytes.Equal(input[0:4], getRawMethodId) { return params.SloadGasEIP2200 + } else if bytes.Equal(input[0:4], getRawMethodId) { + return params.SstoreResetGasEIP2200 } else { - // TODO: remove is not supported yet return 0 } } @@ -784,8 +785,8 @@ func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([ // The execution of remove should not affect the result when validator running without data node. // The remove operation consists of two steps. - // First, remove the data corresponding to kvIdx selected by the operator. - // Second, move the data corresponding to lastKvIdx to the data storage location of kvIdx. + // First, replace the data corresponding to removeKvIdx selected by the operator with data of lastKvIdx + // Second, set the data space of the original lastKvIdx to 0 lastKvIdx := new(big.Int).SetBytes(getData(input, 4, 32)) updateKvIdx := new(big.Int).SetBytes(getData(input, 4+32, 32)) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index fc075f942562..400400c54e82 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -20,10 +20,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/sstorage" "io/ioutil" "math/big" "math/rand" @@ -31,6 +27,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/sstorage" ) // precompiledTest defines the input/output pairs for precompiled contract tests. @@ -643,16 +643,16 @@ func TestRemoveKvData(t *testing.T) { } lastKvIdx := sm.MaxKvSizeEntries()*3 - 2 - // remove the lastKvIdxData + // removeKvIdx is equal to lastKvIdx removeKvIdx := sm.MaxKvSizeEntries()*3 - 2 olr0 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) - // remove the data of kvIdx which shardId is equal with lastKvIdx + // removeKvIdx and lastKvIdx are in the same Shard lastKvIdx-- removeKvIdx = sm.MaxKvSizeEntries()*2 + 1 olr1 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) - // remove the data of kvIdx which shardId is different with lastKvIdx + // removeKvIdx and lastKvIdx are in the different Shard lastKvIdx-- removeKvIdx = 1 olr2 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) From 1fcdebe1faf12028b2f0ff71fc0258788241c4b8 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Fri, 25 Nov 2022 17:36:32 +0800 Subject: [PATCH 5/6] feat: update remove process --- core/vm/contracts.go | 10 ++++++++-- core/vm/contracts_test.go | 26 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 4ce04dfad5cc..30980b3bc96d 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -703,7 +703,7 @@ func (l *systemContractDeployer) RunWith(env *PrecompiledContractCallEnv, input var ( putRawMethodId, _ = hex.DecodeString("4fb03390") // putRaw(uint256,bytes) getRawMethodId, _ = hex.DecodeString("f835367f") // getRaw(bytes32,uint256,uint256,uint256) - removeRawMethodId, _ = hex.DecodeString("6c4b6776") // removeRaw(uint256,uint256) + removeRawMethodId, _ = hex.DecodeString("fbcc2ab1") // removeRaw(uint256,uint256,bytes32) ) type sstoragePisa struct{} @@ -789,13 +789,19 @@ func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([ // Second, set the data space of the original lastKvIdx to 0 lastKvIdx := new(big.Int).SetBytes(getData(input, 4, 32)) updateKvIdx := new(big.Int).SetBytes(getData(input, 4+32, 32)) + lastKvIdxHash := common.BytesToHash(getData(input, 4+32+32, 32)) + perShardKvEntries := sstorage.ShardInfos[0].KVEntries + // only allow removing a KV entry in the last shard + if lastKvIdx.Uint64()/perShardKvEntries != updateKvIdx.Uint64()/perShardKvEntries { + return nil, fmt.Errorf("only allow removing a KV entry in the last shard") + } if lastKvIdx.Cmp(updateKvIdx) == 0 { // Delete the data corresponding to lastKvIdx evm.StateDB.SstorageWrite(caller, lastKvIdx.Uint64(), make([]byte, sstorage.ShardInfos[0].KVSize)) } else { // Read the data corresponding to lastKvIdx - replaceData, _, _ := evm.StateDB.SstorageRead(caller, lastKvIdx.Uint64(), int(sstorage.ShardInfos[0].KVSize), common.Hash{}) + replaceData, _, _ := evm.StateDB.SstorageRead(caller, lastKvIdx.Uint64(), int(sstorage.ShardInfos[0].KVSize), lastKvIdxHash) // Delete the data corresponding to lastKvIdx evm.StateDB.SstorageWrite(caller, lastKvIdx.Uint64(), make([]byte, sstorage.ShardInfos[0].KVSize)) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 400400c54e82..f6f3cdbc5293 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -404,11 +404,13 @@ func constructRemoveOperation(lastKvIdx uint64, updateKvIdx uint64, kvSize uint6 validLastKvIdx := common.BigToHash(big.NewInt(0).SetUint64(lastKvIdx)).Bytes() validUpdateKvIdx := common.BigToHash(big.NewInt(0).SetUint64(updateKvIdx)).Bytes() + lastKvIdxHash := common.BigToHash(big.NewInt(1)).Bytes() removeInput = make([]byte, 0) removeInput = append(removeInput, removeRawMethodId...) removeInput = append(removeInput, validLastKvIdx...) removeInput = append(removeInput, validUpdateKvIdx...) + removeInput = append(removeInput, lastKvIdxHash...) lastKvIdxGetInput = constructGetOperation(lastKvIdx, kvSize) updateKvIdxGetInput = constructGetOperation(updateKvIdx, kvSize) @@ -448,6 +450,7 @@ type operationDetail struct { Output []byte ExpectOutput []byte OtherOperationIdx int // -1 means that do not need to set other operation expectOutput + ExpectErr string } func (o *operationDetail) checkOutput(output []byte) error { @@ -475,6 +478,13 @@ func (ol *operationList) run(db *state.StateDB, env *PrecompiledContractCallEnv, for i, o := range ol.List { output, _, err := RunPrecompiledContract(env, p, o.Input, p.RequiredGas(o.Input)) if err != nil { + if len(o.ExpectErr) != 0 { + if o.ExpectErr == err.Error() { + return nil + } else { + return fmt.Errorf("Error No Match : the %dth operation[%s], actual err:%s , expect err:%s ", i, o.Method, err.Error(), o.ExpectErr) + } + } return fmt.Errorf("the %dth operation[%s], %s", i, o.Method, err.Error()) } @@ -541,6 +551,8 @@ func prepareShardManager() (*sstorage.ShardManager, common.Address, error) { sm := sstorage.NewShardManager(ShardManagerContract, KvSize, KvEntries) // we need to update this ,NewDatabaseWithConfig sstorage.ContractToShardManager[ShardManagerContract] = sm + sstorage.ShardInfos[0] = &sstorage.ShardInfo{ShardManagerContract, KvSize, KvEntries} + err := sm.AddDataShard(0) if err != nil { return nil, common.Address{}, err @@ -652,12 +664,22 @@ func TestRemoveKvData(t *testing.T) { removeKvIdx = sm.MaxKvSizeEntries()*2 + 1 olr1 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + lastKvIdx-- + removeKvIdx = sm.MaxKvSizeEntries() * 2 + olr2 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + // removeKvIdx and lastKvIdx are in the different Shard lastKvIdx-- removeKvIdx = 1 - olr2 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + olr3 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + olr3.List[1].ExpectErr = "only allow removing a KV entry in the last shard" + + lastKvIdx-- + removeKvIdx = sm.MaxKvSizeEntries()*2 - 1 + olr4 := newCheckRemoveOperationList(lastKvIdx, removeKvIdx, 4*1024) + olr4.List[1].ExpectErr = "only allow removing a KV entry in the last shard" - ols = append(ols, olr0, olr1, olr2) + ols = append(ols, olr0, olr1, olr2, olr3, olr4) p := &sstoragePisa{} vmctx := BlockContext{ From 7ee929f8baf18fc897732e2f6482fcb9e51cdab0 Mon Sep 17 00:00:00 2001 From: cyl19970726 <15258378443@163.com> Date: Fri, 25 Nov 2022 17:43:28 +0800 Subject: [PATCH 6/6] refactor: update err --- core/vm/contracts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 30980b3bc96d..a82f33d40555 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -794,7 +794,7 @@ func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([ // only allow removing a KV entry in the last shard if lastKvIdx.Uint64()/perShardKvEntries != updateKvIdx.Uint64()/perShardKvEntries { - return nil, fmt.Errorf("only allow removing a KV entry in the last shard") + return nil, errors.New("only allow removing a KV entry in the last shard") } if lastKvIdx.Cmp(updateKvIdx) == 0 { // Delete the data corresponding to lastKvIdx