eth storage miner part 0 changes

cmd/sstorage/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -63,6 +64,12 @@ var ShardWriteCmd = &cobra.Command{
 	Run:   runShardWrite,
 }
 
+var CheckEmtpyKVsCmd = &cobra.Command{
+	Use:   "check_empty_kvs",
+	Short: "check empty Kvs have been filled",
+	Run:   runCheckEmtpyKVs,
+}
+
 func init() {
 	kvLen = CreateCmd.Flags().Uint64("kv_len", 0, "kv idx len to create")
 	chunkLen = CreateCmd.Flags().Uint64("chunk_len", 0, "Chunk idx len to create")
@@ -250,6 +257,40 @@ func runShardWrite(cmd *cobra.Command, args []string) {
 	log.Info("Write value", "kvIdx", *kvIdx, "bytes", len(bs))
 }
 
+func runCheckEmtpyKVs(cmd *cobra.Command, args []string) {
+	setupLogger()
+
+	if len(*filenames) != 1 {
+		log.Crit("must provide a filename")
+	}
+
+	var err error
+	var df *sstorage.DataFile
+	df, err = sstorage.OpenDataFile((*filenames)[0])
+	if err != nil {
+		log.Crit("open failed", "error", err)
+	}
+
+	commit := common.Hash{}
+	chunkPerKv := df.KVSize() / sstorage.CHUNK_SIZE
+	startChunkIdx := (*kvIdx) * chunkPerKv
+	log.Info("start to verify", "kvidx", *kvIdx, "startChunkIdx", startChunkIdx, "EndChunkIdx", df.EndChunkIdx())
+	for chunkIdx := startChunkIdx; chunkIdx < df.EndChunkIdx(); chunkIdx++ {
+		maskedChunkData, err := df.Read(chunkIdx, int(sstorage.CHUNK_SIZE))
+		if err != nil {
+			log.Warn("read sstorage file failed", "chunkidx", chunkIdx, "error", err)
+		}
+		encodeKey := sstorage.CalcEncodeKey(commit, chunkIdx, df.Miner())
+		unmaskedChunk := sstorage.DecodeChunk(maskedChunkData, 2, encodeKey)
+		if bytes.Compare(unmaskedChunk, make([]byte, sstorage.CHUNK_SIZE)) != 0 {
+			log.Warn("verify empty chunk", "chunkidx", chunkIdx)
+		}
+		if chunkIdx%(chunkPerKv*100) == 0 {
+			log.Info("verify verify state", "chunkidx", chunkIdx)
+		}
+	}
+}
+
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "sstorage",
@@ -262,6 +303,7 @@ func init() {
 	rootCmd.AddCommand(ChunkWriteCmd)
 	rootCmd.AddCommand(ShardReadCmd)
 	rootCmd.AddCommand(ShardWriteCmd)
+	rootCmd.AddCommand(CheckEmtpyKVsCmd)
 }
 
 func main() {

core/blockchain.go

@@ -2458,20 +2458,50 @@ func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageM
 		}
 
 		if meta.KVSize != uint64(len(data)) {
-			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash KVSize: %d", len(val), meta.KVSize)
+			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash kvSize: %d", len(val), meta.KVSize)
 		}
 		data = d
 	}
 
-	hash := crypto.Keccak256Hash(data)
-	if !bytes.Equal(hash[:24], meta.HashInMeta) {
-		return nil, fmt.Errorf("verifyKV fail: size error; Data hash: %s; MetaHash hash (24): %s",
-			common.Bytes2Hex(hash[:24]), common.Bytes2Hex(meta.HashInMeta))
+	root := sstorage.MerkleRootWithMinTree(data)
+	if !bytes.Equal(root[:24], meta.HashInMeta) {
+		return nil, fmt.Errorf("verifyKV fail: Data hash: %s; MetaHash hash (24): %s, providerAddr %s, data %s",
+			common.Bytes2Hex(root[:24]), common.Bytes2Hex(meta.HashInMeta), providerAddr.Hex(), common.Bytes2Hex(data))
 	}
 
 	return data, nil
 }
 
+// FillSstorWithEmptyKV this func is used to fill empty KVs to storage file to make the whole file data encoded.
+// file in the KVs between start and limit(include limit). if the lastKvIdx larger than kv idx to fill, ignore it.
+func (bc *BlockChain) FillSstorWithEmptyKV(contract common.Address, start, limit uint64) (uint64, error) {
+	sm := sstorage.ContractToShardManager[contract]
+	if sm == nil {
+		return start, fmt.Errorf("kv verify fail: contract not support, contract: %s", contract.Hex())
+	}
+
+	bc.chainmu.TryLock()
+	defer bc.chainmu.Unlock()
+
+	empty := make([]byte, 0)
+	lastKvIdx, err := bc.GetSstorageLastKvIdx(contract)
+	if err != nil {
+		return start, fmt.Errorf("get lastKvIdx for FillEmptyKV fail, err: %s", err.Error())
+	}
+	for idx := start; idx <= limit; idx++ {
+		if lastKvIdx > idx {
+			continue
+		}
+		_, err = sm.TryWrite(idx, empty, common.Hash{})
+		if err != nil {
+			err = fmt.Errorf("write empty to kv file fail, index: %d; error: %s", idx, err.Error())
+			return idx, err
+		}
+	}
+
+	return limit + 1, nil
+}
+
 // VerifyAndWriteKV verify a list of raw KV data using the metadata saved in the local level DB and write successfully verified
 // KVs to the sstorage file. And return the inserted KV index list.
 func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64][]byte, providerAddr common.Address) (uint64, uint64, []uint64, error) {

core/genesis.go

@@ -146,10 +146,10 @@ func (e *GenesisMismatchError) Error() string {
 // SetupGenesisBlock writes or updates the genesis block in db.
 // The block that will be used is:
 //
-//                          genesis == nil       genesis != nil
-//                       +------------------------------------------
-//     db has no genesis |  main-net default  |  genesis
-//     db has genesis    |  from DB           |  genesis (if compatible)
+//	                     genesis == nil       genesis != nil
+//	                  +------------------------------------------
+//	db has no genesis |  main-net default  |  genesis
+//	db has genesis    |  from DB           |  genesis (if compatible)
 //
 // The stored chain configuration will be updated if it is compatible (i.e. does not
 // specify a fork block below the local head block). In case of a conflict, the

eth/downloader/downloader.go

@@ -212,13 +212,16 @@ type BlockChain interface {
 	// KVs to the sstorage file. And return the inserted KV index list.
 	VerifyAndWriteKV(contract common.Address, data map[uint64][]byte, provderAddr common.Address) (uint64, uint64, []uint64, error)
 
+	// FillSstorWithEmptyKV get the lastKVIndex and if the kv index need to fill is larger than or equal to lastKVIndex
+	// fill up the kv with empty ([]byte{}), so the data in the file will be filled with encode empty data
+	FillSstorWithEmptyKV(contract common.Address, start, limit uint64) (uint64, error)
+
 	// ReadEncodedKVsByIndexList Read the encoded KVs by a list of KV index.
 	ReadEncodedKVsByIndexList(contract common.Address, shardId uint64, indexes []uint64) (common.Address, []*core.KV, error)
 
 	// ReadEncodedKVsByIndexRange Read encoded KVs sequentially starting from origin until the index exceeds the limit or
 	// the amount of data read is greater than the bytes.
-	ReadEncodedKVsByIndexRange(contract common.Address, shardId uint64, origin uint64,
-		limit uint64, bytes uint64) (common.Address, []*core.KV, error)
+	ReadEncodedKVsByIndexRange(contract common.Address, shardId uint64, origin uint64, limit uint64, bytes uint64) (common.Address, []*core.KV, error)
 
 	// GetSstorageLastKvIdx get LastKvIdx from a sstorage contract with latest stateDB.
 	GetSstorageLastKvIdx(contract common.Address) (uint64, error)
@@ -241,7 +244,7 @@ func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain Bl
 		headerProcCh:   make(chan *headerTask, 1),
 		quitCh:         make(chan struct{}),
 		SnapSyncer:     snap.NewSyncer(stateDb),
-		SstorSyncer:    sstorage.NewSyncer(stateDb, chain, sstor.Shards()),
+		SstorSyncer:    sstorage.NewSyncer(stateDb, chain, mux, sstor.Shards()),
 		stateSyncStart: make(chan *stateSync),
 		sstorSyncStart: make(chan *sstorSync),
 	}