EthStorage Mining

cmd/geth/main.go

@@ -133,6 +133,9 @@ var (
 		utils.MinerNoVerifyFlag,
 		utils.SstorageShardFlag,
 		utils.SstorageFileFlag,
+		utils.SstorageMineFlag,
+		utils.SstorageTXSignerFlag,
+		utils.SstorageMinerContractFlag,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,

cmd/utils/flags.go

@@ -578,6 +578,18 @@ var (
 		Usage: "Add sharded storage data file",
 		Value: nil,
 	}
+	SstorageMineFlag = cli.BoolFlag{
+		Name:  "sstorage.mine",
+		Usage: "Enable sstorage mining",
+	}
+	SstorageTXSignerFlag = cli.StringFlag{
+		Name:  "sstorage.txsigner",
+		Usage: "Account used to sign tx submit to sstorage miner contract",
+	}
+	SstorageMinerContractFlag = cli.StringFlag{
+		Name:  "sstorage.minercontract",
+		Usage: "Sstorage miner contract",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -1137,6 +1149,15 @@ func setSstorage(ctx *cli.Context, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(SstorageFileFlag.Name) {
 		cfg.SstorageFiles = ctx.GlobalStringSlice(SstorageFileFlag.Name)
 	}
+	if ctx.GlobalIsSet(SstorageMineFlag.Name) {
+		cfg.SstorageMine = ctx.GlobalBool(SstorageMineFlag.Name)
+	}
+	if ctx.GlobalIsSet(SstorageTXSignerFlag.Name) {
+		cfg.SstorageTXSigner = ctx.GlobalString(SstorageTXSignerFlag.Name)
+	}
+	if ctx.GlobalIsSet(SstorageMinerContractFlag.Name) {
+		cfg.SstorageMinerContract = ctx.GlobalString(SstorageMinerContractFlag.Name)
+	}
 
 	sstorage.InitializeConfig()
 	for _, s := range cfg.SstorageShards {

core/blockchain.go

@@ -20,6 +20,7 @@ package core
 import (
 	"bytes"
 	"context"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -50,7 +51,6 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/holiman/uint256"
-	"golang.org/x/crypto/sha3"
 )
 
 var (
@@ -2383,7 +2383,9 @@ func (bc *BlockChain) PreExecuteBlock(block *types.Block) (err error) {
 	return
 }
 
-var emptyHash = common.Hash{}
+var (
+	emptyHash = common.Hash{}
+)
 
 type SstorageMetadata struct {
 	KVIdx      uint64
@@ -2410,13 +2412,7 @@ func getSlotHash(slotIdx uint64, key common.Hash) common.Hash {
 	slotdata := slot[:]
 	data := append(keydata, slotdata...)
 
-	hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState)
-	hasher.Write(data)
-
-	hashRes := common.Hash{}
-	hasher.Read(hashRes[:])
-
-	return hashRes
+	return crypto.Keccak256Hash(data)
 }
 
 // GetSstorageMetadata get sstorage metadata for a given kv (specified by contract address and index)
@@ -2426,7 +2422,7 @@ func GetSstorageMetadata(s *state.StateDB, contract common.Address, index uint64
 	// then get SstorageMetadata from kvMap (slot 1) using skey. the SstorageMetadata struct is as following
 	// struct PhyAddr {
 	// 	uint40 KVIdx;
-	// 	uint24 KVSize;
+	// 	uint24 kvSize;
 	// 	bytes24 hash;
 	// }
 	position := getSlotHash(2, uint256.NewInt(index).Bytes32())
@@ -2448,6 +2444,17 @@ func GetSstorageMetadata(s *state.StateDB, contract common.Address, index uint64
 		nil
 }
 
+func GetDefaultMetadata(index uint64) (common.Hash, *SstorageMetadata) {
+	meta := &SstorageMetadata{
+		KVIdx:      index,
+		KVSize:     uint64(0),
+		HashInMeta: make([]byte, 24),
+	}
+	hashBytes := make([]byte, 32)
+	binary.BigEndian.PutUint32(hashBytes[27:], uint32(index))
+	return common.BytesToHash(hashBytes), meta
+}
+
 // VerifyKV verify kv using SstorageMetadata
 func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageMetadata, isEncoded bool, providerAddr common.Address) ([]byte, error) {
 	if idx != meta.KVIdx {
@@ -2465,7 +2472,7 @@ func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageM
 		}
 
 		if meta.KVSize != uint64(len(data)) {
-			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash KVSize: %d", len(val), meta.KVSize)
+			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash kvSize: %d", len(val), meta.KVSize)
 		}
 		data = d
 	}
@@ -2479,6 +2486,34 @@ func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageM
 	return data, nil
 }
 
+func (bc *BlockChain) FillSstorWithEmptyKV(contract common.Address, start, limit uint64) (uint64, error) {
+	sm := sstorage.ContractToShardManager[contract]
+	if sm == nil {
+		return start, fmt.Errorf("kv verify fail: contract not support, contract: %s", contract.Hex())
+	}
+
+	bc.chainmu.TryLock()
+	defer bc.chainmu.Unlock()
+
+	empty := make([]byte, 0)
+	lastKvIdx, err := bc.GetSstorageLastKvIdx(contract)
+	if err != nil {
+		return start, fmt.Errorf("get lastKvIdx for FillEmptyKV fail, err: %s", err.Error())
+	}
+	for idx := start; idx <= limit; idx++ {
+		if lastKvIdx > idx {
+			continue
+		}
+		_, err = sm.TryWrite(idx, empty, common.Hash{})
+		if err != nil {
+			err = fmt.Errorf("write empty to kv file fail, index: %d; error: %s", idx, err.Error())
+			return idx, err
+		}
+	}
+
+	return limit + 1, nil
+}
+
 // VerifyAndWriteKV verify a list of raw KV data using the metadata saved in the local level DB and write successfully verified
 // KVs to the sstorage file. And return the inserted KV index list.
 func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64][]byte, providerAddr common.Address) (uint64, uint64, []uint64, error) {
@@ -2539,7 +2574,7 @@ func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64]
 			continue
 		}
 
-		success, err := sm.TryWrite(vkv.Idx, vkv.Data, vkv.MetaHash)
+		success, err := sm.TryWrite(vkv.Idx, vkv.Data, common.BytesToHash(meta.HashInMeta))
 		if err != nil {
 			log.Warn("write kv fail", "error", err)
 		}
@@ -2550,6 +2585,48 @@ func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64]
 	return synced, syncedBytes, inserted, nil
 }
 
+// ReadKVsByIndexList Read the KVs by a list of KV index.
+func (bc *BlockChain) ReadKVsByIndexList(contract common.Address, indexes []uint64, returnEmpty bool) ([]*KV, error) {
+	stateDB, err := bc.StateAt(bc.CurrentBlock().Root())
+	if err != nil {
+		return nil, err
+	}
+
+	return bc.ReadKVsByIndexListWithState(stateDB, contract, indexes, returnEmpty)
+}
+
+func (bc *BlockChain) ReadKVsByIndexListWithState(stateDB *state.StateDB, contract common.Address, indexes []uint64, returnEmpty bool) ([]*KV, error) {
+	sm := sstorage.ContractToShardManager[contract]
+	if sm == nil {
+		return nil, fmt.Errorf("shard manager for contract %s is not support", contract.Hex())
+	}
+
+	val := stateDB.GetState(contract, uint256.NewInt(0).Bytes32())
+	lastIndex := new(big.Int).SetBytes(val.Bytes()).Uint64()
+
+	res := make([]*KV, 0)
+	for _, idx := range indexes {
+		if idx >= lastIndex {
+			if returnEmpty {
+				kv := KV{idx, make([]byte, 0)}
+				res = append(res, &kv)
+			}
+			continue
+		}
+		_, meta, err := GetSstorageMetadata(stateDB, contract, idx)
+		if err != nil {
+			return nil, fmt.Errorf("get storage metadata fail, err: ", err.Error())
+		}
+		data, ok, err := sm.TryRead(idx, int(meta.KVSize), common.BytesToHash(meta.HashInMeta))
+		if ok && err == nil {
+			kv := KV{idx, data}
+			res = append(res, &kv)
+		}
+	}
+
+	return res, nil
+}
+
 // ReadEncodedKVsByIndexList Read the masked KVs by a list of KV index.
 func (bc *BlockChain) ReadEncodedKVsByIndexList(contract common.Address, shardId uint64, indexes []uint64) (common.Address, []*KV, error) {
 	sm := sstorage.ContractToShardManager[contract]
@@ -2631,6 +2708,58 @@ func (bc *BlockChain) GetSstorageLastKvIdx(contract common.Address) (uint64, err
 	return new(big.Int).SetBytes(val.Bytes()).Uint64(), nil
 }
 
+type MiningInfo struct {
+	MiningHash   common.Hash
+	LastMineTime uint64
+	Difficulty   *big.Int
+	BlockMined   *big.Int
+}
+
+func (a *MiningInfo) Equal(b *MiningInfo) bool {
+	if b == nil {
+		return false
+	}
+	if a.LastMineTime != b.LastMineTime {
+		return false
+	}
+	if !bytes.Equal(a.MiningHash.Bytes(), b.MiningHash.Bytes()) {
+		return false
+	}
+	if a.BlockMined.Cmp(b.BlockMined) != 0 {
+		return false
+	}
+	if a.Difficulty.Cmp(b.Difficulty) != 0 {
+		return false
+	}
+	return true
+}
+
+func (bc *BlockChain) GetSstorageMiningInfo(root common.Hash, contract common.Address, shardId uint64) (*MiningInfo, error) {
+	stateDB, err := bc.StateAt(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return bc.GetSstorageMiningInfoWithStateDB(stateDB, contract, shardId)
+}
+
+func (bc *BlockChain) GetSstorageMiningInfoWithStateDB(stateDB *state.StateDB, contract common.Address, shardId uint64) (*MiningInfo, error) {
+	info := new(MiningInfo)
+	position := getSlotHash(3, uint256.NewInt(shardId).Bytes32())
+	info.MiningHash = stateDB.GetState(contract, position)
+	if info.MiningHash == emptyHash {
+		return nil, fmt.Errorf("fail to get mining info for shard %d", shardId)
+	}
+	info.LastMineTime = stateDB.GetState(contract, hashAdd(position, 1)).Big().Uint64()
+	info.Difficulty = stateDB.GetState(contract, hashAdd(position, 2)).Big()
+	info.BlockMined = stateDB.GetState(contract, hashAdd(position, 3)).Big()
+	return info, nil
+}
+
+func hashAdd(hash common.Hash, i uint64) common.Hash {
+	return common.BytesToHash(new(big.Int).Add(hash.Big(), new(big.Int).SetUint64(i)).Bytes())
+}
+
 func (bc *BlockChain) setMindReading(chainConfig *params.ChainConfig) error {
 	if chainConfig.MindReading != nil {
 		bc.mindReading.EnableBlockNumber = chainConfig.MindReading.EnableBlockNumber

eth/api.go

@@ -94,6 +94,21 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI {
 	return &PrivateMinerAPI{e: e}
 }
 
+// StartSstorMining starts the Sstorage miner. If mining is already running, this method just return.
+func (api *PrivateMinerAPI) StartSstorMining() {
+	api.e.StartSstorMining()
+}
+
+// StopSstorMining terminates the Sstorage miner.
+func (api *PrivateMinerAPI) StopSstorMining() {
+	api.e.StopSstorMining()
+}
+
+// SetSstorRecommitInterval updates the interval for sstorage miner sealing work recommitting.
+func (api *PrivateMinerAPI) SetSstorRecommitInterval(interval int) {
+	api.e.SstorMiner().SetRecommitInterval(time.Duration(interval) * time.Millisecond)
+}
+
 // Start starts the miner with the given number of threads. If threads is nil,
 // the number of workers started is equal to the number of logical CPUs that are
 // usable by this process. If mining is already running, this method adjust the

eth/backend.go

@@ -60,6 +60,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/sstorage"
+	"github.com/ethereum/go-ethereum/sstorminer"
 )
 
 // Config contains the configuration options of the ETH protocol.
@@ -92,9 +93,10 @@ type Ethereum struct {
 
 	APIBackend *EthAPIBackend
 
-	miner     *miner.Miner
-	gasPrice  *big.Int
-	etherbase common.Address
+	miner      *miner.Miner
+	sstorMiner *sstorminer.Miner
+	gasPrice   *big.Int
+	etherbase  common.Address
 
 	networkID     uint64
 	netRPCService *ethapi.PublicNetAPI
@@ -293,6 +295,27 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock)
 	eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
 
+	if config.SstorageMine {
+		if len(sstorage.Shards()) == 0 {
+			return nil, fmt.Errorf("no shards is exist")
+		}
+		if config.SstorageMinerContract == "" {
+			return nil, fmt.Errorf("miner contract is needed when the sstorage mine is enabled.")
+		}
+		minerContract := common.HexToAddress(config.SstorageMinerContract)
+		if config.SstorageTXSigner == "" {
+			return nil, fmt.Errorf("TX signer is needed when the sstorage mine is enabled.")
+		}
+		signer := accounts.Account{Address: common.HexToAddress(config.SstorageTXSigner)}
+		wallet, err := eth.accountManager.Find(signer)
+		if wallet == nil || err != nil {
+			log.Error("sstorage tx signer account unavailable locally", "err", err)
+			return nil, fmt.Errorf("signer missing: %v", err)
+		}
+
+		eth.sstorMiner = sstorminer.New(eth, &config.SStorMiner, chainConfig, eth.EventMux(), &sstorminer.TXSigner{signer, wallet.SignTx}, minerContract)
+	}
+
 	eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
 	if eth.APIBackend.allowUnprotectedTxs {
 		log.Info("Unprotected transactions allowed")
@@ -496,6 +519,26 @@ func (s *Ethereum) SetEtherbase(etherbase common.Address) {
 	s.miner.SetEtherbase(etherbase)
 }
 
+// todo add start / stop mining for special shard
+
+// StartSstorMining starts the sstorage miner. If mining
+// is already running, this method just return.
+func (s *Ethereum) StartSstorMining() {
+	// If the miner was not running, initialize it
+	if !s.IsSstorMining() {
+		go s.sstorMiner.Start()
+	}
+}
+
+// StopSstorMining terminates the sstorage miner.
+func (s *Ethereum) StopSstorMining() {
+	s.sstorMiner.Stop()
+}
+
+func (s *Ethereum) IsSstorMining() bool { return s.sstorMiner.Mining() }
+
+func (s *Ethereum) SstorMiner() *sstorminer.Miner { return s.sstorMiner }
+
 // StartMining starts the miner with the given number of CPU threads. If mining
 // is already running, this method adjust the number of threads allowed to use
 // and updates the minimum price required by the transaction pool.

eth/downloader/downloader.go

@@ -212,13 +212,16 @@ type BlockChain interface {
 	// KVs to the sstorage file. And return the inserted KV index list.
 	VerifyAndWriteKV(contract common.Address, data map[uint64][]byte, provderAddr common.Address) (uint64, uint64, []uint64, error)
 
+	// FillSstorWithEmptyKV get the lastKVIndex and if the kv index need to fill is larger than or equal to lastKVIndex
+	// fill up the kv with empty ([]byte{}), so the data in the file will be filled with encode empty data
+	FillSstorWithEmptyKV(contract common.Address, start, limit uint64) (uint64, error)
+
 	// ReadEncodedKVsByIndexList Read the encoded KVs by a list of KV index.
 	ReadEncodedKVsByIndexList(contract common.Address, shardId uint64, indexes []uint64) (common.Address, []*core.KV, error)
 
 	// ReadEncodedKVsByIndexRange Read encoded KVs sequentially starting from origin until the index exceeds the limit or
 	// the amount of data read is greater than the bytes.
-	ReadEncodedKVsByIndexRange(contract common.Address, shardId uint64, origin uint64,
-		limit uint64, bytes uint64) (common.Address, []*core.KV, error)
+	ReadEncodedKVsByIndexRange(contract common.Address, shardId uint64, origin uint64, limit uint64, bytes uint64) (common.Address, []*core.KV, error)
 
 	// GetSstorageLastKvIdx get LastKvIdx from a sstorage contract with latest stateDB.
 	GetSstorageLastKvIdx(contract common.Address) (uint64, error)