EthStorage Mining

cmd/geth/main.go

@@ -133,6 +133,9 @@ var (
 		utils.MinerNoVerifyFlag,
 		utils.SstorageShardFlag,
 		utils.SstorageFileFlag,
+		utils.SstorageMineFlag,
+		utils.SstorageNodeKeyFlag,
+		utils.SstorageMinerContractFlag,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,

cmd/utils/flags.go

@@ -566,6 +566,18 @@ var (
 		Usage: "Add sharded storage data file",
 		Value: nil,
 	}
+	SstorageMineFlag = cli.BoolFlag{
+		Name:  "sstorage.mine",
+		Usage: "Enable sstorage mining",
+	}
+	SstorageNodeKeyFlag = cli.StringFlag{
+		Name:  "sstorage.nodekey",
+		Usage: "Sstorage miner node key file",
+	}
+	SstorageMinerContractFlag = cli.StringFlag{
+		Name:  "sstorage.minercontract",
+		Usage: "Sstorage miner contract",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -1125,6 +1137,15 @@ func setSstorage(ctx *cli.Context, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(SstorageFileFlag.Name) {
 		cfg.SstorageFiles = ctx.GlobalStringSlice(SstorageFileFlag.Name)
 	}
+	if ctx.GlobalIsSet(SstorageMineFlag.Name) {
+		cfg.SstorageMine = ctx.GlobalBool(SstorageMineFlag.Name)
+	}
+	if ctx.GlobalIsSet(SstorageNodeKeyFlag.Name) {
+		cfg.SstorageNodeKey = ctx.GlobalString(SstorageNodeKeyFlag.Name)
+	}
+	if ctx.GlobalIsSet(SstorageMinerContractFlag.Name) {
+		cfg.SstorageMinerContract = ctx.GlobalString(SstorageMinerContractFlag.Name)
+	}
 
 	sstorage.InitializeConfig()
 	for _, s := range cfg.SstorageShards {

core/blockchain.go

@@ -19,6 +19,7 @@ package core
 
 import (
 	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -48,7 +49,6 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/holiman/uint256"
-	"golang.org/x/crypto/sha3"
 )
 
 var (
@@ -2352,7 +2352,10 @@ func (bc *BlockChain) PreExecuteBlock(block *types.Block) (err error) {
 	return
 }
 
-var emptyHash = common.Hash{}
+var (
+	emptyHash        = common.Hash{}
+	defaultHashBytes = crypto.Keccak256Hash().Bytes()
+)
 
 type SstorageMetadata struct {
 	KVIdx      uint64
@@ -2379,13 +2382,7 @@ func getSlotHash(slotIdx uint64, key common.Hash) common.Hash {
 	slotdata := slot[:]
 	data := append(keydata, slotdata...)
 
-	hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState)
-	hasher.Write(data)
-
-	hashRes := common.Hash{}
-	hasher.Read(hashRes[:])
-
-	return hashRes
+	return crypto.Keccak256Hash(data)
 }
 
 // GetSstorageMetadata get sstorage metadata for a given kv (specified by contract address and index)
@@ -2395,19 +2392,20 @@ func GetSstorageMetadata(s *state.StateDB, contract common.Address, index uint64
 	// then get SstorageMetadata from kvMap (slot 1) using skey. the SstorageMetadata struct is as following
 	// struct PhyAddr {
 	// 	uint40 KVIdx;
-	// 	uint24 KVSize;
+	// 	uint24 kvSize;
 	// 	bytes24 hash;
 	// }
+	defaultHash, defaultMeta := GetDefaultMetadata(index)
 	position := getSlotHash(2, uint256.NewInt(index).Bytes32())
 	skey := s.GetState(contract, position)
 	if skey == emptyHash {
-		return emptyHash, nil, fmt.Errorf("fail to get skey for index %d", index)
+		return defaultHash, defaultMeta, fmt.Errorf("fail to get skey for index %d", index)
 	}
 
 	position = getSlotHash(1, skey)
 	meta := s.GetState(contract, position)
 	if meta == emptyHash {
-		return emptyHash, nil, fmt.Errorf("fail to get SstorageMetadata for skey %s", skey.Hex())
+		return defaultHash, defaultMeta, fmt.Errorf("fail to get SstorageMetadata for skey %s", skey.Hex())
 	}
 
 	return meta, &SstorageMetadata{
@@ -2417,6 +2415,18 @@ func GetSstorageMetadata(s *state.StateDB, contract common.Address, index uint64
 		nil
 }
 
+func GetDefaultMetadata(index uint64) (common.Hash, *SstorageMetadata) {
+	meta := &SstorageMetadata{
+		KVIdx:      index,
+		KVSize:     uint64(0),
+		HashInMeta: defaultHashBytes[:24],
+	}
+	hashBytes := make([]byte, 32)
+	copy(hashBytes[:24], meta.HashInMeta)
+	binary.BigEndian.PutUint32(hashBytes[27:], uint32(index))
+	return common.BytesToHash(hashBytes), meta
+}
+
 // VerifyKV verify kv using SstorageMetadata
 func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageMetadata, isEncoded bool, providerAddr common.Address) ([]byte, error) {
 	if idx != meta.KVIdx {
@@ -2434,7 +2444,7 @@ func VerifyKV(sm *sstorage.ShardManager, idx uint64, val []byte, meta *SstorageM
 		}
 
 		if meta.KVSize != uint64(len(data)) {
-			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash KVSize: %d", len(val), meta.KVSize)
+			return nil, fmt.Errorf("verifyKV fail: size error; Data size: %d; MetaHash kvSize: %d", len(val), meta.KVSize)
 		}
 		data = d
 	}
@@ -2508,7 +2518,7 @@ func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64]
 			continue
 		}
 
-		success, err := sm.TryWrite(vkv.Idx, vkv.Data, vkv.MetaHash)
+		success, err := sm.TryWrite(vkv.Idx, vkv.Data, common.BytesToHash(meta.HashInMeta))
 		if err != nil {
 			log.Warn("write kv fail", "error", err)
 		}
@@ -2519,6 +2529,42 @@ func (bc *BlockChain) VerifyAndWriteKV(contract common.Address, data map[uint64]
 	return synced, syncedBytes, inserted, nil
 }
 
+// ReadKVsByIndexList Read the KVs by a list of KV index.
+func (bc *BlockChain) ReadKVsByIndexList(contract common.Address, indexes []uint64, useMaxKVsize bool) ([]*KV, error) {
+	stateDB, err := bc.StateAt(bc.CurrentBlock().Root())
+	if err != nil {
+		return nil, err
+	}
+
+	return bc.ReadKVsByIndexListWithState(stateDB, contract, indexes, useMaxKVsize)
+}
+
+func (bc *BlockChain) ReadKVsByIndexListWithState(stateDB *state.StateDB, contract common.Address, indexes []uint64, useMaxKVsize bool) ([]*KV, error) {
+	sm := sstorage.ContractToShardManager[contract]
+	if sm == nil {
+		return nil, fmt.Errorf("shard manager for contract %s is not support", contract.Hex())
+	}
+
+	res := make([]*KV, 0)
+	for _, idx := range indexes {
+		_, meta, err := GetSstorageMetadata(stateDB, contract, idx)
+		if err != nil {
+			continue
+		}
+		l := int(meta.KVSize)
+		if useMaxKVsize {
+			l = int(sm.MaxKvSize())
+		}
+		data, ok, err := sm.TryRead(idx, l, common.BytesToHash(meta.HashInMeta))
+		if ok && err == nil {
+			kv := KV{idx, data}
+			res = append(res, &kv)
+		}
+	}
+
+	return res, nil
+}
+
 // ReadEncodedKVsByIndexList Read the masked KVs by a list of KV index.
 func (bc *BlockChain) ReadEncodedKVsByIndexList(contract common.Address, shardId uint64, indexes []uint64) (common.Address, []*KV, error) {
 	sm := sstorage.ContractToShardManager[contract]
@@ -2599,3 +2645,55 @@ func (bc *BlockChain) GetSstorageLastKvIdx(contract common.Address) (uint64, err
 	log.Warn("GetSstorageLastKvIdx", "val", common.Bytes2Hex(val.Bytes()))
 	return new(big.Int).SetBytes(val.Bytes()).Uint64(), nil
 }
+
+type MiningInfo struct {
+	MiningHash   common.Hash
+	LastMineTime uint64
+	Difficulty   *big.Int
+	BlockMined   *big.Int
+}
+
+func (a *MiningInfo) Equal(b *MiningInfo) bool {
+	if b == nil {
+		return false
+	}
+	if a.LastMineTime != b.LastMineTime {
+		return false
+	}
+	if !bytes.Equal(a.MiningHash.Bytes(), b.MiningHash.Bytes()) {
+		return false
+	}
+	if a.BlockMined.Cmp(b.BlockMined) != 0 {
+		return false
+	}
+	if a.Difficulty.Cmp(b.Difficulty) != 0 {
+		return false
+	}
+	return true
+}
+
+func (bc *BlockChain) GetSstorageMiningInfo(root common.Hash, contract common.Address, shardId uint64) (*MiningInfo, error) {
+	stateDB, err := bc.StateAt(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return bc.GetSstorageMiningInfoWithStateDB(stateDB, contract, shardId)
+}
+
+func (bc *BlockChain) GetSstorageMiningInfoWithStateDB(stateDB *state.StateDB, contract common.Address, shardId uint64) (*MiningInfo, error) {
+	info := new(MiningInfo)
+	position := getSlotHash(0, uint256.NewInt(shardId).Bytes32())
+	info.MiningHash = stateDB.GetState(contract, position)
+	if info.MiningHash == emptyHash {
+		return nil, fmt.Errorf("fail to get mining info for shard %d", shardId)
+	}
+	info.LastMineTime = stateDB.GetState(contract, hashAdd(position, 1)).Big().Uint64()
+	info.Difficulty = stateDB.GetState(contract, hashAdd(position, 2)).Big()
+	info.BlockMined = stateDB.GetState(contract, hashAdd(position, 3)).Big()
+	return info, nil
+}
+
+func hashAdd(hash common.Hash, i uint64) common.Hash {
+	return common.BytesToHash(new(big.Int).Add(hash.Big(), new(big.Int).SetUint64(i)).Bytes())
+}

eth/api.go

@@ -94,6 +94,21 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI {
 	return &PrivateMinerAPI{e: e}
 }
 
+// StartSstorMining starts the Sstorage miner. If mining is already running, this method just return.
+func (api *PrivateMinerAPI) StartSstorMining() {
+	api.e.StartSstorMining()
+}
+
+// StopSstorMining terminates the Sstorage miner.
+func (api *PrivateMinerAPI) StopSstorMining() {
+	api.e.StopSstorMining()
+}
+
+// SetSstorRecommitInterval updates the interval for sstorage miner sealing work recommitting.
+func (api *PrivateMinerAPI) SetSstorRecommitInterval(interval int) {
+	api.e.SstorMiner().SetRecommitInterval(time.Duration(interval) * time.Millisecond)
+}
+
 // Start starts the miner with the given number of threads. If threads is nil,
 // the number of workers started is equal to the number of logical CPUs that are
 // usable by this process. If mining is already running, this method adjust the

eth/backend.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/pruner"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -60,6 +61,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/sstorage"
+	"github.com/ethereum/go-ethereum/sstorminer"
 )
 
 // Config contains the configuration options of the ETH protocol.
@@ -92,9 +94,10 @@ type Ethereum struct {
 
 	APIBackend *EthAPIBackend
 
-	miner     *miner.Miner
-	gasPrice  *big.Int
-	etherbase common.Address
+	miner      *miner.Miner
+	sstorMiner *sstorminer.Miner
+	gasPrice   *big.Int
+	etherbase  common.Address
 
 	networkID     uint64
 	netRPCService *ethapi.PublicNetAPI
@@ -273,6 +276,21 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock)
 	eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
 
+	if config.SstorageMine {
+		privKey, err := crypto.LoadECDSA(config.SstorageNodeKey)
+		if err != nil {
+			return nil, err
+		}
+		if len(sstorage.Shards()) == 0 {
+			return nil, fmt.Errorf("no shards is exist")
+		}
+		if config.SstorageMinerContract == "" {
+			return nil, fmt.Errorf("miner contract is needed when the sstorage mine is enabled.")
+		}
+		minerContract := common.HexToAddress(config.SstorageMinerContract)
+		eth.sstorMiner = sstorminer.New(eth, &config.SStorMiner, chainConfig, eth.EventMux(), privKey, minerContract)
+	}
+
 	eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
 	if eth.APIBackend.allowUnprotectedTxs {
 		log.Info("Unprotected transactions allowed")
@@ -476,6 +494,26 @@ func (s *Ethereum) SetEtherbase(etherbase common.Address) {
 	s.miner.SetEtherbase(etherbase)
 }
 
+// todo add start / stop mining for special shard
+
+// StartSstorMining starts the sstorage miner. If mining
+// is already running, this method just return.
+func (s *Ethereum) StartSstorMining() {
+	// If the miner was not running, initialize it
+	if !s.IsSstorMining() {
+		go s.sstorMiner.Start()
+	}
+}
+
+// StopSstorMining terminates the sstorage miner.
+func (s *Ethereum) StopSstorMining() {
+	s.sstorMiner.Stop()
+}
+
+func (s *Ethereum) IsSstorMining() bool { return s.sstorMiner.Mining() }
+
+func (s *Ethereum) SstorMiner() *sstorminer.Miner { return s.sstorMiner }
+
 // StartMining starts the miner with the given number of CPU threads. If mining
 // is already running, this method adjust the number of threads allowed to use
 // and updates the minimum price required by the transaction pool.

eth/ethconfig/config.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/miner"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/sstorminer"
 )
 
 // FullNodeGPO contains default gasprice oracle settings for full node.
@@ -89,6 +90,14 @@ var Defaults = Config{
 		GasPrice: big.NewInt(params.GWei),
 		Recommit: 3 * time.Second,
 	},
+	SStorMiner: sstorminer.Config{
+		RandomChecks:      16,
+		MinimumDiff:       new(big.Int).SetUint64(1),
+		TargetIntervalSec: new(big.Int).SetUint64(60),
+		Cutoff:            new(big.Int).SetUint64(40),
+		DiffAdjDivisor:    new(big.Int).SetUint64(1024),
+		Recommit:          15 * time.Second,
+	},
 	TxPool:        core.DefaultTxPoolConfig,
 	RPCGasCap:     50000000,
 	RPCEVMTimeout: 5 * time.Second,
@@ -174,6 +183,9 @@ type Config struct {
 	// Mining options
 	Miner miner.Config
 
+	// Sstorage Mining options
+	SStorMiner sstorminer.Config
+
 	// Ethash options
 	Ethash ethash.Config
 
@@ -220,8 +232,11 @@ type Config struct {
 	ValidatorChangeEpochId uint64
 
 	// Sstorage config
-	SstorageFiles  []string `toml:",omitempty"`
-	SstorageShards []string `toml:",omitempty"`
+	SstorageFiles         []string `toml:",omitempty"`
+	SstorageShards        []string `toml:",omitempty"`
+	SstorageMine          bool
+	SstorageNodeKey       string
+	SstorageMinerContract string
 }
 
 // CreateConsensusEngine creates a consensus engine for the given chain configuration.