diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index b98597e307e6..d141eaf94b34 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -319,7 +319,8 @@ func importChain(ctx *cli.Context) error { } chain.Stop() fmt.Printf("Import done in %v.\n\n", time.Since(start)) - + // OutPut debug info + common.DebugInfo.Print() // Output pre-compaction stats mostly to see the import trashing stats, err := db.Stat("leveldb.stats") if err != nil { diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 869cf90ea57b..d0ca8b58cdb5 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -164,6 +164,10 @@ func ImportChain(chain *core.BlockChain, fn string) error { if _, err := chain.InsertChain(missing); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } + // only test from 8M to 8.2M + if chain.CurrentBlock().NumberU64() >= 8200000 { + break + } } return nil } diff --git a/common/bytes.go b/common/bytes.go index 634041804d0b..d633e8856543 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -17,7 +17,10 @@ // Package common contains various helper functions. package common -import "encoding/hex" +import ( + "encoding/binary" + "encoding/hex" +) // ToHex returns the hex representation of b, prefixed with '0x'. // For empty slices, the return value is "0x0". @@ -156,3 +159,21 @@ func TrimRightZeroes(s []byte) []byte { } return s[:idx] } + +func Uint64ToBytes(n uint64) []byte { + Bytes := make([]byte, 8) + binary.BigEndian.PutUint64(Bytes, n) + return Bytes +} + +var ( + // CalAccessList + // true:generate access list + // false:use access list + CalAccessList = false +) + +type AccessList struct { + Address Address + Hashs []Hash +} diff --git a/common/debug.go b/common/debug.go index 61acd8ce70f8..96bec0746dd0 100644 --- a/common/debug.go +++ b/common/debug.go @@ -22,6 +22,7 @@ import ( "runtime" "runtime/debug" "strings" + "time" ) // Report gives off a warning requesting the user to submit an issue to the github tracker. @@ -50,3 +51,34 @@ func PrintDepricationWarning(str string) { `, line, emptyLine, str, emptyLine, line) } + +type DebugTime struct { + ExecuteTx time.Duration + ValidateBlock time.Duration + WriteBlock time.Duration + CommitTrie time.Duration + TxLen int +} + +func NewDebugTime() *DebugTime { + d := &DebugTime{ + TxLen: 0, + ExecuteTx: time.Duration(0), + ValidateBlock: time.Duration(0), + WriteBlock: time.Duration(0), + CommitTrie: time.Duration(0), + } + return d +} + +func (d *DebugTime) Print() { + fmt.Println("tx len", d.TxLen) + fmt.Println("process block time", d.ExecuteTx) + fmt.Println("validate block time", d.ValidateBlock) + fmt.Println("write block time", d.WriteBlock) + fmt.Println("write trie time", d.CommitTrie) +} + +var ( + DebugInfo = NewDebugTime() +) diff --git a/core/blockchain.go b/core/blockchain.go index ce1edd9b7fb0..599d442fe99c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1434,6 +1434,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.wg.Add(1) defer bc.wg.Done() + ts := time.Now() // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1456,6 +1457,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. if err := blockBatch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) } + common.DebugInfo.WriteBlock += time.Since(ts) + ts = time.Now() // Commit all cached state changes into underlying memory database. root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { @@ -1566,6 +1569,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } else { bc.chainSideFeed.Send(ChainSideEvent{Block: block}) } + common.DebugInfo.CommitTrie += time.Since(ts) return status, nil } @@ -1819,6 +1823,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er atomic.StoreUint32(&followupInterrupt, 1) return it.index, err } + common.DebugInfo.ExecuteTx += time.Since(substart) // Update the metrics touched during block processing accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them @@ -1841,6 +1846,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er return it.index, err } proctime := time.Since(start) + common.DebugInfo.ValidateBlock += time.Since(substart) + common.DebugInfo.TxLen += len(block.Transactions()) // Update the metrics touched during block validation accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete, we can mark them diff --git a/core/state/state_object.go b/core/state/state_object.go index 26ab67e1adb7..a6f62ab37b98 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -18,17 +18,87 @@ package state import ( "bytes" + "encoding/json" "fmt" - "io" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "io" + "math/big" + "sync" + "time" ) +var ( + // AccessListDB: store generated data(key:blockNumber value:access list) + AccessListDB, _ = leveldb.New("./accessList_eth", 512, 512, "") + + // BlockNumToAccessList: Preprocess the access list corresponding to the block + BlockNumToAccessList = make(map[int][]common.AccessList, 0) +) + +func init() { + if common.CalAccessList { + return + } + + // import block from 800w to 820w + start := 8000000 + end := 8200000 + + fmt.Println("preLoad access list", "from", start, "to", end) + for index := start; index <= end; index++ { + data, err := AccessListDB.Get(common.Uint64ToBytes(uint64(index))) + list := make([]common.AccessList, 0) + if len(data) == 0 || err != nil { + + } else { + if err := json.Unmarshal(data, &list); err != nil { + panic(err) + } + } + BlockNumToAccessList[index] = list + } + fmt.Println("preLoad access list", "size", len(BlockNumToAccessList)) +} + +type OriginStorage struct { + Data map[common.Address]Storage + mu sync.Mutex +} + +func NewOriginStorage() *OriginStorage { + return &OriginStorage{ + Data: make(map[common.Address]Storage, 0), + } +} + +func (o *OriginStorage) SetAccount(addr common.Address) { + o.mu.Lock() + defer o.mu.Unlock() + if _, ok := o.Data[addr]; !ok { + o.Data[addr] = make(Storage) + } +} + +func (o *OriginStorage) SetOrigin(addr common.Address, key, value common.Hash) { + o.mu.Lock() + defer o.mu.Unlock() + if _, ok := o.Data[addr]; !ok { + o.Data[addr] = make(Storage) + } + o.Data[addr][key] = value +} + +func (o *OriginStorage) GetData(addr common.Address, hash common.Hash) common.Hash { + if _, ok := o.Data[addr]; !ok { + return common.Hash{} + } + return o.Data[addr][hash] +} + var emptyCodeHash = crypto.Keccak256(nil) type Code []byte @@ -182,6 +252,17 @@ func (s *stateObject) GetState(db Database, key common.Hash) common.Hash { return s.GetCommittedState(db, key) } +func (s *stateObject) preLoadCommittedStateFromDB(db Database, key common.Hash) { + t := s.getTrie(db) + enc, _ := t.TryGet(key.Bytes()) + var value common.Hash + if len(enc) > 0 { + _, content, _, _ := rlp.Split(enc) + value.SetBytes(content) + } + s.db.OrigForCalAccessList.SetOrigin(s.address, key, value) +} + // GetCommittedState retrieves a value from the committed account storage trie. func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Hash { // If the fake storage is set, only lookup the state here(in the debugging mode) @@ -195,6 +276,13 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has if value, cached := s.originStorage[key]; cached { return value } + + if !common.CalAccessList { + load := s.db.OrigForCalAccessList.GetData(s.address, key) + s.originStorage[key] = load + return load + } + // If no live objects are available, attempt to use snapshots var ( enc []byte @@ -222,6 +310,7 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has } if enc, err = s.getTrie(db).TryGet(key.Bytes()); err != nil { s.setError(err) + s.db.OrigForCalAccessList.SetOrigin(s.address, key, common.Hash{}) return common.Hash{} } } @@ -234,6 +323,7 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has value.SetBytes(content) } s.originStorage[key] = value + s.db.OrigForCalAccessList.SetOrigin(s.address, key, common.Hash{}) return value } diff --git a/core/state/statedb.go b/core/state/statedb.go index 36f7d863af9b..f57798487f55 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "sort" + "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -62,8 +63,9 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - trie Trie + OrigForCalAccessList *OriginStorage + db Database + trie Trie snaps *snapshot.Tree snap snapshot.Snapshot @@ -120,15 +122,16 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) return nil, err } sdb := &StateDB{ - db: db, - trie: tr, - snaps: snaps, - stateObjects: make(map[common.Address]*stateObject), - stateObjectsPending: make(map[common.Address]struct{}), - stateObjectsDirty: make(map[common.Address]struct{}), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), - journal: newJournal(), + OrigForCalAccessList: NewOriginStorage(), + db: db, + trie: tr, + snaps: snaps, + stateObjects: make(map[common.Address]*stateObject), + stateObjectsPending: make(map[common.Address]struct{}), + stateObjectsDirty: make(map[common.Address]struct{}), + logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), + journal: newJournal(), } if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { @@ -492,6 +495,28 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject { return nil } +func (s *StateDB) getStateObjectFromDB(addr common.Address) *stateObject { + var ( + data *Account + err error + ) + + enc, err := s.trie.TryGet(addr.Bytes()) + if err != nil { + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) + return nil + } + if len(enc) == 0 { + return nil + } + data = new(Account) + if err := rlp.DecodeBytes(enc, data); err != nil { + log.Error("Failed to decode state object", "addr", addr, "err", err) + return nil + } + return newObject(s, addr, *data) +} + // getDeletedStateObject is similar to getStateObject, but instead of returning // nil for a deleted state object, it returns the actual object with the deleted // flag set. This is needed by the state journal to revert to the correct s- @@ -555,9 +580,60 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { } func (s *StateDB) setStateObject(object *stateObject) { + s.OrigForCalAccessList.SetAccount(object.address) s.stateObjects[object.Address()] = object } +// PreLoadAccount: preload account from access list +func (s *StateDB) PreLoadAccount(addresses []common.Address) { + ll := len(addresses) + var g sync.WaitGroup + res := make(chan *stateObject, ll) + g.Add(ll) + for _, addr := range addresses { + addr := addr + go func() { + defer g.Done() + res <- s.getStateObjectFromDB(addr) + }() + } + g.Wait() + for index := 0; index < ll; index++ { + v := <-res + if v != nil { + s.stateObjects[v.address] = v + v.getTrie(s.db) + } + } + close(res) +} + +// PreLoadStorage:preLoad storage from access list +func (s *StateDB) PreLoadStorage(addrs []common.Address, hashes []common.Hash) { + lenStorage := len(addrs) + var g sync.WaitGroup + + g.Add(lenStorage) + batch := 64 + if lenStorage < batch { + batch = lenStorage + } + for index := 0; index < batch; index++ { + start := index + go func() { + for i := start; i < lenStorage; i += batch { + obj := s.getStateObject(addrs[i]) + if obj != nil { + obj.preLoadCommittedStateFromDB(s.db, hashes[i]) + } + g.Done() + } + }() + } + g.Wait() + +} + // GetOrNewStateObject retrieves a state object or create a new state object if nil. func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { stateObject := s.getStateObject(addr) diff --git a/core/state_processor.go b/core/state_processor.go index e655d8f3bfba..ecfe2998e62f 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,6 +17,7 @@ package core import ( + "encoding/json" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" @@ -65,6 +66,23 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if p.config.DAOForkSupport && p.config.DAOForkBlock != nil && p.config.DAOForkBlock.Cmp(block.Number()) == 0 { misc.ApplyDAOHardFork(statedb) } + if !common.CalAccessList { + accessList := state.BlockNumToAccessList[int(block.NumberU64())] + if len(accessList) != 0 { + storagePairAddr := make([]common.Address, 0) + storagePairHash := make([]common.Hash, 0) + addresses := make([]common.Address, 0) + for _, v := range accessList { + addresses = append(addresses, v.Address) + for _, hash := range v.Hashs { + storagePairAddr = append(storagePairAddr, v.Address) + storagePairHash = append(storagePairHash, hash) + } + } + statedb.PreLoadAccount(addresses) + statedb.PreLoadStorage(storagePairAddr, storagePairHash) + } + } // Iterate over and process the individual transactions for i, tx := range block.Transactions() { statedb.Prepare(tx.Hash(), block.Hash(), i) @@ -78,6 +96,28 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) p.engine.Finalize(p.bc, header, statedb, block.Transactions(), block.Uncles()) + if common.CalAccessList { + accessList := make([]common.AccessList, 0) + for addr, sts := range statedb.OrigForCalAccessList.Data { + t := common.AccessList{ + Address: addr, + Hashs: make([]common.Hash, 0), + } + for key, _ := range sts { + t.Hashs = append(t.Hashs, key) + } + accessList = append(accessList, t) + } + if len(accessList) != 0 { + data, err := json.Marshal(accessList) + if err != nil { + panic(err) + } + if err := state.AccessListDB.Put(common.Uint64ToBytes(block.NumberU64()), data); err != nil { + panic(err) + } + } + } return receipts, allLogs, *usedGas, nil } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 87b364fb1bba..76a140f623dc 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -181,12 +181,13 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { + var hashKeyBuf [common.HashLength]byte h := newHasher(false) h.sha.Reset() h.sha.Write(key) - h.sha.Read(t.hashKeyBuf[:]) + h.sha.Read(hashKeyBuf[:]) returnHasherToPool(h) - return t.hashKeyBuf[:] + return hashKeyBuf[:] } // getSecKeyCache returns the current secure key cache, creating a new one if