Geth 1.13 (Deneb/Cancun) update (#7)

Reviewed-on: https://git.vdb.to/cerc-io/eth-ipfs-state-validator/pulls/7
Reviewed-by: Thomas E Lackey <telackey@noreply.git.vdb.to>

cmd/validateTrie.go

@@ -108,11 +108,11 @@ func validateTrie() {
 			WithField("storage root", storageRoot).
 			Debug("Validating storage trie")
 		if err = v.ValidateStorageTrie(stateRoot, addr, storageRoot); err != nil {
-			logWithCommand.Fatalf("Validation failed", err)
+			logWithCommand.Fatal("Validation failed", err)
 		}
 		logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr, storageRoot)
 	default:
-		logWithCommand.Fatalf("Invalid traversal level: '%s'", traversal)
+		logWithCommand.Fatal("Invalid traversal level:", traversal)
 	}
 
 	stats := v.GetCacheStats()

pkg/util.go

@@ -19,7 +19,7 @@ package validator
 import (
 	"context"
 
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
 	"github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/repo/fsrepo"
 )

pkg/validator.go

@@ -25,14 +25,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/spf13/viper"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
 	"github.com/jmoiron/sqlx"
 	"github.com/mailgun/groupcache/v2"
 	log "github.com/sirupsen/logrus"
@@ -44,12 +42,13 @@ import (
 	pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
 	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
 	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/triedb"
 )
 
 // Validator is used for validating Ethereum state and storage tries on PG-IPFS
 type Validator struct {
 	kvs           ethdb.KeyValueStore
-	trieDB        *trie.Database
+	trieDB        *triedb.Database
 	stateDatabase state.Database
 	db            *pgipfsethdb.Database
 
@@ -94,7 +93,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
 	normalizeParams(&par)
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
+		trieDB:        triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
 		stateDatabase: state.NewDatabase(database),
 		db:            database.(*pgipfsethdb.Database),
 		params:        par,
@@ -112,7 +111,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
 	normalizeParams(&par)
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
+		trieDB:        triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
 		stateDatabase: state.NewDatabase(database),
 		params:        par,
 	}
@@ -124,7 +123,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
 func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
+		trieDB:        triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
 		stateDatabase: state.NewDatabase(database),
 	}
 }
@@ -146,7 +145,7 @@ func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
 	if err != nil {
 		return err
 	}
-	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, true) }
+	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, true) }
 	return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, fullTraversal), v.params.Workers, iterate)
 }
 
@@ -158,20 +157,21 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
 	if err != nil {
 		return err
 	}
-	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) }
+	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
 	return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, stateTraversal), v.params.Workers, iterate)
 }
 
 // ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
 func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
 	// Generate the state.NodeIterator for this root
 	addrHash := crypto.Keccak256Hash(address.Bytes())
-	t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot)
+	// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
+	storage, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot, nil)
 	if err != nil {
 		return err
 	}
-	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) }
-	return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate)
+	iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
+	return iterateTracked(storage, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate)
 }
 
 // Close implements io.Closer
@@ -184,7 +184,7 @@ func (v *Validator) Close() error {
 
 // Traverses one iterator fully
 // If storage = true, also traverse storage tries for each leaf.
-func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage bool) error {
+func (v *Validator) iterate(ctx context.Context, stateRoot common.Hash, it trie.NodeIterator, storage bool) error {
 	// Iterate through entire state trie. it.Next() will return false when we have
 	// either completed iteration of the entire trie or run into an error (e.g. a
 	// missing node). If we are able to iterate through the entire trie without error
@@ -206,13 +206,18 @@ func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage b
 		if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
 			return err
 		}
-		dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root)
+		// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
+		dataTrie, err := v.stateDatabase.OpenStorageTrie(stateRoot, common.BytesToHash(it.LeafKey()), account.Root, nil)
+		if err != nil {
+			return err
+		}
+		dataIt, err := dataTrie.NodeIterator(nil)
 		if err != nil {
 			return err
 		}
-		dataIt := dataTrie.NodeIterator(nil)
+
 		if !bytes.Equal(account.CodeHash, emptyCodeHash) {
-			_, err := v.stateDatabase.ContractCode(common.BytesToHash(account.CodeHash))
+			_, err := v.stateDatabase.ContractCode(common.Address{}, common.BytesToHash(account.CodeHash))
 			if err != nil {
 				return fmt.Errorf("code hash %x: %w (path %x)", account.CodeHash, err, iterutils.HexToKeyBytes(it.Path()))
 			}
@@ -236,7 +241,6 @@ func iterateTracked(
 ) error {
 	tracker := tracker.New(recoveryFile, iterCount)
 	halt := func() {
-		log.Errorf("writing recovery file: %s", recoveryFile)
 		if err := tracker.CloseAndSave(); err != nil {
 			log.Errorf("failed to write recovery file: %v", err)
 		}
@@ -252,7 +256,10 @@ func iterateTracked(
 	}
 
 	if iters == nil { // nothing restored
-		iters = iterutils.SubtrieIterators(tree.NodeIterator, iterCount)
+		iters, err = iterutils.SubtrieIterators(tree.NodeIterator, iterCount)
+		if err != nil {
+			return err
+		}
 		for i, it := range iters {
 			iters[i] = tracker.Tracked(it)
 		}

pkg/validator_test.go

@@ -17,14 +17,14 @@
 package validator_test
 
 import (
-	"math/big"
 	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/holiman/uint256"
 	cid "github.com/ipfs/go-cid/_rsrch/cidiface"
 	"github.com/jmoiron/sqlx"
 	"github.com/multiformats/go-multihash"
@@ -69,7 +69,7 @@ var (
 	codePath           = common.Hex2Bytes("6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
 	contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: codeHash.Bytes(),
 		Root:     crypto.Keccak256Hash(storageBranchRootNode),
 	})
@@ -80,7 +80,7 @@ var (
 
 	minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(1000),
+		Balance:  uint256.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
@@ -91,7 +91,7 @@ var (
 
 	account1, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(1000),
+		Balance:  uint256.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
@@ -102,7 +102,7 @@ var (
 
 	account2, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(1000),
+		Balance:  uint256.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
@@ -113,7 +113,7 @@ var (
 
 	bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(1000),
+		Balance:  uint256.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
@@ -249,14 +249,14 @@ var _ = Describe("PG-IPFS Validator", func() {
 			err = v.ValidateTrie(stateRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
-			Expect(err.Error()).To(ContainSubstring("%x", missingStateNodePath))
+			Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
 		})
 		It("Returns an error if the storage trie is missing node(s)", func() {
 			loadTrie(trieStateNodes, missingNodeStorageNodes, mockCode)
 			err = v.ValidateTrie(stateRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
-			Expect(err.Error()).To(ContainSubstring("%x", missingStorageNodePath))
+			Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
 		})
 		It("Returns an error if contract code is missing", func() {
 			loadTrie(trieStateNodes, trieStorageNodes)
@@ -288,6 +288,7 @@ var _ = Describe("PG-IPFS Validator", func() {
 			err = v.ValidateStateTrie(stateRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
+			Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
 		})
 		It("Returns no error if the entire state trie can be validated", func() {
 			loadTrie(trieStateNodes, nil)
@@ -301,21 +302,22 @@ var _ = Describe("PG-IPFS Validator", func() {
 			err = ResetTestDB(db)
 			Expect(err).ToNot(HaveOccurred())
 		})
-		It("Returns an error the storage root node is missing", func() {
+		It("Returns an error if the storage root node is missing", func() {
 			loadTrie(nil, missingRootStorageNodes)
-			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
 		})
 		It("Returns an error if the entire storage trie cannot be validated", func() {
 			loadTrie(nil, missingNodeStorageNodes)
-			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
+			Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
 		})
 		It("Returns no error if the entire storage trie can be validated", func() {
 			loadTrie(nil, trieStorageNodes)
-			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

test/compose.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"