fix: add failure retries in testWitness (#1158)

* add more logs in testWitness

* bump version

* add more logs

* use statedb.IntermediateRoot

* add retries

* add more retries

* remove root compute retries

* tweak

* add a fixme

* print error

Author: colinlyguo

eth/api.go

@@ -368,36 +368,63 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles
 	if err := blockchain.Validator().ValidateState(block, statedb, receipts, usedGas); err != nil {
 		return nil, fmt.Errorf("failed to validate block %d: %w", block.Number(), err)
 	}
-	return witness, testWitness(blockchain, block, witness)
+
+	// FIXME: testWitness will fail from time to time, the problem is caused by occasional state root mismatch
+	// after processing the block based on witness. We need to investigate the root cause and fix it.
+	for retries := 0; retries < 5; retries++ {
+		if err = testWitness(blockchain, block, witness); err == nil {
+			return witness, nil
+		} else {
+			log.Warn("Failed to validate witness", "block", block.Number(), "error", err)
+		}
+	}
+	return witness, err
 }
 
 func testWitness(blockchain *core.BlockChain, block *types.Block, witness *stateless.Witness) error {
 	stateRoot := witness.Root()
-	if diskRoot, _ := rawdb.ReadDiskStateRoot(blockchain.Database(), stateRoot); diskRoot != (common.Hash{}) {
+	diskRoot, err := rawdb.ReadDiskStateRoot(blockchain.Database(), stateRoot)
+	if err != nil {
+		return fmt.Errorf("failed to read disk state root for stateRoot %s: %w", stateRoot.Hex(), err)
+	}
+	if diskRoot != (common.Hash{}) {
+		log.Debug("Using disk root for state root", "stateRoot", stateRoot.Hex(), "diskRoot", diskRoot.Hex())
 		stateRoot = diskRoot
 	}
 
 	// Create and populate the state database to serve as the stateless backend
 	statedb, err := state.New(stateRoot, state.NewDatabase(witness.MakeHashDB()), nil)
 	if err != nil {
-		return fmt.Errorf("failed to create state database: %w", err)
+		return fmt.Errorf("failed to create state database with stateRoot %s: %w", stateRoot.Hex(), err)
 	}
 
 	receipts, _, usedGas, err := blockchain.Processor().Process(block, statedb, *blockchain.GetVMConfig())
 	if err != nil {
-		return fmt.Errorf("failed to process block %d: %w", block.Number(), err)
+		return fmt.Errorf("failed to process block %d (hash: %s): %w", block.Number(), block.Hash().Hex(), err)
 	}
 
 	if err := blockchain.Validator().ValidateState(block, statedb, receipts, usedGas); err != nil {
-		return fmt.Errorf("failed to validate block %d: %w", block.Number(), err)
+		return fmt.Errorf("failed to validate block %d (hash: %s): %w", block.Number(), block.Hash().Hex(), err)
 	}
 
 	postStateRoot := block.Root()
-	if diskRoot, _ := rawdb.ReadDiskStateRoot(blockchain.Database(), postStateRoot); diskRoot != (common.Hash{}) {
+	diskRoot, err = rawdb.ReadDiskStateRoot(blockchain.Database(), postStateRoot)
+	if err != nil {
+		return fmt.Errorf("failed to read disk state root for postStateRoot %s: %w", postStateRoot.Hex(), err)
+	}
+	if diskRoot != (common.Hash{}) {
+		log.Debug("Using disk root for post state root", "postStateRoot", postStateRoot.Hex(), "diskRoot", diskRoot.Hex())
 		postStateRoot = diskRoot
 	}
-	if statedb.GetRootHash() != postStateRoot {
-		return fmt.Errorf("failed to commit statelessly %d: %w", block.Number(), err)
+	computedRoot := statedb.GetRootHash()
+	if computedRoot != postStateRoot {
+		log.Debug("State root mismatch", "block", block.Number(), "expected", postStateRoot.Hex(), "got", computedRoot)
+		executionWitness := ToExecutionWitness(witness)
+		jsonStr, err := json.Marshal(executionWitness)
+		if err != nil {
+			return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err)
+		}
+		return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr))
 	}
 	return nil
 }

params/version.go

@@ -24,7 +24,7 @@ import (
 const (
 	VersionMajor = 5         // Major version component of the current release
 	VersionMinor = 8         // Minor version component of the current release
-	VersionPatch = 31        // Patch version component of the current release
+	VersionPatch = 32        // Patch version component of the current release
 	VersionMeta  = "mainnet" // Version metadata to append to the version string
 )