Add E2E tests for AN backup hosts switch functionality

Author: m-Peter

tests/helpers.go

@@ -67,23 +67,18 @@ func testLogWriter() io.Writer {
 	return zerolog.NewConsoleWriter()
 }
 
-func startEmulator(createTestAccounts bool) (*server.EmulatorServer, error) {
+func defaultServerConfig() *server.Config {
 	pkey, err := crypto.DecodePrivateKeyHex(sigAlgo, servicePrivateKey)
 	if err != nil {
-		return nil, err
+		panic(err)
 	}
 
 	genesisToken, err := cadence.NewUFix64("10000.0")
 	if err != nil {
-		return nil, err
+		panic(err)
 	}
 
-	log := logger.With().Timestamp().Str("component", "emulator").Logger().Level(zerolog.DebugLevel)
-	if logOutput == "false" {
-		log = zerolog.Nop()
-	}
-
-	srv := server.NewEmulatorServer(&log, &server.Config{
+	return &server.Config{
 		ServicePrivateKey:      pkey,
 		ServiceKeySigAlgo:      sigAlgo,
 		ServiceKeyHashAlgo:     hashAlgo,
@@ -94,7 +89,19 @@ func startEmulator(createTestAccounts bool) (*server.EmulatorServer, error) {
 		TransactionMaxGasLimit: flow.DefaultMaxTransactionGasLimit,
 		SetupEVMEnabled:        true,
 		SetupVMBridgeEnabled:   false,
-	})
+	}
+}
+
+func startEmulator(createTestAccounts bool, conf *server.Config) (
+	*server.EmulatorServer,
+	error,
+) {
+	log := logger.With().Timestamp().Str("component", "emulator").Logger().Level(zerolog.DebugLevel)
+	if logOutput == "false" {
+		log = zerolog.Nop()
+	}
+
+	srv := server.NewEmulatorServer(&log, conf)
 
 	go func() {
 		srv.Start()
@@ -133,7 +140,7 @@ func runWeb3TestWithSetup(
 // servicesSetup starts up an emulator and the gateway
 // engines required for operation of the evm gateway.
 func servicesSetup(t *testing.T) (emulator.Emulator, func()) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())

tests/integration_test.go

@@ -30,7 +30,7 @@ import (
 // This is using the TxSealValidation mechanism for validating submitted
 // transactions, which blocks until it receives the Flow transaction result.
 func Test_ConcurrentTransactionSubmissionWithTxSeal(t *testing.T) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -141,7 +141,7 @@ func Test_ConcurrentTransactionSubmissionWithTxSeal(t *testing.T) {
 // This is using the LocalIndexValidation mechanism for validating submitted
 // transactions, which is non-blocking and validates based on the local state.
 func Test_ConcurrentTransactionSubmissionWithLocalIndex(t *testing.T) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -246,7 +246,7 @@ func Test_ConcurrentTransactionSubmissionWithLocalIndex(t *testing.T) {
 }
 
 func Test_EthClientTest(t *testing.T) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -325,7 +325,7 @@ func Test_CloudKMSConcurrentTransactionSubmission(t *testing.T) {
 		t.Skip()
 	}
 
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -449,7 +449,7 @@ func Test_CloudKMSConcurrentTransactionSubmission(t *testing.T) {
 // 2. No transactions are lost
 // 3. The state remains consistent
 func Test_ForceStartHeightIdempotency(t *testing.T) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -592,3 +592,112 @@ func Test_ForceStartHeightIdempotency(t *testing.T) {
 		return true
 	}, time.Second*15, time.Second*1, "all transactions were not executed")
 }
+
+// Test_AccessNodeBackupFunctionality verifies that the specified AccessNode
+// backup hosts, are used when the primary `AccessNodeHost` is unavailable
+// or whatever reason.
+func Test_AccessNodeBackupFunctionality(t *testing.T) {
+	srv, err := startEmulator(true, defaultServerConfig())
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer func() {
+		cancel()
+		srv.Stop()
+	}()
+
+	backupConfg := defaultServerConfig()
+	backupConfg.GRPCPort = 3599
+	backupConfg.RESTPort = 9999
+	backupConfg.AdminPort = 9090
+	backupConfg.DebuggerPort = 3456
+	backupSrv, err := startEmulator(true, backupConfg)
+	require.NoError(t, err)
+
+	_, backupCancel := context.WithCancel(context.Background())
+	defer func() {
+		backupCancel()
+		backupSrv.Stop()
+	}()
+
+	grpcHost := "localhost:3569"
+	emu := srv.Emulator()
+	service := emu.ServiceKey()
+
+	client, err := grpc.NewClient(grpcHost)
+	require.NoError(t, err)
+
+	time.Sleep(500 * time.Millisecond) // some time to startup
+
+	// create new account with keys used for key-rotation
+	keyCount := 5
+	createdAddr, privateKey, err := bootstrap.CreateMultiKeyAccount(
+		client,
+		keyCount,
+		service.Address,
+		sc.FungibleToken.Address.HexWithPrefix(),
+		sc.FlowToken.Address.HexWithPrefix(),
+		service.PrivateKey,
+	)
+	require.NoError(t, err)
+
+	backupAccessNodeHost := fmt.Sprintf("localhost:%d", backupConfg.GRPCPort)
+
+	cfg := config.Config{
+		DatabaseDir:           t.TempDir(),
+		AccessNodeHost:        grpcHost,
+		AccessNodeBackupHosts: []string{backupAccessNodeHost},
+		RPCPort:               8545,
+		RPCHost:               "127.0.0.1",
+		FlowNetworkID:         "flow-emulator",
+		EVMNetworkID:          types.FlowEVMPreviewNetChainID,
+		Coinbase:              eoaTestAccount,
+		COAAddress:            *createdAddr,
+		COAKey:                privateKey,
+		GasPrice:              new(big.Int).SetUint64(0),
+		EnforceGasPrice:       true,
+		LogLevel:              zerolog.DebugLevel,
+		LogWriter:             testLogWriter(),
+		TxStateValidation:     config.LocalIndexValidation,
+	}
+
+	boot, err := bootstrap.New(cfg)
+	require.NoError(t, err)
+	defer func() {
+		// Stop the EVM GW service
+		boot.Stop()
+	}()
+
+	ready := make(chan struct{})
+	go func() {
+		err = boot.Run(ctx, cfg, func() {
+			close(ready)
+		})
+		require.NoError(t, err)
+	}()
+
+	<-ready
+
+	time.Sleep(3 * time.Second) // some time to startup
+
+	ethClient, err := ethclient.Dial("http://127.0.0.1:8545")
+	require.NoError(t, err)
+
+	// This endpoint (`eth_syncing`), will make the following gRPC call,
+	// `ExecuteScriptAtLatestBlock`. This gRPC call is served by the
+	// first Emulator process, that is configured as the `AccessNodeHost`.
+	_, err = ethClient.SyncProgress(context.Background())
+	require.NoError(t, err)
+
+	// Shutdown the first Emulator process, that is configured as the
+	// `AccessNodeHost`
+	cancel()
+	srv.Stop()
+
+	// This endpoint (`eth_syncing`), will make the following gRPC call,
+	// `ExecuteScriptAtLatestBlock`. This gRPC call is served by the
+	// first-available AccessNode specified in `AccessNodeBackupHosts`.
+	// In this E2E test, that would be the second Emulator process.
+	_, err = ethClient.SyncProgress(context.Background())
+	require.NoError(t, err)
+}

tests/key_store_release_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 func Test_KeyStoreSigningKeysRelease(t *testing.T) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())

tests/tx_batching_test.go

@@ -515,7 +515,7 @@ func Test_MultipleTransactionSubmissionsWithinNonRecentInterval(t *testing.T) {
 }
 
 func setupGatewayNode(t *testing.T) (emulator.Emulator, config.Config, func()) {
-	srv, err := startEmulator(true)
+	srv, err := startEmulator(true, defaultServerConfig())
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())