turn off server when block received is out of date (#4722)

Author: CoderZhi

pkg/probe/probe.go

@@ -9,10 +9,12 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
 
+	"github.com/iotexproject/iotex-core/v2/blockchain/block"
 	"github.com/iotexproject/iotex-core/v2/pkg/lifecycle"
 	"github.com/iotexproject/iotex-core/v2/pkg/log"
 	"github.com/iotexproject/iotex-core/v2/pkg/util/httputil"
@@ -76,6 +78,20 @@ func (s *Server) Start(_ context.Context) error {
 // Stop shutdown the probe server.
 func (s *Server) Stop(ctx context.Context) error { return s.server.Shutdown(ctx) }
 
+// ReceiveBlock receives the new block and update the readiness status
+func (s *Server) ReceiveBlock(blk *block.Block) error {
+	if time.Now().After(blk.CommitTime().Add(10 * time.Second)) {
+		if s.IsReady() {
+			s.TurnOff()
+		}
+	} else {
+		if !s.IsReady() {
+			s.TurnOn()
+		}
+	}
+	return nil
+}
+
 func successHandleFunc(w http.ResponseWriter, _ *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	if _, err := w.Write([]byte("OK")); err != nil {

pkg/probe/probe_test.go

@@ -13,8 +13,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
+	"github.com/iotexproject/iotex-core/v2/blockchain/block"
 	"github.com/iotexproject/iotex-core/v2/testutil"
+	"github.com/iotexproject/iotex-proto/golang/iotextypes"
 )
 
 type testCase struct {
@@ -73,6 +76,22 @@ func TestBasicProbe(t *testing.T) {
 	require.NoError(t, s.TurnOff())
 	testFunc(t, test1)
 
+	now := time.Now()
+	createFooter := func(ts time.Time) block.Footer {
+		footer := block.Footer{}
+		require.NoError(t, footer.ConvertFromBlockFooterPb(
+			&iotextypes.BlockFooter{
+				Timestamp: timestamppb.New(ts),
+			},
+		))
+		return footer
+	}
+
+	require.NoError(t, s.ReceiveBlock(&block.Block{Footer: createFooter(now.Add(-9 * time.Second))}))
+	require.True(t, s.IsReady())
+	require.NoError(t, s.ReceiveBlock(&block.Block{Footer: createFooter(now.Add(-11 * time.Second))}))
+	require.False(t, s.IsReady())
+
 	require.NoError(t, s.Stop(ctx))
 	_, err := http.Get("http://localhost:7788/liveness")
 	require.Error(t, err)

server/itx/server.go

@@ -277,6 +277,9 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c
 		log.L().Info("Waiting for server to be ready.", zap.Duration("duration", cfg.API.ReadyDuration))
 		time.Sleep(cfg.API.ReadyDuration)
 	}
+	if err := svr.rootChainService.Blockchain().AddSubscriber(probeSvr); err != nil {
+		log.L().Panic("Failed to add probe server as subscriber.", zap.Error(err))
+	}
 	if err := probeSvr.TurnOn(); err != nil {
 		log.L().Panic("Failed to turn on probe server.", zap.Error(err))
 	}
@@ -330,7 +333,9 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c
 	}
 
 	<-ctx.Done()
-	if err := probeSvr.TurnOff(); err != nil {
-		log.L().Panic("Failed to turn off probe server.", zap.Error(err))
+	if probeSvr.IsReady() {
+		if err := probeSvr.TurnOff(); err != nil {
+			log.L().Panic("Failed to turn off probe server.", zap.Error(err))
+		}
 	}
 }