Feature/faster grpc

app/tier1.go

@@ -6,26 +6,27 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
-	"connectrpc.com/connect"
+	connectrpc "connectrpc.com/connect"
 	"github.com/streamingfast/bstream"
 	"github.com/streamingfast/bstream/blockstream"
 	"github.com/streamingfast/bstream/hub"
 	pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1"
-	dauth "github.com/streamingfast/dauth"
+	"github.com/streamingfast/dauth"
 	"github.com/streamingfast/dmetrics"
 	"github.com/streamingfast/dsession"
 	"github.com/streamingfast/dstore"
 	pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2"
 	"github.com/streamingfast/shutter"
 	"github.com/streamingfast/substreams/client"
 	"github.com/streamingfast/substreams/metrics"
-	pbsubstreamsrpcv2connect "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2/pbsubstreamsrpcv2connect"
+	pbsubstreamsrpcv2 "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2"
+	"github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2/pbsubstreamsrpcv2connect"
 	"github.com/streamingfast/substreams/reqctx"
 	"github.com/streamingfast/substreams/service"
 	"github.com/streamingfast/substreams/wasm"
-
 	_ "github.com/streamingfast/substreams/wasm/wasmtime"
 	"github.com/streamingfast/substreams/wasm/wazero"
 	"go.uber.org/atomic"
@@ -54,6 +55,7 @@ func NewDefaultTier1Config() *Tier1Config {
 		MaxSubrequests:        10,
 		StateBundleSize:       1000,
 		BlockExecutionTimeout: 1 * time.Minute,
+		OutputBufferSize:      100,
 	}
 }
 
@@ -86,7 +88,8 @@ type Tier1Config struct {
 	SubrequestsInsecure     bool
 	SubrequestsPlaintext    bool
 
-	SharedCacheSize uint64
+	SharedCacheSize  uint64
+	OutputBufferSize uint64 // Used to bundle execout messages within 'BlockScopedDatas' when using protocol V4
 
 	WASMExtensions wasm.WASMExtensioner
 	Tracing        bool
@@ -249,7 +252,7 @@ func (a *Tier1App) Run() error {
 		FoundationalStoreEndpoints: foundationalStoreEndpoints,
 	}
 
-	svc, err := service.NewTier1(
+	tier1Service, err := service.NewTier1(
 		a.logger,
 		mergedBlocksStore,
 		forkedBlocksStore,
@@ -267,6 +270,7 @@ func (a *Tier1App) Run() error {
 		a.config.ActiveRequestsSoftLimit,
 		a.config.ActiveRequestsHardLimit,
 		a.config.SharedCacheSize,
+		a.config.OutputBufferSize,
 		a.modules.SessionPool,
 		foundationalStoreEndpoints,
 		opts...,
@@ -275,14 +279,16 @@ func (a *Tier1App) Run() error {
 		return err
 	}
 
+	tier1ServiceConnect := service.NewService(tier1Service)
+
 	a.OnTerminating(func(err error) {
 		metrics.AppReadinessTier1.SetNotReady()
 
-		svc.Shutdown(err)
+		tier1Service.Shutdown(err)
 	})
 
 	go func() {
-		var infoServer pbsubstreamsrpcv2connect.EndpointInfoHandler
+		var infoServer pbsubstreamsrpcv2.EndpointInfoServer
 		if a.modules.InfoServer != nil {
 			a.logger.Info("waiting until info server is ready")
 			infoServer = &InfoServerWrapper{a.modules.InfoServer}
@@ -292,6 +298,12 @@ func (a *Tier1App) Run() error {
 			}
 		}
 
+		var infoServerConnect pbsubstreamsrpcv2connect.EndpointInfoHandler
+		if a.modules.InfoServer != nil {
+			infoServerConnect = &InfoServerConnectWrapper{a.modules.InfoServer}
+			a.logger.Info("info server ready")
+		}
+
 		if withLive {
 			a.logger.Info("waiting until hub is real-time synced")
 			select {
@@ -305,7 +317,12 @@ func (a *Tier1App) Run() error {
 		a.logger.Info("launching gRPC server", zap.Bool("live_support", withLive))
 		a.setIsReady(true)
 
-		err := service.ListenTier1(a.config.GRPCListenAddr, svc, infoServer, a.modules.Authenticator, a.logger, a.HealthCheck)
+		addresses := strings.Split(a.config.GRPCListenAddr, ",")
+		if len(addresses) == 0 {
+			a.logger.Error("no gRPC listen addresses provided")
+			return
+		}
+		err := service.ListenTier1(addresses, tier1Service, tier1ServiceConnect, infoServer, infoServerConnect, a.modules.Authenticator, a.logger, a.HealthCheck, a.config.EnforceCompression)
 		a.Shutdown(err)
 	}()
 
@@ -349,17 +366,29 @@ func (config *Tier1Config) Validate() error {
 	return nil
 }
 
-var _ pbsubstreamsrpcv2connect.EndpointInfoHandler = (*InfoServerWrapper)(nil)
+var _ pbsubstreamsrpcv2.EndpointInfoServer = (*InfoServerWrapper)(nil)
 
 type InfoServerWrapper struct {
-	rpcInfoServer InfoServer
+	rpcInfoServer pbsubstreamsrpcv2.EndpointInfoServer
 }
 
 // Info implements pbsubstreamsrpcconnect.EndpointInfoHandler.
-func (i *InfoServerWrapper) Info(ctx context.Context, req *connect.Request[pbfirehose.InfoRequest]) (*connect.Response[pbfirehose.InfoResponse], error) {
+func (i *InfoServerWrapper) Info(ctx context.Context, req *pbfirehose.InfoRequest) (*pbfirehose.InfoResponse, error) {
+	resp, err := i.rpcInfoServer.Info(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+type InfoServerConnectWrapper struct {
+	rpcInfoServer pbsubstreamsrpcv2.EndpointInfoServer
+}
+
+func (i *InfoServerConnectWrapper) Info(ctx context.Context, req *connectrpc.Request[pbfirehose.InfoRequest]) (*connectrpc.Response[pbfirehose.InfoResponse], error) {
 	resp, err := i.rpcInfoServer.Info(ctx, req.Msg)
 	if err != nil {
 		return nil, err
 	}
-	return connect.NewResponse(resp), nil
+	return connectrpc.NewResponse(resp), nil
 }

app/tier2.go

@@ -124,7 +124,7 @@ func (a *Tier2App) Run() error {
 		a.logger.Info("launching gRPC server")
 		a.setIsReady(true)
 
-		err := service.ListenTier2(a.config.GRPCListenAddr, a.config.ServiceDiscoveryURL, svc, trustAuth, a.logger, a.HealthCheck)
+		err := service.ListenTier2(a.config.GRPCListenAddr, a.config.ServiceDiscoveryURL, svc, trustAuth, a.logger, a.HealthCheck, false)
 		a.Shutdown(err)
 	}()
 

buf.gen.yaml

@@ -19,6 +19,15 @@ plugins:
       - Msf/substreams/foundational-store/model/v2/model.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/model/v2;pbmodel
       - Msf/substreams/foundational-store/service/v2/service.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/service/v2;pbservice
       - Msf/substreams/foundational-store/service/v1/service.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/service/v1;pbstore
+  - remote: buf.build/community/planetscale-vtprotobuf:v0.6.0
+    out: pb
+    opt:
+      - paths=source_relative
+      - features=marshal+unmarshal+size+equal+clone
+      - Msf/codegen/conversation/v1/conversation.proto=github.com/streamingfast/substreams/pb/sf/codegen/conversation/v1;pbconvo
+      - Msf/substreams/foundational-store/model/v2/model.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/model/v2;pbmodel
+      - Msf/substreams/foundational-store/service/v2/service.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/service/v2;pbservice
+      - Msf/substreams/foundational-store/service/v1/service.proto=github.com/streamingfast/substreams/pb/sf/substreams/foundational-store/service/v1;pbstore
   - remote: buf.build/connectrpc/go:v1.15.0
     out: pb
     opt:

client/client.go

@@ -7,11 +7,13 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"strconv"
 
+	"github.com/mostynb/go-grpc-compression/experimental/s2"
 	"github.com/streamingfast/dgrpc"
 	networks "github.com/streamingfast/firehose-networks"
 	pbssinternal "github.com/streamingfast/substreams/pb/sf/substreams/intern/v2"
-	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"github.com/streamingfast/substreams/protodecode"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"golang.org/x/oauth2"
@@ -20,10 +22,15 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/oauth"
 	xdscreds "google.golang.org/grpc/credentials/xds"
-	"google.golang.org/grpc/encoding/gzip"
+	"google.golang.org/grpc/encoding"
+	_ "google.golang.org/grpc/encoding/gzip"
 	_ "google.golang.org/grpc/xds"
 )
 
+func init() {
+	encoding.RegisterCodec(protodecode.PreferredVTCodec{})
+}
+
 type AuthType int
 
 const (
@@ -38,6 +45,7 @@ const (
 	ProtocolVersionUnset ProtocolVersion = 0
 	ProtocolVersionV2    ProtocolVersion = 2
 	ProtocolVersionV3    ProtocolVersion = 3
+	ProtocolVersionV4    ProtocolVersion = 4
 )
 
 // String returns the string representation of the protocol version
@@ -49,14 +57,16 @@ func (pv ProtocolVersion) String() string {
 		return "v2"
 	case ProtocolVersionV3:
 		return "v3"
+	case ProtocolVersionV4:
+		return "v4"
 	default:
 		return "unknown"
 	}
 }
 
 // IsValid returns true if the protocol version is valid
 func (pv ProtocolVersion) IsValid() bool {
-	return pv == ProtocolVersionUnset || pv == ProtocolVersionV2 || pv == ProtocolVersionV3
+	return pv == ProtocolVersionUnset || pv == ProtocolVersionV2 || pv == ProtocolVersionV3 || pv == ProtocolVersionV4
 }
 
 // ParseProtocolVersion parses an integer to a ProtocolVersion
@@ -68,8 +78,10 @@ func ParseProtocolVersion(version int) (ProtocolVersion, error) {
 		return ProtocolVersionV2, nil
 	case 3:
 		return ProtocolVersionV3, nil
+	case 4:
+		return ProtocolVersionV4, nil
 	default:
-		return ProtocolVersionUnset, fmt.Errorf("invalid protocol version %d, only 0, 2 and 3 are supported", version)
+		return ProtocolVersionUnset, fmt.Errorf("invalid protocol version %d, only 0, 2, 3 and 4 are supported", version)
 	}
 }
 
@@ -81,6 +93,10 @@ func (pv ProtocolVersion) IsV3() bool {
 	return pv == ProtocolVersionV3
 }
 
+func (pv ProtocolVersion) IsV4() bool {
+	return pv == ProtocolVersionV4
+}
+
 func (pv ProtocolVersion) IsUnset() bool {
 	return pv == ProtocolVersionUnset
 }
@@ -99,7 +115,7 @@ type SubstreamsClientConfigOptions struct {
 	PlainText bool
 	// Agent is the User-Agent string for gRPC requests
 	Agent string
-	// ForceProtocolVersion forces the use of a specific protocol version (2 or 3)
+	// ForceProtocolVersion forces the use of a specific protocol version (2, 3 or 4)
 	ForceProtocolVersion ProtocolVersion
 }
 
@@ -243,6 +259,9 @@ func NewInternalClientFactory(config *SubstreamsClientConfig) InternalClientFact
 
 	noop := func() error { return nil }
 	cli, _, callOpts, headers, err := NewSubstreamsInternalClient(config)
+	if err != nil {
+		zlog.Error("failed to create substreams client", zap.Error(err))
+	}
 	return func() (pbssinternal.SubstreamsClient, func() error, []grpc.CallOption, Headers, error) {
 		return cli, noop, callOpts, headers, err
 	}
@@ -285,7 +304,10 @@ func NewSubstreamsInternalClient(config *SubstreamsClientConfig) (cli pbssintern
 		}
 	}
 
-	dialOptions = append(dialOptions, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
+	if messageLimit, ok := getGRPCSizeLoggerFromEnv(); ok {
+		sizeHandler := dgrpc.NewSizeLoggingHandler(messageLimit, zlog)
+		dialOptions = append(dialOptions, grpc.WithStatsHandler(sizeHandler))
+	}
 
 	zlog.Debug("getting connection", zap.String("endpoint", endpoint))
 	conn, err := dgrpc.NewExternalClientConn(endpoint, dialOptions...)
@@ -354,12 +376,17 @@ func newConnection(config *SubstreamsClientConfig) (conn *grpc.ClientConn, close
 		}
 	}
 
-	dialOptions = append(dialOptions, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
-	dialOptions = append(dialOptions, grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)))
+	if messageLimit, ok := getGRPCSizeLoggerFromEnv(); ok {
+		sizeHandler := dgrpc.NewSizeLoggingHandler(messageLimit, zlog)
+		dialOptions = append(dialOptions, grpc.WithStatsHandler(sizeHandler))
+	}
+
+	dialOptions = append(dialOptions, grpc.WithDefaultCallOptions(grpc.UseCompressor(s2.Name)))
 	dialOptions = append(dialOptions, grpc.WithUserAgent(config.agent))
 
 	zlog.Debug("getting connection", zap.String("endpoint", endpoint))
 	conn, err = dgrpc.NewExternalClient(endpoint, dialOptions...)
+
 	if err != nil {
 		return nil, nil, nil, nil, fmt.Errorf("unable to create external gRPC client: %w", err)
 	}
@@ -382,3 +409,20 @@ func newConnection(config *SubstreamsClientConfig) (conn *grpc.ClientConn, close
 func NewSubstreamsClientConn(config *SubstreamsClientConfig) (conn *grpc.ClientConn, closeFunc func() error, callOpts []grpc.CallOption, headers Headers, err error) {
 	return newConnection(config)
 }
+
+func getGRPCSizeLoggerFromEnv() (limit int, ok bool) {
+	messageLimitString := os.Getenv("GRPC_SIZE_LOGGER_MESSAGE_LIMIT")
+	if messageLimitString == "" {
+		return 0, false
+	}
+	messageLimit := 10000
+	var err error
+	if messageLimitString != "" {
+		messageLimit, err = strconv.Atoi(messageLimitString)
+		if err != nil {
+			zlog.Warn("failed to parse GRPC_SIZE_LOGGER_MESSAGE_LIMIT", zap.Error(err))
+			return 0, false
+		}
+	}
+	return messageLimit, true
+}

go.mod

@@ -9,12 +9,12 @@ require (
 	github.com/jhump/protoreflect v1.14.0
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/streamingfast/bstream v0.0.2-0.20260206144138-e9341a23a897
+	github.com/streamingfast/bstream v0.0.2-0.20260211203220-e95f72b706fa
 	github.com/streamingfast/cli v0.0.4-0.20250815192146-d8a233ec3d0b
 	github.com/streamingfast/dauth v0.0.0-20251218134044-fb716c7172b4
 	github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c
 	github.com/streamingfast/derr v0.0.0-20250814163534-bd7407bd89d7
-	github.com/streamingfast/dgrpc v0.0.0-20251218142640-027692a12722
+	github.com/streamingfast/dgrpc v0.0.0-20260218164858-719a69f1b7b4
 	github.com/streamingfast/dhttp v0.1.3-0.20251218140957-6d46b8f12eb1
 	github.com/streamingfast/dstore v0.1.3-0.20260113210117-94d66eda2027
 	github.com/streamingfast/logging v0.0.0-20260108192805-38f96de0a641
@@ -55,6 +55,7 @@ require (
 	github.com/lithammer/dedent v1.1.0
 	github.com/mattn/go-isatty v0.0.20
 	github.com/mitchellh/go-testing-interface v1.14.1
+	github.com/mostynb/go-grpc-compression v1.2.3
 	github.com/muesli/reflow v0.3.0
 	github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a
 	github.com/orcaman/concurrent-map/v2 v2.0.1
@@ -145,7 +146,6 @@ require (
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/minio/minlz v1.0.1 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
-	github.com/mostynb/go-grpc-compression v1.2.3 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/parquet-go/parquet-go v0.23.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect

go.sum

@@ -549,8 +549,8 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
 github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
-github.com/streamingfast/bstream v0.0.2-0.20260206144138-e9341a23a897 h1:s79Lfp3mtO2oKJ9lLtWywDbJc5B2RDNaiqRetwvaZ2w=
-github.com/streamingfast/bstream v0.0.2-0.20260206144138-e9341a23a897/go.mod h1:0sed+gw+d62tBsGl5omaB1m14Gmfvb3YAs4epHuL1YU=
+github.com/streamingfast/bstream v0.0.2-0.20260211203220-e95f72b706fa h1:ih8oxMICJfr/WVoowKeyATHvZ0lbU8+tQCDCgkwJMGU=
+github.com/streamingfast/bstream v0.0.2-0.20260211203220-e95f72b706fa/go.mod h1:9NVnl3l3Wljoll+cQb8sccVa078E1m1CgBCFzttGha0=
 github.com/streamingfast/cli v0.0.4-0.20250815192146-d8a233ec3d0b h1:ztYeX3/5rg2tV2EU7edcrcHzMz6wUbdJB+LqCrP5W8s=
 github.com/streamingfast/cli v0.0.4-0.20250815192146-d8a233ec3d0b/go.mod h1:o9R/tjNON01X2mgWL5qirl2MV6xQ4EZI5D504ST3K/M=
 github.com/streamingfast/dauth v0.0.0-20251218134044-fb716c7172b4 h1:qJmiXzqs3T7124+FXEnhPgK/3fbSjqjQuBTdIrUk4Oc=
@@ -559,8 +559,8 @@ github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c h1:6WjE2yInE+
 github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c/go.mod h1:dbfiy9ORrL8c6ldSq+L0H9pg8TOqqu/FsghsgUEWK54=
 github.com/streamingfast/derr v0.0.0-20250814163534-bd7407bd89d7 h1:KeyhmbdPhEzot+On1XrHSAOmRoeyIRiM0EkGQAqnVxI=
 github.com/streamingfast/derr v0.0.0-20250814163534-bd7407bd89d7/go.mod h1:QSm/AfaDsE0k1xBYi0lW580YJ/WDV/FKZI628tkZR0Y=
-github.com/streamingfast/dgrpc v0.0.0-20251218142640-027692a12722 h1:gZcPR64H5aWs49bLDoNTjhsedTfva7fAIei891LL0C8=
-github.com/streamingfast/dgrpc v0.0.0-20251218142640-027692a12722/go.mod h1:NbkvenEHfjQpUBRHTCp/tp0Ayjp5hDzzkv/Ve9Uka1I=
+github.com/streamingfast/dgrpc v0.0.0-20260218164858-719a69f1b7b4 h1:C4WiIoGrI3QJzAe4Gcye5L0avOm0lB4yQtk28E0OvNc=
+github.com/streamingfast/dgrpc v0.0.0-20260218164858-719a69f1b7b4/go.mod h1:S6lJ9Dd6M3myzWbxspuhGggHtld9uYVN0xQ/7EGXmX8=
 github.com/streamingfast/dhammer v0.0.0-20220506192416-3797a7906da2 h1:/mcLVdwy6NeHWfJwuh2GD4+FMfPa59fkfM15sl8Jejk=
 github.com/streamingfast/dhammer v0.0.0-20220506192416-3797a7906da2/go.mod h1:MyG3U4ABuf7ANS8tix+e8UUevN7B9juhEnAbslS/X3M=
 github.com/streamingfast/dhttp v0.1.3-0.20251218140957-6d46b8f12eb1 h1:V9oSpY69/OZKTptZEPzJFaC8RS0Y8kffknNTDSZwpME=