Skip to content

Commit 6eff630

Browse files
mateeullahmalikmateeullahmalik
authored
supernode metrics: add active probing (#248)
1 parent a81e3db commit 6eff630

39 files changed

+3137
-2510
lines changed

‎.vscode/launch.json‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,19 @@
2424
"env": {},
2525
"showLog": true,
2626
"trace": "verbose"
27-
},
27+
},
28+
{
29+
"name": "Cascade E2E (test-cascade)",
30+
"type": "go",
31+
"request": "launch",
32+
"mode": "test",
33+
"program": "${workspaceFolder}/tests/system",
34+
"cwd": "${workspaceFolder}/tests/system",
35+
"args": ["-test.run", "TestCascadeE2E", "-test.v"],
36+
"buildFlags": "-tags=system_test",
37+
"env": {},
38+
"showLog": true,
39+
"trace": "verbose"
40+
}
2841
]
29-
}
42+
}

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,4 +192,4 @@ release:
192192
git tag $(NEXT_TAG)
193193
git push upstream $(NEXT_TAG)
194194

195-
@echo "Release complete: $(NEXT_TAG) pushed to upstream"
195+
@echo "Release complete: $(NEXT_TAG) pushed to upstream"

‎go.mod‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ require (
88
cosmossdk.io/math v1.5.3
99
github.com/AlecAivazis/survey/v2 v2.3.7
1010
github.com/DataDog/zstd v1.5.7
11-
github.com/LumeraProtocol/lumera v1.8.6-rc2
11+
github.com/LumeraProtocol/lumera v1.9.0
1212
github.com/LumeraProtocol/rq-go v0.2.1
1313
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
1414
github.com/cenkalti/backoff/v4 v4.3.0
15+
github.com/cometbft/cometbft v0.38.18
1516
github.com/cosmos/btcutil v1.0.5
1617
github.com/cosmos/cosmos-sdk v0.53.0
1718
github.com/cosmos/go-bip39 v1.0.0
@@ -35,11 +36,11 @@ require (
3536
go.uber.org/mock v0.6.0
3637
go.uber.org/ratelimit v0.3.1
3738
go.uber.org/zap v1.27.0
38-
golang.org/x/crypto v0.42.0
39+
golang.org/x/crypto v0.43.0
3940
golang.org/x/sync v0.17.0
40-
golang.org/x/sys v0.36.0
41-
google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4
42-
google.golang.org/grpc v1.76.0
41+
golang.org/x/sys v0.37.0
42+
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8
43+
google.golang.org/grpc v1.77.0
4344
google.golang.org/protobuf v1.36.10
4445
gopkg.in/yaml.v3 v3.0.1
4546
lukechampine.com/blake3 v1.4.1
@@ -60,6 +61,7 @@ require (
6061
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
6162
github.com/99designs/keyring v1.2.2 // indirect
6263
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
64+
github.com/Masterminds/semver/v3 v3.3.1 // indirect
6365
github.com/Microsoft/go-winio v0.6.2 // indirect
6466
github.com/benbjohnson/clock v1.3.0 // indirect
6567
github.com/beorn7/perks v1.0.1 // indirect
@@ -75,7 +77,6 @@ require (
7577
github.com/cockroachdb/pebble v1.1.5 // indirect
7678
github.com/cockroachdb/redact v1.1.6 // indirect
7779
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
78-
github.com/cometbft/cometbft v0.38.18 // indirect
7980
github.com/cometbft/cometbft-db v0.14.1 // indirect
8081
github.com/cosmos/cosmos-db v1.1.2 // indirect
8182
github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
@@ -154,7 +155,7 @@ require (
154155
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
155156
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
156157
github.com/prometheus/client_golang v1.22.0 // indirect
157-
github.com/prometheus/client_model v0.6.1 // indirect
158+
github.com/prometheus/client_model v0.6.2 // indirect
158159
github.com/prometheus/common v0.63.0 // indirect
159160
github.com/prometheus/procfs v0.15.1 // indirect
160161
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
@@ -187,11 +188,12 @@ require (
187188
golang.org/x/arch v0.15.0 // indirect
188189
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect
189190
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d // indirect
190-
golang.org/x/net v0.44.0 // indirect
191-
golang.org/x/term v0.35.0 // indirect
192-
golang.org/x/text v0.29.0 // indirect
191+
golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 // indirect
192+
golang.org/x/term v0.36.0 // indirect
193+
golang.org/x/text v0.30.0 // indirect
193194
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
194-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250929231259-57b25ae835d4 // indirect
195+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
196+
gopkg.in/yaml.v2 v2.4.0 // indirect
195197
gotest.tools/v3 v3.5.2 // indirect
196198
lukechampine.com/uint128 v1.3.0 // indirect
197199
nhooyr.io/websocket v1.8.17 // indirect

‎go.sum‎

Lines changed: 0 additions & 1238 deletions
This file was deleted.

‎p2p/kademlia/conn_pool.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/LumeraProtocol/supernode/v2/pkg/errors"
1313
ltc "github.com/LumeraProtocol/supernode/v2/pkg/net/credentials"
14+
althandshake "github.com/LumeraProtocol/supernode/v2/pkg/net/credentials/alts/handshake"
1415
"google.golang.org/grpc/credentials"
1516
)
1617

@@ -100,8 +101,8 @@ func NewSecureClientConn(ctx context.Context, tc credentials.TransportCredential
100101
}, nil
101102
}
102103

103-
// NewSecureServerConn do server handshake and create a secure connection
104-
func NewSecureServerConn(_ context.Context, tc credentials.TransportCredentials, rawConn net.Conn) (net.Conn, error) {
104+
// NewSecureServerConn does server handshake and returns a secure connection along with the authenticated remote identity (when available).
105+
func NewSecureServerConn(_ context.Context, tc credentials.TransportCredentials, rawConn net.Conn) (net.Conn, string, error) {
105106
if tcp, ok := rawConn.(*net.TCPConn); ok {
106107
_ = tcp.SetKeepAlive(true)
107108
_ = tcp.SetKeepAlivePeriod(2 * time.Minute) // tune: 2–5 min
@@ -111,15 +112,20 @@ func NewSecureServerConn(_ context.Context, tc credentials.TransportCredentials,
111112
_ = tcp.SetNoDelay(true)
112113
}
113114

114-
conn, _, err := tc.ServerHandshake(rawConn)
115+
conn, authInfo, err := tc.ServerHandshake(rawConn)
115116
if err != nil {
116-
return nil, errors.Errorf("server secure establish failed: %w", err)
117+
return nil, "", errors.Errorf("server secure establish failed: %w", err)
118+
}
119+
120+
remoteIdentity := ""
121+
if ai, ok := authInfo.(*althandshake.AuthInfo); ok && ai != nil {
122+
remoteIdentity = ai.RemoteIdentity
117123
}
118124

119125
return &connWrapper{
120126
secureConn: conn,
121127
rawConn: rawConn,
122-
}, nil
128+
}, remoteIdentity, nil
123129
}
124130

125131
// Read implements net.Conn's Read interface

‎p2p/kademlia/dht.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2143,6 +2143,18 @@ func (s *DHT) IterateBatchStore(ctx context.Context, values [][]byte, typ int, i
21432143
successful := 0
21442144

21452145
logtrace.Debug(ctx, "Iterate batch store: dispatching to nodes", logtrace.Fields{"task_id": id, "nodes": len(knownNodes)})
2146+
2147+
// If there are no candidate nodes, there's nothing to fan out to. The caller
2148+
// already persisted the batch locally (see StoreBatch), so treat this as a
2149+
// no-op success rather than an error.
2150+
if len(knownNodes) == 0 {
2151+
logtrace.Info(ctx, "dht: batch store skipped (no candidate nodes)", logtrace.Fields{
2152+
logtrace.FieldModule: "dht",
2153+
"task_id": id,
2154+
"keys": len(values),
2155+
})
2156+
return nil
2157+
}
21462158
storeResponses := s.batchStoreNetwork(ctx, values, knownNodes, storageMap, typ)
21472159
for response := range storeResponses {
21482160
requests++

‎p2p/kademlia/network.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/btcsuite/btcutil/base58"
1616
json "github.com/json-iterator/go"
1717

18+
"github.com/LumeraProtocol/supernode/v2/pkg/reachability"
1819
"github.com/LumeraProtocol/supernode/v2/pkg/utils"
1920

2021
"github.com/google/uuid"
@@ -360,7 +361,8 @@ func (s *Network) handleConn(ctx context.Context, rawConn net.Conn) {
360361
})
361362
// secure handshake
362363
if s.serverTC != nil {
363-
conn, err = NewSecureServerConn(ctx, s.serverTC, rawConn)
364+
var remoteIdentity string
365+
conn, remoteIdentity, err = NewSecureServerConn(ctx, s.serverTC, rawConn)
364366
if err != nil {
365367
_ = rawConn.Close()
366368
logtrace.Warn(ctx, "Server secure handshake failed", logtrace.Fields{
@@ -369,6 +371,11 @@ func (s *Network) handleConn(ctx context.Context, rawConn net.Conn) {
369371
})
370372
return
371373
}
374+
375+
// Record inbound evidence for P2P reachability only after the secure handshake succeeds.
376+
if store := reachability.DefaultStore(); store != nil {
377+
store.RecordInbound(reachability.ServiceP2P, remoteIdentity, rawConn.RemoteAddr(), time.Now())
378+
}
372379
} else {
373380
conn = rawConn
374381
}

‎pkg/lumera/client.go‎

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,21 @@ import (
1010
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/bank"
1111
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/node"
1212
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/supernode"
13+
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/supernode_msg"
1314
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/tx"
1415
)
1516

1617
type lumeraClient struct {
17-
cfg *Config
18-
authMod auth.Module
19-
actionMod action.Module
20-
actionMsgMod action_msg.Module
21-
bankMod bank.Module
22-
supernodeMod supernode.Module
23-
txMod tx.Module
24-
nodeMod node.Module
25-
conn Connection
18+
cfg *Config
19+
authMod auth.Module
20+
actionMod action.Module
21+
actionMsgMod action_msg.Module
22+
bankMod bank.Module
23+
supernodeMod supernode.Module
24+
supernodeMsgMod supernode_msg.Module
25+
txMod tx.Module
26+
nodeMod node.Module
27+
conn Connection
2628
}
2729

2830
func newClient(ctx context.Context, cfg *Config) (Client, error) {
@@ -93,16 +95,31 @@ func newClient(ctx context.Context, cfg *Config) (Client, error) {
9395
return nil, err
9496
}
9597

98+
supernodeMsgModule, err := supernode_msg.NewModule(
99+
conn.GetConn(),
100+
authModule,
101+
txModule,
102+
supernodeModule,
103+
cfg.keyring,
104+
cfg.KeyName,
105+
cfg.ChainID,
106+
)
107+
if err != nil {
108+
conn.Close()
109+
return nil, err
110+
}
111+
96112
return &lumeraClient{
97-
cfg: cfg,
98-
authMod: authModule,
99-
actionMod: actionModule,
100-
actionMsgMod: actionMsgModule,
101-
bankMod: bankModule,
102-
supernodeMod: supernodeModule,
103-
txMod: txModule,
104-
nodeMod: nodeModule,
105-
conn: conn,
113+
cfg: cfg,
114+
authMod: authModule,
115+
actionMod: actionModule,
116+
actionMsgMod: actionMsgModule,
117+
bankMod: bankModule,
118+
supernodeMod: supernodeModule,
119+
supernodeMsgMod: supernodeMsgModule,
120+
txMod: txModule,
121+
nodeMod: nodeModule,
122+
conn: conn,
106123
}, nil
107124
}
108125

@@ -126,6 +143,10 @@ func (c *lumeraClient) SuperNode() supernode.Module {
126143
return c.supernodeMod
127144
}
128145

146+
func (c *lumeraClient) SuperNodeMsg() supernode_msg.Module {
147+
return c.supernodeMsgMod
148+
}
149+
129150
func (c *lumeraClient) Tx() tx.Module {
130151
return c.txMod
131152
}

‎pkg/lumera/codec/encoding.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"sync"
55

66
actiontypes "github.com/LumeraProtocol/lumera/x/action/v1/types"
7+
sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types"
78
"github.com/cosmos/cosmos-sdk/client"
89
"github.com/cosmos/cosmos-sdk/codec"
910
codectypes "github.com/cosmos/cosmos-sdk/codec/types"
@@ -49,7 +50,7 @@ func RegisterInterfaces(registry codectypes.InterfaceRegistry) {
4950
cryptocodec.RegisterInterfaces(registry)
5051
authtypes.RegisterInterfaces(registry)
5152
actiontypes.RegisterInterfaces(registry)
52-
// Add more interface registrations here as you add more modules
53+
sntypes.RegisterInterfaces(registry)
5354
}
5455

5556
// GetEncodingConfig returns the standard encoding config for Lumera client

‎pkg/lumera/interface.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/bank"
1111
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/node"
1212
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/supernode"
13+
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/supernode_msg"
1314
"github.com/LumeraProtocol/supernode/v2/pkg/lumera/modules/tx"
1415
)
1516

@@ -19,6 +20,7 @@ type Client interface {
1920
Action() action.Module
2021
ActionMsg() action_msg.Module
2122
SuperNode() supernode.Module
23+
SuperNodeMsg() supernode_msg.Module
2224
Bank() bank.Module
2325
Tx() tx.Module
2426
Node() node.Module

0 commit comments

Comments
 (0)