diff --git a/code/Cargo.lock b/code/Cargo.lock index 9f88a4d11..11f6d6750 100644 --- a/code/Cargo.lock +++ b/code/Cargo.lock @@ -174,9 +174,9 @@ checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "arbtest" @@ -590,9 +590,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +checksum = "dd0b03af37dad7a14518b7691d81acb0f8222604ad3d1b02f6b4bed5188c0cd5" dependencies = [ "serde", ] @@ -1672,12 +1672,13 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "zerocopy 0.8.38", ] [[package]] @@ -3093,9 +3094,9 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.49.0" +version = "0.49.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba535a5d960cc39c3621cee2941ff570a9476679a4c7ca80982412b14f4a88fd" +checksum = "c7f58e37d8d6848e5c4c9e3c35c6f61133235bff2960c9c00a663b0849301221" dependencies = [ "async-channel", "asynchronous-codec", @@ -3432,6 +3433,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", + "redox_syscall", ] [[package]] @@ -3841,9 +3843,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" dependencies = [ "winapi", ] @@ -4191,7 +4193,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.23", + "zerocopy 0.8.38", ] [[package]] @@ -4480,7 +4482,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.23", + "zerocopy 0.8.38", ] [[package]] @@ -4523,9 +4525,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -4533,9 +4535,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -5917,11 +5919,11 @@ dependencies = [ [[package]] name = "whoami" -version = "1.5.2" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "redox_syscall", + "libredox", "wasite", "web-sys", ] @@ -5950,9 +5952,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ "windows-sys 0.59.0", ] @@ -6390,11 +6392,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.23" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" dependencies = [ - "zerocopy-derive 0.8.23", + "zerocopy-derive 0.8.38", ] [[package]] @@ -6410,9 +6412,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.23" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" dependencies = [ "proc-macro2", "quote", diff --git a/code/Cargo.toml b/code/Cargo.toml index 96f74efa2..e4ed72380 100644 --- a/code/Cargo.toml +++ b/code/Cargo.toml @@ -145,7 +145,7 @@ itf = "0.2.3" libp2p = { version = "0.56.0", features = ["macros", "identify", "tokio", "ed25519", "ecdsa", "tcp", "quic", "noise", "yamux", "gossipsub", "dns", "ping", "metrics", "request-response", "cbor", "serde", "kad"] } libp2p-identity = "0.2.12" libp2p-broadcast = { version = "0.3.0", package = "libp2p-scatter" } -libp2p-gossipsub = { version = "0.49.0", features = ["metrics"] } +libp2p-gossipsub = { version = "0.49.2", features = ["metrics"] } multiaddr = "0.18.2" multihash = { version = "0.19.3", default-features = false } nix = { version = "0.29.0", features = ["signal"] } diff --git a/code/crates/app/src/spawn.rs b/code/crates/app/src/spawn.rs index 12095b1e2..f5916d4cf 100644 --- a/code/crates/app/src/spawn.rs +++ b/code/crates/app/src/spawn.rs @@ -259,6 +259,8 @@ fn make_network_config(cfg: &ConsensusConfig, value_sync_cfg: &ValueSyncConfig) mesh_n_low: config.mesh_n_low(), mesh_outbound_min: config.mesh_outbound_min(), enable_peer_scoring: config.enable_peer_scoring(), + enable_explicit_peering: config.enable_explicit_peering(), + enable_flood_publish: config.enable_flood_publish(), }, config::PubSubProtocol::Broadcast => GossipSubConfig::default(), }, diff --git a/code/crates/config/src/lib.rs b/code/crates/config/src/lib.rs index 6f57509d1..2ed3485d8 100644 --- a/code/crates/config/src/lib.rs +++ b/code/crates/config/src/lib.rs @@ -298,12 +298,23 @@ pub struct GossipSubConfig { /// Enable peer scoring to prioritize nodes based on their type in mesh formation enable_peer_scoring: bool, + + /// Enable explicit peering for persistent peers (validators). + /// When enabled, persistent peers are added as explicit peers in GossipSub, + /// ensuring guaranteed message delivery outside the mesh. + /// This eliminates mesh partitioning and backoff issues between validators. + enable_explicit_peering: bool, + + /// Enable flood publishing (send messages to all known peers, not just mesh peers). + /// When enable_explicit_peering is true, enable_flood_publish is forced to false. + /// Otherwise, this setting controls flood_publish behavior. + enable_flood_publish: bool, } impl Default for GossipSubConfig { fn default() -> Self { - // Peer scoring disabled by default - Self::new(6, 12, 4, 2, false) + // Peer scoring enabled by default, explicit peering disabled by default, flood_publish true by default + Self::new(6, 12, 4, 2, true, false, true) } } @@ -315,17 +326,19 @@ impl GossipSubConfig { mesh_n_low: usize, mesh_outbound_min: usize, enable_peer_scoring: bool, + enable_explicit_peering: bool, + enable_flood_publish: bool, ) -> Self { - let mut result = Self { + // Note: adjust() is disabled to allow mesh_n = 0 for testing gossip-only mode + Self { mesh_n, mesh_n_high, mesh_n_low, mesh_outbound_min, enable_peer_scoring, - }; - - result.adjust(); - result + enable_explicit_peering, + enable_flood_publish, + } } /// Adjust the configuration values. @@ -371,22 +384,60 @@ impl GossipSubConfig { pub fn enable_peer_scoring(&self) -> bool { self.enable_peer_scoring } + + pub fn enable_explicit_peering(&self) -> bool { + self.enable_explicit_peering + } + + pub fn enable_flood_publish(&self) -> bool { + self.enable_flood_publish + } } mod gossipsub { - use super::utils::bool_from_anything; + use super::utils::{bool_from_anything, usize_from_anything}; + + fn default_enable_peer_scoring() -> bool { + true + } + + fn default_enable_explicit_peering() -> bool { + false + } + + fn default_enable_flood_publish() -> bool { + true + } + + fn default_zero() -> usize { + 0 + } + #[derive(serde::Deserialize)] pub struct RawConfig { - #[serde(default)] + #[serde(default = "default_zero", deserialize_with = "usize_from_anything")] mesh_n: usize, - #[serde(default)] + #[serde(default = "default_zero", deserialize_with = "usize_from_anything")] mesh_n_high: usize, - #[serde(default)] + #[serde(default = "default_zero", deserialize_with = "usize_from_anything")] mesh_n_low: usize, - #[serde(default)] + #[serde(default = "default_zero", deserialize_with = "usize_from_anything")] mesh_outbound_min: usize, - #[serde(default, deserialize_with = "bool_from_anything")] + #[serde( + default = "default_enable_peer_scoring", + deserialize_with = "bool_from_anything" + )] enable_peer_scoring: bool, + #[serde( + default = "default_enable_explicit_peering", + deserialize_with = "bool_from_anything" + )] + enable_explicit_peering: bool, + #[serde( + default = "default_enable_flood_publish", + deserialize_with = "bool_from_anything" + )] + enable_flood_publish: bool, } impl From for super::GossipSubConfig { @@ -397,6 +448,8 @@ mod gossipsub { raw.mesh_n_low, raw.mesh_outbound_min, raw.enable_peer_scoring, + raw.enable_explicit_peering, + raw.enable_flood_publish, ) } } @@ -1007,9 +1060,9 @@ mod tests { } #[test] - fn gossipsub_config_default_disables_peer_scoring() { + fn gossipsub_config_default_enables_peer_scoring() { let config = GossipSubConfig::default(); - assert!(!config.enable_peer_scoring()); + assert!(config.enable_peer_scoring()); } #[test] @@ -1022,12 +1075,12 @@ mod tests { let cases = [ TestCase { - name: "missing field defaults to false", + name: "missing field defaults to true", toml: r#" [p2p.protocol] type = "gossipsub" "#, - expected: false, + expected: true, }, TestCase { name: "explicit true", diff --git a/code/crates/config/src/utils.rs b/code/crates/config/src/utils.rs index 457997b00..3cc7777aa 100644 --- a/code/crates/config/src/utils.rs +++ b/code/crates/config/src/utils.rs @@ -35,3 +35,45 @@ where deserializer.deserialize_any(BoolVisitor) } + +/// Deserializes a usize value from either a native integer or a string +pub fn usize_from_anything<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + struct UsizeVisitor; + + impl<'de> de::Visitor<'de> for UsizeVisitor { + type Value = usize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a usize or a string representing a usize") + } + + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + usize::try_from(v) + .map_err(|_| E::custom(format!("u64 value {} out of range for usize", v))) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + usize::try_from(v) + .map_err(|_| E::custom(format!("i64 value {} out of range for usize", v))) + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + v.parse::() + .map_err(|_| E::custom(format!("invalid usize string: {}", v))) + } + } + + deserializer.deserialize_any(UsizeVisitor) +} diff --git a/code/crates/network/src/behaviour.rs b/code/crates/network/src/behaviour.rs index 654e7c3cb..dcaaff9b3 100644 --- a/code/crates/network/src/behaviour.rs +++ b/code/crates/network/src/behaviour.rs @@ -168,6 +168,14 @@ fn message_id(message: &gossipsub::Message) -> gossipsub::MessageId { } fn gossipsub_config(config: GossipSubConfig, max_transmit_size: usize) -> gossipsub::Config { + // When explicit peering is enabled, flood_publish is forced to false since explicit peers + // handle validator-to-validator communication. Otherwise use the configured flood_publish value. + let flood_publish = if config.enable_explicit_peering { + false + } else { + config.enable_flood_publish + }; + gossipsub::ConfigBuilder::default() .max_transmit_size(max_transmit_size) .opportunistic_graft_ticks(peer_scoring::OPPORTUNISTIC_GRAFT_TICKS) @@ -180,6 +188,7 @@ fn gossipsub_config(config: GossipSubConfig, max_transmit_size: usize) -> gossip .mesh_n_low(config.mesh_n_low) .mesh_outbound_min(config.mesh_outbound_min) .mesh_n(config.mesh_n) + .flood_publish(flood_publish) .message_id_fn(message_id) .build() .unwrap() diff --git a/code/crates/network/src/lib.rs b/code/crates/network/src/lib.rs index 7d8731764..df81a3874 100644 --- a/code/crates/network/src/lib.rs +++ b/code/crates/network/src/lib.rs @@ -100,6 +100,8 @@ pub struct GossipSubConfig { pub mesh_n_low: usize, pub mesh_outbound_min: usize, pub enable_peer_scoring: bool, + pub enable_explicit_peering: bool, + pub enable_flood_publish: bool, } impl Default for GossipSubConfig { @@ -111,6 +113,8 @@ impl Default for GossipSubConfig { mesh_n_low: 4, mesh_outbound_min: 2, enable_peer_scoring: false, + enable_explicit_peering: false, + enable_flood_publish: true, } } } @@ -609,6 +613,30 @@ fn set_peer_score(swarm: &mut swarm::Swarm, peer_id: libp2p::PeerId, } } +/// Add a persistent peer as an explicit peer in gossipsub (if explicit peering is enabled). +/// Explicit peers receive messages unconditionally, outside the mesh. +/// This ensures reliable validator-to-validator communication. +fn add_explicit_peer_if_persistent( + swarm: &mut swarm::Swarm, + state: &State, + peer_id: libp2p::PeerId, + connection_id: libp2p::swarm::ConnectionId, + info: &identify::Info, + enable_explicit_peering: bool, +) { + if !enable_explicit_peering { + return; + } + + let peer_type = state.peer_type(&peer_id, connection_id, info); + if peer_type.is_persistent() { + if let Some(gossipsub) = swarm.behaviour_mut().gossipsub.as_mut() { + gossipsub.add_explicit_peer(&peer_id); + info!("Added persistent peer {peer_id} as explicit peer in gossipsub"); + } + } +} + async fn handle_swarm_event( event: SwarmEvent, config: &Config, @@ -730,6 +758,16 @@ async fn handle_swarm_event( let score = state.update_peer(peer_id, connection_id, &info); set_peer_score(swarm, peer_id, score); + // Add persistent peers as explicit peers for guaranteed delivery + add_explicit_peer_if_persistent( + swarm, + state, + peer_id, + connection_id, + &info, + config.gossipsub.enable_explicit_peering, + ); + if !is_already_connected { if let Err(e) = tx_event .send(Event::PeerConnected(PeerId::from_libp2p(&peer_id))) diff --git a/code/crates/network/src/peer_scoring.rs b/code/crates/network/src/peer_scoring.rs index a105f6de9..e2900462e 100644 --- a/code/crates/network/src/peer_scoring.rs +++ b/code/crates/network/src/peer_scoring.rs @@ -133,9 +133,12 @@ pub fn get_peer_score(peer_type: PeerType) -> f64 { /// Constructs the peer score parameters for GossipSub. /// /// Configures application-specific scoring with a weight multiplier to amplify score differences. +/// Disables IP colocation penalty since nodes may share IPs in test/local environments. pub fn peer_score_params() -> gossipsub::PeerScoreParams { gossipsub::PeerScoreParams { app_specific_weight: APP_SPECIFIC_WEIGHT, + // Disable IP colocation penalty (all nodes may be on same IP in test/local environments) + ip_colocation_factor_weight: 0.0, ..Default::default() } } diff --git a/code/crates/starknet/host/src/spawn.rs b/code/crates/starknet/host/src/spawn.rs index 60a881db4..69301bcb5 100644 --- a/code/crates/starknet/host/src/spawn.rs +++ b/code/crates/starknet/host/src/spawn.rs @@ -283,6 +283,8 @@ async fn spawn_network_actor( mesh_n_low: config.mesh_n_low(), mesh_outbound_min: config.mesh_outbound_min(), enable_peer_scoring: config.enable_peer_scoring(), + enable_explicit_peering: config.enable_explicit_peering(), + enable_flood_publish: config.enable_flood_publish(), }, config::PubSubProtocol::Broadcast => gossip::GossipSubConfig::default(), }, diff --git a/code/crates/test/app/config.toml b/code/crates/test/app/config.toml index 5879c15a2..9339c4d2c 100644 --- a/code/crates/test/app/config.toml +++ b/code/crates/test/app/config.toml @@ -164,6 +164,13 @@ mesh_outbound_min = 2 # Override with MALACHITE__CONSENSUS__P2P__PROTOCOL__ENABLE_PEER_SCORING env variable enable_peer_scoring = false +# GossipSub only. Enable explicit peering for persistent peers. +# When enabled, persistent peers are added as explicit peers in GossipSub, ensuring guaranteed +# message delivery outside the mesh. This eliminates mesh partitioning and backoff issues. +# Also sets flood_publish to false when enabled (explicit peers handle delivery). +# Override with MALACHITE__CONSENSUS__P2P__PROTOCOL__ENABLE_EXPLICIT_PEERING env variable +enable_explicit_peering = false + ####################################################### ### ValueSync Configuration Options ### ####################################################### diff --git a/code/examples/channel/config.toml b/code/examples/channel/config.toml index 852abbcf8..adc4c7866 100644 --- a/code/examples/channel/config.toml +++ b/code/examples/channel/config.toml @@ -163,6 +163,13 @@ mesh_outbound_min = 2 # Override with MALACHITE__CONSENSUS__P2P__PROTOCOL__ENABLE_PEER_SCORING env variable enable_peer_scoring = false +# GossipSub only. Enable explicit peering for persistent peers. +# When enabled, persistent peers are added as explicit peers in GossipSub, ensuring guaranteed +# message delivery outside the mesh. This eliminates mesh partitioning and backoff issues. +# Also sets flood_publish to false when enabled (explicit peers handle delivery). +# Override with MALACHITE__CONSENSUS__P2P__PROTOCOL__ENABLE_EXPLICIT_PEERING env variable +enable_explicit_peering = false + ####################################################### ### Mempool Configuration Options ### ####################################################### diff --git a/code/examples/channel/src/node.rs b/code/examples/channel/src/node.rs index 0aff256c0..743b17809 100644 --- a/code/examples/channel/src/node.rs +++ b/code/examples/channel/src/node.rs @@ -114,6 +114,17 @@ impl Node for App { async fn start(&self) -> eyre::Result { let config = self.load_config()?; + if let malachitebft_test_cli::config::PubSubProtocol::GossipSub(gs_config) = + &config.consensus.p2p.protocol + { + dbg!(gs_config.enable_explicit_peering()); + dbg!(gs_config.enable_flood_publish()); + dbg!(gs_config.enable_peer_scoring()); + dbg!(gs_config.mesh_n()); + dbg!(gs_config.mesh_n_high()); + dbg!(gs_config.mesh_n_low()); + dbg!(gs_config.mesh_outbound_min()); + } let span = tracing::error_span!("node", moniker = %config.moniker); let _enter = span.enter(); diff --git a/code/examples/channel/src/state.rs b/code/examples/channel/src/state.rs index c86e1c684..a8504f364 100644 --- a/code/examples/channel/src/state.rs +++ b/code/examples/channel/src/state.rs @@ -493,16 +493,19 @@ impl State { /// Returns the validator set for the given height. /// The validator set is rotated every 10 heights, selecting floor((n+1)/2) /// validators from the genesis validator set. - pub fn get_validator_set(&self, height: Height) -> ValidatorSet { + pub fn get_validator_set(&self, _height: Height) -> ValidatorSet { let num_validators = self.genesis.validator_set.len(); - let selection_size = num_validators.div_ceil(2); + let selection_size = num_validators.div_ceil(1); if num_validators <= selection_size { return self.genesis.validator_set.clone(); } + // Disable rotation for easier debugging + let rotation_index = 0; + // Rotate every 10 heights for easier debugging - let rotation_index = (height.as_u64() / 10) as usize % num_validators; + //let rotation_index = (height.as_u64() / 10) as usize % num_validators; ValidatorSet::new( self.genesis