diff --git a/deny.toml b/deny.toml index afae4114..dbe97ba4 100644 --- a/deny.toml +++ b/deny.toml @@ -33,5 +33,18 @@ allow = [ # r-efi (LGPL-2.1-or-later) is only compiled for UEFI targets, which are # outside our target set. It is allowed above for completeness. +# smoldot (the embedded light client behind subxt's `light-client` feature) +# is GPL-3.0-or-later with the Classpath exception. Compatible with this +# repo's GPL-3.0-only provider node — the exception only grants additional +# linking permissions — but scoped to these two crates rather than allowing +# the expression workspace-wide. +[[licenses.exceptions]] +allow = ["GPL-3.0-or-later WITH Classpath-exception-2.0"] +crate = "smoldot" + +[[licenses.exceptions]] +allow = ["GPL-3.0-or-later WITH Classpath-exception-2.0"] +crate = "smoldot-light" + [licenses.private] ignore = true diff --git a/provider-node/Cargo.toml b/provider-node/Cargo.toml index c74b58a8..9d49086f 100644 --- a/provider-node/Cargo.toml +++ b/provider-node/Cargo.toml @@ -19,7 +19,7 @@ serde = { workspace = true, features = ["std"] } serde_json = { workspace = true } sp-core = { workspace = true, features = ["std"] } sp-runtime = { workspace = true, features = ["std"] } -subxt = { workspace = true } +subxt = { workspace = true, features = ["light-client"] } subxt-signer = { workspace = true } base64 = { workspace = true } thiserror = { workspace = true } diff --git a/provider-node/src/chain_connection.rs b/provider-node/src/chain_connection.rs index b93ef986..44fe8574 100644 --- a/provider-node/src/chain_connection.rs +++ b/provider-node/src/chain_connection.rs @@ -10,12 +10,11 @@ //! borrow the live client and nobody else carries reconnect logic. use crate::Error; +use std::path::PathBuf; +use subxt::lightclient::LightClient; use subxt::{OnlineClient, PolkadotConfig}; /// How the provider node talks to the chain. -/// -/// A `Light` (embedded smoldot) variant is planned as a follow-up; keeping -/// construction behind this enum means adding it only touches this module. #[derive(Clone, Debug)] pub enum ChainTransport { /// External RPC node reached over WebSocket. @@ -23,6 +22,56 @@ pub enum ChainTransport { /// `ws://` / `wss://` URL of the parachain RPC endpoint. url: String, }, + /// Embedded smoldot light client following the relay chain and deriving + /// parachain finality from it — no operated RPC infrastructure needed. + Light { + /// Relay-chain spec (with reachable boot nodes). + relay_spec: SpecSource, + /// Parachain spec (with boot nodes serving the light-client + /// request-response protocols). + para_spec: SpecSource, + }, +} + +/// Where a chain spec for the light client comes from. +#[derive(Clone, Debug)] +pub enum SpecSource { + /// A chain-spec JSON file shipped with the deployment. This is the + /// trust-preserving option: the spec (genesis + boot nodes) is vetted + /// ahead of time rather than trusted from a node at runtime. + File(PathBuf), + /// Fetch the spec from a running node's RPC at startup. Convenient for + /// local development (zombienet regenerates genesis every run), but it + /// reintroduces trust in that node — dev use only. + FetchFromRpc(String), +} + +impl SpecSource { + async fn load(&self, what: &str) -> Result { + match self { + SpecSource::File(path) => std::fs::read_to_string(path).map_err(|e| { + Error::Internal(format!( + "Failed to read {what} chain spec {}: {e}", + path.display() + )) + }), + SpecSource::FetchFromRpc(url) => { + tracing::warn!( + "Fetching the {what} chain spec from {url}: this trusts that node and \ + defeats the light client's verification purpose — use spec files in \ + production" + ); + let spec = subxt::utils::fetch_chainspec_from_rpc_node(url) + .await + .map_err(|e| { + Error::Internal(format!( + "Failed to fetch {what} chain spec from {url}: {e}" + )) + })?; + Ok(spec.get().to_string()) + } + } + } } /// A live chain connection, cheap to clone (`OnlineClient` is `Arc`-backed). @@ -30,6 +79,18 @@ pub enum ChainTransport { pub struct ChainHandle { /// The subxt client for storage reads, event decoding, and tx submission. pub api: OnlineClient, + /// Keeps the embedded smoldot instance alive for the handle's lifetime; + /// dropping the last clone tears the light client down. `None` on the + /// RPC transport. + _light: Option, +} + +impl ChainHandle { + /// Handle over an existing client with no embedded light client to keep + /// alive: the RPC transport, and tests driving a mock connection. + pub(crate) fn from_api(api: OnlineClient) -> Self { + Self { api, _light: None } + } } /// Receiver side of the connection watch channel. `None` until the first @@ -37,13 +98,39 @@ pub struct ChainHandle { pub type ChainWatch = tokio::sync::watch::Receiver>; /// Build a fresh connection for the given transport. +/// +/// For `Light` this boots a new embedded smoldot instance (relay + para); +/// the watchdog's rebuild path therefore recovers even from a wedged smoldot +/// background task, not just a dropped subscription. pub async fn connect(transport: &ChainTransport) -> Result { match transport { ChainTransport::Rpc { url } => { let api = OnlineClient::::from_url(url) .await .map_err(|e| Error::Internal(format!("Failed to connect to chain: {e}")))?; - Ok(ChainHandle { api }) + Ok(ChainHandle::from_api(api)) + } + ChainTransport::Light { + relay_spec, + para_spec, + } => { + let relay = relay_spec.load("relay").await?; + let para = para_spec.load("parachain").await?; + + let (light, _relay_rpc) = LightClient::relay_chain(relay.as_str()) + .map_err(|e| Error::Internal(format!("Failed to start light client: {e}")))?; + let para_rpc = light.parachain(para.as_str()).map_err(|e| { + Error::Internal(format!("Failed to add parachain to light client: {e}")) + })?; + let api = OnlineClient::::from_rpc_client(para_rpc) + .await + .map_err(|e| Error::Internal(format!("Failed to connect via light client: {e}")))?; + + tracing::info!("Embedded light client started (relay + parachain)"); + Ok(ChainHandle { + api, + _light: Some(light), + }) } } } @@ -65,13 +152,12 @@ mod tests { #[tokio::test] async fn rpc_connect_to_unreachable_chain_errors() { // Port 1 on loopback refuses immediately. - let result = connect(&ChainTransport::Rpc { + let err = connect(&ChainTransport::Rpc { url: "ws://127.0.0.1:1".to_string(), }) - .await; - let Err(err) = result else { - panic!("connect must fail against a closed port"); - }; + .await + .map(|_| ()) + .expect_err("connect must fail against a closed port"); assert!(err.to_string().contains("Failed to connect to chain")); } @@ -81,4 +167,60 @@ mod tests { let err = current_api(&rx).expect_err("no connection published yet"); assert!(err.to_string().contains("not established")); } + + #[tokio::test] + async fn light_with_missing_spec_file_errors() { + let err = connect(&ChainTransport::Light { + relay_spec: SpecSource::File(PathBuf::from("/nonexistent/relay.json")), + para_spec: SpecSource::File(PathBuf::from("/nonexistent/para.json")), + }) + .await + .map(|_| ()) + .expect_err("connect must fail when the relay spec file is missing"); + assert!( + err.to_string().contains("Failed to read relay chain spec"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn light_with_invalid_spec_errors() { + // The file loads (covering the File happy path) but smoldot rejects + // the contents as a chain spec. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.json"); + std::fs::write(&path, "{\"not\": \"a chain spec\"}").unwrap(); + + let err = connect(&ChainTransport::Light { + relay_spec: SpecSource::File(path), + para_spec: SpecSource::File(PathBuf::from("/nonexistent/para.json")), + }) + .await + .map(|_| ()) + .expect_err("connect must fail on an invalid relay spec"); + // The para spec is never reached: either the relay spec parse fails, + // or (if smoldot were lenient) the missing para file errors next. + assert!( + err.to_string().contains("Failed to start light client") + || err + .to_string() + .contains("Failed to read parachain chain spec"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn light_spec_fetch_from_unreachable_node_errors() { + let err = connect(&ChainTransport::Light { + relay_spec: SpecSource::FetchFromRpc("ws://127.0.0.1:1".to_string()), + para_spec: SpecSource::File(PathBuf::from("/nonexistent/para.json")), + }) + .await + .map(|_| ()) + .expect_err("connect must fail when the spec fetch node is unreachable"); + assert!( + err.to_string().contains("Failed to fetch relay chain spec"), + "unexpected error: {err}" + ); + } } diff --git a/provider-node/src/chain_state_coordinator.rs b/provider-node/src/chain_state_coordinator.rs index ce2eb281..44f3e470 100644 --- a/provider-node/src/chain_state_coordinator.rs +++ b/provider-node/src/chain_state_coordinator.rs @@ -308,6 +308,10 @@ impl ChainStateCoordinator { /// backend resubscriptions), so this is several times the block time; /// a genuinely stalled stream otherwise hangs forever with no error. const STALL_TIMEOUT: Duration = Duration::from_secs(60); + /// The first block gets a longer budget: a fresh embedded light client + /// has to discover peers and warp-sync before anything is finalized, + /// and rebuilding mid-sync would throw that progress away. + const FIRST_BLOCK_TIMEOUT: Duration = Duration::from_secs(300); let api = handle.api.clone(); let mut blocks = api @@ -339,18 +343,20 @@ impl ChainStateCoordinator { .load(std::sync::atomic::Ordering::Relaxed), }); + let mut stall_timeout = FIRST_BLOCK_TIMEOUT; loop { - let next = match tokio::time::timeout(STALL_TIMEOUT, blocks.next()).await { + let next = match tokio::time::timeout(stall_timeout, blocks.next()).await { Ok(Some(next)) => next, Ok(None) => break, Err(_) => { tracing::warn!( "chain-state coordinator: no finalized block for {}s; rebuilding connection", - STALL_TIMEOUT.as_secs() + stall_timeout.as_secs() ); break; } }; + stall_timeout = STALL_TIMEOUT; let block = match next { Ok(block) => block, Err(e) => { @@ -1309,7 +1315,7 @@ mod tests { // `follow` bootstraps, processes the block (decoding the // ProviderRegistered event and refreshing state), and returns. coordinator - .follow(ChainHandle { api }) + .follow(ChainHandle::from_api(api)) .await .expect("follow runs to stream end"); diff --git a/provider-node/src/cli.rs b/provider-node/src/cli.rs index ca8b7a8c..9a226317 100644 --- a/provider-node/src/cli.rs +++ b/provider-node/src/cli.rs @@ -2,6 +2,7 @@ //! CLI argument parsing for the storage provider node. +use crate::chain_connection::{ChainTransport, SpecSource}; use clap::Parser; use std::path::PathBuf; @@ -76,6 +77,33 @@ pub struct RpcParams { )] pub chain_rpc: String, + /// How to talk to the chain: an external RPC node or the embedded smoldot + /// light client (which needs no operated RPC infrastructure). + #[arg( + long, + value_enum, + value_name = "TRANSPORT", + default_value_t = TransportKind::Rpc, + env = "CHAIN_TRANSPORT" + )] + pub chain_transport: TransportKind, + + /// Relay-chain spec file for the light transport (a raw spec with + /// reachable boot nodes). Falls back to fetching from --relay-rpc. + #[arg(long, value_name = "FILE", env = "RELAY_CHAIN_SPEC")] + pub relay_chain_spec: Option, + + /// Parachain spec file for the light transport (a raw spec with boot + /// nodes serving the light request-response protocols). Falls back to + /// fetching from --chain-rpc. + #[arg(long, value_name = "FILE", env = "PARA_CHAIN_SPEC")] + pub para_chain_spec: Option, + + /// Relay-chain RPC URL used only to fetch the relay spec at startup when + /// --relay-chain-spec is not given (dev convenience; trusts that node). + #[arg(long, value_name = "URL", env = "RELAY_RPC")] + pub relay_rpc: Option, + /// Public multiaddr to advertise on chain instead of the bind-derived one. /// /// On hosted deployments the bind address (e.g. `0.0.0.0:3333`) is not @@ -99,6 +127,52 @@ pub struct RpcParams { pub cors_allowed_origins: Option>, } +/// Chain transport selection for `--chain-transport`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub enum TransportKind { + /// External RPC node over WebSocket (`--chain-rpc`). + Rpc, + /// Embedded smoldot light client. Chain specs come from + /// `--relay-chain-spec` / `--para-chain-spec` files, falling back to + /// fetching from `--relay-rpc` / `--chain-rpc` (dev only). + Light, +} + +impl RpcParams { + /// Resolve the CLI flags into a concrete [`ChainTransport`]. + /// + /// Errors when the light transport has no way to obtain the relay spec + /// (neither a spec file nor a relay RPC to fetch it from). + pub fn chain_transport(&self) -> Result { + match self.chain_transport { + TransportKind::Rpc => Ok(ChainTransport::Rpc { + url: self.chain_rpc.clone(), + }), + TransportKind::Light => { + let relay_spec = match (&self.relay_chain_spec, &self.relay_rpc) { + (Some(path), _) => SpecSource::File(path.clone()), + (None, Some(url)) => SpecSource::FetchFromRpc(url.clone()), + (None, None) => { + return Err( + "--chain-transport light needs --relay-chain-spec (or, for dev, \ + --relay-rpc to fetch it from a node)" + .to_string(), + ) + } + }; + let para_spec = match &self.para_chain_spec { + Some(path) => SpecSource::File(path.clone()), + None => SpecSource::FetchFromRpc(self.chain_rpc.clone()), + }; + Ok(ChainTransport::Light { + relay_spec, + para_spec, + }) + } + } + } +} + /// Parameters for provider identity and signing keys. #[derive(Debug, clap::Args)] pub struct KeyParams { @@ -330,6 +404,75 @@ mod tests { assert_eq!(cli.replica_sync.replica_max_concurrent, 5); } + #[test] + fn transport_defaults_to_rpc() { + let cli = Cli::try_parse_from(["storage-provider-node"]).unwrap(); + assert!(matches!(cli.rpc.chain_transport, TransportKind::Rpc)); + let transport = cli.rpc.chain_transport().unwrap(); + assert!(matches!(transport, ChainTransport::Rpc { url } if url == "ws://127.0.0.1:2222")); + } + + #[test] + fn light_transport_resolves_spec_sources() { + // Spec files win over RPC fetching. + let cli = Cli::try_parse_from([ + "storage-provider-node", + "--chain-transport", + "light", + "--relay-chain-spec", + "/specs/relay.json", + "--para-chain-spec", + "/specs/para.json", + ]) + .unwrap(); + let ChainTransport::Light { + relay_spec, + para_spec, + } = cli.rpc.chain_transport().unwrap() + else { + panic!("expected light transport"); + }; + assert!( + matches!(relay_spec, SpecSource::File(p) if p == PathBuf::from("/specs/relay.json").as_path()) + ); + assert!( + matches!(para_spec, SpecSource::File(p) if p == PathBuf::from("/specs/para.json").as_path()) + ); + + // Without spec files, the relay spec fetches from --relay-rpc and the + // para spec from --chain-rpc. + let cli = Cli::try_parse_from([ + "storage-provider-node", + "--chain-transport", + "light", + "--relay-rpc", + "ws://127.0.0.1:9900", + ]) + .unwrap(); + let ChainTransport::Light { + relay_spec, + para_spec, + } = cli.rpc.chain_transport().unwrap() + else { + panic!("expected light transport"); + }; + assert!( + matches!(relay_spec, SpecSource::FetchFromRpc(url) if url == "ws://127.0.0.1:9900") + ); + assert!(matches!(para_spec, SpecSource::FetchFromRpc(url) if url == "ws://127.0.0.1:2222")); + } + + #[test] + fn light_transport_without_relay_source_errors() { + let cli = + Cli::try_parse_from(["storage-provider-node", "--chain-transport", "light"]).unwrap(); + let err = cli.rpc.chain_transport().unwrap_err(); + assert!( + err.contains("--relay-chain-spec"), + "unexpected error: {err}" + ); + } + #[test] fn load_seed_missing_file() { let params = KeyParams { diff --git a/provider-node/src/command.rs b/provider-node/src/command.rs index 49abfe02..be928230 100644 --- a/provider-node/src/command.rs +++ b/provider-node/src/command.rs @@ -40,9 +40,7 @@ pub async fn run() -> Result<(), Box> { // channel. The chain-state coordinator owns the sender and rebuilds the // connection on loss or stall; every consumer (HTTP auth, the signing // client, coordinators) borrows the current handle from the receiver. - let transport = ChainTransport::Rpc { - url: cli.rpc.chain_rpc.clone(), - }; + let transport = cli.rpc.chain_transport()?; let (chain_tx, chain_rx) = watch::channel::>(None); // Per-block event fan-out from the chain-state coordinator to the // background coordinators. diff --git a/scripts/test-light-client.sh b/scripts/test-light-client.sh new file mode 100755 index 00000000..07d8bc51 --- /dev/null +++ b/scripts/test-light-client.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0-only +# +# Manual e2e check for the provider node's embedded light-client transport. +# +# Starts the provider with --chain-transport light against an already-running +# local zombienet (relay + parachain, e.g. `just start-chain`), using the +# chain specs from zombienet's network directory: the relay's raw spec as-is, +# and the parachain's raw spec with the live boot nodes injected (zombienet +# only writes boot nodes into the non-raw variant, which smoldot cannot use). +# Waits until the provider's chain-state coordinator reports synced provider +# info on /info. +# +# Prerequisites: +# - `just start-chain` running +# - provider registered on-chain (e.g. `just register-provider`), otherwise +# the check passes on /health + block-following alone +# - `cargo build --release -p storage-provider-node` +# +# Usage: scripts/test-light-client.sh [timeout-seconds] + +set -euo pipefail + +PROVIDER_PORT="${PROVIDER_PORT:-3433}" +TIMEOUT="${1:-300}" +BIN="${BIN:-./target/release/storage-provider-node}" +# `ls` exits non-zero when one of the globs has no match (it still prints the +# matches of the other); tolerate that or set -e kills the script here. +ZOMBIE_DIR="${ZOMBIE_DIR:-$(ls -dt /var/folders/*/*/T/zombie-* /tmp/zombie-* 2>/dev/null | head -1 || true)}" + +if [ -z "$ZOMBIE_DIR" ] || [ ! -d "$ZOMBIE_DIR" ]; then + echo "No zombienet network directory found — is 'just start-chain' running?" + exit 1 +fi +echo "Using zombienet network dir: $ZOMBIE_DIR" + +RELAY_SPEC=$(ls "$ZOMBIE_DIR"/*-local.json 2>/dev/null | grep -v '^.*/[0-9]' | head -1 || true) +PARA_LIVE=$(ls "$ZOMBIE_DIR"/[0-9]*-local.json 2>/dev/null | grep -v raw | grep -v plain | head -1 || true) +if [ -z "$RELAY_SPEC" ] || [ -z "$PARA_LIVE" ]; then + echo "Could not locate chain specs in $ZOMBIE_DIR" + exit 1 +fi + +# Para spec for smoldot: the collator's spec converted to raw genesis storage +# (smoldot cannot execute runtimeGenesis; converting the very spec the +# network runs preserves the genesis hash) plus the advertised boot nodes. +# chain-spec-builder writes `chain_spec.json` into the working directory. +CONVERT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/para-raw-spec.XXXXXX") +(cd "$CONVERT_DIR" && "$OLDPWD/.bin/chain-spec-builder" convert-to-raw "$PARA_LIVE") +PARA_RAW="$CONVERT_DIR/chain_spec.json" +PARA_SPEC=$(mktemp "${TMPDIR:-/tmp}/para-light-spec.XXXXXX.json") +python3 - "$PARA_RAW" "$PARA_LIVE" "$RELAY_SPEC" "$PARA_SPEC" <<'EOF' +import json, sys +raw = json.load(open(sys.argv[1])) +live = json.load(open(sys.argv[2])) +relay = json.load(open(sys.argv[3])) +raw["bootNodes"] = live.get("bootNodes", []) +assert raw["bootNodes"], "no boot nodes found in the live parachain spec" +assert "raw" in raw.get("genesis", {}), "parachain spec conversion did not produce raw genesis" +# smoldot matches a parachain to its relay by exact id; zombienet's para spec +# says e.g. "westend-local" while the relay spec's id is "westend_local_testnet". +raw["relay_chain"] = relay["id"] +json.dump(raw, open(sys.argv[4], "w")) +EOF +echo "Relay spec: $RELAY_SPEC" +echo "Para spec: $PARA_SPEC (raw genesis + $(python3 -c "import json,sys;print(len(json.load(open('$PARA_SPEC'))['bootNodes']))") boot node(s))" + +KEYFILE=$(mktemp) +echo "//Alice" > "$KEYFILE" && chmod 600 "$KEYFILE" +LOG=$(mktemp "${TMPDIR:-/tmp}/light-client-provider.XXXXXX") + +cleanup() { + [ -n "${PROVIDER_PID:-}" ] && kill "$PROVIDER_PID" 2>/dev/null || true + rm -f "$KEYFILE" +} +trap cleanup EXIT + +echo "Starting provider with the embedded light client (log: $LOG)..." +RUST_LOG=info "$BIN" \ + --keyfile "$KEYFILE" --storage-mode inmemory \ + --bind-addr "0.0.0.0:$PROVIDER_PORT" \ + --chain-transport light \ + --relay-chain-spec "$RELAY_SPEC" \ + --para-chain-spec "$PARA_SPEC" \ + --disable-auth-i-know-what-i-am-doing > "$LOG" 2>&1 & +PROVIDER_PID=$! + +echo "Waiting up to ${TIMEOUT}s for the light client to sync..." +DEADLINE=$(( $(date +%s) + TIMEOUT )) +while [ "$(date +%s)" -lt "$DEADLINE" ]; do + if ! kill -0 "$PROVIDER_PID" 2>/dev/null; then + echo "FAILED: provider exited early. Last log lines:" + tail -20 "$LOG" + exit 1 + fi + if curl -sf "http://127.0.0.1:$PROVIDER_PORT/info" 2>/dev/null \ + | grep -q '"provider_registration_info":{'; then + echo "PASSED: provider synced its on-chain registration via the light client." + grep -E "light client|following finalized" "$LOG" | tail -3 + exit 0 + fi + sleep 5 +done + +echo "FAILED: /info did not report synced provider info within ${TIMEOUT}s." +echo "Last log lines:" +tail -30 "$LOG" +exit 1