From e17fbc8e0b0bb4485d56a29f720395fcb6285467 Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Tue, 29 Sep 2026 19:30:54 -0700 Subject: [PATCH 01/10] Carry uplink relay sessions over WebSockets where UDP is blocked (#80) --- Cargo.lock | 1 + crates/cli/src/cli.rs | 9 + crates/cli/src/main.rs | 3 +- crates/cli/src/uplink.rs | 915 +------------- crates/cli/tests/support/uplink.rs | 390 +++++- crates/cli/tests/uplink_e2e.rs | 56 +- crates/client/src/lib.rs | 3 + crates/proxy/Cargo.toml | 1 + crates/proxy/src/lib.rs | 2 + crates/proxy/src/uplink_producer.rs | 1361 +++++++++++++++++++++ crates/proxy/src/uplink_producer_tests.rs | 915 ++++++++++++++ docs/uplink.md | 150 ++- 12 files changed, 2829 insertions(+), 977 deletions(-) create mode 100644 crates/proxy/src/uplink_producer.rs create mode 100644 crates/proxy/src/uplink_producer_tests.rs diff --git a/Cargo.lock b/Cargo.lock index a65f33ae6..597a01ce1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8334,6 +8334,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tokio-rustls", "tokio-tungstenite 0.30.0", "url", "web-transport-quinn", diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index 265485825..5b70cd9af 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -539,6 +539,15 @@ pub enum Command { required = true )] allow_client: Vec, + + /// What carries the relay session: auto (WebTransport, else WebSocket + /// where UDP is blocked), webtransport or websocket + #[arg( + long, + env = "YAS_UPLINK_TRANSPORT", + value_parser = ["auto", "webtransport", "websocket"] + )] + transport: Option, }, /// Print an X25519 key pair as JSON (private_key and public_key, base64url) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 80547857c..492994f62 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -947,8 +947,9 @@ async fn async_main() { url, identity, allow_client, + transport, } => { - if let Err(e) = uplink::cmd_uplink(url, identity, allow_client).await { + if let Err(e) = uplink::cmd_uplink(url, identity, allow_client, transport).await { eprintln!("yas: {e}"); std::process::exit(1); } diff --git a/crates/cli/src/uplink.rs b/crates/cli/src/uplink.rs index c1d403c72..e4d8f5172 100644 --- a/crates/cli/src/uplink.rs +++ b/crates/cli/src/uplink.rs @@ -1,25 +1,12 @@ //! `yas uplink` — expose the local yas server through a relay. //! //! Authenticates to an HTTPS control endpoint with `YAS_UPLINK_TOKEN`, -//! receives a pool of WebTransport relays, -//! establishes a session with one, authenticates each consumer using pinned -//! X25519 keys over Noise IK, then bridges decrypted streams to local YAS. +//! receives a pool of relays (WebTransport, WebSocket), holds a session with +//! one, authenticates each consumer using pinned X25519 keys over Noise IK, +//! then bridges decrypted streams to local YAS. The producer itself is +//! [`yas_proxy::uplink_producer`]; this is its command line. -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use tokio::io::AsyncWriteExt; -use web_transport_quinn as wt; - -/// Stream application error codes. -const CODE_SHUTDOWN: u32 = 2; - -const INITIAL_BACKOFF: Duration = Duration::from_secs(1); -const MAX_BACKOFF: Duration = Duration::from_secs(60); -const MAX_DATAGRAM_ROUTES: usize = 4_096; -const DATAGRAM_QUEUE: usize = 64; - -type DatagramRoutes = yas_composite_transport::RoutedDatagramRoutes; +use yas_proxy::uplink_producer::{Local, Producer, Transport}; /// Derive only the public identity; private input never appears in diagnostics. pub fn public_key_from_env() -> Result { @@ -62,21 +49,13 @@ pub fn connection_url(control: &str, client_token: &str) -> Result>, -} - +/// Publish the local server until a fatal error (the control endpoint +/// refusing the token) or Ctrl-C, which closes the relay session. pub async fn cmd_uplink( url: String, identity: String, allow_client: Vec, + transport: Option, ) -> Result<(), String> { let identity = yas_uplink::Identity::from_base64(&identity)?; let allowed = allow_client @@ -84,871 +63,27 @@ pub async fn cmd_uplink( .map(|key| key.parse()) .collect::, _>>()?; let crypto = identity.server_config(allowed)?; - let control = url::Url::parse(&url).map_err(|_| "invalid uplink control URL")?; - if control.scheme() != "https" - || !control.username().is_empty() - || control.password().is_some() - || control.fragment().is_some() - { - return Err("uplink control URL must be HTTPS without userinfo or a fragment".into()); - } let token = std::env::var("YAS_UPLINK_TOKEN").unwrap_or_default(); - if token.is_empty() { - return Err("YAS_UPLINK_TOKEN is not set".into()); - } - - // The active session, shared with the ctrl-c arm so shutdown can close - // it with CODE_SHUTDOWN instead of letting it idle out on the relay. - let current: Arc>> = Arc::new(Mutex::new(None)); - let current2 = current.clone(); - - tokio::select! { - _ = tokio::signal::ctrl_c() => { - if let Some(session) = current2.lock().unwrap().take() { - session.close(CODE_SHUTDOWN, b"uplink shutting down"); - } - Ok(()) - } - result = run_loop(&url, &token, current, crypto) => result, - } -} - -async fn run_loop( - url: &str, - token: &str, - current: Arc>>, - crypto: Arc, -) -> Result<(), String> { - let http = yas_proxy::uplink_http_client()?; - let mut backoff = INITIAL_BACKOFF; - - loop { - let mut pool = match fetch_pool(&http, url, token).await? { - FetchOutcome::Pool(pool) => pool, - FetchOutcome::Retry { after, reason } => { - let delay = after.unwrap_or(backoff); - eprintln!("[uplink] {reason}; retrying in {}s", delay.as_secs()); - tokio::time::sleep(jittered(delay)).await; - backoff = (backoff * 2).min(MAX_BACKOFF); - continue; - } - }; - - { - use rand::seq::SliceRandom; - pool.shuffle(&mut rand::rng()); - } - - // Try relays in (shuffled) order; a session that was actually - // established ends with a fresh control-plane query, per - // docs/uplink.md. - let mut established = false; - for relay in &pool { - match run_session(relay, ¤t, crypto.clone()).await { - SessionEnd::Ended(reason) => { - eprintln!("[uplink] session ended: {reason}"); - established = true; - break; - } - SessionEnd::NeverConnected(e) => { - eprintln!("[uplink] relay {}: {e}", relay.label); - } - } - } - - if established { - backoff = INITIAL_BACKOFF; - } else { - eprintln!( - "[uplink] relay pool exhausted; re-querying in {}s", - backoff.as_secs() - ); - tokio::time::sleep(jittered(backoff)).await; - backoff = (backoff * 2).min(MAX_BACKOFF); - } - } -} - -// --------------------------------------------------------------------------- -// Control plane -// --------------------------------------------------------------------------- - -enum FetchOutcome { - Pool(Vec), - Retry { - after: Option, - reason: String, - }, -} - -/// Query the control endpoint. `Err` is fatal (bad token); every other -/// failure is a retryable `FetchOutcome::Retry`. -async fn fetch_pool( - http: &reqwest::Client, - url: &str, - token: &str, -) -> Result { - let retry = |after, reason| Ok(FetchOutcome::Retry { after, reason }); - - let resp = match http - .get(url) - .header("authorization", format!("Bearer {token}")) - .header("accept", "application/json") - .send() - .await - { - Ok(resp) => resp, - Err(e) => return retry(None, format!("control endpoint unreachable: {e}")), - }; - - let status = resp.status(); - if status.as_u16() == 401 || status.as_u16() == 403 { - return Err(format!( - "control endpoint rejected YAS_UPLINK_TOKEN ({status})" - )); - } - if !status.is_success() { - let after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.trim().parse::().ok()) - .map(Duration::from_secs); - return retry(after, format!("control endpoint returned {status}")); - } - - let body = match resp.text().await { - Ok(body) => body, - Err(e) => return retry(None, format!("error reading relay pool: {e}")), - }; - match parse_pool(&body) { - Ok(pool) => Ok(FetchOutcome::Pool(pool)), - Err(e) => retry(None, format!("bad relay pool: {e}")), - } -} - -fn parse_pool(body: &str) -> Result, String> { - let v: serde_json::Value = - serde_json::from_str(body).map_err(|e| format!("invalid JSON: {e}"))?; - let relays = v - .get("relays") - .and_then(|r| r.as_array()) - .ok_or("missing \"relays\" array")?; - - let mut pool = Vec::new(); - for relay in relays { - let s = relay.as_str().ok_or("relay entries must be URL strings")?; - pool.push(parse_relay(s)?); - } - if pool.is_empty() { - return Err("empty \"relays\" array".into()); - } - Ok(pool) -} - -fn parse_relay(s: &str) -> Result { - let mut url = url::Url::parse(s).map_err(|e| format!("bad relay URL: {e}"))?; - if url.scheme() != "https" { - return Err(format!( - "relay URL scheme must be https, got {}", - url.scheme() - )); - } - let host = url.host_str().ok_or("relay URL has no host")?.to_string(); - let label = format!("{}:{}", host, url.port().unwrap_or(443)); - - // A malformed pin must be an error, never a silent fall-back to system - // roots — that would defeat the pinning. - let cert_hash = match url.fragment() { - None | Some("") => None, - Some(frag) => { - let hash = frag - .strip_prefix("sha256=") - .ok_or("relay URL fragment must be sha256=")?; - let bytes = - base64url_decode(hash).ok_or("relay certificate pin is not valid base64url")?; - if bytes.len() != 32 { - return Err("relay certificate pin must be a SHA-256 (32 bytes)".into()); - } - Some(bytes) - } - }; - // Fragments are client-side only; strip before connecting. - url.set_fragment(None); - - Ok(Relay { - url, - label, - cert_hash, - }) -} - -fn base64url_decode(s: &str) -> Option> { - let s = s.trim_end_matches('='); - let mut out = Vec::with_capacity(s.len() * 3 / 4); - let mut acc: u32 = 0; - let mut bits = 0u32; - for b in s.bytes() { - let v = match b { - b'A'..=b'Z' => b - b'A', - b'a'..=b'z' => b - b'a' + 26, - b'0'..=b'9' => b - b'0' + 52, - b'-' => 62, - b'_' => 63, - _ => return None, - }; - acc = (acc << 6) | u32::from(v); - bits += 6; - if bits >= 8 { - bits -= 8; - out.push((acc >> bits) as u8); - } - } - // Leftover bits must be padding zeros of a valid encoding. - if bits > 0 && (acc & ((1 << bits) - 1)) != 0 { - return None; - } - Some(out) -} - -// --------------------------------------------------------------------------- -// Relay session -// --------------------------------------------------------------------------- - -enum SessionEnd { - /// Handshake or CONNECT failed — try the next relay in the pool. - NeverConnected(String), - /// The session was established and later died — re-query the control - /// plane for a fresh pool. - Ended(String), -} - -async fn run_session( - relay: &Relay, - current: &Arc>>, - crypto: Arc, -) -> SessionEnd { - let client = match build_client(relay.cert_hash.as_deref()) { - Ok(client) => client, - Err(e) => return SessionEnd::NeverConnected(e), - }; - // Careful: the URL is the credential — log `relay.label` only. - let session = match client.connect(relay.url.clone()).await { - Ok(session) => session, - Err(e) => return SessionEnd::NeverConnected(format!("connect failed: {e}")), - }; - eprintln!("[uplink] connected to relay {}", relay.label); - *current.lock().unwrap() = Some(session.clone()); - - let routes = DatagramRoutes::new(MAX_DATAGRAM_ROUTES); - tokio::spawn(distribute_datagrams(session.clone(), routes.clone())); - let pending = Arc::new(tokio::sync::Semaphore::new(64)); - let local_socket = crate::transport::default_local_socket(); - - let reason = loop { - match session.accept_bi().await { - Ok((send, recv)) => { - let Ok(permit) = pending.clone().try_acquire_owned() else { - continue; - }; - tokio::spawn(bridge( - session.clone(), - routes.clone(), - send, - recv, - crypto.clone(), - permit, - local_socket.clone(), - )); - } - Err(e) => break format!("{e}"), - } - }; - current.lock().unwrap().take(); - SessionEnd::Ended(reason) -} - -/// Bridge one relay-initiated stream to one local yas server connection. -/// Authentication precedes both ingress classification and local IPC. The -/// relay sees only Noise records; plaintext selectors cannot bypass admission. -async fn bridge( - session: wt::Session, - routes: DatagramRoutes, - send: wt::SendStream, - recv: wt::RecvStream, - crypto: Arc, - permit: tokio::sync::OwnedSemaphorePermit, - local_socket: String, -) { - let relay = tokio::io::join(recv, send); - let relay = match yas_uplink::accept(relay, crypto).await { - Ok(relay) => relay, - Err(_) => return, - }; - // Bind each sideband's keys to its authenticated main stream. The random - // route token remains authenticated as AEAD AAD on every datagram. - let material = relay.datagram_key_material(); - let ingress = tokio::time::timeout( - Duration::from_secs(5), - yas_composite_transport::classify(relay), - ) - .await; - drop(permit); - match ingress { - Ok(Ok(yas_composite_transport::Ingress::Direct(relay))) => { - bridge_direct(relay, &local_socket).await - } - Ok(Ok(yas_composite_transport::Ingress::Composite { offer, stream })) - if offer.role == yas_composite_transport::Role::Main => - { - let (sender, receiver) = yas_uplink::datagram_pair(material, offer.token, false); - bridge_composite( - session, - routes, - offer, - stream, - sender, - receiver, - &local_socket, - ) - .await; - } - _ => {} - } -} - -async fn bridge_direct(relay: S, path: &str) -where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, -{ - let transport = match crate::transport::connect_ipc(path).await { - Ok(transport) => transport, - Err(e) => { - eprintln!("[uplink] local yas server unavailable at {path}: {e}"); - return; - } - }; - let (mut sock_read, mut sock_write) = transport.split(); - let (mut recv, mut send) = tokio::io::split(relay); - - let down = async move { - tokio::io::copy(&mut recv, &mut sock_write).await?; - sock_write.shutdown().await - }; - let up = async move { - tokio::io::copy(&mut sock_read, &mut send).await?; - send.shutdown().await - }; - let _ = tokio::try_join!(down, up); -} - -async fn distribute_datagrams(session: wt::Session, routes: DatagramRoutes) { - while let Ok(bytes) = session.read_datagram().await { - let _ = routes.route(&bytes); - } - routes.clear(); -} - -async fn bridge_composite( - session: wt::Session, - routes: DatagramRoutes, - offer: yas_composite_transport::Offer, - relay: S, - mut sender: yas_uplink::DatagramSender, - mut receiver: yas_uplink::DatagramReceiver, - path: &str, -) where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, -{ - let physical_maximum = session - .max_datagram_size() - .saturating_sub(yas_composite_transport::ROUTED_DATAGRAM_HEADER) - .saturating_sub(yas_uplink::DATAGRAM_OVERHEAD) - .min(yas_composite_transport::HARD_MAX_DATAGRAM as usize - yas_uplink::DATAGRAM_OVERHEAD); - if offer.max_datagram as usize > physical_maximum { - eprintln!( - "[uplink] rejected composite maximum {} above path maximum {physical_maximum}", - offer.max_datagram - ); - return; - } - - let main = match crate::transport::connect_ipc(path).await { - Ok(transport) => transport, - Err(error) => { - eprintln!("[uplink] local yas server unavailable at {path}: {error}"); - return; - } - }; - let side = match crate::transport::connect_ipc(path).await { - Ok(transport) => transport, - Err(error) => { - eprintln!("[uplink] local YAS datagram sideband unavailable at {path}: {error}"); - return; - } - }; - let (mut main_read, mut main_write) = main.split(); - let (mut side_read, mut side_write) = side.split(); - let main_offer = yas_composite_transport::Offer::new( - yas_composite_transport::Role::Main, - offer.token, - offer.max_datagram, - ) - .expect("the classified offer is valid"); - let side_offer = yas_composite_transport::Offer::new( - yas_composite_transport::Role::Datagram, - offer.token, - offer.max_datagram, - ) - .expect("the classified offer is valid"); - if yas_composite_transport::write_offer(&mut main_write, main_offer) - .await - .is_err() - || yas_composite_transport::write_offer(&mut side_write, side_offer) - .await - .is_err() - { - return; - } - - let encrypted_maximum = offer.max_datagram + yas_uplink::DATAGRAM_OVERHEAD as u32; - let Ok(mut route_rx) = routes.register(offer.token, encrypted_maximum, DATAGRAM_QUEUE) else { - return; - }; - - let (mut relay_read, mut relay_write) = tokio::io::split(relay); - let down = async { - tokio::io::copy(&mut relay_read, &mut main_write).await?; - main_write.shutdown().await - }; - let up = async { - tokio::io::copy(&mut main_read, &mut relay_write).await?; - relay_write.shutdown().await - }; - let side_routes = routes.clone(); - let side_token = offer.token; - let side_task = tokio::spawn(async move { - let side_out = async { - loop { - let Ok(frame) = - yas_composite_transport::read_datagram(&mut side_read, offer.max_datagram) - .await - else { - break; - }; - let Some(encrypted) = sender.seal(&frame) else { - break; - }; - let Ok(routed) = yas_composite_transport::encode_routed_datagram( - offer.token, - &encrypted, - encrypted_maximum, - ) else { - break; - }; - // Congestion is ordinary datagram loss; never await reliable - // transport capacity or fall back to the main stream here. - let _ = session.send_datagram(routed.into()); - } - }; - let side_in = async { - while let Some(frame) = route_rx.recv().await { - let Some(frame) = receiver.open(&frame) else { - continue; - }; - if yas_composite_transport::write_datagram( - &mut side_write, - &frame, - offer.max_datagram, - ) + let transport: Transport = match transport { + Some(transport) => transport.parse()?, + None => Transport::default(), + }; + let socket = crate::transport::default_local_socket(); + let local = Local::custom(socket.clone(), move || { + let socket = socket.clone(); + async move { + let transport = crate::transport::connect_ipc(&socket) .await - .is_err() - { - break; - } - } - }; - tokio::select! { - _ = side_out => {} - _ = side_in => {} + .map_err(std::io::Error::other)?; + let (reader, writer) = transport.split(); + Ok(tokio::io::join(reader, writer)) } - side_routes.remove(side_token); }); - - // The authoritative reliable splice owns the connection lifetime. An - // optional sideband ending only removes its route and cannot drop `main`. - let _ = tokio::try_join!(down, up); - routes.remove(offer.token); - side_task.abort(); -} - -/// Build a WebTransport client with the liveness settings from -/// docs/uplink.md (10s keepalive, 30s idle timeout) and either -/// system-root or pinned TLS verification. `wt::ClientBuilder` doesn't expose the quinn transport -/// config, so this mirrors its setup by hand. -fn build_client(cert_hash: Option<&[u8]>) -> Result { - let provider = Arc::new(rustls::crypto::ring::default_provider()); - let builder = rustls::ClientConfig::builder_with_provider(provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13]) - .map_err(|e| format!("TLS config: {e}"))?; - - let mut crypto = match cert_hash { - Some(hash) => builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(PinnedCert { - hash: hash.to_vec(), - provider, - })) - .with_no_client_auth(), - None => { - let mut roots = rustls::RootCertStore::empty(); - for cert in rustls_native_certs::load_native_certs().certs { - let _ = roots.add(cert); - } - builder.with_root_certificates(roots).with_no_client_auth() - } - }; - crypto.alpn_protocols = vec![wt::ALPN.as_bytes().to_vec()]; - - let quic_crypto = wt::quinn::crypto::rustls::QuicClientConfig::try_from(crypto) - .map_err(|e| format!("QUIC TLS config: {e}"))?; - let mut config = wt::quinn::ClientConfig::new(Arc::new(quic_crypto)); - - let mut transport = wt::quinn::TransportConfig::default(); - transport.keep_alive_interval(Some(Duration::from_secs(10))); - transport.max_idle_timeout(Some( - wt::quinn::IdleTimeout::try_from(Duration::from_secs(30)) - .expect("30s fits in an idle timeout"), - )); - config.transport_config(Arc::new(transport)); - - let endpoint = wt::quinn::Endpoint::client("[::]:0".parse().unwrap()) - .or_else(|_| wt::quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())) - .map_err(|e| format!("UDP socket: {e}"))?; - Ok(wt::Client::new(endpoint, config)) -} - -/// Pins the relay's end-entity certificate to a SHA-256 hash from the pool -/// (the edge's `serverCertificateHashes` flow, client side). Chain and -/// expiry are deliberately not checked — the hash is the trust anchor. -#[derive(Debug)] -struct PinnedCert { - hash: Vec, - provider: Arc, -} - -impl rustls::client::danger::ServerCertVerifier for PinnedCert { - fn verify_server_cert( - &self, - end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - let digest = ring::digest::digest(&ring::digest::SHA256, end_entity.as_ref()); - if digest.as_ref() == self.hash.as_slice() { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } else { - Err(rustls::Error::InvalidCertificate( - rustls::CertificateError::ApplicationVerificationFailure, - )) - } - } - - fn verify_tls12_signature( - &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, - ) -> Result { - Err(rustls::Error::PeerIncompatible( - rustls::PeerIncompatible::Tls12NotOffered, - )) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.provider - .signature_verification_algorithms - .supported_schemes() - } -} - -fn jittered(base: Duration) -> Duration { - use rand::RngExt as _; - let ms = base.as_millis().max(1) as u64; - // 0.75x–1.25x - Duration::from_millis(rand::rng().random_range(ms * 3 / 4..=ms * 5 / 4)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(unix)] - #[tokio::test] - async fn producer_authenticates_before_ipc_and_encrypts_datagrams() { - use tokio::io::AsyncReadExt; - use yas_composite_transport::{Offer, Role}; - let _ = rustls::crypto::ring::default_provider().install_default(); - tokio::time::timeout(Duration::from_secs(15), async { - let dir = tempfile::tempdir().unwrap(); - let (server_key, server_public) = yas_uplink::Identity::generate().unwrap(); - let (client_key, client_public) = yas_uplink::Identity::generate().unwrap(); - let (attacker_key, _) = yas_uplink::Identity::generate().unwrap(); - let crypto = yas_uplink::Identity::from_base64(&server_key) - .unwrap() - .server_config(vec![client_public]) - .unwrap(); - let client_crypto = yas_uplink::Identity::from_base64(&client_key) - .unwrap() - .client_config(server_public) - .unwrap(); - let attacker_crypto = yas_uplink::Identity::from_base64(&attacker_key) - .unwrap() - .client_config(server_public) - .unwrap(); - let socket = dir.path().join("local.sock"); - let listener = tokio::net::UnixListener::bind(&socket).unwrap(); - - let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); - let hash = ring::digest::digest(&ring::digest::SHA256, cert.cert.der()); - let mut worker = wt::ServerBuilder::new() - .with_addr("127.0.0.1:0".parse().unwrap()) - .with_certificate( - vec![cert.cert.der().clone()], - rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()) - .into(), - ) - .unwrap(); - let outer = build_client(Some(hash.as_ref())).unwrap(); - let url: url::Url = - format!("https://127.0.0.1:{}/", worker.local_addr().unwrap().port()) - .parse() - .unwrap(); - let (local, remote) = tokio::join!(outer.connect(url), async { - worker.accept().await.unwrap().ok().await.unwrap() - }); - let local = local.unwrap(); - let routes = DatagramRoutes::new(16); - let datagrams = tokio::spawn(distribute_datagrams(local.clone(), routes.clone())); - - for case in 0..4 { - let local = local.clone(); - let routes = routes.clone(); - let crypto = crypto.clone(); - let socket = socket.to_str().unwrap().to_string(); - let producer = tokio::spawn(async move { - let (send, recv) = local.accept_bi().await.unwrap(); - let permit = Arc::new(tokio::sync::Semaphore::new(1)) - .acquire_owned() - .await - .unwrap(); - bridge(local, routes, send, recv, crypto, permit, socket).await; - }); - let (send, recv) = remote.open_bi().await.unwrap(); - let mut stream = tokio::io::join(recv, send); - if case == 0 { - // A relay can synthesize protocol bytes, but those bytes - // must never open a local socket before Noise authentication. - stream - .write_all(yas_wire::PREFACE.as_slice()) - .await - .unwrap(); - stream.shutdown().await.unwrap(); - drop(stream); - producer.await.unwrap(); - assert!( - tokio::time::timeout(Duration::from_millis(25), listener.accept()) - .await - .is_err() - ); - continue; - } - if case == 1 { - assert!( - yas_uplink::connect(stream, attacker_crypto.clone()) - .await - .is_err() - ); - producer.await.unwrap(); - assert!( - tokio::time::timeout(Duration::from_millis(25), listener.accept()) - .await - .is_err() - ); - continue; - } - let mut stream = yas_uplink::connect(stream, client_crypto.clone()) - .await - .unwrap(); - // Authentication alone also does not open IPC: the encrypted - // YAS preface or composite selector must follow first. - assert!( - tokio::time::timeout(Duration::from_millis(25), listener.accept()) - .await - .is_err() - ); - if case == 2 { - stream - .write_all(yas_wire::PREFACE.as_slice()) - .await - .unwrap(); - stream.write_all(b"command").await.unwrap(); - stream.flush().await.unwrap(); - let (mut ipc, _) = listener.accept().await.unwrap(); - let mut received = vec![0; yas_wire::PREFACE.len() + 7]; - ipc.read_exact(&mut received).await.unwrap(); - assert_eq!( - received, - [yas_wire::PREFACE.as_slice(), b"command"].concat() - ); - ipc.write_all(b"reply").await.unwrap(); - ipc.shutdown().await.unwrap(); - let mut reply = Vec::new(); - stream.read_to_end(&mut reply).await.unwrap(); - assert_eq!(reply, b"reply"); - stream.shutdown().await.unwrap(); - producer.await.unwrap(); - continue; - } - - let token = [0x35; 16]; - let maximum = 512; - let material = stream.datagram_key_material(); - let (mut sender, mut receiver) = yas_uplink::datagram_pair(material, token, true); - yas_composite_transport::write_offer( - &mut stream, - Offer::new(Role::Main, token, maximum).unwrap(), - ) - .await - .unwrap(); - stream.flush().await.unwrap(); - let (mut main, _) = listener.accept().await.unwrap(); - let (mut side, _) = listener.accept().await.unwrap(); - for (socket, expected) in [(&mut main, Role::Main), (&mut side, Role::Datagram)] { - match yas_composite_transport::classify(socket).await.unwrap() { - yas_composite_transport::Ingress::Composite { offer, .. } => { - assert_eq!(offer.role, expected); - assert_eq!(offer.token, token); - assert_eq!(offer.max_datagram, maximum); - } - _ => panic!("missing local composite selector"), - } - } - // A reliable reply also synchronizes route registration. - main.write_all(b"ready").await.unwrap(); - let mut ready = [0; 5]; - stream.read_exact(&mut ready).await.unwrap(); - assert_eq!(&ready, b"ready"); - - let encrypted = sender.seal(b"private datagram").unwrap(); - let wire_maximum = maximum + yas_uplink::DATAGRAM_OVERHEAD as u32; - let mut forged = encrypted.clone(); - *forged.last_mut().unwrap() ^= 1; - for bytes in [&forged, &encrypted, &encrypted] { - let routed = - yas_composite_transport::encode_routed_datagram(token, bytes, wire_maximum) - .unwrap(); - remote.send_datagram(routed.into()).unwrap(); - } - assert_eq!( - yas_composite_transport::read_datagram(&mut side, maximum) - .await - .unwrap(), - b"private datagram" - ); - assert!( - tokio::time::timeout( - Duration::from_millis(50), - yas_composite_transport::read_datagram(&mut side, maximum) - ) - .await - .is_err() - ); - - yas_composite_transport::write_datagram(&mut side, b"private reply", maximum) - .await - .unwrap(); - let packet = remote.read_datagram().await.unwrap(); - let (received_token, ciphertext) = - yas_composite_transport::split_routed_datagram(&packet).unwrap(); - assert_eq!(received_token, token); - assert!(!ciphertext.windows(13).any(|w| w == b"private reply")); - assert_eq!(receiver.open(ciphertext).unwrap(), b"private reply"); - stream.shutdown().await.unwrap(); - main.shutdown().await.unwrap(); - producer.await.unwrap(); - } - datagrams.abort(); - local.close(0, b"test complete"); + Producer::new(&url, &token, crypto, local)? + .transport(transport) + .on_event(|event| eprintln!("[uplink] {event}")) + .run_until(async { + let _ = tokio::signal::ctrl_c().await; }) .await - .expect("producer uplink test stalled"); - } - - #[test] - fn parse_pool_accepts_plain_and_pinned_relay_urls() { - let pool = parse_pool( - r#"{"relays":[ - "https://relay-1.indent.com:4443/t/kfV3aB", - "https://[2001:db8::7]/session?key=x#sha256=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - ],"ttl":60}"#, - ) - .unwrap(); - assert_eq!(pool.len(), 2); - assert_eq!( - pool[0].url.as_str(), - "https://relay-1.indent.com:4443/t/kfV3aB" - ); - assert_eq!(pool[0].label, "relay-1.indent.com:4443"); - assert!(pool[0].cert_hash.is_none()); - assert_eq!(pool[1].label, "[2001:db8::7]:443"); - assert_eq!(pool[1].cert_hash.as_ref().unwrap().len(), 32); - // The pin must be stripped before the URL is used to connect. - assert_eq!(pool[1].url.fragment(), None); - assert_eq!(pool[1].url.query(), Some("key=x")); - } - - #[test] - fn parse_pool_rejects_bad_input() { - assert!(parse_pool("not json").is_err()); - assert!(parse_pool(r#"{"relays":[]}"#).is_err()); - assert!(parse_pool(r#"{"relays":[{"host":"h"}]}"#).is_err()); - assert!( - parse_relay("http://relay.example/t/x").is_err(), - "non-https scheme must be rejected" - ); - assert!( - parse_relay("https://relay.example/t/x#sha256=AAAA").is_err(), - "a pin of the wrong length must be rejected, not ignored" - ); - assert!( - parse_relay("https://relay.example/t/x#pin=abc").is_err(), - "an unrecognized fragment must be rejected, not ignored" - ); - } - - #[test] - fn base64url_decodes() { - assert_eq!(base64url_decode("aGVsbG8").unwrap(), b"hello"); - assert_eq!(base64url_decode("aGVsbG8=").unwrap(), b"hello"); - assert_eq!(base64url_decode("_-8").unwrap(), vec![0xff, 0xef]); - assert!(base64url_decode("a+b").is_none()); - assert_eq!(base64url_decode("").unwrap(), Vec::::new()); - } } diff --git a/crates/cli/tests/support/uplink.rs b/crates/cli/tests/support/uplink.rs index 0116544f9..92194b18a 100644 --- a/crates/cli/tests/support/uplink.rs +++ b/crates/cli/tests/support/uplink.rs @@ -1,5 +1,9 @@ +// Shared by the uplink E2E test and the browser fixture, which use different parts. +#![allow(dead_code)] + use futures_util::{SinkExt, StreamExt}; use std::{ + collections::HashMap, path::Path, process::Stdio, sync::{Arc, Mutex}, @@ -9,11 +13,18 @@ use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, process::Command, + sync::{mpsc, oneshot}, }; use tokio_rustls::TlsAcceptor; -use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::tungstenite::{ + Message, + handshake::server::{Request, Response}, +}; use web_transport_quinn as wt; +/// The WebSocket relay protocol's subprotocol (docs/uplink.md). +const WEBSOCKET_SUBPROTOCOL: &str = "yas-uplink.v1"; + pub fn cli(binary: &Path, directory: &Path, ca: &Path) -> Command { let mut command = Command::new(binary); command @@ -42,10 +53,41 @@ pub fn cli(binary: &Path, directory: &Path, ca: &Path) -> Command { .env("YAS_CHANNEL", "0") .env("YAS_REMOTES", directory.join("remotes")) .env_remove("YAS_TARGET") + .env_remove("YAS_UPLINK_TRANSPORT") .stdin(Stdio::null()); command } +/// What carries the producer's relay session in a fixture. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Carrier { + /// A pool of WebTransport relays only, as older control endpoints give. + WebTransport, + /// A pool of both kinds, the producer forced onto WebSockets. + WebSocket, + /// A pool of both kinds whose WebTransport relay's UDP goes nowhere: the + /// producer (on its default transport) must fall back to WebSockets. + Fallback, +} + +/// The producer's relay session, as the relay holds it. +#[derive(Clone)] +enum ProducerLink { + WebTransport(Box), + /// Stream requests to send on the session WebSocket. + WebSocket(mpsc::UnboundedSender), +} + +type StreamSocket = + tokio_tungstenite::WebSocketStream>; + +#[derive(Default)] +struct RelayState { + producer: Option, + next_stream: u64, + streams: HashMap>, +} + pub struct Fixture { pub directory: tempfile::TempDir, pub ca: std::path::PathBuf, @@ -55,12 +97,19 @@ pub struct Fixture { pub capture: Arc>>, pub requests: Arc>>, pub producer: tokio::process::Child, + /// What the producer printed on stderr (it goes to the test's stderr too). + pub producer_log: Arc>>, _server: tokio::process::Child, + _black_hole: Option, tasks: Vec>, } impl Fixture { pub async fn start(binary: &Path) -> Self { + Self::start_with(binary, Carrier::WebTransport).await + } + + pub async fn start_with(binary: &Path, carrier: Carrier) -> Self { let _ = rustls::crypto::ring::default_provider().install_default(); let directory = tempfile::tempdir().unwrap(); let root = directory.path(); @@ -82,14 +131,35 @@ impl Fixture { .with_addr("127.0.0.1:0".parse().unwrap()) .with_certificate(vec![cert.cert.der().clone()], key()) .unwrap(); - let relay_url = format!( - "https://127.0.0.1:{}/producer", - worker.local_addr().unwrap().port() - ); + // A UDP socket that never answers stands for a network dropping UDP. + let black_hole = (carrier == Carrier::Fallback) + .then(|| std::net::UdpSocket::bind("127.0.0.1:0").unwrap()) + .map(|socket| { + socket.set_nonblocking(true).unwrap(); + tokio::net::UdpSocket::from_std(socket).unwrap() + }); + let relay_url = match &black_hole { + Some(hole) => format!( + "https://127.0.0.1:{}/producer", + hole.local_addr().unwrap().port() + ), + None => format!( + "https://127.0.0.1:{}/producer", + worker.local_addr().unwrap().port() + ), + }; let control = TcpListener::bind("127.0.0.1:0").await.unwrap(); let control_url = format!("https://{}", control.local_addr().unwrap()); let websocket = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let ws_url = format!("wss://{}/consumer", websocket.local_addr().unwrap()); + let websocket_origin = format!("wss://{}", websocket.local_addr().unwrap()); + let ws_url = format!("{websocket_origin}/consumer"); + let pool = match carrier { + Carrier::WebTransport => serde_json::json!({ "relays": [relay_url] }), + Carrier::WebSocket | Carrier::Fallback => serde_json::json!({ + "relays": [relay_url], + "websockets": [format!("{websocket_origin}/producer")], + }), + }; let requests = Arc::new(Mutex::new(Vec::new())); let seen = requests.clone(); let control_tls = tls.clone(); @@ -97,7 +167,7 @@ impl Fixture { loop { let (stream, _) = control.accept().await.unwrap(); let tls = control_tls.clone(); - let relay_url = relay_url.clone(); + let pool = pool.to_string(); let ws_url = ws_url.clone(); let seen = seen.clone(); tokio::spawn(async move { @@ -114,10 +184,7 @@ impl Fixture { && header.contains("authorization: bearer producer-token\r\n") { seen.lock().unwrap().push("allocate"); - ( - "200 OK", - serde_json::json!({"relays": [relay_url]}).to_string(), - ) + ("200 OK", pool) } else if header.starts_with("get /attach ") && header.contains("authorization: bearer consumer-token\r\n") { @@ -158,7 +225,21 @@ impl Fixture { ); tokio::time::sleep(Duration::from_millis(20)).await; } - let mut producer = cli(binary, root, &ca) + + let state = Arc::new(Mutex::new(RelayState::default())); + let connected = Arc::new(tokio::sync::Notify::new()); + let capture = Arc::new(Mutex::new(Vec::new())); + let relay_task = tokio::spawn(serve_websockets( + websocket, + tls, + websocket_origin, + state.clone(), + connected.clone(), + capture.clone(), + )); + + let mut command = cli(binary, root, &ca); + command .env("YAS_UPLINK_IDENTITY", &producer_key) .env("YAS_UPLINK_TOKEN", "producer-token") .args([ @@ -168,61 +249,47 @@ impl Fixture { &consumer_public.to_string(), ]) .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn() - .unwrap(); - let session = tokio::select! { - session = async { worker.accept().await.unwrap().ok().await.unwrap() } => session, - status = producer.wait() => panic!("uplink producer exited before connecting: {status:?}"), - }; - let capture = Arc::new(Mutex::new(Vec::new())); - let captured = capture.clone(); - let relay_task = tokio::spawn(async move { - loop { - let (stream, _) = websocket.accept().await.unwrap(); - let tls = tls.clone(); - let session = session.clone(); - let captured = captured.clone(); - tokio::spawn(async move { - let stream = tls.accept(stream).await.unwrap(); - let mut ws = tokio_tungstenite::accept_async(stream).await.unwrap(); - assert_eq!( - ws.next().await.unwrap().unwrap(), - Message::Text("consumer-token".into()) - ); - ws.send(Message::Text("ok".into())).await.unwrap(); - let (mut send, mut recv) = session.open_bi().await.unwrap(); - let (mut ws_send, mut ws_recv) = ws.split(); - let upstream = async { - while let Some(Ok(Message::Binary(bytes))) = ws_recv.next().await { - captured.lock().unwrap().extend_from_slice(&bytes); - if send.write_all(&bytes).await.is_err() { - break; - } - } - let _ = send.shutdown().await; - }; - let downstream = async { - let mut buffer = [0; 16384]; - while let Ok(Some(count)) = recv.read(&mut buffer).await { - captured.lock().unwrap().extend_from_slice(&buffer[..count]); - if count == 0 { - break; - } - if ws_send - .send(Message::Binary(buffer[..count].to_vec().into())) - .await - .is_err() - { - break; - } - } - let _ = ws_send.close().await; - }; - tokio::join!(upstream, downstream); - }); + .stderr(Stdio::piped()); + if carrier == Carrier::WebSocket { + command.env("YAS_UPLINK_TRANSPORT", "websocket"); + } + let mut producer = command.spawn().unwrap(); + let producer_log = Arc::new(Mutex::new(Vec::new())); + let log = producer_log.clone(); + let stderr = producer.stderr.take().unwrap(); + let log_task = tokio::spawn(async move { + use tokio::io::AsyncBufReadExt; + let mut lines = tokio::io::BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + eprintln!("{line}"); + log.lock().unwrap().push(line); } }); + match carrier { + Carrier::WebTransport => { + let session = tokio::select! { + session = async { worker.accept().await.unwrap().ok().await.unwrap() } => session, + status = producer.wait() => panic!("uplink producer exited before connecting: {status:?}"), + }; + state.lock().unwrap().producer = + Some(ProducerLink::WebTransport(Box::new(session))); + } + Carrier::WebSocket | Carrier::Fallback => { + let waiting = async { + loop { + let notified = connected.notified(); + if state.lock().unwrap().producer.is_some() { + break; + } + notified.await; + } + }; + tokio::select! { + () = waiting => {} + status = producer.wait() => panic!("uplink producer exited before connecting: {status:?}"), + } + } + } // Invoke the documented URL generator; the consumer resolves /attach // and uses the normal WSS connector, with no test-only transport hooks. let generated = cli(binary, root, &ca) @@ -255,12 +322,203 @@ impl Fixture { capture, requests, producer, + producer_log, _server: server, - tasks: vec![control_task, relay_task], + _black_hole: black_hole, + tasks: vec![control_task, relay_task, log_task], } } } +/// The relay's WebSocket side: consumers (`/consumer`), the producer's session +/// (`/producer`) and the stream WebSockets it opens (`/stream/`). +async fn serve_websockets( + listener: TcpListener, + tls: TlsAcceptor, + origin: String, + state: Arc>, + connected: Arc, + capture: Arc>>, +) { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let tls = tls.clone(); + let origin = origin.clone(); + let state = state.clone(); + let connected = connected.clone(); + let capture = capture.clone(); + tokio::spawn(async move { + let Ok(stream) = tls.accept(stream).await else { + return; + }; + let path = Arc::new(Mutex::new(String::new())); + let seen = path.clone(); + // tungstenite's callback signature: its error is an HTTP response. + #[allow(clippy::result_large_err)] + let callback = move |request: &Request, mut response: Response| { + *seen.lock().unwrap() = request.uri().path().to_owned(); + let offered = request + .headers() + .get("sec-websocket-protocol") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if offered + .split(',') + .any(|each| each.trim() == WEBSOCKET_SUBPROTOCOL) + { + response.headers_mut().insert( + "sec-websocket-protocol", + WEBSOCKET_SUBPROTOCOL.parse().unwrap(), + ); + } + Ok(response) + }; + let Ok(ws) = tokio_tungstenite::accept_hdr_async(stream, callback).await else { + return; + }; + let path = path.lock().unwrap().clone(); + if path == "/producer" { + let (mut sink, mut source) = ws.split(); + let (requests, mut pending) = mpsc::unbounded_channel(); + state.lock().unwrap().producer = Some(ProducerLink::WebSocket(requests)); + connected.notify_waiters(); + loop { + tokio::select! { + request = pending.recv() => match request { + Some(message) => if sink.send(message).await.is_err() { break }, + None => break, + }, + message = source.next() => match message { + Some(Ok(_)) => {} + _ => break, + }, + } + } + } else if let Some(id) = path.strip_prefix("/stream/") { + let waiting = state.lock().unwrap().streams.remove(id); + if let Some(waiting) = waiting { + let _ = waiting.send(ws); + } + } else if path == "/consumer" { + serve_consumer(ws, &origin, &state, &capture).await; + } + }); + } +} + +async fn serve_consumer( + mut ws: StreamSocket, + origin: &str, + state: &Arc>, + captured: &Arc>>, +) { + assert_eq!( + ws.next().await.unwrap().unwrap(), + Message::Text("consumer-token".into()) + ); + ws.send(Message::Text("ok".into())).await.unwrap(); + let link = state.lock().unwrap().producer.clone().expect("a producer"); + let (mut recv, mut send): ( + Box, + Box, + ) = match link { + ProducerLink::WebTransport(session) => { + let (send, recv) = session.open_bi().await.unwrap(); + (Box::new(recv), Box::new(send)) + } + ProducerLink::WebSocket(requests) => { + let (tx, rx) = oneshot::channel(); + let id = { + let mut state = state.lock().unwrap(); + state.next_stream += 1; + let id = state.next_stream.to_string(); + state.streams.insert(id.clone(), tx); + id + }; + let url = format!("{origin}/stream/{id}"); + requests + .send(Message::Text( + serde_json::json!({ "stream": url }).to_string().into(), + )) + .unwrap(); + let stream = tokio::time::timeout(Duration::from_secs(10), rx) + .await + .expect("the producer opened no stream WebSocket") + .unwrap(); + let (ours, theirs) = tokio::io::duplex(128 << 10); + tokio::spawn(pump(stream, ours)); + let (read, write) = tokio::io::split(theirs); + (Box::new(read), Box::new(write)) + } + }; + let (mut ws_send, mut ws_recv) = ws.split(); + let upstream = async { + while let Some(Ok(Message::Binary(bytes))) = ws_recv.next().await { + captured.lock().unwrap().extend_from_slice(&bytes); + if send.write_all(&bytes).await.is_err() { + break; + } + } + let _ = send.shutdown().await; + }; + let downstream = async { + let mut buffer = [0; 16384]; + loop { + match recv.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(count) => { + captured.lock().unwrap().extend_from_slice(&buffer[..count]); + if ws_send + .send(Message::Binary(buffer[..count].to_vec().into())) + .await + .is_err() + { + break; + } + } + } + } + let _ = ws_send.close().await; + }; + tokio::join!(upstream, downstream); +} + +/// Bytes between a stream WebSocket and a duplex pipe, as a relay forwards them. +async fn pump(socket: StreamSocket, pipe: tokio::io::DuplexStream) { + let (mut sink, mut source) = socket.split(); + let (mut read, mut write) = tokio::io::split(pipe); + let up = async { + while let Some(Ok(message)) = source.next().await { + match message { + Message::Binary(bytes) => { + if write.write_all(&bytes).await.is_err() { + break; + } + } + Message::Close(_) => break, + _ => {} + } + } + let _ = write.shutdown().await; + }; + let down = async { + let mut buffer = vec![0; 16 << 10]; + loop { + match read.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(count) => { + let chunk = buffer[..count].to_vec(); + if sink.send(Message::Binary(chunk.into())).await.is_err() { + break; + } + } + } + } + let _ = sink.close().await; + }; + tokio::join!(up, down); +} + impl Drop for Fixture { fn drop(&mut self) { let _ = self.producer.start_kill(); diff --git a/crates/cli/tests/uplink_e2e.rs b/crates/cli/tests/uplink_e2e.rs index 1c08c5419..61fe12d86 100644 --- a/crates/cli/tests/uplink_e2e.rs +++ b/crates/cli/tests/uplink_e2e.rs @@ -4,18 +4,64 @@ #[path = "support/uplink.rs"] mod uplink; use std::{path::Path, time::Duration}; -use uplink::{Fixture, cli}; +use uplink::{Carrier, Fixture, cli}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn uplink_cli_remote_execution_and_authentication() { - tokio::time::timeout(Duration::from_secs(45), exercise_uplink()) + tokio::time::timeout( + Duration::from_secs(45), + exercise_uplink(Carrier::WebTransport), + ) + .await + .expect("uplink E2E stalled"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn uplink_over_websockets_when_forced() { + tokio::time::timeout(Duration::from_secs(45), exercise_uplink(Carrier::WebSocket)) + .await + .expect("uplink E2E over WebSockets stalled"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn uplink_falls_back_to_websockets_where_udp_goes_nowhere() { + tokio::time::timeout(Duration::from_secs(60), exercise_uplink(Carrier::Fallback)) .await - .expect("uplink E2E stalled"); + .expect("uplink E2E falling back to WebSockets stalled"); } -async fn exercise_uplink() { +async fn exercise_uplink(carrier: Carrier) { let binary = Path::new(env!("CARGO_BIN_EXE_yas")); - let mut fixture = Fixture::start(binary).await; + let mut fixture = Fixture::start_with(binary, carrier).await; + { + // The relay sees the session before the producer's line about it is read. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let log = loop { + let log = fixture.producer_log.lock().unwrap().clone(); + let connected = log + .iter() + .any(|line| line.starts_with("[uplink] connected to relay")); + if connected || std::time::Instant::now() > deadline { + assert!(connected, "producer log: {log:?}"); + break log; + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let over_websocket = log.iter().any(|line| { + line.starts_with("[uplink] connected to relay") && line.ends_with(" over WebSocket") + }); + assert_eq!( + over_websocket, + carrier != Carrier::WebTransport, + "producer log: {log:?}" + ); + let gave_up_on_udp = log.iter().any(|line| line.contains("(is UDP blocked?)")); + assert_eq!( + gave_up_on_udp, + carrier == Carrier::Fallback, + "producer log: {log:?}" + ); + } let root = fixture.directory.path(); let ca = &fixture.ca; let uri = &fixture.uri; diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 2ecfeed3a..67ae4488e 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -75,6 +75,9 @@ pub use client::{Client, DEFAULT_REQUEST_TIMEOUT}; pub use error::{Error, Result, format_result_detail}; pub use options::{ConnectOptions, HelloOptions, read_only_extension}; +/// The producer side of YAS uplinks (`yas uplink`): publish a YAS server +/// through a relay, over WebTransport or WebSockets. +pub use yas_proxy::uplink_producer; /// The wire codecs this crate speaks (`yas-wire`), for the escape hatches /// ([`Client::request`], [`Client::request_raw`]). pub use yas_wire as wire; diff --git a/crates/proxy/Cargo.toml b/crates/proxy/Cargo.toml index 0ad40c22f..6eb4f038a 100644 --- a/crates/proxy/Cargo.toml +++ b/crates/proxy/Cargo.toml @@ -54,3 +54,4 @@ libc = "0.2" [dev-dependencies] tempfile = "3" rcgen = "0.14" +tokio-rustls = { version = "0.26", default-features = false, features = ["tls12"] } diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index 42cb43eb2..07d17136d 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -782,6 +782,8 @@ fn parse_uplink_uri(rest: &str) -> Result { }) } +pub mod uplink_producer; + /// HTTPS control client with the same explicit CA override semantics as the /// WSS and WebTransport legs. Reqwest's platform verifier otherwise ignores /// SSL_CERT_FILE/SSL_CERT_DIR on macOS. diff --git a/crates/proxy/src/uplink_producer.rs b/crates/proxy/src/uplink_producer.rs new file mode 100644 index 000000000..773db6a4d --- /dev/null +++ b/crates/proxy/src/uplink_producer.rs @@ -0,0 +1,1361 @@ +//! The producer side of a YAS uplink (`yas uplink`, docs/uplink.md), as a +//! library: publish a local YAS server through an untrusted relay. +//! +//! A [`Producer`] authenticates to an HTTPS control endpoint, receives a pool +//! of relays, holds a session with one of them, and bridges each consumer the +//! relay sends it to a fresh connection to the local server, once the consumer +//! has passed end-to-end Noise IK authentication against pinned keys. The +//! session is carried either by WebTransport (HTTP/3 over UDP) or by +//! WebSockets (over TCP, where networks block UDP); [`Transport`] chooses, and +//! [`Transport::Auto`] falls back from one to the other by itself. +//! +//! ```no_run +//! # async fn run() -> Result<(), String> { +//! use yas_proxy::uplink_producer::{Local, Producer, Transport}; +//! +//! let identity = yas_uplink::Identity::from_base64(&std::env::var("YAS_UPLINK_IDENTITY").unwrap())?; +//! let allowed = vec!["CLIENT_PUBLIC_KEY_43_CHARACTERS_OF_BASE64URL".parse()?]; +//! Producer::new( +//! "https://relay.example/uplink/control", +//! &std::env::var("YAS_UPLINK_TOKEN").unwrap(), +//! identity.server_config(allowed)?, +//! Local::socket("/run/user/1000/yas/yas.sock"), +//! )? +//! .transport(Transport::from_env()?) +//! .on_event(|event| eprintln!("[uplink] {event}")) +//! .run_until(async { tokio::signal::ctrl_c().await.ok(); }) +//! .await +//! # } +//! ``` + +use std::fmt; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use futures_util::{SinkExt, StreamExt}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; +use web_transport_quinn as wt; + +/// The keys a producer is configured with. +pub use yas_uplink::{Identity, PublicKey, ServerConfig}; + +/// The WebSocket subprotocol of the uplink's WebSocket relay sessions and +/// streams (docs/uplink.md, "WebSocket relay session"), with its version. +pub const WEBSOCKET_SUBPROTOCOL: &str = "yas-uplink.v1"; + +/// The environment variable [`Transport::from_env`] reads. +pub const TRANSPORT_ENV: &str = "YAS_UPLINK_TRANSPORT"; + +/// WebTransport application code closing a session on shutdown. +const CODE_SHUTDOWN: u32 = 2; +const INITIAL_BACKOFF: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(60); +const MAX_DATAGRAM_ROUTES: usize = 4_096; +const DATAGRAM_QUEUE: usize = 64; +/// Consumer streams authenticating at once on one session. +const MAX_PENDING: usize = 64; +/// Liveness of a session: a keepalive this often, dead after this long silent. +const KEEPALIVE: Duration = Duration::from_secs(10); +const IDLE_TIMEOUT: Duration = Duration::from_secs(30); +/// How long a WebSocket (session or stream) may take to open. +const WEBSOCKET_CONNECT: Duration = Duration::from_secs(10); +/// How long [`Transport::Auto`] waits for a WebTransport session before it +/// tries the pool's WebSocket relays. Where UDP is dropped rather than +/// refused, QUIC would otherwise wait out its whole idle timeout. +const WEBTRANSPORT_CONNECT_AUTO: Duration = Duration::from_secs(5); +/// A WebTransport session shorter than this counts as WebTransport failing. +const SHORT_SESSION: Duration = Duration::from_secs(60); +/// How long [`Transport::Auto`] tries WebSockets first after WebTransport failed. +const PREFER_WEBSOCKET_FOR: Duration = Duration::from_secs(600); +/// The largest WebSocket message either way. +const MAX_WEBSOCKET_MESSAGE: usize = 64 << 10; + +type DatagramRoutes = yas_composite_transport::RoutedDatagramRoutes; +type BoxRead = Box; +type BoxWrite = Box; +type Socket = + tokio_tungstenite::WebSocketStream>; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +/// What carries the relay session. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Transport { + /// WebTransport when it works, else WebSockets (when the control endpoint + /// offers WebSocket relays): the default. + #[default] + Auto, + /// WebTransport (HTTP/3 over UDP) only. + WebTransport, + /// WebSockets (over TCP) only. + WebSocket, +} + +impl Transport { + /// `YAS_UPLINK_TRANSPORT` (`auto`, `webtransport` or `websocket`), else + /// [`Transport::Auto`]. + pub fn from_env() -> Result { + match std::env::var(TRANSPORT_ENV) { + Ok(value) if !value.trim().is_empty() => value.parse(), + Ok(_) | Err(std::env::VarError::NotPresent) => Ok(Self::Auto), + Err(std::env::VarError::NotUnicode(_)) => Err(format!( + "{TRANSPORT_ENV} must be auto, webtransport or websocket" + )), + } + } +} + +impl std::str::FromStr for Transport { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "auto" => Ok(Self::Auto), + "webtransport" => Ok(Self::WebTransport), + "websocket" => Ok(Self::WebSocket), + other => Err(format!( + "unknown uplink transport {other:?}: auto, webtransport or websocket" + )), + } + } +} + +impl fmt::Display for Transport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Auto => "auto", + Self::WebTransport => "webtransport", + Self::WebSocket => "websocket", + }) + } +} + +/// What carries one relay session. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Carrier { + WebTransport, + WebSocket, +} + +impl fmt::Display for Carrier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::WebTransport => "WebTransport", + Self::WebSocket => "WebSocket", + }) + } +} + +type Connect = dyn Fn() -> Pin> + Send>> + + Send + + Sync; + +/// How the producer reaches the local YAS server, once per consumer (twice for +/// a consumer with a datagram lane). +#[derive(Clone)] +pub struct Local { + label: String, + connect: Arc, +} + +impl Local { + /// The YAS server listening at `path`: a Unix socket, or a named pipe on + /// Windows. + pub fn socket(path: impl Into) -> Self { + let path = path.into(); + let label = path.clone(); + Self { + label, + connect: Arc::new(move || { + let path = path.clone(); + Box::pin(async move { connect_socket_split(&path).await }) + }), + } + } + + /// Any other way to reach the server (an in-process one, say): `connect` + /// opens a fresh byte stream to it each time; `label` names it in events. + pub fn custom(label: impl Into, connect: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + Self { + label: label.into(), + connect: Arc::new(move || { + let connecting = connect(); + Box::pin(async move { + let stream = connecting.await.map_err(|error| error.to_string())?; + let (reader, writer) = tokio::io::split(stream); + Ok((Box::new(reader) as BoxRead, Box::new(writer) as BoxWrite)) + }) + }), + } + } + + async fn connect(&self) -> Result<(BoxRead, BoxWrite), String> { + (self.connect)().await + } +} + +impl fmt::Debug for Local { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Local").field("label", &self.label).finish() + } +} + +async fn connect_socket_split(path: &str) -> Result<(BoxRead, BoxWrite), String> { + #[cfg(unix)] + { + let stream = tokio::net::UnixStream::connect(path) + .await + .map_err(|error| error.to_string())?; + let (reader, writer) = tokio::io::split(stream); + Ok((Box::new(reader), Box::new(writer))) + } + #[cfg(windows)] + { + // Every instance may be taken for a while (the server makes its next + // one only once it accepts the last): wait out ERROR_PIPE_BUSY. + use tokio::net::windows::named_pipe::ClientOptions; + const ERROR_PIPE_BUSY: i32 = 231; + let deadline = Instant::now() + Duration::from_secs(10); + let pipe = loop { + match ClientOptions::new().open(path) { + Err(error) + if error.raw_os_error() == Some(ERROR_PIPE_BUSY) + && Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(20)).await; + } + opened => break opened.map_err(|error| error.to_string())?, + } + }; + let (reader, writer) = tokio::io::split(pipe); + Ok((Box::new(reader), Box::new(writer))) + } +} + +/// What happens to a producer, for logs and status. Displayed, each is the +/// text `yas uplink` prints after `[uplink] `. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum Event { + /// A relay session is up: consumers can reach the server. `relay` is the + /// relay's `host:port` (its URL is a credential and never shown). + Connected { relay: String, carrier: Carrier }, + /// The session that was up ended; the producer asks the control endpoint + /// for a new pool at once. + SessionEnded { carrier: Carrier, reason: String }, + /// A relay of the pool didn't take a session; the next one is tried. + RelayFailed { + relay: String, + carrier: Carrier, + error: String, + }, + /// No relay of the pool took a session. + PoolExhausted { retry_in: Duration }, + /// The control endpoint couldn't give a pool (unreachable, an error, a + /// pool naming no relay the transport may use). + ControlRetry { reason: String, retry_in: Duration }, + /// An authenticated consumer couldn't reach the local server. + LocalUnavailable { local: String, error: String }, + /// A consumer stream couldn't be served (a WebSocket stream the relay + /// asked for that didn't open, a datagram lane this session can't carry). + StreamFailed { error: String }, +} + +impl fmt::Display for Event { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let over = |carrier: &Carrier| match carrier { + Carrier::WebTransport => "", + Carrier::WebSocket => " over WebSocket", + }; + match self { + Self::Connected { relay, carrier } => { + write!(f, "connected to relay {relay}{}", over(carrier)) + } + Self::SessionEnded { reason, .. } => write!(f, "session ended: {reason}"), + Self::RelayFailed { + relay, + carrier, + error, + } => write!(f, "relay {relay}{}: {error}", over(carrier)), + Self::PoolExhausted { retry_in } => write!( + f, + "relay pool exhausted; re-querying in {}s", + retry_in.as_secs() + ), + Self::ControlRetry { reason, retry_in } => { + write!(f, "{reason}; retrying in {}s", retry_in.as_secs()) + } + Self::LocalUnavailable { local, error } => { + write!(f, "local yas server unavailable at {local}: {error}") + } + Self::StreamFailed { error } => write!(f, "consumer stream failed: {error}"), + } + } +} + +type Events = Arc; + +/// A YAS server published through a relay. See the module documentation. +#[derive(Clone)] +pub struct Producer { + control: url::Url, + shared: Arc, + local: Local, + transport: Transport, + events: Events, +} + +/// What a running producer takes from its [`Handle`]. +struct Shared { + token: Mutex, + crypto: Mutex>, +} + +impl Shared { + fn token(&self) -> String { + self.token.lock().unwrap_or_else(|p| p.into_inner()).clone() + } + + fn crypto(&self) -> Arc { + self.crypto + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } +} + +/// Changes a running [`Producer`] takes without restarting its session. +#[derive(Clone)] +pub struct Handle(Arc); + +impl Handle { + /// The token the next control request presents (a renewed one, say). + pub fn set_token(&self, token: impl Into) { + *self.0.token.lock().unwrap_or_else(|p| p.into_inner()) = token.into(); + } + + /// Who may connect from now on: consumers that start their handshake + /// afterwards are checked against `crypto`'s allowlist (and answered with + /// its identity). Consumers already connected keep their authority until + /// they disconnect, as they do across a restart. + pub fn set_server_config(&self, crypto: Arc) { + *self.0.crypto.lock().unwrap_or_else(|p| p.into_inner()) = crypto; + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Handle").finish_non_exhaustive() + } +} + +impl fmt::Debug for Producer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The token and keys are credentials: never shown. + f.debug_struct("Producer") + .field("control", &label(&self.control)) + .field("local", &self.local) + .field("transport", &self.transport) + .finish_non_exhaustive() + } +} + +impl Producer { + /// A producer authenticating to `control` (HTTPS, without userinfo or a + /// fragment) with `token`, admitting the consumers `crypto` allows + /// ([`yas_uplink::Identity::server_config`]), bridged to `local`. + pub fn new( + control: &str, + token: &str, + crypto: Arc, + local: Local, + ) -> Result { + let control = url::Url::parse(control).map_err(|_| "invalid uplink control URL")?; + if control.scheme() != "https" + || control.host_str().is_none() + || !control.username().is_empty() + || control.password().is_some() + || control.fragment().is_some() + { + return Err("uplink control URL must be HTTPS without userinfo or a fragment".into()); + } + if token.is_empty() { + return Err("YAS_UPLINK_TOKEN is not set".into()); + } + Ok(Self { + control, + shared: Arc::new(Shared { + token: Mutex::new(token.to_owned()), + crypto: Mutex::new(crypto), + }), + local, + transport: Transport::Auto, + events: Arc::new(|_| {}), + }) + } + + /// Where to change the token and the allowlist while it runs. + pub fn handle(&self) -> Handle { + Handle(self.shared.clone()) + } + + /// What may carry the relay session ([`Transport::Auto`] by default). + pub fn transport(mut self, transport: Transport) -> Self { + self.transport = transport; + self + } + + /// Hear what happens (nothing is printed otherwise). + pub fn on_event(mut self, events: impl Fn(&Event) + Send + Sync + 'static) -> Self { + self.events = Arc::new(events); + self + } + + /// Publish the server until a fatal error: the control endpoint refusing + /// the token. Every other failure is retried. + pub async fn run(self) -> Result<(), String> { + self.run_until(std::future::pending()).await + } + + /// [`Producer::run`] until `shutdown` completes, then close the relay + /// session rather than let it idle out on the relay. + pub async fn run_until(self, shutdown: impl Future) -> Result<(), String> { + let active = Active::default(); + tokio::select! { + () = shutdown => { + active.close().await; + Ok(()) + } + result = self.run_loop(&active) => result, + } + } + + fn emit(&self, event: Event) { + (self.events)(&event); + } + + async fn run_loop(&self, active: &Active) -> Result<(), String> { + let http = crate::uplink_http_client()?; + let mut backoff = INITIAL_BACKOFF; + let mut prefer_websocket_until: Option = None; + loop { + let token = self.shared.token(); + let pool = match fetch_pool(&http, &self.control, &token).await? { + FetchOutcome::Pool(pool) => pool, + FetchOutcome::Retry { after, reason } => { + let delay = after.unwrap_or(backoff); + self.emit(Event::ControlRetry { + reason, + retry_in: delay, + }); + tokio::time::sleep(jittered(delay)).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + continue; + } + }; + let websocket_first = + prefer_websocket_until.is_some_and(|until| Instant::now() < until); + let relays = order(pool, self.transport, websocket_first); + if relays.is_empty() { + let reason = format!( + "the relay pool has no relay for the {} transport", + self.transport + ); + self.emit(Event::ControlRetry { + reason, + retry_in: backoff, + }); + tokio::time::sleep(jittered(backoff)).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + continue; + } + let has_websocket = relays + .iter() + .any(|relay| relay.carrier == Carrier::WebSocket); + // A session that was actually established ends with a fresh + // control-plane query, per docs/uplink.md. + let mut established = false; + for relay in &relays { + let end = match relay.carrier { + Carrier::WebTransport => { + let limit = (self.transport == Transport::Auto && has_websocket) + .then_some(WEBTRANSPORT_CONNECT_AUTO); + self.webtransport_session(relay, limit, active).await + } + Carrier::WebSocket => self.websocket_session(relay, active).await, + }; + match end { + SessionEnd::Ended { reason, lasted } => { + self.emit(Event::SessionEnded { + carrier: relay.carrier, + reason, + }); + if relay.carrier == Carrier::WebTransport && lasted < SHORT_SESSION { + prefer_websocket_until = Some(Instant::now() + PREFER_WEBSOCKET_FOR); + } + established = true; + break; + } + SessionEnd::NeverConnected(error) => { + if relay.carrier == Carrier::WebTransport { + prefer_websocket_until = Some(Instant::now() + PREFER_WEBSOCKET_FOR); + } + self.emit(Event::RelayFailed { + relay: relay.label.clone(), + carrier: relay.carrier, + error, + }); + } + } + } + if established { + backoff = INITIAL_BACKOFF; + } else { + self.emit(Event::PoolExhausted { retry_in: backoff }); + tokio::time::sleep(jittered(backoff)).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + } + } + } + + async fn webtransport_session( + &self, + relay: &Relay, + limit: Option, + active: &Active, + ) -> SessionEnd { + let client = match webtransport_client(relay.cert_hash.as_deref()) { + Ok(client) => client, + Err(error) => return SessionEnd::NeverConnected(error), + }; + // Careful: the URL is the credential — show `relay.label` only. + let connecting = client.connect(relay.url.clone()); + let connected = match limit { + Some(limit) => match tokio::time::timeout(limit, connecting).await { + Ok(connected) => connected, + Err(_) => { + return SessionEnd::NeverConnected(format!( + "no WebTransport session within {}s (is UDP blocked?)", + limit.as_secs() + )); + } + }, + None => connecting.await, + }; + let session = match connected { + Ok(session) => session, + Err(error) => return SessionEnd::NeverConnected(format!("connect failed: {error}")), + }; + let started = Instant::now(); + self.emit(Event::Connected { + relay: relay.label.clone(), + carrier: Carrier::WebTransport, + }); + active.set(Session::WebTransport(Box::new(session.clone()))); + + let routes = DatagramRoutes::new(MAX_DATAGRAM_ROUTES); + let distributing = tokio::spawn(distribute_datagrams(session.clone(), routes.clone())); + let pending = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING)); + let reason = loop { + match session.accept_bi().await { + Ok((send, recv)) => { + let Ok(permit) = pending.clone().try_acquire_owned() else { + continue; + }; + let lane = Lane::WebTransport { + session: Box::new(session.clone()), + routes: routes.clone(), + }; + tokio::spawn(bridge( + tokio::io::join(recv, send), + lane, + self.shared.clone(), + permit, + self.local.clone(), + self.events.clone(), + )); + } + Err(error) => break error.to_string(), + } + }; + distributing.abort(); + active.clear(); + SessionEnd::Ended { + reason, + lasted: started.elapsed(), + } + } + + async fn websocket_session(&self, relay: &Relay, active: &Active) -> SessionEnd { + let socket = match connect_websocket(&relay.url, relay.cert_hash.as_deref()).await { + Ok(socket) => socket, + Err(error) => return SessionEnd::NeverConnected(error), + }; + let started = Instant::now(); + self.emit(Event::Connected { + relay: relay.label.clone(), + carrier: Carrier::WebSocket, + }); + let (sink, mut source) = socket.split(); + let sink = Arc::new(tokio::sync::Mutex::new(sink)); + active.set(Session::WebSocket(sink.clone())); + + let pending = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING)); + // Stream WebSockets are connections of their own: they end with the + // session (dropped, the set aborts them), as WebTransport streams end + // with their QUIC connection. + let mut streams = tokio::task::JoinSet::new(); + let mut keepalive = tokio::time::interval(KEEPALIVE); + keepalive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + keepalive.tick().await; + let mut heard = Instant::now(); + let reason = loop { + tokio::select! { + message = source.next() => match message { + Some(Ok(Message::Text(text))) => { + heard = Instant::now(); + let Some(stream) = stream_request(&text, &relay.url) else { + continue; + }; + let Ok(permit) = pending.clone().try_acquire_owned() else { + continue; + }; + streams.spawn(websocket_stream( + stream, + relay.cert_hash.clone(), + self.shared.clone(), + permit, + self.local.clone(), + self.events.clone(), + )); + } + Some(Ok(Message::Close(frame))) => { + break match frame { + Some(frame) if !frame.reason.is_empty() => { + format!("closed by the relay: {}", frame.reason) + } + _ => "closed by the relay".to_owned(), + }; + } + Some(Ok(_)) => heard = Instant::now(), + Some(Err(error)) => break error.to_string(), + None => break "the relay closed the connection".to_owned(), + }, + Some(_) = streams.join_next(), if !streams.is_empty() => {} + _ = keepalive.tick() => { + if heard.elapsed() >= IDLE_TIMEOUT { + break format!("nothing from the relay for {}s", IDLE_TIMEOUT.as_secs()); + } + let ping = Message::Ping(Default::default()); + if sink.lock().await.send(ping).await.is_err() { + break "the connection to the relay failed".to_owned(); + } + } + } + }; + active.clear(); + SessionEnd::Ended { + reason, + lasted: started.elapsed(), + } + } +} + +/// The relays to try, in order: WebTransport's then WebSocket's for +/// [`Transport::Auto`] (the other way round while WebTransport is failing), +/// each carrier's shuffled. +fn order(pool: Pool, transport: Transport, websocket_first: bool) -> Vec { + use rand::seq::SliceRandom; + let Pool { + mut webtransport, + mut websocket, + } = pool; + webtransport.shuffle(&mut rand::rng()); + websocket.shuffle(&mut rand::rng()); + match transport { + Transport::WebTransport => webtransport, + Transport::WebSocket => websocket, + Transport::Auto if websocket_first => websocket.into_iter().chain(webtransport).collect(), + Transport::Auto => webtransport.into_iter().chain(websocket).collect(), + } +} + +// --------------------------------------------------------------------------- +// The active session, for a graceful shutdown +// --------------------------------------------------------------------------- + +type WsSink = futures_util::stream::SplitSink; + +enum Session { + WebTransport(Box), + WebSocket(Arc>), +} + +#[derive(Clone, Default)] +struct Active(Arc>>); + +impl Active { + fn set(&self, session: Session) { + *self.0.lock().unwrap_or_else(|p| p.into_inner()) = Some(session); + } + + fn clear(&self) { + self.0.lock().unwrap_or_else(|p| p.into_inner()).take(); + } + + async fn close(&self) { + let session = self.0.lock().unwrap_or_else(|p| p.into_inner()).take(); + match session { + Some(Session::WebTransport(session)) => { + session.close(CODE_SHUTDOWN, b"uplink shutting down"); + } + Some(Session::WebSocket(sink)) => { + let frame = tokio_tungstenite::tungstenite::protocol::CloseFrame { + code: CloseCode::Away, + reason: "uplink shutting down".into(), + }; + let _ = tokio::time::timeout(Duration::from_secs(1), async { + sink.lock().await.send(Message::Close(Some(frame))).await + }) + .await; + } + None => {} + } + } +} + +// --------------------------------------------------------------------------- +// Control plane +// --------------------------------------------------------------------------- + +struct Relay { + /// Connection URL as minted by the control plane (fragment stripped). + /// The URL is the session credential — never show it; use `label`. + url: url::Url, + /// `host:port`, for events. + label: String, + /// SHA-256 of the relay's certificate (DER), from the URL's `#sha256=` + /// fragment; pins TLS verification instead of using system trust roots. + cert_hash: Option>, + carrier: Carrier, +} + +#[derive(Default)] +struct Pool { + webtransport: Vec, + websocket: Vec, +} + +enum FetchOutcome { + Pool(Pool), + Retry { + after: Option, + reason: String, + }, +} + +/// Query the control endpoint. `Err` is fatal (bad token); every other +/// failure is a retryable `FetchOutcome::Retry`. +async fn fetch_pool( + http: &reqwest::Client, + url: &url::Url, + token: &str, +) -> Result { + let retry = |after, reason| Ok(FetchOutcome::Retry { after, reason }); + let response = match http + .get(url.clone()) + .header("authorization", format!("Bearer {token}")) + .header("accept", "application/json") + .send() + .await + { + Ok(response) => response, + Err(error) => return retry(None, format!("control endpoint unreachable: {error}")), + }; + let status = response.status(); + if status.as_u16() == 401 || status.as_u16() == 403 { + return Err(format!( + "control endpoint rejected YAS_UPLINK_TOKEN ({status})" + )); + } + if !status.is_success() { + let after = response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .map(Duration::from_secs); + return retry(after, format!("control endpoint returned {status}")); + } + let body = match response.text().await { + Ok(body) => body, + Err(error) => return retry(None, format!("error reading relay pool: {error}")), + }; + match parse_pool(&body) { + Ok(pool) => Ok(FetchOutcome::Pool(pool)), + Err(error) => retry(None, format!("bad relay pool: {error}")), + } +} + +/// `relays` (WebTransport, `https`) and `websockets` (`wss`): either may be +/// absent or empty, not both. +fn parse_pool(body: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(body).map_err(|error| format!("invalid JSON: {error}"))?; + let list = |name: &str| -> Result, String> { + match value.get(name) { + None | Some(serde_json::Value::Null) => Ok(Vec::new()), + Some(serde_json::Value::Array(items)) => items + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| format!("\"{name}\" entries must be URL strings")) + }) + .collect(), + Some(_) => Err(format!("\"{name}\" must be an array")), + } + }; + let pool = Pool { + webtransport: list("relays")? + .iter() + .map(|url| parse_relay(url, Carrier::WebTransport)) + .collect::>()?, + websocket: list("websockets")? + .iter() + .map(|url| parse_relay(url, Carrier::WebSocket)) + .collect::>()?, + }; + if pool.webtransport.is_empty() && pool.websocket.is_empty() { + return Err("the pool names no relay (\"relays\" and \"websockets\" are empty)".into()); + } + Ok(pool) +} + +fn parse_relay(value: &str, carrier: Carrier) -> Result { + let mut url = url::Url::parse(value).map_err(|error| format!("bad relay URL: {error}"))?; + let scheme = match carrier { + Carrier::WebTransport => "https", + Carrier::WebSocket => "wss", + }; + if url.scheme() != scheme { + return Err(format!( + "{carrier} relay URL scheme must be {scheme}, got {}", + url.scheme() + )); + } + if url.host_str().is_none() { + return Err("relay URL has no host".into()); + } + if !url.username().is_empty() || url.password().is_some() { + return Err("relay URL must not have userinfo".into()); + } + let label = label(&url); + // A malformed pin must be an error, never a silent fall-back to system + // roots — that would defeat the pinning. + let cert_hash = match url.fragment() { + None | Some("") => None, + Some(fragment) => { + let hash = fragment + .strip_prefix("sha256=") + .ok_or("relay URL fragment must be sha256=")?; + let bytes = + base64url_decode(hash).ok_or("relay certificate pin is not valid base64url")?; + if bytes.len() != 32 { + return Err("relay certificate pin must be a SHA-256 (32 bytes)".into()); + } + Some(bytes) + } + }; + // Fragments are client-side only; strip before connecting. + url.set_fragment(None); + Ok(Relay { + url, + label, + cert_hash, + carrier, + }) +} + +/// `host:port`: what may be shown of a URL that is a credential. +fn label(url: &url::Url) -> String { + let host = match url.host() { + Some(url::Host::Ipv6(address)) => format!("[{address}]"), + Some(host) => host.to_string(), + None => String::new(), + }; + format!("{host}:{}", url.port_or_known_default().unwrap_or(443)) +} + +fn base64url_decode(value: &str) -> Option> { + let value = value.trim_end_matches('='); + let mut out = Vec::with_capacity(value.len() * 3 / 4); + let mut acc: u32 = 0; + let mut bits = 0u32; + for byte in value.bytes() { + let digit = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'-' => 62, + b'_' => 63, + _ => return None, + }; + acc = (acc << 6) | u32::from(digit); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + } + } + // Leftover bits must be padding zeros of a valid encoding. + if bits > 0 && (acc & ((1 << bits) - 1)) != 0 { + return None; + } + Some(out) +} + +fn jittered(base: Duration) -> Duration { + use rand::RngExt as _; + let ms = base.as_millis().max(1) as u64; + // 0.75x–1.25x + Duration::from_millis(rand::rng().random_range(ms * 3 / 4..=ms * 5 / 4)) +} + +// --------------------------------------------------------------------------- +// Relay sessions +// --------------------------------------------------------------------------- + +enum SessionEnd { + /// Handshake or CONNECT failed — try the next relay in the pool. + NeverConnected(String), + /// The session was established and later died — re-query the control + /// plane for a fresh pool. + Ended { reason: String, lasted: Duration }, +} + +/// A WebTransport client with the liveness settings of docs/uplink.md (10 s +/// keepalive, 30 s idle timeout), on YAS's rustls provider, verifying the +/// relay with the platform's roots or a certificate pin. +fn webtransport_client(cert_hash: Option<&[u8]>) -> Result { + let provider = yas_webrtc_forwarder::tls::provider(); + let builder = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| format!("TLS config: {error}"))?; + let mut crypto = match cert_hash { + Some(hash) => builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(crate::CertificateHash { + provider, + hash: hash.to_vec(), + })) + .with_no_client_auth(), + None => builder + .with_root_certificates(yas_webrtc_forwarder::tls::native_roots()) + .with_no_client_auth(), + }; + crypto.alpn_protocols = vec![wt::ALPN.as_bytes().to_vec()]; + let crypto = wt::quinn::crypto::rustls::QuicClientConfig::try_from(crypto) + .map_err(|error| format!("QUIC TLS config: {error}"))?; + let mut config = wt::quinn::ClientConfig::new(Arc::new(crypto)); + let mut transport = wt::quinn::TransportConfig::default(); + transport.keep_alive_interval(Some(KEEPALIVE)); + transport.max_idle_timeout(Some( + wt::quinn::IdleTimeout::try_from(IDLE_TIMEOUT).expect("30s fits in an idle timeout"), + )); + config.transport_config(Arc::new(transport)); + let endpoint = wt::quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()) + .or_else(|_| wt::quinn::Endpoint::client((std::net::Ipv4Addr::UNSPECIFIED, 0).into())) + .map_err(|error| format!("UDP socket: {error}"))?; + Ok(wt::Client::new(endpoint, config)) +} + +/// A WebSocket to `url` (`wss`, a session's or a stream's) speaking +/// [`WEBSOCKET_SUBPROTOCOL`], TLS verified by the platform's roots or a pin. +async fn connect_websocket(url: &url::Url, cert_hash: Option<&[u8]>) -> Result { + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + let mut request = url + .as_str() + .into_client_request() + .map_err(|_| "bad WebSocket URL".to_owned())?; + request.headers_mut().insert( + "sec-websocket-protocol", + tokio_tungstenite::tungstenite::http::HeaderValue::from_static(WEBSOCKET_SUBPROTOCOL), + ); + let connector = match cert_hash { + Some(hash) => { + let provider = yas_webrtc_forwarder::tls::provider(); + let config = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions() + .map_err(|error| format!("TLS config: {error}"))? + .dangerous() + .with_custom_certificate_verifier(Arc::new(crate::CertificateHash { + provider, + hash: hash.to_vec(), + })) + .with_no_client_auth(); + tokio_tungstenite::Connector::Rustls(Arc::new(config)) + } + None => yas_webrtc_forwarder::tls::websocket_connector(), + }; + let config = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default() + .max_message_size(Some(MAX_WEBSOCKET_MESSAGE)) + .max_frame_size(Some(MAX_WEBSOCKET_MESSAGE)); + let connecting = tokio_tungstenite::connect_async_tls_with_config( + request, + Some(config), + true, + Some(connector), + ); + // Errors may quote the URL, a credential: say what failed, not where. + let (socket, response) = match tokio::time::timeout(WEBSOCKET_CONNECT, connecting).await { + Err(_) => { + return Err(format!( + "no WebSocket within {}s", + WEBSOCKET_CONNECT.as_secs() + )); + } + Ok(Err(tokio_tungstenite::tungstenite::Error::Http(response))) => { + return Err(format!("the relay answered HTTP {}", response.status())); + } + // tungstenite checks the answer's subprotocol against the one offered. + Ok(Err(tokio_tungstenite::tungstenite::Error::Protocol( + tokio_tungstenite::tungstenite::error::ProtocolError::SecWebSocketSubProtocolError(_), + ))) => { + return Err(format!("the relay doesn't speak {WEBSOCKET_SUBPROTOCOL}")); + } + Ok(Err(error)) => return Err(websocket_error(&error)), + Ok(Ok(connected)) => connected, + }; + let selected = response + .headers() + .get("sec-websocket-protocol") + .and_then(|value| value.to_str().ok()); + if selected != Some(WEBSOCKET_SUBPROTOCOL) { + return Err(format!("the relay doesn't speak {WEBSOCKET_SUBPROTOCOL}")); + } + Ok(socket) +} + +fn websocket_error(error: &tokio_tungstenite::tungstenite::Error) -> String { + use tokio_tungstenite::tungstenite::Error; + match error { + Error::Io(error) => format!("connect failed: {error}"), + Error::Tls(error) => format!("TLS: {error}"), + Error::Url(_) => "bad WebSocket URL".to_owned(), + Error::Protocol(error) => format!("WebSocket handshake: {error}"), + other => format!("WebSocket: {other}"), + } +} + +/// The stream URL of a relay's request (`{"stream": "wss://…"}`), when it is +/// one: a `wss` URL on the session's own origin (so the same TLS trust +/// applies), without userinfo or a fragment. Anything else is ignored, for +/// relays newer than this producer. +fn stream_request(text: &str, session: &url::Url) -> Option { + let value: serde_json::Value = serde_json::from_str(text).ok()?; + let stream = url::Url::parse(value.get("stream")?.as_str()?).ok()?; + (stream.scheme() == "wss" + && stream.host() == session.host() + && stream.port_or_known_default() == session.port_or_known_default() + && stream.username().is_empty() + && stream.password().is_none() + && stream.fragment().is_none()) + .then_some(stream) +} + +/// One consumer on a WebSocket session: open the stream the relay asked for, +/// then serve it as a WebTransport stream is served, without datagrams. +async fn websocket_stream( + url: url::Url, + cert_hash: Option>, + shared: Arc, + permit: tokio::sync::OwnedSemaphorePermit, + local: Local, + events: Events, +) { + let socket = match connect_websocket(&url, cert_hash.as_deref()).await { + Ok(socket) => socket, + Err(error) => { + events(&Event::StreamFailed { error }); + return; + } + }; + let (sink, source) = socket.split(); + // Opaque chunks of Noise records both ways. A stream's write side + // finishing doesn't close the WebSocket (Noise carries the half-close). + let relay = tokio::io::join( + crate::WsFrameReader { + inner: source, + buf: bytes::Bytes::new(), + frame_lengths: false, + }, + crate::WsByteWriter { inner: sink }, + ); + bridge(relay, Lane::None, shared, permit, local, events).await; +} + +async fn distribute_datagrams(session: wt::Session, routes: DatagramRoutes) { + while let Ok(bytes) = session.read_datagram().await { + let _ = routes.route(&bytes); + } + routes.clear(); +} + +// --------------------------------------------------------------------------- +// Bridging a consumer to the local server +// --------------------------------------------------------------------------- + +/// Where a consumer's datagrams go: a WebTransport session's datagrams, or +/// nowhere (WebSocket sessions carry none). +enum Lane { + WebTransport { + session: Box, + routes: DatagramRoutes, + }, + None, +} + +/// Bridge one consumer stream to one local YAS server connection. +/// Authentication precedes both ingress classification and local IPC. The +/// relay sees only Noise records; plaintext selectors cannot bypass admission. +async fn bridge( + relay: S, + lane: Lane, + shared: Arc, + permit: tokio::sync::OwnedSemaphorePermit, + local: Local, + events: Events, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let relay = match yas_uplink::accept(relay, shared.crypto()).await { + Ok(relay) => relay, + Err(_) => return, + }; + // Bind each sideband's keys to its authenticated main stream. The random + // route token remains authenticated as AEAD AAD on every datagram. + let material = relay.datagram_key_material(); + let ingress = tokio::time::timeout( + Duration::from_secs(5), + yas_composite_transport::classify(relay), + ) + .await; + drop(permit); + match ingress { + Ok(Ok(yas_composite_transport::Ingress::Direct(relay))) => { + bridge_direct(relay, &local, &events).await + } + Ok(Ok(yas_composite_transport::Ingress::Composite { offer, stream })) + if offer.role == yas_composite_transport::Role::Main => + { + let Lane::WebTransport { session, routes } = lane else { + events(&Event::StreamFailed { + error: "rejected a datagram lane: a WebSocket session carries no datagrams" + .into(), + }); + return; + }; + let (sender, receiver) = yas_uplink::datagram_pair(material, offer.token, false); + let lane = Sideband { + session: *session, + routes, + sender, + receiver, + }; + bridge_composite(lane, offer, stream, &local, &events).await; + } + _ => {} + } +} + +async fn bridge_direct(relay: S, local: &Local, events: &Events) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let (mut sock_read, mut sock_write) = match local.connect().await { + Ok(halves) => halves, + Err(error) => { + events(&Event::LocalUnavailable { + local: local.label.clone(), + error, + }); + return; + } + }; + let (mut recv, mut send) = tokio::io::split(relay); + let down = async move { + tokio::io::copy(&mut recv, &mut sock_write).await?; + sock_write.shutdown().await + }; + let up = async move { + tokio::io::copy(&mut sock_read, &mut send).await?; + send.shutdown().await + }; + let _ = tokio::try_join!(down, up); +} + +/// A consumer's datagram lane on a WebTransport session. +struct Sideband { + session: wt::Session, + routes: DatagramRoutes, + sender: yas_uplink::DatagramSender, + receiver: yas_uplink::DatagramReceiver, +} + +async fn bridge_composite( + lane: Sideband, + offer: yas_composite_transport::Offer, + relay: S, + local: &Local, + events: &Events, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let Sideband { + session, + routes, + mut sender, + mut receiver, + } = lane; + let physical_maximum = session + .max_datagram_size() + .saturating_sub(yas_composite_transport::ROUTED_DATAGRAM_HEADER) + .saturating_sub(yas_uplink::DATAGRAM_OVERHEAD) + .min(yas_composite_transport::HARD_MAX_DATAGRAM as usize - yas_uplink::DATAGRAM_OVERHEAD); + if offer.max_datagram as usize > physical_maximum { + events(&Event::StreamFailed { + error: format!( + "rejected composite maximum {} above path maximum {physical_maximum}", + offer.max_datagram + ), + }); + return; + } + let (mut main_read, mut main_write) = match local.connect().await { + Ok(halves) => halves, + Err(error) => { + events(&Event::LocalUnavailable { + local: local.label.clone(), + error, + }); + return; + } + }; + let (mut side_read, mut side_write) = match local.connect().await { + Ok(halves) => halves, + Err(error) => { + events(&Event::LocalUnavailable { + local: format!("{} (datagram sideband)", local.label), + error, + }); + return; + } + }; + let main_offer = yas_composite_transport::Offer::new( + yas_composite_transport::Role::Main, + offer.token, + offer.max_datagram, + ) + .expect("the classified offer is valid"); + let side_offer = yas_composite_transport::Offer::new( + yas_composite_transport::Role::Datagram, + offer.token, + offer.max_datagram, + ) + .expect("the classified offer is valid"); + if yas_composite_transport::write_offer(&mut main_write, main_offer) + .await + .is_err() + || yas_composite_transport::write_offer(&mut side_write, side_offer) + .await + .is_err() + { + return; + } + + let encrypted_maximum = offer.max_datagram + yas_uplink::DATAGRAM_OVERHEAD as u32; + let Ok(mut route_rx) = routes.register(offer.token, encrypted_maximum, DATAGRAM_QUEUE) else { + return; + }; + + let (mut relay_read, mut relay_write) = tokio::io::split(relay); + let down = async { + tokio::io::copy(&mut relay_read, &mut main_write).await?; + main_write.shutdown().await + }; + let up = async { + tokio::io::copy(&mut main_read, &mut relay_write).await?; + relay_write.shutdown().await + }; + let side_routes = routes.clone(); + let side_token = offer.token; + let side_task = tokio::spawn(async move { + let side_out = async { + loop { + let Ok(frame) = + yas_composite_transport::read_datagram(&mut side_read, offer.max_datagram) + .await + else { + break; + }; + let Some(encrypted) = sender.seal(&frame) else { + break; + }; + let Ok(routed) = yas_composite_transport::encode_routed_datagram( + offer.token, + &encrypted, + encrypted_maximum, + ) else { + break; + }; + // Congestion is ordinary datagram loss; never await reliable + // transport capacity or fall back to the main stream here. + let _ = session.send_datagram(routed.into()); + } + }; + let side_in = async { + while let Some(frame) = route_rx.recv().await { + let Some(frame) = receiver.open(&frame) else { + continue; + }; + if yas_composite_transport::write_datagram( + &mut side_write, + &frame, + offer.max_datagram, + ) + .await + .is_err() + { + break; + } + } + }; + tokio::select! { + _ = side_out => {} + _ = side_in => {} + } + side_routes.remove(side_token); + }); + + // The authoritative reliable splice owns the connection lifetime. An + // optional sideband ending only removes its route and cannot drop `main`. + let _ = tokio::try_join!(down, up); + routes.remove(offer.token); + side_task.abort(); +} + +#[cfg(test)] +#[path = "uplink_producer_tests.rs"] +mod tests; diff --git a/crates/proxy/src/uplink_producer_tests.rs b/crates/proxy/src/uplink_producer_tests.rs new file mode 100644 index 000000000..35550e698 --- /dev/null +++ b/crates/proxy/src/uplink_producer_tests.rs @@ -0,0 +1,915 @@ +// The WebSocket and local-socket tests are Unix-only: on Windows the helpers +// only they use would fail CI's `-D warnings`. +#![cfg_attr(not(unix), allow(dead_code, unused_imports))] + +use super::*; +use tokio::io::AsyncReadExt; + +fn events() -> (Events, Arc>>) { + let seen = Arc::new(Mutex::new(Vec::new())); + let log = seen.clone(); + let events: Events = Arc::new(move |event: &Event| { + log.lock().unwrap().push(event.to_string()); + }); + (events, seen) +} + +fn shared(crypto: Arc) -> Arc { + Arc::new(Shared { + token: Mutex::new("token".into()), + crypto: Mutex::new(crypto), + }) +} + +struct Keys { + server: Arc, + server_public: yas_uplink::PublicKey, + server_identity: yas_uplink::Identity, + client: Arc, + client_public: yas_uplink::PublicKey, + other: Arc, + other_public: yas_uplink::PublicKey, +} + +fn keys() -> Keys { + let (server_key, server_public) = yas_uplink::Identity::generate().unwrap(); + let (client_key, client_public) = yas_uplink::Identity::generate().unwrap(); + let (other_key, other_public) = yas_uplink::Identity::generate().unwrap(); + let server_identity = yas_uplink::Identity::from_base64(&server_key).unwrap(); + Keys { + server: server_identity.server_config(vec![client_public]).unwrap(), + server_public, + server_identity, + client: yas_uplink::Identity::from_base64(&client_key) + .unwrap() + .client_config(server_public) + .unwrap(), + client_public, + other: yas_uplink::Identity::from_base64(&other_key) + .unwrap() + .client_config(server_public) + .unwrap(), + other_public, + } +} + +#[cfg(unix)] +fn local_socket(dir: &std::path::Path) -> (Local, tokio::net::UnixListener) { + let socket = dir.join("local.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + (Local::socket(socket.to_str().unwrap()), listener) +} + +#[cfg(unix)] +async fn nothing_accepted(listener: &tokio::net::UnixListener) -> bool { + tokio::time::timeout(Duration::from_millis(25), listener.accept()) + .await + .is_err() +} + +#[cfg(unix)] +#[tokio::test] +async fn producer_authenticates_before_ipc_and_encrypts_datagrams() { + use yas_composite_transport::{Offer, Role}; + yas_webrtc_forwarder::tls::install_default_provider(); + tokio::time::timeout(Duration::from_secs(15), async { + let dir = tempfile::tempdir().unwrap(); + let keys = keys(); + let (local, listener) = local_socket(dir.path()); + let (events, _) = events(); + + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let hash = wt::crypto::sha256(&yas_webrtc_forwarder::tls::provider(), cert.cert.der()); + let mut worker = wt::ServerBuilder::new() + .with_addr("127.0.0.1:0".parse().unwrap()) + .with_certificate( + vec![cert.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()) + .into(), + ) + .unwrap(); + let outer = webtransport_client(Some(hash.as_ref())).unwrap(); + let url: url::Url = format!("https://127.0.0.1:{}/", worker.local_addr().unwrap().port()) + .parse() + .unwrap(); + let (connected, remote) = tokio::join!(outer.connect(url), async { + worker.accept().await.unwrap().ok().await.unwrap() + }); + let session = connected.unwrap(); + let routes = DatagramRoutes::new(16); + let datagrams = tokio::spawn(distribute_datagrams(session.clone(), routes.clone())); + + for case in 0..4 { + let lane = Lane::WebTransport { + session: Box::new(session.clone()), + routes: routes.clone(), + }; + let shared = shared(keys.server.clone()); + let (local, events) = (local.clone(), events.clone()); + let accepting = session.clone(); + let producer = tokio::spawn(async move { + let (send, recv) = accepting.accept_bi().await.unwrap(); + let permit = Arc::new(tokio::sync::Semaphore::new(1)) + .acquire_owned() + .await + .unwrap(); + bridge( + tokio::io::join(recv, send), + lane, + shared, + permit, + local, + events, + ) + .await; + }); + let (send, recv) = remote.open_bi().await.unwrap(); + let mut stream = tokio::io::join(recv, send); + if case == 0 { + // A relay can synthesize protocol bytes, but those bytes must + // never open a local socket before Noise authentication. + stream + .write_all(yas_wire::PREFACE.as_slice()) + .await + .unwrap(); + stream.shutdown().await.unwrap(); + drop(stream); + producer.await.unwrap(); + assert!(nothing_accepted(&listener).await); + continue; + } + if case == 1 { + assert!( + yas_uplink::connect(stream, keys.other.clone()) + .await + .is_err() + ); + producer.await.unwrap(); + assert!(nothing_accepted(&listener).await); + continue; + } + let mut stream = yas_uplink::connect(stream, keys.client.clone()) + .await + .unwrap(); + // Authentication alone also does not open IPC: the encrypted YAS + // preface or composite selector must follow first. + assert!(nothing_accepted(&listener).await); + if case == 2 { + stream + .write_all(yas_wire::PREFACE.as_slice()) + .await + .unwrap(); + stream.write_all(b"command").await.unwrap(); + stream.flush().await.unwrap(); + let (mut ipc, _) = listener.accept().await.unwrap(); + let mut received = vec![0; yas_wire::PREFACE.len() + 7]; + ipc.read_exact(&mut received).await.unwrap(); + assert_eq!( + received, + [yas_wire::PREFACE.as_slice(), b"command"].concat() + ); + ipc.write_all(b"reply").await.unwrap(); + ipc.shutdown().await.unwrap(); + let mut reply = Vec::new(); + stream.read_to_end(&mut reply).await.unwrap(); + assert_eq!(reply, b"reply"); + stream.shutdown().await.unwrap(); + producer.await.unwrap(); + continue; + } + + let token = [0x35; 16]; + let maximum = 512; + let material = stream.datagram_key_material(); + let (mut sender, mut receiver) = yas_uplink::datagram_pair(material, token, true); + yas_composite_transport::write_offer( + &mut stream, + Offer::new(Role::Main, token, maximum).unwrap(), + ) + .await + .unwrap(); + stream.flush().await.unwrap(); + let (mut main, _) = listener.accept().await.unwrap(); + let (mut side, _) = listener.accept().await.unwrap(); + for (socket, expected) in [(&mut main, Role::Main), (&mut side, Role::Datagram)] { + match yas_composite_transport::classify(socket).await.unwrap() { + yas_composite_transport::Ingress::Composite { offer, .. } => { + assert_eq!(offer.role, expected); + assert_eq!(offer.token, token); + assert_eq!(offer.max_datagram, maximum); + } + _ => panic!("missing local composite selector"), + } + } + // A reliable reply also synchronizes route registration. + main.write_all(b"ready").await.unwrap(); + let mut ready = [0; 5]; + stream.read_exact(&mut ready).await.unwrap(); + assert_eq!(&ready, b"ready"); + + let encrypted = sender.seal(b"private datagram").unwrap(); + let wire_maximum = maximum + yas_uplink::DATAGRAM_OVERHEAD as u32; + let mut forged = encrypted.clone(); + *forged.last_mut().unwrap() ^= 1; + for bytes in [&forged, &encrypted, &encrypted] { + let routed = + yas_composite_transport::encode_routed_datagram(token, bytes, wire_maximum) + .unwrap(); + remote.send_datagram(routed.into()).unwrap(); + } + assert_eq!( + yas_composite_transport::read_datagram(&mut side, maximum) + .await + .unwrap(), + b"private datagram" + ); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + yas_composite_transport::read_datagram(&mut side, maximum) + ) + .await + .is_err() + ); + + yas_composite_transport::write_datagram(&mut side, b"private reply", maximum) + .await + .unwrap(); + let packet = remote.read_datagram().await.unwrap(); + let (received_token, ciphertext) = + yas_composite_transport::split_routed_datagram(&packet).unwrap(); + assert_eq!(received_token, token); + assert!(!ciphertext.windows(13).any(|w| w == b"private reply")); + assert_eq!(receiver.open(ciphertext).unwrap(), b"private reply"); + stream.shutdown().await.unwrap(); + main.shutdown().await.unwrap(); + producer.await.unwrap(); + } + datagrams.abort(); + session.close(0, b"test complete"); + }) + .await + .expect("producer uplink test stalled"); +} + +/// A relay for WebSocket sessions, pinned by its certificate's hash: it +/// takes one session, then answers `/stream/…` WebSockets, handing each to the +/// test as a byte stream the consumer's side of Noise runs on. +#[cfg(unix)] +struct WebSocketRelay { + url: url::Url, + hash: Vec, + /// Stream requests sent on the session: what the test writes there. + requests: tokio::sync::mpsc::UnboundedSender, + /// Stream WebSockets the producer opened, as duplex byte streams. + streams: tokio::sync::mpsc::UnboundedReceiver<(String, tokio::io::DuplexStream)>, + /// What the relay offered as the session's subprotocol, and saw. + session_protocols: Arc>>, + _task: tokio::task::JoinHandle<()>, +} + +#[cfg(unix)] +impl WebSocketRelay { + async fn start() -> Self { + use tokio_tungstenite::tungstenite::handshake::server::{Request, Response}; + yas_webrtc_forwarder::tls::install_default_provider(); + let cert = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap(); + let hash = wt::crypto::sha256(&yas_webrtc_forwarder::tls::provider(), cert.cert.der()) + .as_ref() + .to_vec(); + let tls = tokio_rustls::TlsAcceptor::from(Arc::new( + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![cert.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()) + .into(), + ) + .unwrap(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url: url::Url = format!( + "wss://127.0.0.1:{}/session", + listener.local_addr().unwrap().port() + ) + .parse() + .unwrap(); + let (requests, mut pending) = tokio::sync::mpsc::unbounded_channel::(); + let (streams_tx, streams) = tokio::sync::mpsc::unbounded_channel(); + let session_protocols = Arc::new(Mutex::new(Vec::new())); + let protocols = session_protocols.clone(); + let task = tokio::spawn(async move { + let mut session_taken = false; + loop { + let (tcp, _) = listener.accept().await.unwrap(); + let Ok(tls) = tls.accept(tcp).await else { + continue; + }; + let path = Arc::new(Mutex::new(String::new())); + let seen = path.clone(); + let offered = protocols.clone(); + // tungstenite's callback signature: its error is an HTTP response. + #[allow(clippy::result_large_err)] + let callback = move |request: &Request, mut response: Response| { + *seen.lock().unwrap() = request.uri().path().to_owned(); + let protocol = request + .headers() + .get("sec-websocket-protocol") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); + offered.lock().unwrap().push(protocol.clone()); + if protocol + .split(',') + .any(|each| each.trim() == WEBSOCKET_SUBPROTOCOL) + { + response.headers_mut().insert( + "sec-websocket-protocol", + WEBSOCKET_SUBPROTOCOL.parse().unwrap(), + ); + } + Ok(response) + }; + let Ok(socket) = tokio_tungstenite::accept_hdr_async(tls, callback).await else { + continue; + }; + let path = path.lock().unwrap().clone(); + if path == "/session" && !session_taken { + session_taken = true; + let (mut sink, mut source) = socket.split(); + let mut requests = + std::mem::replace(&mut pending, tokio::sync::mpsc::unbounded_channel().1); + tokio::spawn(async move { + loop { + tokio::select! { + request = requests.recv() => match request { + Some(message) => { + if sink.send(message).await.is_err() { + break; + } + } + None => break, + }, + message = source.next() => match message { + Some(Ok(_)) => {} + _ => break, + }, + } + } + }); + } else if let Some(stream) = path.strip_prefix("/stream/") { + let (ours, theirs) = tokio::io::duplex(128 << 10); + tokio::spawn(pump(socket, ours)); + let _ = streams_tx.send((stream.to_owned(), theirs)); + } + } + }); + Self { + url, + hash, + requests, + streams, + session_protocols, + _task: task, + } + } + + fn relay(&self) -> Relay { + Relay { + url: self.url.clone(), + label: label(&self.url), + cert_hash: Some(self.hash.clone()), + carrier: Carrier::WebSocket, + } + } + + fn ask(&self, stream: &str) { + let url = format!( + "wss://127.0.0.1:{}/stream/{stream}", + self.url.port().unwrap() + ); + self.requests + .send(Message::text( + serde_json::json!({ "stream": url }).to_string(), + )) + .unwrap(); + } + + async fn stream(&mut self) -> (String, tokio::io::DuplexStream) { + tokio::time::timeout(Duration::from_secs(5), self.streams.recv()) + .await + .expect("the producer opened no stream WebSocket") + .unwrap() + } +} + +/// Bytes between a stream WebSocket and a duplex pipe, as a relay forwards them. +#[cfg(unix)] +async fn pump(socket: tokio_tungstenite::WebSocketStream, pipe: tokio::io::DuplexStream) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let (mut sink, mut source) = socket.split(); + let (mut read, mut write) = tokio::io::split(pipe); + let up = async { + while let Some(Ok(message)) = source.next().await { + match message { + Message::Binary(bytes) => { + if write.write_all(&bytes).await.is_err() { + break; + } + } + Message::Close(_) => break, + _ => {} + } + } + let _ = write.shutdown().await; + }; + let down = async { + let mut buffer = vec![0; 16 << 10]; + loop { + match read.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(count) => { + let chunk = bytes::Bytes::copy_from_slice(&buffer[..count]); + if sink.send(Message::Binary(chunk)).await.is_err() { + break; + } + } + } + } + let _ = sink.close().await; + }; + tokio::join!(up, down); +} + +#[cfg(unix)] +#[tokio::test] +async fn websocket_session_serves_consumers_and_takes_allowlist_changes() { + use yas_composite_transport::{Offer, Role}; + tokio::time::timeout(Duration::from_secs(20), async { + let dir = tempfile::tempdir().unwrap(); + let keys = keys(); + let (local, listener) = local_socket(dir.path()); + let mut relay = WebSocketRelay::start().await; + let producer = Producer::new( + "https://127.0.0.1:1/control", + "token", + keys.server.clone(), + local, + ) + .unwrap(); + let (events, seen) = events(); + let producer = Producer { events, ..producer }; + let handle = producer.handle(); + let active = Active::default(); + let session = { + let (producer, active, relay) = (producer.clone(), active.clone(), relay.relay()); + tokio::spawn(async move { producer.websocket_session(&relay, &active).await }) + }; + // The session is up once it reports it. + for _ in 0..200 { + if seen + .lock() + .unwrap() + .iter() + .any(|line| line.starts_with("connected")) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!( + seen.lock().unwrap()[0], + format!( + "connected to relay 127.0.0.1:{} over WebSocket", + relay.url.port().unwrap() + ) + ); + assert_eq!( + relay.session_protocols.lock().unwrap()[0], + WEBSOCKET_SUBPROTOCOL + ); + + // A stream elsewhere than the session's origin is ignored. + relay + .requests + .send(Message::text( + r#"{"stream":"wss://127.0.0.2:1/stream/elsewhere"}"#.to_owned(), + )) + .unwrap(); + relay + .requests + .send(Message::text("{\"hello\":1}".to_owned())) + .unwrap(); + + // A consumer: authenticated, then its bytes reach the local server. + relay.ask("first"); + let (name, stream) = relay.stream().await; + assert_eq!(name, "first"); + assert_eq!( + relay.session_protocols.lock().unwrap()[1], + WEBSOCKET_SUBPROTOCOL + ); + assert!(nothing_accepted(&listener).await); + let mut first = yas_uplink::connect(stream, keys.client.clone()) + .await + .unwrap(); + first.write_all(yas_wire::PREFACE.as_slice()).await.unwrap(); + first.write_all(b"command").await.unwrap(); + first.flush().await.unwrap(); + let (mut ipc, _) = listener.accept().await.unwrap(); + let mut received = vec![0; yas_wire::PREFACE.len() + 7]; + ipc.read_exact(&mut received).await.unwrap(); + assert_eq!( + received, + [yas_wire::PREFACE.as_slice(), b"command"].concat() + ); + + // A key it doesn't allow is refused before the local server hears of it. + relay.ask("stranger"); + let (_, stream) = relay.stream().await; + assert!( + yas_uplink::connect(stream, keys.other.clone()) + .await + .is_err() + ); + assert!(nothing_accepted(&listener).await); + + // The allowlist changes while it runs: the other key only, now. + handle.set_server_config( + keys.server_identity + .server_config(vec![keys.other_public]) + .unwrap(), + ); + relay.ask("revoked"); + let (_, stream) = relay.stream().await; + assert!( + yas_uplink::connect(stream, keys.client.clone()) + .await + .is_err() + ); + relay.ask("added"); + let (_, stream) = relay.stream().await; + let mut added = yas_uplink::connect(stream, keys.other.clone()) + .await + .unwrap(); + added.write_all(yas_wire::PREFACE.as_slice()).await.unwrap(); + added.flush().await.unwrap(); + let (mut second, _) = listener.accept().await.unwrap(); + let mut preface = vec![0; yas_wire::PREFACE.len()]; + second.read_exact(&mut preface).await.unwrap(); + assert_eq!(preface, yas_wire::PREFACE.as_slice()); + drop(second); + drop(added); + let _ = keys.client_public; + + // The consumer connected before keeps its session, both ways, + // including the local server finishing first (Noise's half-close). + ipc.write_all(b"reply").await.unwrap(); + ipc.shutdown().await.unwrap(); + let mut reply = [0; 5]; + first.read_exact(&mut reply).await.unwrap(); + assert_eq!(&reply, b"reply"); + first.write_all(b" and more").await.unwrap(); + first.flush().await.unwrap(); + let mut more = [0; 9]; + ipc.read_exact(&mut more).await.unwrap(); + assert_eq!(&more, b" and more"); + first.shutdown().await.unwrap(); + drop(ipc); + + // A datagram lane can't be carried: refused, nothing opened locally. + relay.ask("composite"); + let (_, stream) = relay.stream().await; + let mut composite = yas_uplink::connect(stream, keys.other.clone()) + .await + .unwrap(); + yas_composite_transport::write_offer( + &mut composite, + Offer::new(Role::Main, [7; 16], 512).unwrap(), + ) + .await + .unwrap(); + composite.flush().await.unwrap(); + for _ in 0..200 { + if seen + .lock() + .unwrap() + .iter() + .any(|line| line.contains("datagram lane")) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(nothing_accepted(&listener).await); + assert!( + seen.lock() + .unwrap() + .iter() + .any(|line| line.contains("a WebSocket session carries no datagrams")) + ); + + // A consumer still connected when the session ends goes with it, as + // a WebTransport stream goes with its connection. + relay.ask("lingering"); + let (_, stream) = relay.stream().await; + let mut lingering = yas_uplink::connect(stream, keys.other.clone()) + .await + .unwrap(); + lingering + .write_all(yas_wire::PREFACE.as_slice()) + .await + .unwrap(); + lingering.flush().await.unwrap(); + let (mut held, _) = listener.accept().await.unwrap(); + let mut preface = vec![0; yas_wire::PREFACE.len()]; + held.read_exact(&mut preface).await.unwrap(); + + // Shutting down closes the session with a close frame. + active.close().await; + match tokio::time::timeout(Duration::from_secs(5), session).await { + Ok(Ok(SessionEnd::Ended { reason, .. })) => { + assert!(!reason.is_empty()); + } + _ => panic!("the session didn't end"), + } + let mut rest = Vec::new(); + assert!( + tokio::time::timeout(Duration::from_secs(5), held.read_to_end(&mut rest)) + .await + .is_ok(), + "the lingering consumer's local connection closes with the session" + ); + drop(lingering); + let _ = keys.server_public; + }) + .await + .expect("WebSocket session test stalled"); +} + +#[tokio::test] +async fn websocket_relay_must_speak_the_subprotocol() { + // A WebSocket server that selects no subprotocol is not an uplink relay. + yas_webrtc_forwarder::tls::install_default_provider(); + let cert = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap(); + let hash = wt::crypto::sha256(&yas_webrtc_forwarder::tls::provider(), cert.cert.der()) + .as_ref() + .to_vec(); + let tls = tokio_rustls::TlsAcceptor::from(Arc::new( + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![cert.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()) + .into(), + ) + .unwrap(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + let tls = tls.accept(tcp).await.unwrap(); + let _ = tokio_tungstenite::accept_async(tls).await; + }); + let url: url::Url = format!("wss://127.0.0.1:{port}/session").parse().unwrap(); + let error = connect_websocket(&url, Some(&hash)).await.err().unwrap(); + assert!(error.contains("doesn't speak yas-uplink.v1"), "{error}"); + // A wrong pin fails TLS, and the error never quotes the URL. + let error = connect_websocket(&url, Some(&[0; 32])).await.err().unwrap(); + assert!(!error.contains("/session"), "{error}"); +} + +#[tokio::test] +async fn webtransport_gives_up_quickly_where_udp_goes_nowhere() { + yas_webrtc_forwarder::tls::install_default_provider(); + // A UDP socket that never answers: what a network dropping UDP looks like. + let hole = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let url: url::Url = format!( + "https://127.0.0.1:{}/t/x", + hole.local_addr().unwrap().port() + ) + .parse() + .unwrap(); + let relay = Relay { + label: label(&url), + url, + cert_hash: Some(vec![0; 32]), + carrier: Carrier::WebTransport, + }; + let keys = keys(); + let dir = tempfile::tempdir().unwrap(); + let producer = Producer::new( + "https://127.0.0.1:1/control", + "token", + keys.server, + Local::socket(dir.path().join("none").to_str().unwrap()), + ) + .unwrap(); + let started = Instant::now(); + let end = producer + .webtransport_session(&relay, Some(Duration::from_millis(300)), &Active::default()) + .await; + match end { + SessionEnd::NeverConnected(error) => assert!(error.contains("is UDP blocked?"), "{error}"), + SessionEnd::Ended { .. } => panic!("a session through a black hole"), + } + assert!(started.elapsed() < Duration::from_secs(5)); +} + +#[test] +fn parse_pool_accepts_plain_and_pinned_relay_urls() { + let pool = parse_pool( + r#"{"relays":[ + "https://relay-1.indent.com:4443/t/kfV3aB", + "https://[2001:db8::7]/session?key=x#sha256=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + ],"ttl":60}"#, + ) + .unwrap(); + assert_eq!(pool.webtransport.len(), 2); + assert!(pool.websocket.is_empty()); + let relays = &pool.webtransport; + assert_eq!( + relays[0].url.as_str(), + "https://relay-1.indent.com:4443/t/kfV3aB" + ); + assert_eq!(relays[0].label, "relay-1.indent.com:4443"); + assert!(relays[0].cert_hash.is_none()); + assert_eq!(relays[1].label, "[2001:db8::7]:443"); + assert_eq!(relays[1].cert_hash.as_ref().unwrap().len(), 32); + // The pin must be stripped before the URL is used to connect. + assert_eq!(relays[1].url.fragment(), None); + assert_eq!(relays[1].url.query(), Some("key=x")); +} + +#[test] +fn parse_pool_takes_websocket_relays() { + let pool = parse_pool( + r#"{"relays":["https://relay.example:4433/t/a"], + "websockets":["wss://relay.example/uplink/producer/b", + "wss://relay.example:8443/p#sha256=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]}"#, + ) + .unwrap(); + assert_eq!(pool.webtransport.len(), 1); + assert_eq!(pool.websocket.len(), 2); + assert_eq!(pool.websocket[0].label, "relay.example:443"); + assert_eq!(pool.websocket[0].carrier, Carrier::WebSocket); + assert_eq!(pool.websocket[1].cert_hash.as_ref().unwrap().len(), 32); + assert_eq!(pool.websocket[1].url.fragment(), None); + // WebSocket relays alone make a pool; so does an empty WebTransport list. + let pool = parse_pool(r#"{"relays":[],"websockets":["wss://relay.example/p"]}"#).unwrap(); + assert!(pool.webtransport.is_empty()); + let pool = parse_pool(r#"{"websockets":["wss://relay.example/p"]}"#).unwrap(); + assert_eq!(pool.websocket.len(), 1); +} + +#[test] +fn parse_pool_rejects_bad_input() { + assert!(parse_pool("not json").is_err()); + assert!(parse_pool(r#"{"relays":[]}"#).is_err()); + assert!(parse_pool(r#"{"relays":[],"websockets":[]}"#).is_err()); + assert!(parse_pool(r#"{"relays":[{"host":"h"}]}"#).is_err()); + assert!(parse_pool(r#"{"relays":"https://relay.example/t"}"#).is_err()); + assert!( + parse_relay("http://relay.example/t/x", Carrier::WebTransport).is_err(), + "non-https scheme must be rejected" + ); + assert!( + parse_relay( + "https://relay.example/t/x#sha256=AAAA", + Carrier::WebTransport + ) + .is_err(), + "a pin of the wrong length must be rejected, not ignored" + ); + assert!( + parse_relay("https://relay.example/t/x#pin=abc", Carrier::WebTransport).is_err(), + "an unrecognized fragment must be rejected, not ignored" + ); + assert!( + parse_relay("ws://relay.example/p", Carrier::WebSocket).is_err(), + "a plaintext WebSocket relay must be rejected: its URL is a credential" + ); + assert!(parse_relay("https://relay.example/p", Carrier::WebSocket).is_err()); + assert!(parse_relay("wss://user:pass@relay.example/p", Carrier::WebSocket).is_err()); + // One bad WebSocket relay spoils the pool, as a bad WebTransport one does. + assert!( + parse_pool( + r#"{"relays":["https://relay.example/t"],"websockets":["ws://relay.example/p"]}"# + ) + .is_err() + ); +} + +fn pool(webtransport: usize, websocket: usize) -> Pool { + let relay = |carrier, n| { + let url = match carrier { + Carrier::WebTransport => format!("https://wt{n}.example/t"), + Carrier::WebSocket => format!("wss://ws{n}.example/p"), + }; + parse_relay(&url, carrier).unwrap() + }; + Pool { + webtransport: (0..webtransport) + .map(|n| relay(Carrier::WebTransport, n)) + .collect(), + websocket: (0..websocket) + .map(|n| relay(Carrier::WebSocket, n)) + .collect(), + } +} + +fn carriers(relays: &[Relay]) -> Vec { + relays.iter().map(|relay| relay.carrier).collect() +} + +#[test] +fn order_puts_webtransport_first_unless_it_is_failing() { + use Carrier::{WebSocket as S, WebTransport as T}; + assert_eq!( + carriers(&order(pool(2, 1), Transport::Auto, false)), + [T, T, S] + ); + assert_eq!( + carriers(&order(pool(2, 1), Transport::Auto, true)), + [S, T, T] + ); + assert_eq!( + carriers(&order(pool(2, 1), Transport::WebTransport, true)), + [T, T] + ); + assert_eq!( + carriers(&order(pool(2, 1), Transport::WebSocket, false)), + [S] + ); + assert!(order(pool(0, 1), Transport::WebTransport, false).is_empty()); + assert!(order(pool(1, 0), Transport::WebSocket, false).is_empty()); +} + +#[test] +fn transport_parses() { + assert_eq!("auto".parse::().unwrap(), Transport::Auto); + assert_eq!( + " WebSocket ".parse::().unwrap(), + Transport::WebSocket + ); + assert_eq!( + "webtransport".parse::().unwrap(), + Transport::WebTransport + ); + assert!("udp".parse::().is_err()); + assert_eq!(Transport::WebSocket.to_string(), "websocket"); +} + +#[test] +fn stream_requests_stay_on_the_session_origin() { + let session: url::Url = "wss://relay.example/uplink/producer/abc".parse().unwrap(); + let ask = + |url: &str| stream_request(&serde_json::json!({ "stream": url }).to_string(), &session); + assert_eq!( + ask("wss://relay.example/uplink/stream/1").unwrap().as_str(), + "wss://relay.example/uplink/stream/1" + ); + assert!(ask("wss://relay.example:443/uplink/stream/1").is_some()); + assert!(ask("wss://relay.example:8443/uplink/stream/1").is_none()); + assert!(ask("wss://elsewhere.example/uplink/stream/1").is_none()); + assert!(ask("ws://relay.example/uplink/stream/1").is_none()); + assert!(ask("wss://user@relay.example/uplink/stream/1").is_none()); + assert!(ask("wss://relay.example/uplink/stream/1#x").is_none()); + assert!(stream_request("not json", &session).is_none()); + assert!(stream_request(r#"{"other":"wss://relay.example/x"}"#, &session).is_none()); +} + +#[test] +fn events_read_as_yas_uplink_prints_them() { + let connected = |carrier| Event::Connected { + relay: "relay.example:443".into(), + carrier, + }; + assert_eq!( + connected(Carrier::WebTransport).to_string(), + "connected to relay relay.example:443" + ); + assert_eq!( + connected(Carrier::WebSocket).to_string(), + "connected to relay relay.example:443 over WebSocket" + ); + assert_eq!( + Event::PoolExhausted { + retry_in: Duration::from_secs(4) + } + .to_string(), + "relay pool exhausted; re-querying in 4s" + ); +} + +#[test] +fn base64url_decodes() { + assert_eq!(base64url_decode("aGVsbG8").unwrap(), b"hello"); + assert_eq!(base64url_decode("aGVsbG8=").unwrap(), b"hello"); + assert_eq!(base64url_decode("_-8").unwrap(), vec![0xff, 0xef]); + assert!(base64url_decode("a+b").is_none()); + assert_eq!(base64url_decode("").unwrap(), Vec::::new()); +} diff --git a/docs/uplink.md b/docs/uplink.md index 49207b1a3..fa1003f54 100644 --- a/docs/uplink.md +++ b/docs/uplink.md @@ -2,13 +2,15 @@ `yas uplink --allow-client PUBLIC_KEY`, with the private key in `YAS_UPLINK_IDENTITY`, makes -the local YAS server reachable outside NAT. It holds an outbound WebTransport -session to a relay. Each relay-initiated stream must complete end-to-end +the local YAS server reachable outside NAT. It holds an outbound session to a +relay: WebTransport (HTTP/3 over UDP), or WebSockets (over TCP) where a network +blocks UDP. Each relay-initiated stream must complete end-to-end mutual authentication before it can reach the local server socket. This document specifies the protocol between the uplink and its control endpoint and relay. It leaves the relay side abstract: a WebTransport server that opens -one bidirectional stream per consumer and -forwards opaque bytes can act as a relay. The inner stream carries Noise records. +one bidirectional stream per consumer, or a WebSocket server that asks for +one WebSocket per consumer, and forwards opaque bytes can act as a relay. The +inner stream carries Noise records. ## Roles @@ -16,7 +18,7 @@ forwards opaque bytes can act as a relay. The inner stream carries Noise records | ---------------- | -------------------------------------------------------------------- | | uplink | `yas uplink` — connects out, bridges streams to the local YAS server | | control endpoint | HTTPS URL that authenticates the uplink and allocates it a relay | -| relay | WebTransport server the uplink stays connected to | +| relay | WebTransport or WebSocket server the uplink stays connected to | | consumer | A YAS client reaching the server through the relay | ## End-to-end trust and setup @@ -150,11 +152,20 @@ Accept: application/json A success response is the **relay pool**: ```json -{ "relays": ["https://relay-1.example.com:4443/t/kfV3aB#sha256="] } +{ + "relays": ["https://relay-1.example.com:4443/t/kfV3aB#sha256="], + "websockets": ["wss://relay-1.example.com/uplink/producer/Qm9vdA"] +} ``` -- `relays` is a non-empty array of `https` URLs. Any other scheme is an - error. +- `relays` is an array of `https` URLs: WebTransport relays. Any other + scheme is an error. +- `websockets` is an array of `wss` URLs: WebSocket relays (see + [WebSocket relay session](#websocket-relay-session-version-1)). Any other + scheme, plaintext `ws` included, is an error. Uplinks older than this + field ignore it, so a control endpoint can offer both kinds to every uplink. +- Either array may be empty or absent, not both. An uplink only uses the + relays its transport allows (see [Choosing a transport](#choosing-a-transport)). - **A relay URL is a credential.** Whatever authenticates the uplink to the relay (a token in the path, a capability URL) is embedded in it. Implementations MUST NOT log relay URLs; log `host:port` instead. @@ -229,10 +240,33 @@ Both producers and consumers must upgrade and configure keys. Old `uplink:https://relay.example#CLIENT_TOKEN` URIs and unencrypted consumer streams are intentionally rejected. Relays must forward Noise bytes unchanged. +## Choosing a transport + +`yas uplink --transport auto|webtransport|websocket` (or +`YAS_UPLINK_TRANSPORT`) chooses what carries the relay session: + +- `auto` (the default) tries the pool's WebTransport relays first, then its + WebSocket relays. While the pool also names WebSocket relays, a WebTransport + session that isn't up within **5 seconds** counts as failed: a network that + drops UDP gives no error, and QUIC would otherwise wait out its idle + timeout. After WebTransport fails to connect, or a WebTransport session + ends within 60 seconds of starting, the uplink tries WebSocket relays first + for the next 10 minutes, then WebTransport first again. +- `webtransport` uses only the pool's `relays`; `websocket` only its + `websockets`. A pool with none of them is retried like a failed control + request. + +Both carry the same Noise streams to the same local server; only WebTransport +carries datagrams. + ## Relay session -The uplink shuffles the pool and tries each relay in order: a -WebTransport (HTTP/3 CONNECT) session to the relay URL. Liveness settings +The uplink shuffles the pool's relays of each transport and tries them in the +order above. + +### WebTransport relay session + +A WebTransport (HTTP/3 CONNECT) session to the relay URL. Liveness settings are a **10s keepalive** and a **30s idle timeout**, so a dead relay is noticed within 30 seconds without any application-level pings. @@ -241,6 +275,52 @@ per consumer**. After Noise authentication, the uplink bridges decrypted bytes to a fresh local YAS socket. Direct streams carry the normal YAS preface and length-prefixed frames, unparsed and unreframed inside Noise. +### WebSocket relay session (version 1) + +A relay can't open streams over one WebSocket, so it asks the uplink to open +one WebSocket per consumer instead. Both kinds of WebSocket use the +subprotocol `yas-uplink.v1`: the uplink offers it in +`Sec-WebSocket-Protocol` and fails the relay unless the relay selects it. A +later version of this carrier gets a new subprotocol, which the uplink offers +alongside. + +- **TLS.** `wss` only. A `#sha256=` fragment pins the relay's certificate + exactly as for WebTransport relays (and is stripped before connecting); + otherwise the system roots (or `SSL_CERT_FILE`/`SSL_CERT_DIR`) verify it. + The relay URL is a credential: implementations MUST NOT log it. +- **Session.** The uplink holds one WebSocket to the relay URL. The relay + sends a **text message per consumer**, a JSON object naming a stream URL: + + ```json + { "stream": "wss://relay-1.example.com/uplink/stream/c2Vzc2lvbg" } + ``` + + The stream URL is a single-use credential, and must be `wss` on the + session's own origin (scheme, host and port), without userinfo or a + fragment, so the session's TLS trust applies to it. The uplink ignores + messages that aren't such an object (unknown fields included, for later + relays) and never sends data messages on the session. + +- **Streams.** For each request the uplink opens a WebSocket to the stream + URL (same subprotocol, same pin) and treats it as a WebTransport stream: + Noise authentication, then the local socket. Binary messages carry opaque + chunks of at most 64 KiB whose boundaries mean nothing; text messages are + ignored. Noise's authenticated FIN carries the half-close, so neither end + closes a stream WebSocket before both directions are done; closing it ends + both. At most 64 streams may be authenticating per session; the uplink + ignores requests beyond that, and the relay gives up on a stream the uplink + doesn't open within 10 seconds. +- **Liveness.** The uplink pings every **10 seconds** and gives the session up + after **30 seconds** without hearing anything from the relay. A relay should + likewise drop a session it hears nothing on for 30 seconds. +- **Datagrams.** None: an encrypted composite selector asking for a datagram + lane is refused on a WebSocket session (the stream closes), and consumers + on reliable carriers never send one. + +A relay pairs a consumer's attachment (`/attach` and its WSS worker) with a +stream WebSocket just as it would with a WebTransport stream, forwarding the +bytes unchanged either way. + ### Encrypted native datagrams A consumer with an unreliable carrier sends a composite-main selector **inside @@ -293,8 +373,9 @@ move audio onto unreliable datagrams. ### Closure Failed authentication or unavailable local IPC closes the consumer stream. -On SIGINT the outer session closes with application code 2. The worker should -close its consumer connection when the corresponding stream closes. +On SIGINT the outer session closes: a WebTransport session with application +code 2, a WebSocket session with close code 1001 (going away). The worker +should close its consumer connection when the corresponding stream closes. ## Reconnection @@ -305,13 +386,47 @@ close its consumer connection when the corresponding stream closes. re-balanced on every reconnect — and reset the backoff. - Pool exhausted with no session established: back off (same schedule as the control endpoint) and re-query. -- On SIGINT the uplink closes the active session with code 2 instead of - letting it idle out on the relay. +- On SIGINT the uplink closes the active session (code 2, or WebSocket close + code 1001) instead of letting it idle out on the relay. Relay URLs stay valid for as long as their embedded credential does; the uplink treats each pool response as single-use and re-queries rather than caching it. +## Embedding the uplink + +The uplink is a library as well as a command: +`yas_proxy::uplink_producer` (re-exported as `yas_client::uplink_producer`). +It installs no process-wide TLS provider, changes no environment and never +exits the process. + +```rust +use yas_client::uplink_producer::{Event, Identity, Local, Producer, Transport}; + +let identity = Identity::from_base64(&private_key)?; +let producer = Producer::new( + "https://relay.example/uplink/control", + &token, + identity.server_config(allowed_client_keys)?, + Local::socket(socket_path), // a Unix socket, or a named pipe on Windows +)? +.transport(Transport::Auto) +.on_event(|event: &Event| eprintln!("[uplink] {event}")); +let handle = producer.handle(); +tokio::spawn(producer.run_until(shutdown)); +// Later, without restarting the session: +handle.set_server_config(identity.server_config(new_allowlist)?); +handle.set_token(renewed_token); +``` + +`Local::custom` reaches a server some other way. `Event` values (a relay +session up, with its carrier; a session ended; a relay failing…) display as +the lines `yas uplink` prints. A new allowlist applies to consumers that start +their handshake afterwards; consumers already connected keep their authority +until they disconnect, as across a restart. A new token applies from the next +control request. `run_until` returns an error only when the control endpoint +refuses the token. + ## Browser embedding Serve the browser client from an origin trusted independently of the relay. @@ -372,7 +487,12 @@ in process arguments. Both endpoints must upgrade together. `direnv exec . cargo test -p yas-cli --test uplink_e2e` exercises the complete CLI path through a local HTTPS control endpoint and WSS/WebTransport relay to -an isolated YAS server. CI includes this test in the Rust workspace test suite. +an isolated YAS server: over WebTransport, over WebSockets when forced, and +falling back to WebSockets when the WebTransport relay's UDP goes nowhere. CI +includes this test in the Rust workspace test suite. +`cargo test -p yas-proxy uplink_producer` covers the library: pools, the +transport order, WebSocket sessions and streams, allowlist changes while +running, and refusals before local IPC. For private relay infrastructure, `SSL_CERT_FILE` and `SSL_CERT_DIR` select outer TLS trust roots consistently for HTTPS, WSS, and WebTransport (unless a From 1f7552e49b121ab970d18407558111fb58c9b4ad Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Tue, 29 Sep 2026 19:31:15 -0700 Subject: [PATCH 02/10] Turn off Nagle on ssh and WebSocket upstream sockets (#81) --- crates/proxy/src/lib.rs | 4 +++- crates/ssh/src/lib.rs | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/proxy/src/lib.rs b/crates/proxy/src/lib.rs index 07d17136d..fe4e860a2 100644 --- a/crates/proxy/src/lib.rs +++ b/crates/proxy/src/lib.rs @@ -1057,10 +1057,12 @@ async fn connect_ws_mode( .max_message_size(Some(64 * 1024)) .max_frame_size(Some(64 * 1024)) }); + // Nagle's algorithm off (`disable_nagle`), as for tcp: upstreams: a request's small frames + // must not wait for the ACK of the one before, which the relay delays (40 ms on Linux). let (mut ws, response) = tokio_tungstenite::connect_async_tls_with_config( request, config, - false, + true, Some(yas_webrtc_forwarder::tls::websocket_connector()), ) .await diff --git a/crates/ssh/src/lib.rs b/crates/ssh/src/lib.rs index 64a4352d7..744ee4a96 100644 --- a/crates/ssh/src/lib.rs +++ b/crates/ssh/src/lib.rs @@ -1053,6 +1053,10 @@ async fn establish_connection( // every 15 s and give up after 3 consecutive misses (~45 s). keepalive_interval: Some(std::time::Duration::from_secs(15)), keepalive_max: 3, + // YAS requests are small frames, often several in a row: with Nagle's algorithm, one + // waits for the ACK of the one before, which the server delays (40 ms on Linux) since it + // has nothing to answer until the whole request is in. + nodelay: true, ..Default::default() }; From 09bc64f200c0c0f4554f2fccfe2362823ce180a0 Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Tue, 29 Sep 2026 19:31:40 -0700 Subject: [PATCH 03/10] Wake the compositor for a surface capture (#82) --- crates/compositor/src/imp.rs | 7 +++++++ crates/compositor/src/lib.rs | 3 +++ crates/server/src/lib.rs | 27 ++++++++++++++++++++++++--- crates/server/src/yas.rs | 13 ++++++++----- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/crates/compositor/src/imp.rs b/crates/compositor/src/imp.rs index fdcf2dc44..31759d6d7 100644 --- a/crates/compositor/src/imp.rs +++ b/crates/compositor/src/imp.rs @@ -12228,6 +12228,13 @@ impl CompositorCommandSender { ) -> Result<(), mpsc::SendError> { send_command_with_wake(&self.command_tx, command, || self.loop_signal.wakeup()) } + + /// Wake the compositor loop, after a command admitted on the raw channel + /// (`try_send`): an idle loop otherwise sees it only at its next dispatch + /// timeout, up to a second later. + pub fn wake(&self) { + self.loop_signal.wakeup(); + } } fn send_command_with_wake( diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index 50b37ad02..1854954fe 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -709,6 +709,9 @@ mod stub { ) -> Result<(), mpsc::SendError> { self.command_tx.send(command) } + + /// Wake the compositor event loop immediately. + pub fn wake(&self) {} } impl CompositorHandle { diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 2f99755a4..c8d9d2c99 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -2419,8 +2419,12 @@ fn downscale_target_color_mode( } } +/// Ask the compositor for a surface's pixels: the command, then `wake` for its +/// loop (an idle one would see the command only at its next dispatch timeout, a +/// second later), then the reply, or None after `timeout`. async fn request_surface_capture_with_timeout( command_tx: std::sync::mpsc::SyncSender, + wake: impl FnOnce(), surface_id: u16, scale_120: u16, timeout: Duration, @@ -2433,6 +2437,7 @@ async fn request_surface_capture_with_timeout( reply: tx, }) .ok()?; + wake(); // The compositor replies through a blocking std::sync::mpsc channel. // Wait for it off the async runtime so this request never stalls the @@ -15718,12 +15723,22 @@ mod tests { }) .unwrap(); + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let wake = { + let woken = woken.clone(); + move || woken.store(true, std::sync::atomic::Ordering::SeqCst) + }; let result = - request_surface_capture_with_timeout(command_tx, 7, 0, Duration::from_millis(50)).await; + request_surface_capture_with_timeout(command_tx, wake, 7, 0, Duration::from_millis(50)) + .await; let (w, h, pixels) = result.unwrap(); assert_eq!((w, h), (2, 3)); assert_eq!(pixels.to_rgba(w, h), vec![1, 2, 3, 4]); + assert!( + woken.load(std::sync::atomic::Ordering::SeqCst), + "the compositor loop is woken for the command" + ); } #[tokio::test] @@ -15736,8 +15751,14 @@ mod tests { }) .unwrap(); - let result = - request_surface_capture_with_timeout(command_tx, 7, 0, Duration::from_millis(50)).await; + let result = request_surface_capture_with_timeout( + command_tx, + || {}, + 7, + 0, + Duration::from_millis(50), + ) + .await; assert!(result.is_none()); } diff --git a/crates/server/src/yas.rs b/crates/server/src/yas.rs index 09d20bc0d..2798aa6e1 100644 --- a/crates/server/src/yas.rs +++ b/crates/server/src/yas.rs @@ -9753,16 +9753,19 @@ impl Session { .max_by_key(|(_, pixels)| u64::from(pixels.width) * u64::from(pixels.height)) .map(|(_, pixels)| (pixels.width, pixels.height, pixels.pixels.clone())) }); - let command_tx = shared - .compositor - .as_ref() - .map(|compositor| compositor.handle.command_tx.clone()); + let command_tx = shared.compositor.as_ref().map(|compositor| { + ( + compositor.handle.command_tx.clone(), + compositor.handle.command_sender(), + ) + }); (snapshot, command_tx) }; let mut captured = match command_tx { - Some(command_tx) => { + Some((command_tx, loop_waker)) => { super::request_surface_capture_with_timeout( command_tx, + move || loop_waker.wake(), surface_id, 0, Duration::from_secs(5), From 9d5fa98e964592afa86972b63c7deba94411de51 Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 02:02:25 -0700 Subject: [PATCH 04/10] Capture a surface at the revision its caller listed, in one round trip (#90) * Expect TIMEOUT from a terminal command wait nothing started Since #54 a command wait that runs out before any command it could report on has started answers TIMEOUT (NOT_FOUND stays for an exited terminal or an evicted index). The client_host test still expected NOT_FOUND, so it failed on main. * Say what a terminal command wait answers when no command started Since #54 the server answers TIMEOUT when a command wait runs out before any command it could report on has started; wait_terminal_command's documentation still said NOT_FOUND, which stays for an evicted index or an exited terminal. * Capture a surface at the revision its caller listed, in one round trip capture_surface looks the window's revision up (a WATCH snapshot) before CAPTURE: two round trips, three for a caller that listed the windows first to choose one, as Ultimator's screenshots do. capture_surface_at(id, revision, format) sends CAPTURE with the revision the caller has; when the window changed since, the server answers STALE and it looks the window up again and captures that, as capture_surface does. --- crates/cli/tests/client_host.rs | 53 +++++++++++++++++++++++++++++++-- crates/client/src/surface.rs | 32 ++++++++++++++++++-- crates/client/src/terminal.rs | 3 +- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/crates/cli/tests/client_host.rs b/crates/cli/tests/client_host.rs index f5befd7e3..eca8e90c0 100644 --- a/crates/cli/tests/client_host.rs +++ b/crates/cli/tests/client_host.rs @@ -1465,12 +1465,17 @@ exec sleep 600"#; std::fs::canonicalize(String::from_utf8(cwd).unwrap()).unwrap(), std::fs::canonicalize(directory.path()).unwrap() ); - // Nothing else starts: waiting for the next command finds none. + // Nothing else starts: waiting for the next command times out (TIMEOUT + // since #54; NOT_FOUND is for an exited terminal or an evicted index). let waited = client .wait_terminal_command(id, None, Duration::from_millis(200)) .await .unwrap_err(); - assert!(waited.is_not_found(), "{waited:?}"); + assert_eq!( + waited.status(), + Some(yas_client::wire::core::Status::Timeout), + "{waited:?}" + ); client.close_terminal(id).await.unwrap(); } @@ -1485,6 +1490,11 @@ async fn surfaces_are_none_without_the_compositor() { .await .unwrap_err(); assert!(missing.is_not_found(), "{missing:?}"); + let missing = client + .capture_surface_at(1, 1, CaptureFormat::Png) + .await + .unwrap_err(); + assert!(missing.is_not_found(), "{missing:?}"); } /// Against a real window: build the compositor's probe client @@ -1569,7 +1579,46 @@ async fn surfaces_capture_take_input_and_close_with_the_paste_probe() { .await .unwrap(); client.type_surface_text(surface.id, "é").await.unwrap(); + // At the revision listed, the capture goes at once. + let listed = client.surface(surface.id).await.unwrap(); + let png = client + .capture_surface_at(surface.id, listed.revision, CaptureFormat::Png) + .await + .unwrap(); + assert!(png.starts_with(b"\x89PNG\r\n\x1a\n"), "{} bytes", png.len()); client.resize_surface(surface.id, 320, 240).await.unwrap(); + // Once the window changed, that revision is stale: the server says so, and + // capture_surface_at looks the window up again rather than failing. + let deadline = Instant::now() + TIMEOUT; + while client.surface(surface.id).await.unwrap().revision == listed.revision { + assert!( + Instant::now() < deadline, + "the resize never changed the window" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let stale = client + .request_raw( + yas_client::wire::family::SURFACE, + yas_client::wire::surface::request_kind::CAPTURE, + yas_client::wire::Encode::encode(&yas_client::wire::surface::Capture { + surface_handle: surface.id, + revision: listed.revision, + initial_receive_credit: 0, + formats: vec![yas_client::wire::schema::surface::CAPTURE_PNG as u8], + extensions: Default::default(), + }) + .unwrap(), + None, + ) + .await + .unwrap(); + assert_eq!(stale.status, yas_client::wire::core::Status::Stale); + let png = client + .capture_surface_at(surface.id, listed.revision, CaptureFormat::Png) + .await + .unwrap(); + assert!(png.starts_with(b"\x89PNG\r\n\x1a\n"), "{} bytes", png.len()); // A read-only session sees the window but cannot send it input. let viewer = server diff --git a/crates/client/src/surface.rs b/crates/client/src/surface.rs index 3049b7e8d..e6ab0ea43 100644 --- a/crates/client/src/surface.rs +++ b/crates/client/src/surface.rs @@ -4,7 +4,8 @@ //! `YAS_SKIP_COMPOSITOR=1`; [`crate::host::HostOptions::compositor`] for a //! hosted one), the programs it starts get a `WAYLAND_DISPLAY`, and each //! window they map is a surface in the catalogue ([`Client::surfaces`]). A -//! client can capture one as an image ([`Client::capture_surface`]), click, +//! client can capture one as an image ([`Client::capture_surface`], or +//! [`Client::capture_surface_at`] with the revision it listed), click, //! scroll and type into it, resize, focus and close it: what `yas surface` //! does, for programs that drive GUIs (a screenshot, then a click). //! @@ -116,9 +117,34 @@ impl Client { Ok(SurfaceInfo::from_record(&self.surface_record(id).await?)) } - /// What the window shows now, as an image. + /// What the window shows now, as an image: its revision looked up, then `CAPTURE`, two + /// round trips. [`Client::capture_surface_at`] takes one with a revision already listed. pub async fn capture_surface(&self, id: u64, format: CaptureFormat) -> Result> { let record = self.surface_record(id).await?; + self.capture(id, record.revision, format).await + } + + /// What the window shows now, as an image, given the revision the caller last saw it at + /// ([`SurfaceInfo::revision`], from [`Client::surfaces`] or [`Client::surface`]): one + /// round trip. When the window changed since (the server answers `STALE`), it is looked up + /// again and captured at its current revision, as [`Client::capture_surface`] does. + pub async fn capture_surface_at( + &self, + id: u64, + revision: u64, + format: CaptureFormat, + ) -> Result> { + match self.capture(id, revision, format).await { + Err(error) if error.status() == Some(yas_wire::core::Status::Stale) => { + self.capture_surface(id, format).await + } + captured => captured, + } + } + + /// `CAPTURE` of the window at `revision`, which the server answers `STALE` unless it + /// is the window's current one. + async fn capture(&self, id: u64, revision: u64, format: CaptureFormat) -> Result> { let hook: Hook = Box::new(|prefix: &ResultPrefix| { InlineOrTransfer::decode(&prefix.body) .map(|result| delivery_routes(&result)) @@ -130,7 +156,7 @@ impl Client { request_kind::CAPTURE, wire::Capture { surface_handle: id, - revision: record.revision, + revision, initial_receive_credit: CAPTURE_CREDIT, formats: vec![match format { CaptureFormat::Png => schema::CAPTURE_PNG as u8, diff --git a/crates/client/src/terminal.rs b/crates/client/src/terminal.rs index 72bca5d1e..d0317995e 100644 --- a/crates/client/src/terminal.rs +++ b/crates/client/src/terminal.rs @@ -381,7 +381,8 @@ impl Client { /// one with this journal `index`, or the one running now (else the next /// one to start). Answers its journal record (exit code, times, command /// line). When it still runs at the timeout: a `Timeout` error; when - /// none started by then (or it left the backlog): a `NotFound` status. + /// none started by then: a `Timeout` status; when it left the backlog, + /// or the terminal exited: a `NotFound` status. /// Needs shell integration. pub async fn wait_terminal_command( &self, From b4dfaa058f007056db79ee7abc64447be9d8d63c Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 02:04:07 -0700 Subject: [PATCH 05/10] Write small files in place in one APPLY round trip (#89) * Expect TIMEOUT from a terminal command wait nothing started Since #54 a command wait that runs out before any command it could report on has started answers TIMEOUT (NOT_FOUND stays for an exited terminal or an evicted index). The client_host test still expected NOT_FOUND, so it failed on main. * Say what a terminal command wait answers when no command started Since #54 the server answers TIMEOUT when a command wait runs out before any command it could report on has started; wait_terminal_command's documentation still said NOT_FOUND, which stays for an evicted index or an exited terminal. * Write small files in place in one APPLY round trip APPLY WRITE_INLINE items take item flag APPLY_ITEM_IN_PLACE (2), offered with CAPABILITY_APPLY_IN_PLACE (32): the item writes as COMMIT of a STAGE_IN_PLACE stage does (open with create and truncate through a final symlink, no temporary, no rename), through one shared write_in_place. An in-place write onto a directory is CONFLICT with {EISDIR, open} in the item's ApplyOsErrors entry, as COMMIT's. The flag is INVALID on other item kinds and with APPLY_ITEM_CREATE_PARENTS; APPLY_ITEM_EXTENDED_FLAGS lists it beside the v1 baseline's APPLY_ITEM_FLAGS. yas-client's write_in_place sends it for content within the server's inline limit and stages the rest, so a write tool call that already spends a round trip resolving or reading the file takes two in all, not three. TypeScript codec parity (inPlace) and golden vector fs.apply.in_place.payload. * Allow InPlace::mode unread on Windows; wrap the capabilities paragraph Review of #89: the mode an in-place write gives a file it creates is read under cfg(unix) alone, so Windows' -D warnings build failed on dead_code (Stage::mode had atomic_replace to read it). Windows files take their directory's ACL; say so on the field. Rewrap yas.md's opt-in paragraph at 80 columns. --- crates/cli/src/yas_fs.rs | 2 + crates/cli/tests/client_host.rs | 50 +++--- crates/client/src/fs.rs | 37 +++- crates/server/src/yas_fs.rs | 264 +++++++++++++++++++++++----- crates/yas/schema_codegen.rs | 22 +++ crates/yas/src/fs.rs | 84 +++++++-- crates/yas/src/generated.rs | 12 +- crates/yas/src/lib.rs | 3 + docs/design/fs-write.md | 10 ++ docs/design/yas.md | 19 +- js/core/src/__tests__/yas.test.ts | 9 + js/core/src/__tests__/yasFs.test.ts | 4 + js/core/src/yas/fs.ts | 25 ++- js/core/src/yas/generated.ts | 27 ++- protocol/yas/families/fs.toml | 14 +- protocol/yas/schema.json | 18 +- protocol/yas/vectors.json | 4 + protocol/yas/wire.md | 1 + 18 files changed, 513 insertions(+), 92 deletions(-) diff --git a/crates/cli/src/yas_fs.rs b/crates/cli/src/yas_fs.rs index c4f730884..91aaadde7 100644 --- a/crates/cli/src/yas_fs.rs +++ b/crates/cli/src/yas_fs.rs @@ -462,6 +462,7 @@ async fn cmd_write( create_parents: parents, mode, content, + in_place: false, }, json, ) @@ -1241,6 +1242,7 @@ mod tests { create_parents: false, mode: 0, content: b"contents".to_vec(), + in_place: false, }, ApplyItem::Mkdir { path: wire_path("directory").unwrap(), diff --git a/crates/cli/tests/client_host.rs b/crates/cli/tests/client_host.rs index eca8e90c0..744c94b40 100644 --- a/crates/cli/tests/client_host.rs +++ b/crates/cli/tests/client_host.rs @@ -888,28 +888,38 @@ async fn files_answer_as_the_os_does() { pair("ENOENT", "lstat") ); - // In place through a symlink: the same inode, the new bytes. + // In place through a symlink: the same inode, the new bytes. Inline-sized content goes in + // one APPLY (CAPABILITY_APPLY_IN_PLACE), bigger content is staged: both answer alike. use std::os::unix::fs::MetadataExt; let inode = std::fs::metadata(base.join("file")).unwrap().ino(); - root.write_in_place("to-file", b"new content") - .await - .unwrap(); - assert_eq!(std::fs::read(base.join("file")).unwrap(), b"new content"); - assert_eq!(std::fs::metadata(base.join("file")).unwrap().ino(), inode); - assert!( - std::fs::symlink_metadata(base.join("to-file")) - .unwrap() - .file_type() - .is_symlink() - ); - assert_eq!( - named(root.write_in_place("dir", b"x").await.unwrap_err()), - pair("EISDIR", "open") - ); - assert_eq!( - named(root.write_in_place("missing/file", b"x").await.unwrap_err()), - pair("ENOENT", "open") - ); + let staged = vec![b's'; yas_client::wire::fs::MAX_INLINE_BYTES + 1]; + for content in [&b"new content"[..], &staged] { + let written = root.write_in_place("to-file", content).await.unwrap(); + assert_eq!(written.hash, *blake3::hash(content).as_bytes()); + assert_eq!(std::fs::read(base.join("file")).unwrap(), content); + assert_eq!(std::fs::metadata(base.join("file")).unwrap().ino(), inode); + assert!( + std::fs::symlink_metadata(base.join("to-file")) + .unwrap() + .file_type() + .is_symlink() + ); + let error = root.write_in_place("dir", content).await.unwrap_err(); + assert!(error.is_conflict(), "{error:?}"); + assert_eq!(named(error), pair("EISDIR", "open")); + assert_eq!( + named( + root.write_in_place("missing/file", content) + .await + .unwrap_err() + ), + pair("ENOENT", "open") + ); + // A new file is made where it is named. + let _ = std::fs::remove_file(base.join("fresh")); + root.write_in_place("fresh", content).await.unwrap(); + assert_eq!(std::fs::read(base.join("fresh")).unwrap(), content); + } assert_eq!( named(root.read("missing").await.unwrap_err()), diff --git a/crates/client/src/fs.rs b/crates/client/src/fs.rs index 3d5799448..2062ccc11 100644 --- a/crates/client/src/fs.rs +++ b/crates/client/src/fs.rs @@ -208,6 +208,16 @@ impl Client { .and_then(|limits| wire::Limits::from_extensions(&limits).ok()) .map_or(0, |limits| limits.capabilities) } + + /// The most content one inline FS write carries on this server (`MAX_INLINE_BYTES`, as it + /// advertises it): 0 before the FS family is negotiated. + fn fs_inline_bytes(&self) -> usize { + self.family_limits(family::FS) + .and_then(|limits| wire::Limits::from_extensions(&limits).ok()) + .map_or(0, |limits| { + (limits.max_inline_bytes as usize).min(wire::MAX_INLINE_BYTES) + }) + } } /// File content read by [`FsRoot::read`] and friends. @@ -797,6 +807,7 @@ impl FsRoot { create_parents: options.create_parents, mode: options.mode, content: content.to_vec(), + in_place: false, }) .await; } @@ -819,11 +830,13 @@ impl FsRoot { .await } - /// Write a whole file as `open(2)` with `O_WRONLY|O_CREAT|O_TRUNC` and `write(2)` would - /// (`STAGE_IN_PLACE`), as Node's `writeFile` does: through a final symlink, an existing - /// file keeping its inode, owner and mode, a new one mode 0666 less the server's umask. Not - /// atomic: a failure can leave the file truncated. Needs `CAPABILITY_STAGE_IN_PLACE`; a - /// failure carries its [`os_error`]. + /// Write a whole file as `open(2)` with `O_WRONLY|O_CREAT|O_TRUNC` and `write(2)` would, + /// as Node's `writeFile` does: through a final symlink, an existing file keeping its inode, + /// owner and mode, a new one mode 0666 less the server's umask. Not atomic: a failure can + /// leave the file truncated. Content up to the server's inline limit goes in one `APPLY` + /// (one round trip) where it offers `CAPABILITY_APPLY_IN_PLACE`; other content is staged + /// (`STAGE_IN_PLACE`, needing `CAPABILITY_STAGE_IN_PLACE`; two round trips). A failure + /// carries its [`os_error`] either way. pub async fn write_in_place(&self, path: &str, content: &[u8]) -> Result { if content.len() as u64 > MAX_FILE_BYTES { return Err(Error::invalid(format!( @@ -831,6 +844,20 @@ impl FsRoot { content.len() ))); } + if content.len() <= self.client.fs_inline_bytes() + && self.client.fs_capabilities() & schema::CAPABILITY_APPLY_IN_PLACE as u32 != 0 + { + return self + .apply_one(ApplyItem::WriteInline { + path: wire_path(path)?, + precondition: Precondition::Any, + create_parents: false, + mode: 0, + content: content.to_vec(), + in_place: true, + }) + .await; + } self.stage_and_commit( wire_path(path)?, content, diff --git a/crates/server/src/yas_fs.rs b/crates/server/src/yas_fs.rs index e5ccfdaa9..cfc3a0e8b 100644 --- a/crates/server/src/yas_fs.rs +++ b/crates/server/src/yas_fs.rs @@ -2536,18 +2536,67 @@ fn in_place_destination(root: &Root, target: &OsPath) -> Result })) } -/// COMMIT of a `STAGE_IN_PLACE` stage: open(2) the target write-only with -/// O_CREAT|O_TRUNC through a final symlink and write the bytes, as Node's -/// writeFile does. The file keeps its inode, owner and mode; no temporary -/// file, no rename, so a failure can leave it truncated or partly written. +/// COMMIT of a `STAGE_IN_PLACE` stage: its staged bytes written in place. fn commit_in_place( stage: &Stage, target: &OsPath, operation_id: [u8; 16], flags: u16, ) -> Result { - let destination = in_place_destination(&stage.root, target)?; - let is_directory = || synthetic_os(conflict_for(&stage.root, &stage.path), EISDIR, "open"); + let mut source = File::open(&stage.temp_path).map_err(os_io("open"))?; + let written = write_in_place( + &stage.root, + &stage.path, + target, + InPlace { + mode: stage.mode, + content_hash: stage.content_hash, + operation_id, + flags, + }, + |file| io::copy(&mut source, file).map(|_| ()), + )?; + Ok(wire::CommitResult { + root_revision: stage.root.revision.load(Ordering::Acquire).max(1), + entry_revision: written.entry_revision, + modified_unix_ns: written.modified_unix_ns, + content_hash: stage.content_hash, + }) +} + +/// How an in-place write goes: the mode a file it creates gets (0: 0o666), +/// the content's hash, the operation marking the entry, and COMMIT flags +/// (`COMMIT_SYNC_DATA`, `COMMIT_SYNC_DIRECTORY`; APPLY gives none). +struct InPlace { + /// Read on Unix alone: a file Windows creates takes its directory's ACL. + #[cfg_attr(not(unix), allow(dead_code))] + mode: u32, + content_hash: [u8; 32], + operation_id: [u8; 16], + flags: u16, +} + +/// What an in-place write reports: the entry written (through a symlink +/// inside the root, the link's destination) and the file's modification time. +struct WrittenInPlace { + entry_revision: u64, + modified_unix_ns: i64, +} + +/// An in-place write (COMMIT of a `STAGE_IN_PLACE` stage, APPLY WRITE_INLINE +/// with `APPLY_ITEM_IN_PLACE`): open(2) `target` write-only with +/// O_CREAT|O_TRUNC through a final symlink and `write` the bytes, as Node's +/// writeFile does. The file keeps its inode, owner and mode; no temporary +/// file, no rename, so a failure can leave it truncated or partly written. +fn write_in_place( + root: &Root, + path: &wire::Path, + target: &OsPath, + how: InPlace, + write: impl FnOnce(&mut File) -> io::Result<()>, +) -> Result { + let destination = in_place_destination(root, target)?; + let is_directory = || synthetic_os(conflict_for(root, path), EISDIR, "open"); if fs::metadata(&destination).is_ok_and(|metadata| metadata.is_dir()) { return Err(is_directory()); } @@ -2556,7 +2605,7 @@ fn commit_in_place( #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; - options.mode(if stage.mode != 0 { stage.mode } else { 0o666 }); + options.mode(if how.mode != 0 { how.mode } else { 0o666 }); } let mut file = match options.open(target) { Ok(file) => file, @@ -2567,34 +2616,32 @@ fn commit_in_place( } Err(error) => return Err(os_io("open")(error)), }; - let mut source = File::open(&stage.temp_path).map_err(os_io("open"))?; - io::copy(&mut source, &mut file).map_err(os_io("write"))?; + write(&mut file).map_err(os_io("write"))?; file.flush().map_err(os_io("write"))?; - if flags & schema::fs::COMMIT_SYNC_DATA as u16 != 0 { + if how.flags & schema::fs::COMMIT_SYNC_DATA as u16 != 0 { file.sync_all().map_err(os_io("fsync"))?; } drop(file); - if flags & schema::fs::COMMIT_SYNC_DIRECTORY as u16 != 0 + if how.flags & schema::fs::COMMIT_SYNC_DIRECTORY as u16 != 0 && let Some(parent) = destination.parent() { sync_directory(parent)?; } // Report the file written: through a symlink inside the root that is // the link's destination entry, not the link. - let written = if stage.root.single_file { - stage.path.clone() + let written = if root.single_file { + path.clone() } else { - relative_path(&stage.root, &destination).unwrap_or_else(|_| stage.path.clone()) + relative_path(root, &destination).unwrap_or_else(|_| path.clone()) }; - mark_operation(&stage.root, &written, operation_id); + mark_operation(root, &written, how.operation_id); // The bytes are known and verified, so nothing is read back: a // write-only file answers as Node's writeFile would. let metadata = fs::metadata(&destination).map_err(os_io("stat"))?; - let named = - fs::symlink_metadata(joined_path(&stage.root, &written)?).map_err(os_io("lstat"))?; + let named = fs::symlink_metadata(joined_path(root, &written)?).map_err(os_io("lstat"))?; let entry_revision = if named.is_file() { observe_entry( - &stage.root, + root, wire::EntryRecord { path: written.clone(), entry_revision: 1, @@ -2603,7 +2650,7 @@ fn commit_in_place( modified_unix_ns: metadata_time(&named), body: wire::EntryBody::File { byte_len: named.len(), - content_hash: stage.content_hash, + content_hash: how.content_hash, inline_content: None, }, extensions: Extensions::default(), @@ -2612,13 +2659,11 @@ fn commit_in_place( .entry_revision } else { // A single-file root that is itself a symlink: the link's entry. - stat_entry(&stage.root, &written, 0, false)?.entry_revision + stat_entry(root, &written, 0, false)?.entry_revision }; - Ok(wire::CommitResult { - root_revision: stage.root.revision.load(Ordering::Acquire).max(1), + Ok(WrittenInPlace { entry_revision, modified_unix_ns: metadata_time(&metadata), - content_hash: stage.content_hash, }) } @@ -2632,8 +2677,9 @@ fn apply_items( let mut os_errors = Vec::new(); for (index, item) in items.iter().enumerate() { let outcome = apply_one(root, operation_id, item); + // A conflict carries one only where a check stood for a system call + // (an in-place write onto a directory is EISDIR, as COMMIT's). if let Err(error) = &outcome - && !matches!(error.kind(), Error::Conflict(_)) && let Some(os) = error.os_error() { os_errors.push((index as u16, os.clone())); @@ -2686,12 +2732,53 @@ fn apply_one( item: &wire::ApplyItem, ) -> Result { match item { + wire::ApplyItem::WriteInline { + path, + precondition, + create_parents: false, + mode, + content, + in_place: true, + } => { + check_precondition(root, path, precondition)?; + let target = mutation_target(root, path, false, "open")?; + let content_hash = *blake3::hash(content).as_bytes(); + let written = write_in_place( + root, + path, + &target, + InPlace { + mode: *mode, + content_hash, + operation_id, + flags: 0, + }, + |file| file.write_all(content), + )?; + Ok(wire::EntryRecord { + path: path.clone(), + entry_revision: written.entry_revision, + flags: 0, + mode: 0, + modified_unix_ns: written.modified_unix_ns, + body: wire::EntryBody::File { + byte_len: content.len() as u64, + content_hash, + inline_content: None, + }, + extensions: Extensions::default(), + }) + } + wire::ApplyItem::WriteInline { in_place: true, .. } => { + Err(Error::Invalid("FS APPLY in-place write creating parents")) + } wire::ApplyItem::WriteInline { path, precondition, create_parents, mode, content, + in_place: false, } => { check_precondition(root, path, precondition)?; atomic_write(root, path, *create_parents, *mode, content)?; @@ -3287,6 +3374,7 @@ mod tests { create_parents: false, mode: 0, content: b"one".to_vec(), + in_place: false, }, wire::ApplyItem::Rename { from: path(&[b"a"]), @@ -3826,9 +3914,108 @@ mod tests { } } + /// An in-place write of `bytes` to `components` with `mode` under + /// `operation_id`, answering the content hash it reports. + #[cfg(unix)] + type WriteInPlace = + dyn Fn(&Arc, &[&[u8]], u32, &[u8], [u8; 16]) -> Result<[u8; 32], Error>; + #[cfg(unix)] #[test] fn stage_in_place_keeps_the_inode_and_writes_through_symlinks() { + in_place_keeps_the_inode_and_writes_through_symlinks( + &|root, components, mode, bytes, id| { + let stage = sealed_stage(root, components, mode, bytes); + commit_stage(stage, id, schema::fs::COMMIT_SYNC_DATA as u16) + .map(|result| result.content_hash) + }, + ); + } + + #[cfg(unix)] + #[test] + fn apply_in_place_writes_as_commit_in_place_does() { + in_place_keeps_the_inode_and_writes_through_symlinks( + &|root, components, mode, bytes, id| { + let item = wire::ApplyItem::WriteInline { + path: path(components), + precondition: wire::Precondition::Any, + create_parents: false, + mode, + content: bytes.to_vec(), + in_place: true, + }; + let entry = apply_one(root, id, &item)?; + assert_eq!(entry.path, path(components)); + Ok(entry_hash(&entry).unwrap()) + }, + ); + } + + #[cfg(unix)] + #[test] + fn an_apply_in_place_onto_a_directory_is_a_conflict_saying_eisdir() { + let directory = TestDir::new(); + let root = test_root(&directory); + fs::create_dir(directory.0.join("dir")).unwrap(); + fs::write(directory.0.join("file"), b"old").unwrap(); + let write = |name: &[u8], in_place| wire::ApplyItem::WriteInline { + path: path(&[name]), + precondition: wire::Precondition::Any, + create_parents: false, + mode: 0, + content: b"x".to_vec(), + in_place, + }; + let result = apply_items( + &root, + [9; 16], + &[ + write(b"file", true), + write(b"dir", true), + write(b"dir", false), + ], + ) + .unwrap(); + let statuses: Vec = result.items.iter().map(|item| item.status).collect(); + assert_eq!( + statuses, + [ + schema::core::status::OK, + schema::core::status::CONFLICT, + schema::core::status::CONFLICT + ] + ); + assert_eq!(fs::read(directory.0.join("file")).unwrap(), b"x"); + // The in-place write says EISDIR, as COMMIT's; the replacing write + // keeps its bare conflict. + let os_errors = result.os_errors().unwrap(); + assert_eq!(os_errors.len(), 1); + assert_eq!(os_errors[0].0, 1); + assert_eq!( + ( + os_errors[0].1.name.as_str(), + os_errors[0].1.operation.as_str() + ), + ("EISDIR", "open") + ); + // Creating parents is never in place. + let item = wire::ApplyItem::WriteInline { + path: path(&[b"new", b"file"]), + precondition: wire::Precondition::Any, + create_parents: true, + mode: 0, + content: b"x".to_vec(), + in_place: true, + }; + assert!(matches!( + apply_one(&root, [10; 16], &item).unwrap_err().kind(), + Error::Invalid(_) + )); + } + + #[cfg(unix)] + fn in_place_keeps_the_inode_and_writes_through_symlinks(write: &WriteInPlace) { use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; let directory = TestDir::new(); let root = Arc::new(test_root(&directory)); @@ -3838,8 +4025,7 @@ mod tests { let inode = fs::metadata(&target).unwrap().ino(); symlink("target.txt", directory.0.join("link")).unwrap(); - let stage = sealed_stage(&root, &[b"link"], 0o600, b"new"); - let result = commit_stage(stage, [4; 16], schema::fs::COMMIT_SYNC_DATA as u16).unwrap(); + let hash = write(&root, &[b"link"], 0o600, b"new", [4; 16]).unwrap(); assert_eq!(fs::read(&target).unwrap(), b"new"); let metadata = fs::metadata(&target).unwrap(); assert_eq!(metadata.ino(), inode); @@ -3850,7 +4036,7 @@ mod tests { .file_type() .is_symlink() ); - assert_eq!(result.content_hash, *blake3::hash(b"new").as_bytes()); + assert_eq!(hash, *blake3::hash(b"new").as_bytes()); assert_eq!( root.operation_echoes .lock() @@ -3860,13 +4046,11 @@ mod tests { ); // A new file is created where it is named, without a temporary. - let stage = sealed_stage(&root, &[b"fresh"], 0, b"fresh bytes"); - commit_stage(stage, [5; 16], 0).unwrap(); + write(&root, &[b"fresh"], 0, b"fresh bytes", [5; 16]).unwrap(); assert_eq!(fs::read(directory.0.join("fresh")).unwrap(), b"fresh bytes"); // Through a dangling symlink, the file it names is created. symlink("made-by-link", directory.0.join("pending")).unwrap(); - let stage = sealed_stage(&root, &[b"pending"], 0, b"via link"); - commit_stage(stage, [6; 16], 0).unwrap(); + write(&root, &[b"pending"], 0, b"via link", [6; 16]).unwrap(); assert_eq!( fs::read(directory.0.join("made-by-link")).unwrap(), b"via link" @@ -3876,13 +4060,8 @@ mod tests { let write_only = directory.0.join("write-only"); fs::write(&write_only, b"previous").unwrap(); fs::set_permissions(&write_only, fs::Permissions::from_mode(0o200)).unwrap(); - let result = commit_stage( - sealed_stage(&root, &[b"write-only"], 0, b"blind"), - [11; 16], - 0, - ) - .unwrap(); - assert_eq!(result.content_hash, *blake3::hash(b"blind").as_bytes()); + let hash = write(&root, &[b"write-only"], 0, b"blind", [11; 16]).unwrap(); + assert_eq!(hash, *blake3::hash(b"blind").as_bytes()); let metadata = fs::metadata(&write_only).unwrap(); assert_eq!( (metadata.permissions().mode() & 0o7777, metadata.len()), @@ -3894,17 +4073,14 @@ mod tests { fs::create_dir(directory.0.join("dir")).unwrap(); symlink("dir", directory.0.join("dir-link")).unwrap(); for components in [&[&b"dir"[..]][..], &[b"dir-link"]] { - let stage = sealed_stage(&root, components, 0, b"x"); - let error = commit_stage(stage, [7; 16], 0).unwrap_err(); + let error = write(&root, components, 0, b"x", [7; 16]).unwrap_err(); assert!(matches!(error.kind(), Error::Conflict(_))); assert_eq!(os_of(&error), (libc::EISDIR, "EISDIR", "open")); } - let stage = sealed_stage(&root, &[b"nope", b"file"], 0, b"x"); - let error = commit_stage(stage, [8; 16], 0).unwrap_err(); + let error = write(&root, &[b"nope", b"file"], 0, b"x", [8; 16]).unwrap_err(); assert_eq!(*error.kind(), Error::NotFound); assert_eq!(os_of(&error), (libc::ENOENT, "ENOENT", "open")); - let stage = sealed_stage(&root, &[b"target.txt", b"file"], 0, b"x"); - let error = commit_stage(stage, [10; 16], 0).unwrap_err(); + let error = write(&root, &[b"target.txt", b"file"], 0, b"x", [10; 16]).unwrap_err(); assert_eq!(os_of(&error), (libc::ENOTDIR, "ENOTDIR", "open")); } } diff --git a/crates/yas/schema_codegen.rs b/crates/yas/schema_codegen.rs index 0a952e24c..13527f146 100644 --- a/crates/yas/schema_codegen.rs +++ b/crates/yas/schema_codegen.rs @@ -2511,6 +2511,28 @@ fn vectors(artifact: &Artifact) -> VectorArtifact { push_u32(&mut fs_apply, 0); push_vector(&mut vectors, "fs.apply.payload", &fs_apply); + let mut fs_apply_in_place = Vec::new(); + push_u64(&mut fs_apply_in_place, 1); + fs_apply_in_place.extend_from_slice(&[5; 16]); + push_u16(&mut fs_apply_in_place, 0); + push_u16(&mut fs_apply_in_place, 1); + push_u32(&mut fs_apply_in_place, 4 + fs_apply_item_body.len() as u32); + push_u16( + &mut fs_apply_in_place, + family_constant(artifact, "yas.fs", "APPLY_WRITE_INLINE") as u16, + ); + push_u16( + &mut fs_apply_in_place, + family_constant(artifact, "yas.fs", "APPLY_ITEM_IN_PLACE") as u16, + ); + fs_apply_in_place.extend_from_slice(&fs_apply_item_body); + push_u32(&mut fs_apply_in_place, 0); + push_vector( + &mut vectors, + "fs.apply.in_place.payload", + &fs_apply_in_place, + ); + let mut fs_entry = Vec::new(); push_bytes_u32(&mut fs_entry, &fs_path_a); push_u64(&mut fs_entry, 1); diff --git a/crates/yas/src/fs.rs b/crates/yas/src/fs.rs index c942e0cb1..5c31887f0 100644 --- a/crates/yas/src/fs.rs +++ b/crates/yas/src/fs.rs @@ -2528,6 +2528,10 @@ pub enum ApplyItem { create_parents: bool, mode: u32, content: Vec, + /// `APPLY_ITEM_IN_PLACE` (with `CAPABILITY_APPLY_IN_PLACE`): write as `open(2)` with + /// `O_WRONLY|O_CREAT|O_TRUNC` would, through a final symlink, rather than replace the + /// file; never with `create_parents`. + in_place: bool, }, Mkdir { path: Path, @@ -2569,8 +2573,12 @@ impl ApplyItem { precondition, mode, content, - .. + create_parents, + in_place, } => { + if *in_place && *create_parents { + return Err(Error::Invalid("FS APPLY in-place write creating parents")); + } if content.len() > MAX_INLINE_BYTES { return Err(limit( "FS inline apply bytes", @@ -2668,16 +2676,16 @@ impl ApplyItem { let len = 4usize .checked_add(body.len()) .ok_or(Error::LengthOverflow)?; + let mut item_flags = 0; + if create_parents { + item_flags |= crate::schema::fs::APPLY_ITEM_CREATE_PARENTS as u16; + } + if matches!(self, Self::WriteInline { in_place: true, .. }) { + item_flags |= crate::schema::fs::APPLY_ITEM_IN_PLACE as u16; + } put_len_u32(out, len)?; put_u16(out, kind); - put_u16( - out, - if create_parents { - crate::schema::fs::APPLY_ITEM_CREATE_PARENTS as u16 - } else { - 0 - }, - ); + put_u16(out, item_flags); out.extend_from_slice(&body); Ok(()) } @@ -2687,10 +2695,18 @@ impl ApplyItem { let mut item = Decoder::new(bytes); let kind = item.u16()?; let item_flags = item.u16()?; - if item_flags & !(crate::schema::fs::APPLY_ITEM_FLAGS as u16) != 0 { + let known = + crate::schema::fs::APPLY_ITEM_FLAGS | crate::schema::fs::APPLY_ITEM_EXTENDED_FLAGS; + if item_flags & !(known as u16) != 0 { return Err(Error::Invalid("FS APPLY item flags")); } let create_parents = item_flags & crate::schema::fs::APPLY_ITEM_CREATE_PARENTS as u16 != 0; + let in_place = item_flags & crate::schema::fs::APPLY_ITEM_IN_PLACE as u16 != 0; + if in_place && kind != crate::schema::fs::APPLY_WRITE_INLINE as u16 { + return Err(Error::Invalid( + "FS APPLY in-place flag on an item that writes no file", + )); + } let value = match kind { value if value == crate::schema::fs::APPLY_WRITE_INLINE as u16 => Self::WriteInline { path: Path::decode(item.len_bytes_u32()?)?, @@ -2698,6 +2714,7 @@ impl ApplyItem { create_parents, mode: item.u32()?, content: item.len_bytes_u32()?.to_vec(), + in_place, }, value if value == crate::schema::fs::APPLY_MKDIR as u16 => Self::Mkdir { path: Path::decode(item.len_bytes_u32()?)?, @@ -3269,12 +3286,59 @@ mod tests { create_parents: true, mode: 0o644, content: b"yas".to_vec(), + in_place: false, }], extensions: Extensions::default(), }; truncations::(&apply.encode().unwrap()); } + #[test] + fn an_in_place_apply_write_is_a_write_inline_flag_without_create_parents() { + let write = |create_parents, in_place| Apply { + root_handle: 1, + operation_id: [3; 16], + flags: 0, + items: vec![ApplyItem::WriteInline { + path: path(b"file"), + precondition: Precondition::Any, + create_parents, + mode: 0, + content: b"yas".to_vec(), + in_place, + }], + extensions: Extensions::default(), + }; + let apply = write(false, true); + let bytes = apply.encode().unwrap(); + assert_eq!(Apply::decode(&bytes).unwrap(), apply); + truncations::(&bytes); + // The item's flags follow the root handle, operation ID, flags, count and item length. + let flags_at = 8 + 16 + 2 + 2 + 4 + 2; + assert_eq!( + u16::from_le_bytes([bytes[flags_at], bytes[flags_at + 1]]), + crate::schema::fs::APPLY_ITEM_IN_PLACE as u16 + ); + assert!(write(true, true).encode().is_err()); + let mut both = bytes.clone(); + both[flags_at] |= crate::schema::fs::APPLY_ITEM_CREATE_PARENTS as u8; + assert!(Apply::decode(&both).is_err()); + // Only a write may be in place. + let mkdir = Apply { + items: vec![ApplyItem::Mkdir { + path: path(b"dir"), + precondition: Precondition::Any, + create_parents: false, + mode: 0, + }], + ..apply + }; + let mut bytes = mkdir.encode().unwrap(); + assert_eq!(Apply::decode(&bytes).unwrap(), mkdir); + bytes[flags_at] |= crate::schema::fs::APPLY_ITEM_IN_PLACE as u8; + assert!(Apply::decode(&bytes).is_err()); + } + #[test] fn capabilities_are_an_optional_forward_compatible_limit() { assert_eq!( diff --git a/crates/yas/src/generated.rs b/crates/yas/src/generated.rs index 97a3e8bda..a3b9caf40 100644 --- a/crates/yas/src/generated.rs +++ b/crates/yas/src/generated.rs @@ -2346,6 +2346,8 @@ pub const APPLY_SYMLINK: u64 = 4; pub const APPLY_HARDLINK: u64 = 5; pub const APPLY_ITEM_CREATE_PARENTS: u64 = 1; pub const APPLY_ITEM_FLAGS: u64 = 1; +pub const APPLY_ITEM_IN_PLACE: u64 = 2; +pub const APPLY_ITEM_EXTENDED_FLAGS: u64 = 2; pub const REMOVE_RECURSIVE: u64 = 1; pub const REMOVE_FLAGS: u64 = 1; pub const FILE_CONTENT_KIND: u64 = 0; @@ -2388,7 +2390,8 @@ pub const CAPABILITY_READ_LIST: u64 = 2; pub const CAPABILITY_READ_REALPATH: u64 = 4; pub const CAPABILITY_READ_STAT_ONLY: u64 = 8; pub const CAPABILITY_STAGE_IN_PLACE: u64 = 16; -pub const CAPABILITY_FLAGS: u64 = 31; +pub const CAPABILITY_APPLY_IN_PLACE: u64 = 32; +pub const CAPABILITY_FLAGS: u64 = 63; pub static OPERATIONS: &[super::OperationMetadata] = &[ super::OperationMetadata { name: "OPEN", class: 1, kind: 0, direction: 0, sensitive: 1, compression: 0, datagram: 0, layout: "flags:u16,reserved:u16=0,source:bytes_u32 containing RootSource,Extensions; ResultPrefix + root_handle:u64,root_revision:u64,path_model:u8,case_behavior:u8,reserved:u16=0,canonical_path:bytes_u32,Extensions" }, super::OperationMetadata { name: "CLOSE", class: 1, kind: 1, direction: 0, sensitive: 1, compression: 0, datagram: 0, layout: "root_handle:u64,Extensions; ResultPrefix; idempotent and invalidates root-scoped watches/stages" }, @@ -2432,6 +2435,7 @@ super::TypeMetadata { name: "query_read_extended_record", layout: "QueryReadReco super::TypeMetadata { name: "query_list_entries", layout: "repeated kind:u8,name:bytes_u16; one directory level without dot and dot-dot, hidden names included, in no defined order; kind ENTRY_FILE, ENTRY_DIRECTORY, ENTRY_SYMLINK or ENTRY_OTHER describes the entry itself, so a symlink to a directory is ENTRY_SYMLINK; name is one nonempty raw platform-name component without NUL or slash" }, super::TypeMetadata { name: "query_stat_only", layout: "kind:u8,reserved:u8=0,reserved:u16=0,mode:u32,size:u64,modified_unix_ns:i64; kind ENTRY_FILE, ENTRY_DIRECTORY, ENTRY_SYMLINK or ENTRY_OTHER; follows the final symlink unless READ_NO_FOLLOW; no content is read or hashed" }, super::TypeMetadata { name: "stage_in_place", layout: "STAGE_WRITE flag STAGE_IN_PLACE: COMMIT opens the target write-only with create and truncate, following a final symlink, writes the staged bytes and optionally syncs them; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; no temporary file and no rename; STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID" }, +super::TypeMetadata { name: "apply_in_place", layout: "APPLY WRITE_INLINE item flag APPLY_ITEM_IN_PLACE, offered with CAPABILITY_APPLY_IN_PLACE: the item writes its content as COMMIT of a STAGE_IN_PLACE stage writes, opening the target write-only with create and truncate, following a final symlink, with no temporary file and no rename; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; the item result describes the file written; a directory at the destination is CONFLICT whose ApplyOsErrors entry is EISDIR open; APPLY_ITEM_CREATE_PARENTS with APPLY_ITEM_IN_PLACE is INVALID" }, ]; pub static LIMITS: &[super::LimitMetadata] = &[ super::LimitMetadata { name: "MAX_ROOTS_PER_SESSION", tag: 1, value_type: super::LimitValueType::U32, required: true, hard_min: 1, hard_max: 64 }, @@ -2550,6 +2554,8 @@ super::ConstantMetadata { name: "APPLY_SYMLINK", value: 4 }, super::ConstantMetadata { name: "APPLY_HARDLINK", value: 5 }, super::ConstantMetadata { name: "APPLY_ITEM_CREATE_PARENTS", value: 1 }, super::ConstantMetadata { name: "APPLY_ITEM_FLAGS", value: 1 }, +super::ConstantMetadata { name: "APPLY_ITEM_IN_PLACE", value: 2 }, +super::ConstantMetadata { name: "APPLY_ITEM_EXTENDED_FLAGS", value: 2 }, super::ConstantMetadata { name: "REMOVE_RECURSIVE", value: 1 }, super::ConstantMetadata { name: "REMOVE_FLAGS", value: 1 }, super::ConstantMetadata { name: "FILE_CONTENT_KIND", value: 0 }, @@ -2592,7 +2598,8 @@ super::ConstantMetadata { name: "CAPABILITY_READ_LIST", value: 2 }, super::ConstantMetadata { name: "CAPABILITY_READ_REALPATH", value: 4 }, super::ConstantMetadata { name: "CAPABILITY_READ_STAT_ONLY", value: 8 }, super::ConstantMetadata { name: "CAPABILITY_STAGE_IN_PLACE", value: 16 }, -super::ConstantMetadata { name: "CAPABILITY_FLAGS", value: 31 }, +super::ConstantMetadata { name: "CAPABILITY_APPLY_IN_PLACE", value: 32 }, +super::ConstantMetadata { name: "CAPABILITY_FLAGS", value: 63 }, ]; } pub mod git { @@ -5160,6 +5167,7 @@ GoldenVector { name: "fs.grep.payload", hex: "0100000000000000020014000500000009 GoldenVector { name: "fs.stage_write.payload", hex: "0100000000000000050000000100010061040000000100000001000000a401000003000000000000000303030303030303030303030303030303030303030303030303030303030303000400000000000000000000" }, GoldenVector { name: "fs.commit.payload", hex: "0200000000000000040404040404040404040404040404040100000000000000" }, GoldenVector { name: "fs.apply.payload", hex: "0100000000000000050505050505050505050505050505050100010020000000000000000500000001000100610400000001000000a40100000300000079617300000000" }, +GoldenVector { name: "fs.apply.in_place.payload", hex: "0100000000000000050505050505050505050505050505050000010020000000000002000500000001000100610400000001000000a40100000300000079617300000000" }, GoldenVector { name: "fs.entry.inline.payload", hex: "050000000100010061010000000000000000000000a4010000010000000000000003000000000000000606060606060606060606060606060606060606060606060606060606060606010000000300000079617300000000" }, GoldenVector { name: "fs.query.path_record.payload", hex: "05000000010001006100000000" }, GoldenVector { name: "fs.query.read_record.payload", hex: "000000000100000005000000010001006103000000796173" }, diff --git a/crates/yas/src/lib.rs b/crates/yas/src/lib.rs index 3e5925dc3..18ede38fb 100644 --- a/crates/yas/src/lib.rs +++ b/crates/yas/src/lib.rs @@ -541,6 +541,9 @@ mod generated_artifact_tests { ("fs.apply.payload", true, |bytes| { fs::Apply::decode(bytes).map(|_| ()) }), + ("fs.apply.in_place.payload", true, |bytes| { + fs::Apply::decode(bytes).map(|_| ()) + }), ("fs.entry.inline.payload", true, |bytes| { fs::EntryRecord::decode(bytes).map(|_| ()) }), diff --git a/docs/design/fs-write.md b/docs/design/fs-write.md index 80e6adabe..f55f2fab1 100644 --- a/docs/design/fs-write.md +++ b/docs/design/fs-write.md @@ -142,6 +142,16 @@ destination is `CONFLICT` carrying OsError `{EISDIR, open}`. Other failures carry OsError with operation `open` (resolving or opening the target) or `write`. `STAGE_CREATE_PARENTS` with `STAGE_IN_PLACE` is `INVALID`. +Content that fits inline can be written in place in one round trip rather than +two: an APPLY `WRITE_INLINE` item with item flag `APPLY_ITEM_IN_PLACE` (2), +advertised by `CAPABILITY_APPLY_IN_PLACE`, writes exactly as COMMIT of an +in-place stage does, with no sync flags. Its item result describes the file +written, and a directory at the destination is `CONFLICT` whose +ApplyOsErrors entry is `{EISDIR, open}`. The flag is `INVALID` on other +item kinds and with `APPLY_ITEM_CREATE_PARENTS`. yas-client's +`write_in_place` sends it for content within the server's inline limit and +stages the rest. + ## OS error detail Every failed top-level FS Result caused by an OS error carries the optional diff --git a/docs/design/yas.md b/docs/design/yas.md index 8ce0bc6e0..0ac6e4f15 100644 --- a/docs/design/yas.md +++ b/docs/design/yas.md @@ -2861,11 +2861,11 @@ extensions remain forbidden by Core. Optional family limit `CAPABILITIES` (tag `LIMIT_CAPABILITIES` = 14, `u32`) advertises the opt-in values a server implements: `CAPABILITY_OS_ERROR` (1), `CAPABILITY_READ_LIST` (2), `CAPABILITY_READ_REALPATH` (4), -`CAPABILITY_READ_STAT_ONLY` (8), and `CAPABILITY_STAGE_IN_PLACE` (16); -`CAPABILITY_FLAGS` (31) is their union. Absent means zero and receivers ignore -unknown bits. A client sends an opt-in question kind or flag only when its bit -is set; an older server rejects them as INVALID. Nothing changes for a client -that sends none of them. +`CAPABILITY_READ_STAT_ONLY` (8), `CAPABILITY_STAGE_IN_PLACE` (16), and +`CAPABILITY_APPLY_IN_PLACE` (32); `CAPABILITY_FLAGS` (63) is their union. +Absent means zero and receivers ignore unknown bits. A client sends an opt-in +question kind or flag only when its bit is set; an older server rejects them as +INVALID. Nothing changes for a client that sends none of them. A failed top-level FS Result whose failure came from an OS error adds optional ResultPrefix `detail` tag `RESULT_OS_ERROR_EXTENSION` (2), whose exact value is @@ -2924,6 +2924,15 @@ destination entry when that lies inside the root. STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID. `STAGE_EXTENDED_FLAGS` (2) lists the stage flags added after the v1 baseline `STAGE_FLAGS`. +APPLY item flag `APPLY_ITEM_IN_PLACE` (2) on a `WRITE_INLINE` item writes +its content in place in the same way, in one round trip: open with create and +truncate through a final symlink, and no temporary file or rename. APPLY +carries no sync flags. The item result's revision, time, and hash describe the +file written. A directory at the destination is CONFLICT, and its +ApplyOsErrors entry is `{EISDIR, open}`. The flag on another item kind, or +with `APPLY_ITEM_CREATE_PARENTS`, is INVALID. `APPLY_ITEM_EXTENDED_FLAGS` +(2) lists the item flags added after the v1 baseline `APPLY_ITEM_FLAGS`. + ## Git family Git is family `0x0031`, version 1. It preserves the useful split between small diff --git a/js/core/src/__tests__/yas.test.ts b/js/core/src/__tests__/yas.test.ts index d088a1854..1735fa6b1 100644 --- a/js/core/src/__tests__/yas.test.ts +++ b/js/core/src/__tests__/yas.test.ts @@ -1752,6 +1752,15 @@ describe("YAS v1", () => { matches("fs.conflict_detail.payload", encodeFsConflictDetail(fsConflict)); const fsApply = decodeFsApply(fromHex(vector("fs.apply.payload"))); matches("fs.apply.payload", encodeFsApply(fsApply)); + const fsApplyInPlace = decodeFsApply( + fromHex(vector("fs.apply.in_place.payload")), + ); + expect(fsApplyInPlace.items[0]).toMatchObject({ + kind: "write-inline", + inPlace: true, + createParents: false, + }); + matches("fs.apply.in_place.payload", encodeFsApply(fsApplyInPlace)); const fsApplyResult = decodeFsApplyResult( fromHex(vector("fs.apply_result.payload")), ); diff --git a/js/core/src/__tests__/yasFs.test.ts b/js/core/src/__tests__/yasFs.test.ts index 0b14afecf..b1c9d599f 100644 --- a/js/core/src/__tests__/yasFs.test.ts +++ b/js/core/src/__tests__/yasFs.test.ts @@ -133,6 +133,10 @@ const cases: readonly [string, (payload: Uint8Array) => Uint8Array][] = [ (payload) => encodeFsConflictDetail(decodeFsConflictDetail(payload)), ], ["fs.apply.payload", (payload) => encodeFsApply(decodeFsApply(payload))], + [ + "fs.apply.in_place.payload", + (payload) => encodeFsApply(decodeFsApply(payload)), + ], [ "fs.apply_result.payload", (payload) => encodeFsApplyResult(decodeFsApplyResult(payload)), diff --git a/js/core/src/yas/fs.ts b/js/core/src/yas/fs.ts index 29350a690..29b2ee24c 100644 --- a/js/core/src/yas/fs.ts +++ b/js/core/src/yas/fs.ts @@ -244,6 +244,11 @@ export type YasFsApplyItem = createParents?: boolean; mode: number; content: Uint8Array; + /** + * APPLY_ITEM_IN_PLACE (offered with CAPABILITY_APPLY_IN_PLACE): write + * through the file as open(2) would; never with createParents. + */ + inPlace?: boolean; } | { kind: "mkdir"; @@ -1637,11 +1642,18 @@ export function decodeFsCommitResult(bytes: Uint8Array): YasFsCommitResult { function encodeFsApplyItem(value: YasFsApplyItem): Uint8Array { const body = new YasWriter(); let kind: number; - const itemFlags = + let itemFlags = value.kind !== "remove" && value.createParents ? g.YAS_FS_APPLY_ITEM_CREATE_PARENTS : 0; if (value.kind === "write-inline") { + if (value.inPlace) { + if (value.createParents) + throw new YasProtocolError( + "FS APPLY in-place write cannot create parents", + ); + itemFlags |= g.YAS_FS_APPLY_ITEM_IN_PLACE; + } if (value.content.length > g.YAS_FS_MAX_INLINE_BYTES) throw new YasProtocolError("FS inline apply content exceeds its limit"); kind = g.YAS_FS_APPLY_WRITE_INLINE; @@ -1708,9 +1720,17 @@ function decodeFsApplyItem(cursor: YasCursor): YasFsApplyItem { const item = new YasCursor(bytes); const kind = item.u16("FS APPLY item kind"); const itemFlags = item.u16("FS APPLY item flags"); - if (itemFlags & ~g.YAS_FS_APPLY_ITEM_FLAGS) + if ( + itemFlags & + ~(g.YAS_FS_APPLY_ITEM_FLAGS | g.YAS_FS_APPLY_ITEM_EXTENDED_FLAGS) + ) throw new YasProtocolError("FS APPLY item flags are invalid"); const createParents = Boolean(itemFlags & g.YAS_FS_APPLY_ITEM_CREATE_PARENTS); + const inPlace = Boolean(itemFlags & g.YAS_FS_APPLY_ITEM_IN_PLACE); + if (inPlace && kind !== g.YAS_FS_APPLY_WRITE_INLINE) + throw new YasProtocolError( + "FS APPLY in-place flag on an item that writes no file", + ); let value: YasFsApplyItem; if (kind === g.YAS_FS_APPLY_WRITE_INLINE) value = { @@ -1720,6 +1740,7 @@ function decodeFsApplyItem(cursor: YasCursor): YasFsApplyItem { createParents, mode: item.u32("FS mode"), content: new Uint8Array(item.bytesU32("FS inline content")), + ...(inPlace ? { inPlace } : {}), }; else if (kind === g.YAS_FS_APPLY_MKDIR) value = { diff --git a/js/core/src/yas/generated.ts b/js/core/src/yas/generated.ts index fc834f0ad..1fe937672 100644 --- a/js/core/src/yas/generated.ts +++ b/js/core/src/yas/generated.ts @@ -1076,6 +1076,8 @@ export const YAS_FS_APPLY_SYMLINK = 4 as const; export const YAS_FS_APPLY_HARDLINK = 5 as const; export const YAS_FS_APPLY_ITEM_CREATE_PARENTS = 1 as const; export const YAS_FS_APPLY_ITEM_FLAGS = 1 as const; +export const YAS_FS_APPLY_ITEM_IN_PLACE = 2 as const; +export const YAS_FS_APPLY_ITEM_EXTENDED_FLAGS = 2 as const; export const YAS_FS_REMOVE_RECURSIVE = 1 as const; export const YAS_FS_REMOVE_FLAGS = 1 as const; export const YAS_FS_FILE_CONTENT_KIND = 0 as const; @@ -1118,7 +1120,8 @@ export const YAS_FS_CAPABILITY_READ_LIST = 2 as const; export const YAS_FS_CAPABILITY_READ_REALPATH = 4 as const; export const YAS_FS_CAPABILITY_READ_STAT_ONLY = 8 as const; export const YAS_FS_CAPABILITY_STAGE_IN_PLACE = 16 as const; -export const YAS_FS_CAPABILITY_FLAGS = 31 as const; +export const YAS_FS_CAPABILITY_APPLY_IN_PLACE = 32 as const; +export const YAS_FS_CAPABILITY_FLAGS = 63 as const; export const YAS_FAMILY_GIT = 49 as const; export const YAS_GIT_VERSION = 1 as const; export const YAS_GIT_OPEN = 0 as const; @@ -8145,6 +8148,10 @@ export const YAS_SCHEMA = { { "name": "stage_in_place", "layout": "STAGE_WRITE flag STAGE_IN_PLACE: COMMIT opens the target write-only with create and truncate, following a final symlink, writes the staged bytes and optionally syncs them; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; no temporary file and no rename; STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID" + }, + { + "name": "apply_in_place", + "layout": "APPLY WRITE_INLINE item flag APPLY_ITEM_IN_PLACE, offered with CAPABILITY_APPLY_IN_PLACE: the item writes its content as COMMIT of a STAGE_IN_PLACE stage writes, opening the target write-only with create and truncate, following a final symlink, with no temporary file and no rename; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; the item result describes the file written; a directory at the destination is CONFLICT whose ApplyOsErrors entry is EISDIR open; APPLY_ITEM_CREATE_PARENTS with APPLY_ITEM_IN_PLACE is INVALID" } ], "constants": [ @@ -8548,6 +8555,14 @@ export const YAS_SCHEMA = { "name": "APPLY_ITEM_FLAGS", "value": 1 }, + { + "name": "APPLY_ITEM_IN_PLACE", + "value": 2 + }, + { + "name": "APPLY_ITEM_EXTENDED_FLAGS", + "value": 2 + }, { "name": "REMOVE_RECURSIVE", "value": 1 @@ -8716,9 +8731,13 @@ export const YAS_SCHEMA = { "name": "CAPABILITY_STAGE_IN_PLACE", "value": 16 }, + { + "name": "CAPABILITY_APPLY_IN_PLACE", + "value": 32 + }, { "name": "CAPABILITY_FLAGS", - "value": 31 + "value": 63 } ] }, @@ -15910,6 +15929,10 @@ export const YAS_GOLDEN_VECTORS = { "name": "fs.apply.payload", "hex": "0100000000000000050505050505050505050505050505050100010020000000000000000500000001000100610400000001000000a40100000300000079617300000000" }, + { + "name": "fs.apply.in_place.payload", + "hex": "0100000000000000050505050505050505050505050505050000010020000000000002000500000001000100610400000001000000a40100000300000079617300000000" + }, { "name": "fs.entry.inline.payload", "hex": "050000000100010061010000000000000000000000a4010000010000000000000003000000000000000606060606060606060606060606060606060606060606060606060606060606010000000300000079617300000000" diff --git a/protocol/yas/families/fs.toml b/protocol/yas/families/fs.toml index 4ea289f6f..7551b0197 100644 --- a/protocol/yas/families/fs.toml +++ b/protocol/yas/families/fs.toml @@ -210,6 +210,9 @@ layout = "kind:u8,reserved:u8=0,reserved:u16=0,mode:u32,size:u64,modified_unix_n [[type]] name = "stage_in_place" layout = "STAGE_WRITE flag STAGE_IN_PLACE: COMMIT opens the target write-only with create and truncate, following a final symlink, writes the staged bytes and optionally syncs them; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; no temporary file and no rename; STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID" +[[type]] +name = "apply_in_place" +layout = "APPLY WRITE_INLINE item flag APPLY_ITEM_IN_PLACE, offered with CAPABILITY_APPLY_IN_PLACE: the item writes its content as COMMIT of a STAGE_IN_PLACE stage writes, opening the target write-only with create and truncate, following a final symlink, with no temporary file and no rename; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; the item result describes the file written; a directory at the destination is CONFLICT whose ApplyOsErrors entry is EISDIR open; APPLY_ITEM_CREATE_PARENTS with APPLY_ITEM_IN_PLACE is INVALID" [[constant]] name = "SOURCE_PLATFORM_PATH" @@ -516,6 +519,12 @@ value = 1 name = "APPLY_ITEM_FLAGS" value = 1 [[constant]] +name = "APPLY_ITEM_IN_PLACE" +value = 2 +[[constant]] +name = "APPLY_ITEM_EXTENDED_FLAGS" +value = 2 +[[constant]] name = "REMOVE_RECURSIVE" value = 1 [[constant]] @@ -646,5 +655,8 @@ value = 8 name = "CAPABILITY_STAGE_IN_PLACE" value = 16 [[constant]] +name = "CAPABILITY_APPLY_IN_PLACE" +value = 32 +[[constant]] name = "CAPABILITY_FLAGS" -value = 31 +value = 63 diff --git a/protocol/yas/schema.json b/protocol/yas/schema.json index decf6fa2e..1cea9b7ba 100644 --- a/protocol/yas/schema.json +++ b/protocol/yas/schema.json @@ -5232,6 +5232,10 @@ { "name": "stage_in_place", "layout": "STAGE_WRITE flag STAGE_IN_PLACE: COMMIT opens the target write-only with create and truncate, following a final symlink, writes the staged bytes and optionally syncs them; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; no temporary file and no rename; STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID" + }, + { + "name": "apply_in_place", + "layout": "APPLY WRITE_INLINE item flag APPLY_ITEM_IN_PLACE, offered with CAPABILITY_APPLY_IN_PLACE: the item writes its content as COMMIT of a STAGE_IN_PLACE stage writes, opening the target write-only with create and truncate, following a final symlink, with no temporary file and no rename; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; the item result describes the file written; a directory at the destination is CONFLICT whose ApplyOsErrors entry is EISDIR open; APPLY_ITEM_CREATE_PARENTS with APPLY_ITEM_IN_PLACE is INVALID" } ], "constants": [ @@ -5635,6 +5639,14 @@ "name": "APPLY_ITEM_FLAGS", "value": 1 }, + { + "name": "APPLY_ITEM_IN_PLACE", + "value": 2 + }, + { + "name": "APPLY_ITEM_EXTENDED_FLAGS", + "value": 2 + }, { "name": "REMOVE_RECURSIVE", "value": 1 @@ -5803,9 +5815,13 @@ "name": "CAPABILITY_STAGE_IN_PLACE", "value": 16 }, + { + "name": "CAPABILITY_APPLY_IN_PLACE", + "value": 32 + }, { "name": "CAPABILITY_FLAGS", - "value": 31 + "value": 63 } ] }, diff --git a/protocol/yas/vectors.json b/protocol/yas/vectors.json index 749ecac01..27f1e0fbc 100644 --- a/protocol/yas/vectors.json +++ b/protocol/yas/vectors.json @@ -1289,6 +1289,10 @@ "name": "fs.apply.payload", "hex": "0100000000000000050505050505050505050505050505050100010020000000000000000500000001000100610400000001000000a40100000300000079617300000000" }, + { + "name": "fs.apply.in_place.payload", + "hex": "0100000000000000050505050505050505050505050505050000010020000000000002000500000001000100610400000001000000a40100000300000079617300000000" + }, { "name": "fs.entry.inline.payload", "hex": "050000000100010061010000000000000000000000a4010000010000000000000003000000000000000606060606060606060606060606060606060606060606060606060606060606010000000300000079617300000000" diff --git a/protocol/yas/wire.md b/protocol/yas/wire.md index e65809e7e..1d2f075fe 100644 --- a/protocol/yas/wire.md +++ b/protocol/yas/wire.md @@ -654,6 +654,7 @@ Every Request kind has a correlated Result with the same family and kind. | `query_list_entries` | repeated kind:u8,name:bytes_u16; one directory level without dot and dot-dot, hidden names included, in no defined order; kind ENTRY_FILE, ENTRY_DIRECTORY, ENTRY_SYMLINK or ENTRY_OTHER describes the entry itself, so a symlink to a directory is ENTRY_SYMLINK; name is one nonempty raw platform-name component without NUL or slash | | `query_stat_only` | kind:u8,reserved:u8=0,reserved:u16=0,mode:u32,size:u64,modified_unix_ns:i64; kind ENTRY_FILE, ENTRY_DIRECTORY, ENTRY_SYMLINK or ENTRY_OTHER; follows the final symlink unless READ_NO_FOLLOW; no content is read or hashed | | `stage_in_place` | STAGE_WRITE flag STAGE_IN_PLACE: COMMIT opens the target write-only with create and truncate, following a final symlink, writes the staged bytes and optionally syncs them; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; no temporary file and no rename; STAGE_CREATE_PARENTS with STAGE_IN_PLACE is INVALID | +| `apply_in_place` | APPLY WRITE_INLINE item flag APPLY_ITEM_IN_PLACE, offered with CAPABILITY_APPLY_IN_PLACE: the item writes its content as COMMIT of a STAGE_IN_PLACE stage writes, opening the target write-only with create and truncate, following a final symlink, with no temporary file and no rename; an existing file keeps its inode, owner and mode; a new file gets mode, or 0o666 when mode is zero, less the server umask; the item result describes the file written; a directory at the destination is CONFLICT whose ApplyOsErrors entry is EISDIR open; APPLY_ITEM_CREATE_PARENTS with APPLY_ITEM_IN_PLACE is INVALID | ## `yas.git` (`0x0031`/v1) From b62c65c3d0c4889cd545ab4619fd0e54cd20b84b Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 02:07:57 -0700 Subject: [PATCH 06/10] Pace process output by its owner; drop a lagging watcher alone (#85) * Expect TIMEOUT from a terminal command wait nothing started Since #54 a command wait that runs out before any command it could report on has started answers TIMEOUT (NOT_FOUND stays for an exited terminal or an evicted index). The client_host test still expected NOT_FOUND, so it failed on main. * Say what a terminal command wait answers when no command started Since #54 the server answers TIMEOUT when a command wait runs out before any command it could report on has started; wait_terminal_command's documentation still said NOT_FOUND, which stays for an evicted index or an exited terminal. * Pace process output by its owner; drop a lagging watcher alone A command that wrote faster than its spawning session read lost its output and wedged that session's Process endpoint: - output_reader read each pipe with no backpressure, so once the owner's binding was 1 MiB (or 1024 frames) behind, it was evicted like any watcher. Its ACKs then found no binding, the stdout Transfer was reset NOT_FOUND "process not found", and the exit never came. - The eviction kicked the whole endpoint: route_outbound closed the session and returned, dropping the endpoint's event receiver. The session still accepted SPAWNs, but no later process on it ever reported output or an exit. Now the owner paces the child: before each read, output_reader waits until the owner's binding has a frame of room in its window (and at most 32 unacknowledged frames), and reserves its queue slot, so the owner is never evicted and a fast writer blocks on its pipe. Watchers of other sessions are still dropped when they fall a window behind, but alone: the endpoint reports evicted process IDs, their routes fail, and their attachments reset their Transfers RESOURCE_EXHAUSTED. A full route queue fails that route rather than the session, output for a route that already left is dropped, and a WAIT whose route left looks again instead of failing. close_session keeps its error (send_replace: nobody subscribes to it). Server-only; clients are unchanged. * Keep a paced owner's output past the exit; free an evicted watcher's slot Review of #85 (four paths it left): - An evicted watcher's binding left its endpoint slot behind, so a session that kept watching ran out of its 16 process slots. The eviction now frees the slot. - Detach answered Conflict once the child had exited, while its output could still be draining. A route that failed then (its queue full) left a binding whose exit found no route and closed the session; an owner that dropped its stream then left a binding nobody acknowledged, and its WAIT answered Conflict. Detach now goes through until the exit is queued, and an exit for a route that already failed is dropped rather than closing the session. - Once the child exited, the cleanup stopped waiting for its streams after the kill grace (or a LEAVE_RESIDUE grace), and dropped what the pipe still held for an owner slow to take its window: a slow link, or a client that waits for the exit first. With nothing of the group left, the readers the owner paces now go on until the pipes close, however slowly it takes them; the cleanup stops waiting only when no reader has waited for the owner for 250 ms (a holder outside the group, nothing coming) or a stream has given 1 MiB more (such a holder writing on). The graces still bound the group's residue. Each has a test in yas_process::tests that fails without the change (from the review's probes), plus one that a residue writing on still ends its grace though the owner reads nothing. * Say what a WAIT answers when its route failed as the exit was queued --- crates/cli/tests/client_host.rs | 149 +++++++++++ crates/server/src/process.rs | 234 +++++++++++++++-- crates/server/src/yas.rs | 18 +- crates/server/src/yas_process.rs | 425 ++++++++++++++++++++++++++----- docs/design/processes.md | 18 ++ docs/design/yas.md | 5 +- 6 files changed, 768 insertions(+), 81 deletions(-) diff --git a/crates/cli/tests/client_host.rs b/crates/cli/tests/client_host.rs index 744c94b40..dd1895ef3 100644 --- a/crates/cli/tests/client_host.rs +++ b/crates/cli/tests/client_host.rs @@ -419,6 +419,155 @@ async fn a_default_server_refuses_a_seventeenth_process_and_extra_environment() assert!(matches!(error, Error::Invalid(_)), "{error}"); } +/// Waits at most [`TIMEOUT`] for `future`, naming what it waited for. +async fn within(what: &str, future: impl std::future::Future) -> T { + tokio::time::timeout(TIMEOUT, future) + .await + .unwrap_or_else(|_| panic!("{what} took over {TIMEOUT:?}")) +} + +/// The session still runs commands: the Process family was not taken down. +async fn still_runs_commands(client: &Client) { + let output = within( + "a command after it", + client + .spawn(Command::new("echo").arg("after")) + .await + .unwrap() + .output(), + ) + .await + .unwrap(); + assert_eq!(output.stdout, b"after\n"); + assert!(output.status.success(), "{}", output.status); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_command_writing_faster_than_its_output_is_read_loses_none_of_it() { + use tokio::io::AsyncReadExt; + let server = start().await; + let client = server.connect().await.unwrap(); + // Three times the largest stream buffer a default server keeps, written as fast as the + // pipe takes it: the server reads it no faster than this client does (it used to drop the + // stream NOT_FOUND about a window in, and the exit never came). + const BYTES: u64 = 3 * 8 * 1024 * 1024; + let mut process = client + .spawn(Command::new("sh").args([ + "-c".to_owned(), + format!("head -c {BYTES} /dev/zero; printf end; echo tail >&2; exit 9"), + ])) + .await + .unwrap(); + let stdout = process.take_stdout().unwrap(); + let stderr = process.take_stderr().unwrap(); + // Read as Ultimator's bash does: the exit is awaited while the output streams in. + let reader = tokio::spawn(async move { + tokio::join!(stdout.read_to_end(BYTES + 1024), stderr.read_to_end(1024)) + }); + let status = within("the exit", process.wait()).await.unwrap(); + assert_eq!(status.code(), Some(9), "{status}\n{}", server_log(&server)); + let (stdout, stderr) = within("the output", reader).await.unwrap(); + let stdout = stdout.unwrap_or_else(|error| panic!("{error}\n{}", server_log(&server))); + assert_eq!(stdout.len() as u64, BYTES + 3); + assert!(stdout.ends_with(b"end")); + assert!(stdout[..BYTES as usize].iter().all(|byte| *byte == 0)); + assert_eq!(stderr.unwrap(), b"tail\n"); + + // A reader slower than the writer, a few KiB at a time: the writer waits for it. + const SLOW: usize = 4 * 1024 * 1024; + let mut process = client + .spawn( + Command::new("head") + .args(["-c".to_owned(), SLOW.to_string()]) + .arg("/dev/zero"), + ) + .await + .unwrap(); + let mut stdout = process.take_stdout().unwrap(); + let mut total = 0; + let mut buffer = vec![0u8; 64 * 1024]; + within("the slow reads", async { + loop { + let n = stdout.read(&mut buffer).await.unwrap(); + if n == 0 { + break; + } + total += n; + tokio::time::sleep(Duration::from_millis(2)).await; + } + }) + .await; + assert_eq!(total, SLOW); + assert!(within("the exit", process.wait()).await.unwrap().success()); + + still_runs_commands(&client).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_dropped_output_stream_leaves_its_process_and_session_working() { + use tokio::io::AsyncReadExt; + let server = start().await; + let client = server.connect().await.unwrap(); + let mut process = client + .spawn(Command::new("sh").args(["-c", "head -c 25165824 /dev/zero; exit 4"])) + .await + .unwrap(); + let mut stdout = process.take_stdout().unwrap(); + let mut some = vec![0u8; 1024 * 1024]; + within("the first MiB", stdout.read_exact(&mut some)) + .await + .unwrap(); + // Dropped half-way: what the command writes next goes nowhere, and it still finishes. + drop(stdout); + let status = within("the exit", process.wait()).await.unwrap(); + assert_eq!(status.code(), Some(4), "{status}\n{}", server_log(&server)); + still_runs_commands(&client).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_watcher_that_falls_behind_is_dropped_alone() { + let server = start().await; + let owner = server.connect().await.unwrap(); + let watcher = server.connect().await.unwrap(); + const BYTES: u64 = 3 * 8 * 1024 * 1024; + let mut process = owner + .spawn(Command::new("sh").args([ + "-c".to_owned(), + format!("sleep 1; head -c {BYTES} /dev/zero; exit 3"), + ])) + .await + .unwrap(); + // Another session watches, and reads nothing until the command is done. + let mut watched = within("the attach", watcher.attach(process.handle(), false)) + .await + .unwrap(); + let watched_stdout = watched.take_stdout().unwrap(); + let stdout = process.take_stdout().unwrap(); + let stdout = within("the owner's output", stdout.read_to_end(BYTES + 1024)) + .await + .unwrap_or_else(|error| panic!("{error}\n{}", server_log(&server))); + assert_eq!(stdout.len() as u64, BYTES); + assert_eq!( + within("the exit", process.wait()).await.unwrap().code(), + Some(3) + ); + // The watcher fell a window behind: its stream was dropped, and says why. + let dropped = within( + "the watcher's output", + watched_stdout.read_to_end(BYTES + 1024), + ) + .await + .unwrap_err(); + assert_eq!( + dropped.status(), + Some(yas_client::wire::core::Status::ResourceExhausted), + "{dropped}" + ); + // Both sessions still run commands (the watcher's used to lose its Process family). + still_runs_commands(&watcher).await; + still_runs_commands(&owner).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_commands_background_children_die_with_it() { let server = start().await; diff --git a/crates/server/src/process.rs b/crates/server/src/process.rs index 624806c8e..daeff6725 100644 --- a/crates/server/src/process.rs +++ b/crates/server/src/process.rs @@ -21,7 +21,7 @@ use std::sync::{Arc, Mutex as StdMutex, Weak}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; -use tokio::sync::{Notify, Semaphore, mpsc, oneshot, watch}; +use tokio::sync::{Notify, Semaphore, mpsc, oneshot}; use tokio::task::AbortHandle; use yas_wire::process as wire; use yas_wire::schema::process as process_schema; @@ -311,6 +311,14 @@ const RESIDUE_LEFT_RUNNING: &str = "residual process group left running"; /// that are already zombies to be reaped (see `kill_group_until_gone`). #[cfg(unix)] const RESIDUAL_ZOMBIE_WAIT: Duration = Duration::from_secs(1); +/// Once nothing of a finished process's group is left, how long its streams may stay open with +/// no reader waiting for the owner to take its window before the cleanup stops waiting for them +/// (a holder outside the group keeps a pipe open with nothing coming): see `drain_paced`. +const DRAIN_IDLE: Duration = Duration::from_millis(250); +/// How much more a stream may give while the cleanup waits for its owner (`drain_paced`): more +/// than a pipe holds (1 MiB at most unless root raised /proc/sys/fs/pipe-max-size), so it bounds +/// only a holder outside the group that writes on. +const DRAIN_BUDGET: u64 = 1024 * 1024; const PROCESS_STREAM_STDOUT: u8 = process_schema::STREAM_STDOUT_CONTENT_KIND as u8; const PROCESS_STREAM_STDERR: u8 = process_schema::STREAM_STDERR_CONTENT_KIND as u8; const PROCESS_STREAM_STDIN_ACCEPTING: u8 = 1 << 0; @@ -334,6 +342,10 @@ const PROCESS_KILL_TERMINATE_TIMEOUT: u8 = process_schema::EXIT_REASON_TERMINATE const PROCESS_KILL_SERVER_SHUTDOWN: u8 = process_schema::EXIT_REASON_SERVER_SHUTDOWN as u8; const PROCESS_MAX_UNACKED_PACKETS: usize = 1_024; const PROCESS_DEFAULT_STREAM_WINDOW: u64 = 1024 * 1024; +/// Unacknowledged frames a process's owner may have on one stream: its window in whole frames, +/// so a burst of small writes fits the adapter's queues (80 route events) as a full window does. +const PROCESS_OWNER_UNACKED_FRAMES: usize = + (PROCESS_DEFAULT_STREAM_WINDOW / OUTPUT_FRAME_PAYLOAD as u64) as usize; pub(crate) const NATIVE_STREAM_STDOUT: u8 = PROCESS_STREAM_STDOUT; pub(crate) const NATIVE_STREAM_STDERR: u8 = PROCESS_STREAM_STDERR; @@ -630,14 +642,35 @@ pub(crate) enum NativeControl { #[derive(Clone)] struct EndpointOutput { events: mpsc::Sender, - closed: watch::Sender>, + evictions: Arc, +} + +/// The processes of one endpoint whose binding the output readers dropped: a watcher that fell +/// a window behind, or one whose queue was full. Its adapter fails those attachments alone; the +/// endpoint and its other processes go on. +#[derive(Default)] +pub(crate) struct Evictions { + process_ids: StdMutex>, + notify: Notify, +} + +impl Evictions { + /// Waits for evictions (one waiter: the endpoint's adapter). + pub(crate) async fn notified(&self) { + self.notify.notified().await; + } + + /// The process IDs evicted since the last call. + pub(crate) fn take(&self) -> Vec { + std::mem::take(&mut *self.process_ids.lock().unwrap()) + } } impl EndpointOutput { - fn kick(&self, reason: &str) { - if self.closed.borrow().is_none() { - let _ = self.closed.send(Some(reason.to_owned())); - } + fn evict(&self, process_id: u32) { + self.evictions.process_ids.lock().unwrap().push(process_id); + // A permit is kept when the adapter is not waiting yet. + self.evictions.notify.notify_one(); } fn send_native(&self, event: NativeEvent, guard: Option) -> bool { @@ -740,19 +773,22 @@ impl Server { &self, session_id: [u8; 16], event_capacity: usize, - ) -> ( - Manager, - mpsc::Receiver, - watch::Receiver>, - ) { + ) -> (Manager, mpsc::Receiver, Arc) { debug_assert!(session_id.iter().any(|byte| *byte != 0)); let id = self.0.next_endpoint.fetch_add(1, Ordering::Relaxed); let (events, receiver) = mpsc::channel(event_capacity.max(1)); - let (closed, closed_receiver) = watch::channel(None); + let evictions = Arc::new(Evictions::default()); ( - self.endpoint_with_id(EndpointOutput { events, closed }, id, session_id), + self.endpoint_with_id( + EndpointOutput { + events, + evictions: evictions.clone(), + }, + id, + session_id, + ), receiver, - closed_receiver, + evictions, ) } @@ -1157,6 +1193,8 @@ struct RecordInner { stderr: Option, stdout_readers: u8, stderr_readers: u8, + /// Output readers waiting for the owner to take its window (`owner_with_room`). + paced_readers: u8, child_outcome: Option, tree_cleanup_done: bool, exit_override: Option, @@ -1491,6 +1529,7 @@ impl Manager { stderr: (!merged).then_some(StreamState { next: 0 }), stdout_readers: 1, stderr_readers: if merged { 0 } else { 1 }, + paced_readers: 0, child_outcome: None, tree_cleanup_done: false, exit_override: None, @@ -1735,7 +1774,12 @@ impl Manager { return Err(NativeError::NotFound); }; let residual_running = residual_running(&record, &inner); - if inner.terminal_queued || (inner.child_outcome.is_some() && !residual_running) { + // Detach goes through until the exit is queued: its output may still be draining, + // and a binding nobody acknowledges would hold the readers the owner paces. + let detach = matches!(action, NativeControl::Detach); + if inner.terminal_queued + || (inner.child_outcome.is_some() && !residual_running && !detach) + { return Err(NativeError::Conflict); } match action { @@ -2855,6 +2899,11 @@ async fn stdin_writer( async fn output_reader(record: Arc, stream: u8, mut reader: impl AsyncRead + Unpin) { let mut buffer = vec![0u8; OUTPUT_FRAME_PAYLOAD]; loop { + // The process's owner takes all of its output: the pipe is read no further ahead of + // what the owner has taken than its window, so a writer faster than the owner's + // Transfer blocks on its pipe. (It used to be evicted, which closed the owner's whole + // Process endpoint.) Other watchers are still dropped when they fall a window behind. + let owner = owner_with_room(&record, stream).await; match reader.read(&mut buffer).await { Ok(0) => break, Err(_) => { @@ -2862,6 +2911,17 @@ async fn output_reader(record: Arc, stream: u8, mut reader: impl AsyncRe break; } Ok(n) => { + // Room in the owner's event queue too, taken before the lock: its frame never + // finds the queue full. + let owner = match owner { + Some((endpoint_id, process_id, events)) => events + .reserve_owned() + .await + .ok() + .map(|permit| (endpoint_id, process_id, permit)), + None => None, + }; + let mut owner = owner; let mut inner = record.inner.lock().unwrap(); let state = if stream == PROCESS_STREAM_STDOUT { &mut inner.stdout @@ -2878,6 +2938,30 @@ async fn output_reader(record: Arc, stream: u8, mut reader: impl AsyncRe let mut evicted = Vec::new(); let mut index = 0; while index < inner.bindings.len() { + if let Some((endpoint_id, process_id, _)) = owner + && inner.bindings[index].endpoint_id == endpoint_id + && inner.bindings[index].process_id == process_id + { + let (_, _, permit) = owner.take().expect("owner permit"); + permit.send(NativeEventEnvelope { + event: NativeEvent::Output { + process_id, + stream, + offset, + data: buffer[..n].to_vec(), + }, + _guard: None, + }); + let binding = &mut inner.bindings[index]; + let credit = if stream == PROCESS_STREAM_STDOUT { + &mut binding.stdout + } else { + binding.stderr.as_mut().expect("separate stderr binding") + }; + credit.frames.push_back(next); + index += 1; + continue; + } let has_credit = { let binding = &inner.bindings[index]; let credit = if stream == PROCESS_STREAM_STDOUT { @@ -2917,9 +3001,11 @@ async fn output_reader(record: Arc, stream: u8, mut reader: impl AsyncRe } drop(inner); for binding in evicted { - binding - .out - .kick("native process watcher exceeded its output window"); + // Its endpoint goes on: free the slot, or the process holds one for good. + if let Some(endpoint) = binding.endpoint.upgrade() { + remove_bound_slot(&endpoint, binding.process_id, &record); + } + binding.out.evict(binding.process_id); } } } @@ -2927,6 +3013,109 @@ async fn output_reader(record: Arc, stream: u8, mut reader: impl AsyncRe stream_closed(&record, stream); } +/// Waits until the process's owner, while it is bound, can take another whole frame of +/// `stream` (its window and unacknowledged frames); answers where to send it. None when the +/// owner is not bound (it left, or detached): the output then goes to the watchers alone. +async fn owner_with_room( + record: &Record, + stream: u8, +) -> Option<(u64, u32, mpsc::Sender)> { + // Counted in `paced_readers` while it waits, however it stops (an abort drops it). + let mut paced = Paced { + record, + counted: false, + }; + loop { + // Created before the check: an acknowledgement in between still wakes it. + let changed = record.changed.notified(); + { + let mut inner = record.inner.lock().unwrap(); + let owner = inner + .bindings + .iter() + .find(|binding| Weak::ptr_eq(&binding.endpoint, &record.owner))?; + let (next, credit) = if stream == PROCESS_STREAM_STDOUT { + (inner.stdout.next, &owner.stdout) + } else { + match (inner.stderr.as_ref(), owner.stderr.as_ref()) { + (Some(state), Some(credit)) => (state.next, credit), + _ => return None, + } + }; + let debt = next.saturating_sub(credit.acked); + if debt.saturating_add(OUTPUT_FRAME_PAYLOAD as u64) <= PROCESS_DEFAULT_STREAM_WINDOW + && credit.frames.len() < PROCESS_OWNER_UNACKED_FRAMES + { + return Some(( + owner.endpoint_id, + owner.process_id, + owner.out.events.clone(), + )); + } + if !paced.counted { + paced.counted = true; + inner.paced_readers += 1; + drop(inner); + record.changed.notify_waiters(); + } + } + changed.await; + } +} + +/// An output reader waiting for its owner (`owner_with_room`), as `paced_readers` counts it. +struct Paced<'a> { + record: &'a Record, + counted: bool, +} + +impl Drop for Paced<'_> { + fn drop(&mut self) { + if self.counted { + self.record.inner.lock().unwrap().paced_readers -= 1; + self.record.changed.notify_waiters(); + } + } +} + +/// The cleanup of a finished process whose streams are still open once nothing of its group is +/// left: what holds them is the child's own output that its owner has not taken yet (it paces +/// the readers), so they go on until the pipes close, however slowly the owner takes its window. +/// It stops waiting when no reader has waited for the owner for `DRAIN_IDLE` (a holder outside +/// the group keeps a pipe open with nothing coming) or a stream has given `DRAIN_BUDGET` more (one +/// that writes on). True when the streams closed. +async fn drain_paced(record: &Record) -> bool { + let next = |inner: &RecordInner| { + ( + inner.stdout.next, + inner.stderr.as_ref().map_or(0, |state| state.next), + ) + }; + let start = next(&record.inner.lock().unwrap()); + loop { + let changed = record.changed.notified(); + let paced = { + let inner = record.inner.lock().unwrap(); + if io_tasks_done(&inner) { + return true; + } + let (stdout, stderr) = next(&inner); + if inner.tree_cleanup_done + || stdout - start.0 > DRAIN_BUDGET + || stderr - start.1 > DRAIN_BUDGET + { + return false; + } + inner.paced_readers > 0 + }; + if paced { + changed.await; + } else if tokio::time::timeout(DRAIN_IDLE, changed).await.is_err() { + return false; + } + } +} + fn stream_closed(record: &Arc, stream: u8) { { let mut inner = record.inner.lock().unwrap(); @@ -3021,6 +3210,9 @@ fn schedule_residual_cleanup(record: Arc) { _ = &mut deadline => break false, } }; + // The grace is for the group's residue: with none left, the streams hold the + // child's own output, which its owner takes at its own pace. + let closed = closed || (process_group_absent(&record) && drain_paced(&record).await); if closed { { let mut inner = record.inner.lock().unwrap(); @@ -3086,6 +3278,12 @@ fn schedule_residual_cleanup(record: Arc) { let cleanup_failed = force_kill(&record) .err() .is_some_and(|error| !process_tree_already_absent(&error)); + // Nothing of the group writes any more: what the pipes still hold is the child's own + // output, which its owner takes at its own pace. (Only a holder outside the group, or a + // group that could not be killed, stops the readers.) + if !cleanup_failed { + drain_paced(&record).await; + } let (stdin_abort, output_aborts) = { let mut inner = record.inner.lock().unwrap(); if inner.tree_cleanup_done { diff --git a/crates/server/src/yas.rs b/crates/server/src/yas.rs index 2798aa6e1..178306f9e 100644 --- a/crates/server/src/yas.rs +++ b/crates/server/src/yas.rs @@ -31120,7 +31120,23 @@ fn spawn_process_attachment( .await; break; } - None => break, + None => { + // The route failed (its output fell a window behind): the session resets + // this attachment's Transfers with why, rather than end them cleanly + // short, and cancels this task and its streams meanwhile. + if let Some(detail) = events.failure() { + let _ = internal + .send(Internal::ProcessFailed { + attachment_id, + status: Status::ResourceExhausted, + detail, + }) + .await; + cancellation.cancelled().await; + return; + } + break; + } } } drop(stdout_tx); diff --git a/crates/server/src/yas_process.rs b/crates/server/src/yas_process.rs index 8f4714421..9c7637cc3 100644 --- a/crates/server/src/yas_process.rs +++ b/crates/server/src/yas_process.rs @@ -186,6 +186,7 @@ pub(crate) struct AttachmentControl { pub(crate) struct AttachmentEvents { events: mpsc::Receiver, + failed: watch::Receiver>, } #[derive(Clone)] @@ -251,8 +252,14 @@ struct Route { auto_ack_output: bool, events: mpsc::Sender, exit: watch::Sender>, + /// Why the process's output left this route: its native binding fell a window behind or + /// its queue filled. Only this attachment fails; the session and its other processes go on. + failed: watch::Sender>, } +/// What a route dropped for falling behind tells its attachment and its WAITs. +const ROUTE_EVICTED: &str = "Process output fell a window behind its reader and was dropped"; + enum WatchOutcome { Running(Attachment), Exited(ExitInfo), @@ -308,7 +315,7 @@ impl Runtime { if owner_session.iter().all(|byte| *byte == 0) { return Err(Error::Invalid("zero Process owner session".to_owned())); } - let (manager, events, endpoint_closed) = self + let (manager, events, evictions) = self .server .native_endpoint_with_session(owner_session, self.server.maxima().endpoint_events()); let (closed, _) = watch::channel(None); @@ -324,11 +331,7 @@ impl Runtime { #[cfg(test)] operation_gate: self.operation_gate.clone(), }); - tokio::spawn(route_outbound( - Arc::downgrade(&inner), - events, - endpoint_closed, - )); + tokio::spawn(route_outbound(Arc::downgrade(&inner), events, evictions)); Ok(Session { inner }) } @@ -512,6 +515,26 @@ impl Session { } pub(crate) async fn wait(&self, request: &wire::Wait) -> Result { + let deadline = (request.timeout_ns != 0) + .then(|| tokio::time::Instant::now() + Duration::from_nanos(request.timeout_ns)); + // The route waited on can leave first (its stream was dropped, or it fell behind and + // was failed): the process goes on, so the wait looks again. + loop { + if let Some(exit) = self.wait_route(request, deadline).await? { + return Ok(exit); + } + if let Some(error) = self.inner.closed.borrow().clone() { + return Err(error); + } + } + } + + /// Waits on the process's route of now; None when the route left before the exit. + async fn wait_route( + &self, + request: &wire::Wait, + deadline: Option, + ) -> Result, Error> { if let Some(exit) = self .inner .exits @@ -520,11 +543,11 @@ impl Session { .get(request.process_handle) .cloned() { - return Ok(exit); + return Ok(Some(exit)); } - let (mut exit, temporary) = + let (mut exit, mut failed, temporary) = if let Some(route) = self.route_by_handle(request.process_handle) { - (route.exit.subscribe(), None) + (route.exit.subscribe(), route.failed.subscribe(), None) } else if let Some(exit) = self .inner .exits @@ -535,36 +558,42 @@ impl Session { { // It exited between the first look and the route lookup; the // replay is recorded before the route is removed. - return Ok(exit); + return Ok(Some(exit)); } else { match self .watch_process(request.process_handle, false, true) .await? { - WatchOutcome::Exited(exit) => return Ok(exit), + WatchOutcome::Exited(exit) => return Ok(Some(exit)), WatchOutcome::Running(attachment) => { let exit = attachment.route.exit.subscribe(); - (exit, Some(attachment)) + let failed = attachment.route.failed.subscribe(); + (exit, failed, Some(attachment)) } } }; let wait = async { loop { if let Some(exit) = exit.borrow().clone() { - return Ok(exit); + return Some(exit); + } + if failed.borrow().is_some() { + return None; + } + tokio::select! { + changed = exit.changed() => if changed.is_err() { + // The route is gone: with its exit, or before it. + return exit.borrow().clone(); + }, + _ = failed.changed() => {} } - exit.changed() - .await - .map_err(|_| Error::Closed("Process attachment closed".to_owned()))?; } }; - let result = if request.timeout_ns == 0 { - wait.await - } else { - match tokio::time::timeout(Duration::from_nanos(request.timeout_ns), wait).await { - Ok(result) => result, - Err(_) => Err(Error::Timeout), - } + let result = match deadline { + None => Ok(wait.await), + Some(deadline) => tokio::time::timeout_at(deadline, wait) + .await + .map_err(|_| Error::Timeout), }; if let Some(attachment) = temporary { let _ = self @@ -691,12 +720,14 @@ impl Session { ) -> Result<(Arc, mpsc::Receiver), Error> { let (events, receiver) = mpsc::channel(ROUTE_EVENTS); let (exit, _) = watch::channel(None); + let (failed, _) = watch::channel(None); let route = Arc::new(Route { process_id, process_handle: AtomicU64::new(0), auto_ack_output, events, exit, + failed, }); if self .inner @@ -743,10 +774,12 @@ impl Attachment { stdin_window: self.stdin_window, merged_stderr: self.merged_stderr, }; + let failed = control.route.failed.subscribe(); ( control, AttachmentEvents { events: self.events, + failed, }, ) } @@ -768,8 +801,21 @@ impl Attachment { } impl AttachmentEvents { + /// The next event; None once the route is closed or failed ([`AttachmentEvents::failure`]). pub(crate) async fn next(&mut self) -> Option { - self.events.recv().await + if self.failed.borrow().is_some() { + return None; + } + tokio::select! { + biased; + _ = self.failed.changed() => None, + event = self.events.recv() => event, + } + } + + /// Why the route failed, when it did. + pub(crate) fn failure(&self) -> Option { + self.failed.borrow().clone() } } @@ -814,26 +860,19 @@ impl AttachmentControl { async fn route_outbound( inner: std::sync::Weak, mut events: mpsc::Receiver, - mut endpoint_closed: watch::Receiver>, + evictions: Arc, ) { loop { let event = tokio::select! { event = events.recv() => event, - changed = endpoint_closed.changed() => { - let detail = if changed.is_ok() { - endpoint_closed.borrow().clone() - } else { - None + () = evictions.notified() => { + let Some(inner) = inner.upgrade() else { + return; }; - if let Some(inner) = inner.upgrade() { - close_session( - &inner, - Error::Closed(detail.unwrap_or_else(|| { - "Process endpoint writer closed".to_owned() - })), - ); + for process_id in evictions.take() { + fail_route(&inner, process_id, ROUTE_EVICTED); } - return; + continue; } }; let Some(event) = event else { @@ -863,13 +902,10 @@ fn dispatch_outbound(inner: &Arc, event: process::NativeEvent) -> offset, data, } => { - let route = inner - .routes - .lock() - .unwrap() - .get(&process_id) - .cloned() - .ok_or_else(|| Error::Closed("output for unknown Process binding".to_owned()))?; + // Queued before its route failed or detached: nobody takes it now. + let Some(route) = inner.routes.lock().unwrap().get(&process_id).cloned() else { + return Ok(()); + }; let semantic_stream = match stream { process::NATIVE_STREAM_STDOUT => Stream::Stdout, process::NATIVE_STREAM_STDERR => Stream::Stderr, @@ -885,40 +921,53 @@ fn dispatch_outbound(inner: &Arc, event: process::NativeEvent) -> .map_err(backend_error)?; return Ok(()); } - route + if route .events .try_send(Event::Output { stream: semantic_stream, lifetime_offset: offset, data, }) - .map_err(|_| Error::Closed("Process semantic stream queue overflowed".to_owned())) + .is_err() + { + // This attachment's reader fell behind: it alone fails. + fail_route(inner, process_id, ROUTE_EVICTED); + let _ = inner + .manager + .control_native(process_id, process::NativeControl::Detach); + } + Ok(()) } process::NativeEvent::StdinProgress { process_id, consumed, open, } => { - let route = inner - .routes - .lock() - .unwrap() - .get(&process_id) - .cloned() - .ok_or_else(|| Error::Closed("stdin ACK for unknown Process binding".to_owned()))?; - route + let Some(route) = inner.routes.lock().unwrap().get(&process_id).cloned() else { + return Ok(()); + }; + if route .events .try_send(Event::StdinProgress { consumed, open }) - .map_err(|_| Error::Closed("Process semantic stream queue overflowed".to_owned())) + .is_err() + { + fail_route(inner, process_id, ROUTE_EVICTED); + let _ = inner + .manager + .control_native(process_id, process::NativeControl::Detach); + } + Ok(()) } process::NativeEvent::Exit { process_id, exit } => { let exit = native_exit_info(exit); let route = { let mut routes = inner.routes.lock().unwrap(); - let route = routes - .get(&process_id) - .cloned() - .ok_or_else(|| Error::Closed("exit for unknown Process binding".to_owned()))?; + // Its route failed as the exit was queued: that attachment is gone already. No + // replay is recorded (the handle left with the route), so a WAIT in this session + // answers CONFLICT while the exit is in flight, then NOT_FOUND. + let Some(route) = routes.get(&process_id).cloned() else { + return Ok(()); + }; let process_handle = route.process_handle.load(Ordering::Acquire); if process_handle != 0 { // Record the replay before the route disappears: a WAIT @@ -939,11 +988,21 @@ fn dispatch_outbound(inner: &Arc, event: process::NativeEvent) -> } } +/// The route of `process_id` leaves the session and its attachment and WAITs fail with +/// `reason`; the session and its other processes go on. +fn fail_route(inner: &SessionInner, process_id: u32, reason: &str) { + let route = inner.routes.lock().unwrap().remove(&process_id); + if let Some(route) = route { + route.failed.send_replace(Some(reason.to_owned())); + } +} + fn close_session(inner: &SessionInner, error: Error) { if inner.closed.borrow().is_some() { return; } - let _ = inner.closed.send(Some(error)); + // send_replace: nobody subscribes, and send would drop the value. + inner.closed.send_replace(Some(error)); inner.routes.lock().unwrap().clear(); inner.exits.lock().unwrap().clear(); } @@ -1352,6 +1411,250 @@ mod tests { server.shutdown().await; } + fn watch_request(handle: u64) -> wire::Attach { + wire::Attach { + process_handle: handle, + flags: 0, + stdout_receive_credit: 1 << 20, + stderr_receive_credit: 1 << 20, + extensions: Extensions::default(), + } + } + + /// `sh -c SCRIPT`, SCRIPT's programs named by absolute path (spawn_request's environment is + /// empty). + fn sh(script: &str) -> Vec> { + let mut script = script.to_owned(); + for name in ["head", "sleep", "yes"] { + let path = String::from_utf8(executable(name)).unwrap(); + script = script.replace(&format!("{{{name}}}"), &path); + } + vec![executable("sh"), b"-c".to_vec(), script.into_bytes()] + } + + /// What a command a session starts then says: its output and exit. + async fn runs_a_command(session: &Session) -> (Vec, i32) { + let mut after = session + .spawn( + &spawn_request(vec![executable("echo"), b"after".to_vec()], Vec::new()), + None, + ) + .await + .unwrap(); + let (output, exit) = output_and_exit(&mut after, Duration::from_secs(5), 0).await; + (output, exit.code) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn an_evicted_watcher_gives_back_its_process_slot() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([4; 16], None).unwrap(); + let watcher = runtime.session([5; 16], None).unwrap(); + // More rounds than a session has process slots (16). + for round in 0..20 { + let mut attachment = owner + .spawn( + &spawn_request(sh("{sleep} 0.2; {head} -c 3145728 /dev/zero"), Vec::new()), + None, + ) + .await + .unwrap(); + let watched = watcher + .attach(&watch_request(attachment.process_handle)) + .await + .unwrap_or_else(|error| panic!("round {round}: {error:?}")); + let (control, mut events) = watched.split(); + let (output, exit) = output_and_exit(&mut attachment, Duration::from_secs(10), 0).await; + assert_eq!((output.len(), exit.code), (3145728, 0), "round {round}"); + // The watcher never acknowledges: it falls a window behind and is dropped. + while tokio::time::timeout(Duration::from_secs(5), events.next()) + .await + .unwrap() + .is_some() + {} + assert!(events.failure().is_some(), "round {round}"); + // What the adapter does next. + let _ = control.detach().await; + } + assert_eq!(runs_a_command(&watcher).await, (b"after\n".to_vec(), 0)); + owner.shutdown().await; + watcher.shutdown().await; + server.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_watcher_whose_queue_fills_after_the_exit_fails_alone() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([6; 16], None).unwrap(); + let watcher = runtime.session([7; 16], None).unwrap(); + let mut attachment = owner + .spawn( + &spawn_request( + sh( + "{sleep} 0.3; i=0; while [ $i -lt 80 ]; do echo x; {sleep} 0.02; \ + i=$((i+1)); done; printf z; exit 0", + ), + Vec::new(), + ), + None, + ) + .await + .unwrap(); + // Never read: its route queue (80 events) fills with the 80 lines, and the last + // frame comes after the child exits. + let _watched = watcher + .attach(&watch_request(attachment.process_handle)) + .await + .unwrap(); + // The owner acknowledges its first 48 frames only: with 32 unacknowledged, the reader + // waits for it, and `z` stays in the pipe as the child exits. + let (mut frames, mut end) = (0, 0u64); + while frames < 80 { + match tokio::time::timeout(Duration::from_secs(5), attachment.next()) + .await + .unwrap() + .unwrap() + { + Event::Output { + stream, + lifetime_offset, + data, + } => { + frames += 1; + end = lifetime_offset + data.len() as u64; + if frames <= 48 { + attachment.acknowledge_output(stream, end).unwrap(); + } + } + other => panic!("{other:?}"), + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + attachment.acknowledge_output(Stream::Stdout, end).unwrap(); + let (rest, exit) = output_and_exit(&mut attachment, Duration::from_secs(5), end).await; + assert_eq!((rest, exit.code), (b"z".to_vec(), 0)); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(watcher.inner.closed.borrow().is_none()); + assert_eq!(runs_a_command(&watcher).await, (b"after\n".to_vec(), 0)); + owner.shutdown().await; + watcher.shutdown().await; + server.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn an_owner_that_drops_its_stream_after_the_exit_still_waits_for_it() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([10; 16], None).unwrap(); + // The reader takes a window (1 MiB) and waits for the owner; the last 32 KiB stay in + // the pipe as the child exits. + let attachment = owner + .spawn( + &spawn_request(sh("{head} -c 1081344 /dev/zero; exit 4"), Vec::new()), + None, + ) + .await + .unwrap(); + let handle = attachment.process_handle; + let (control, _events) = attachment.split(); + tokio::time::sleep(Duration::from_millis(800)).await; + // What the adapter does when its client drops the stream. + control.detach().await.unwrap(); + let wait = wire::Wait { + process_handle: handle, + timeout_ns: 5_000_000_000, + extensions: Extensions::default(), + }; + let exit = owner.wait(&wait).await.unwrap(); + assert_eq!((exit.code, exit.detail.as_slice()), (4, &b""[..])); + owner.shutdown().await; + server.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn an_owner_slow_to_take_its_window_after_the_exit_gets_all_its_output() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([12; 16], None).unwrap(); + let mut attachment = owner + .spawn( + &spawn_request(sh("{head} -c 1081344 /dev/zero; exit 4"), Vec::new()), + None, + ) + .await + .unwrap(); + // Longer than the kill grace (2 s): a slow link, or a client that waits for the exit + // before it reads. + tokio::time::sleep(Duration::from_millis(2_500)).await; + let (output, exit) = output_and_exit(&mut attachment, Duration::from_secs(5), 0).await; + assert_eq!(output.len(), 1081344); + assert_eq!((exit.code, exit.detail.as_slice()), (4, &b""[..])); + owner.shutdown().await; + server.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_residue_owner_slow_to_take_its_window_after_the_exit_gets_all_its_output() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([13; 16], None).unwrap(); + let mut request = residue_request(String::new(), Some(Duration::from_millis(300))); + request.argv = sh("{head} -c 1081344 /dev/zero; exit 4"); + let mut attachment = owner.spawn(&request, None).await.unwrap(); + // Longer than its grace: nothing of the group is left, so the grace does not cut it. + tokio::time::sleep(Duration::from_millis(1_500)).await; + let (output, exit) = output_and_exit(&mut attachment, Duration::from_secs(5), 0).await; + assert_eq!(output.len(), 1081344); + assert_eq!((exit.code, exit.detail.as_slice()), (4, &b""[..])); + owner.shutdown().await; + server.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_residue_writing_on_ends_its_grace_though_the_owner_reads_nothing() { + let server = Server::new(false, true); + let runtime = Runtime::new(server.clone()); + let owner = runtime.session([14; 16], None).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let pid_file = dir.path().join("pid"); + let mut request = residue_request(String::new(), Some(Duration::from_millis(300))); + request.argv = sh(&format!( + "{{yes}} & echo $! > '{}'; exit 0", + pid_file.display() + )); + let started = std::time::Instant::now(); + let mut attachment = owner.spawn(&request, None).await.unwrap(); + // The owner takes nothing until the exit: the residue fills its window and blocks. + let exit = loop { + if let Event::Exit(exit) = + tokio::time::timeout(Duration::from_secs(5), attachment.next()) + .await + .unwrap() + .unwrap() + { + break exit; + } + }; + assert!( + started.elapsed() < Duration::from_secs(3), + "{:?}", + started.elapsed() + ); + assert_eq!(exit.code, 0); + assert_eq!(exit.detail, b"residual process group left running"); + let pid: i32 = std::fs::read_to_string(&pid_file) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(alive(pid), "the residue is left running"); + unsafe { libc::kill(pid, libc::SIGKILL) }; + owner.shutdown().await; + server.shutdown().await; + } + #[tokio::test] async fn stdin_null_gives_the_child_the_null_device() { let server = Server::new(false, true); diff --git a/docs/design/processes.md b/docs/design/processes.md index 2a0d857c8..c0ad004c9 100644 --- a/docs/design/processes.md +++ b/docs/design/processes.md @@ -175,6 +175,24 @@ for stdin, stdout, and stderr; MESSAGE/frame counts are bounded by the common Transfer limits. A slow attachment cannot force unbounded process-wide output retention. +The spawning session's credit paces the child itself: the server reads a pipe no +more than 1 MiB (32 frames) ahead of what that session's Transfer has taken, so a +command that writes faster than its owner reads blocks on its pipe and loses +nothing. Other sessions' attachments are not waited for. One that falls 1 MiB +behind is dropped alone: its Transfers are reset with `RESOURCE_EXHAUSTED`, and a +`WAIT` it had pending still answers at the exit. Its session and its other +processes go on. (Servers up to 0.4.0 dropped the owner too when it fell behind, +and closed the lagging session's whole Process endpoint with it.) + +The pacing outlasts the child. Once it has exited and nothing of its group is left, +what its pipes still hold reaches the owner however slowly the owner takes it: the +kill grace, and a `LEAVE_RESIDUE` grace, bound only the time the group's residue +keeps the streams open. The server stops waiting for the owner when no reader has +waited for it for 250 ms (a process outside the group holds a pipe open with nothing +coming) or when a stream has given 1 MiB more (such a process writes on). An owner +that stops reading keeps its process's slot until it reads, drops the streams, or +its session ends, as a local pipe's reader would. + Catalogue State is coalesced under its subscription credit. Output offsets are lifetime counters, so a later `ATTACH` reports the exact skipped prefix rather than pretending to replay bytes that were not retained. A detachable final diff --git a/docs/design/yas.md b/docs/design/yas.md index 0ac6e4f15..7f7cbd1eb 100644 --- a/docs/design/yas.md +++ b/docs/design/yas.md @@ -3277,7 +3277,10 @@ discoverable until its retained exit result expires. ATTACH returns new stdout/stderr Transfers beginning at the process's current lifetime offsets; earlier output is explicitly reported as a gap and is not replayed. At most one attachment owns the stdin Transfer at a time, while any -number may observe output. CONTROL provides portable signal, terminate, kill, +number may observe output. The spawning session's credit paces the child (its +pipe is read at most a window ahead); an attachment of another session that falls +a window behind has its Transfers reset `RESOURCE_EXHAUSTED`, and nothing else +changes. CONTROL provides portable signal, terminate, kill, and detach actions with operation IDs. Closing the stdin Transfer half-closes stdin. WAIT returns the final exit record or TIMEOUT. From c812b11596d7ae5950dae3c31d7508ae9847a647 Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 02:08:19 -0700 Subject: [PATCH 07/10] Expect TIMEOUT from a terminal command wait nothing started (#83) * Expect TIMEOUT from a terminal command wait nothing started Since #54 a command wait that runs out before any command it could report on has started answers TIMEOUT (NOT_FOUND stays for an exited terminal or an evicted index). The client_host test still expected NOT_FOUND, so it failed on main. * Say what a terminal command wait answers when no command started Since #54 the server answers TIMEOUT when a command wait runs out before any command it could report on has started; wait_terminal_command's documentation still said NOT_FOUND, which stays for an evicted index or an exited terminal. From 622109a048f73349deb0db8cc4bbc2d329a1bcc7 Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 04:54:30 +0000 Subject: [PATCH 08/10] Report a spawned process's exit unasked: bash in one round trip A SPAWN with the new opt-in flag REPORT_EXIT (16) gets its process's exit as an EXIT Event (Process 0x0002, sensitive) once it is final: the record a WAIT would return then. A command's SPAWN Result, output and exit thus all come from one request, where a client had to send WAIT and wait a round trip more. - Servers advertise it in a new optional family limit, LAUNCHER_FLAGS_EXTENDED (tag 19), since clients from before accept at most 12 in tag 11; a client sets REPORT_EXIT only when tag 19 offers it, so either side may be older. - The report takes none of the session's pending WAITs, is sent once (a SPAWN retried under its operation ID shares it), and outlives the streams: when the spawning session's attachment goes before the exit (its streams dropped, their route failed), the server waits for the exit itself and still sends it. - The streams go on at the pace of their credit, so the EXIT may arrive before their last bytes and CLOSE. - yas-client sets the flag whenever offered; Process::wait/wait_timeout then take no request and keep the reported exit. Attached processes, and servers without the flag, still WAIT. Tests (client_host): a_spawned_process_reports_its_exit_without_a_wait (with the only WAIT held, a non-zero exit with stderr, output beyond the stream buffer read while the exit is awaited, a signal, and another session's attachment that still WAITs), and a_server_from_before_report_exit_is_waited_for (ignored; YAS_OLD_SERVER=). --- crates/cli/tests/client_host.rs | 220 +++++++++++++++++++++++++- crates/client/src/client.rs | 14 ++ crates/client/src/process.rs | 91 +++++++++-- crates/server/src/yas.rs | 243 +++++++++++++++++++++++++++-- crates/server/src/yas_process.rs | 8 +- crates/yas/src/generated.rs | 11 ++ crates/yas/src/process.rs | 219 ++++++++++++++++++++++++-- docs/design/processes.md | 8 +- docs/design/yas.md | 26 ++- js/core/src/yas/generated.ts | 44 ++++++ protocol/yas/families/process.toml | 26 +++ protocol/yas/inspection.json | 17 ++ protocol/yas/schema.json | 33 ++++ protocol/yas/vectors.json | 4 + protocol/yas/wire.md | 3 + 15 files changed, 928 insertions(+), 39 deletions(-) diff --git a/crates/cli/tests/client_host.rs b/crates/cli/tests/client_host.rs index dd1895ef3..034cd17d0 100644 --- a/crates/cli/tests/client_host.rs +++ b/crates/cli/tests/client_host.rs @@ -17,7 +17,11 @@ use yas_client::{Client, Error, HelloOptions}; const TIMEOUT: Duration = Duration::from_secs(30); fn options() -> HostOptions { - HostOptions::new(env!("CARGO_BIN_EXE_yas")) + options_for(env!("CARGO_BIN_EXE_yas")) +} + +fn options_for(binary: impl Into) -> HostOptions { + HostOptions::new(binary) .arg("--no-persistent-extensions") .env("YAS_EXT", "0") .env("YAS_CHANNEL", "0") @@ -568,6 +572,218 @@ async fn a_watcher_that_falls_behind_is_dropped_alone() { still_runs_commands(&owner).await; } +/// A server started with `--process-max-waits 1`, with that one WAIT held on a +/// `sleep` and shown to be held: a second WAIT is refused. +async fn hold_the_only_wait( + client: &Client, +) -> (yas_client::process::Process, tokio::task::JoinHandle<()>) { + let sleeper = client.spawn(Command::new("sleep").arg("30")).await.unwrap(); + let handle = sleeper.handle(); + let holder = { + let client = client.clone(); + tokio::spawn(async move { + let _ = client.wait_process(handle, None).await; + }) + }; + // The holder's WAIT goes first on the one connection. + tokio::time::sleep(Duration::from_millis(200)).await; + let refused = client + .wait_process(handle, Some(Duration::from_millis(1))) + .await + .unwrap_err(); + assert_eq!( + refused.status(), + Some(yas_client::wire::core::Status::ResourceExhausted), + "{refused}" + ); + (sleeper, holder) +} + +/// Commands whose exits must come whole: a non-zero exit with stderr, output +/// beyond the stream buffer read while the exit is awaited, a signal. +async fn exits_come_with_all_their_output(server: &HostedServer, client: &Client) { + let output = client + .spawn(Command::new("sh").args(["-c", "echo out; echo 'went wrong' >&2; exit 42"])) + .await + .unwrap() + .output() + .await + .unwrap(); + assert_eq!(output.status.code(), Some(42), "{}", output.status); + assert_eq!(output.stdout, b"out\n"); + assert_eq!(output.stderr, b"went wrong\n"); + + // Three times the largest stream buffer a default server keeps: the + // command blocks on its pipe until this reads, and its exit arrives while + // the tail is still on its way, as Ultimator's bash reads it. + const BYTES: u64 = 3 * 8 * 1024 * 1024; + let mut process = client + .spawn(Command::new("sh").args([ + "-c".to_owned(), + format!("head -c {BYTES} /dev/zero; printf end; echo tail >&2; exit 9"), + ])) + .await + .unwrap(); + let stdout = process.take_stdout().unwrap(); + let stderr = process.take_stderr().unwrap(); + let reader = tokio::spawn(async move { + ( + stdout.read_to_end(BYTES + 1024).await.unwrap(), + stderr.read_to_end(1024).await, + ) + }); + let status = tokio::time::timeout(TIMEOUT, process.wait()) + .await + .unwrap_or_else(|_| panic!("no exit\n{}", server_log(server))) + .unwrap(); + assert_eq!(status.code(), Some(9), "{status}"); + let (stdout, stderr) = tokio::time::timeout(TIMEOUT, reader) + .await + .expect("output after the exit") + .unwrap(); + let stderr = stderr.unwrap_or_else(|error| panic!("{error}\n{}", server_log(server))); + assert_eq!(stdout.len() as u64, BYTES + 3); + assert!(stdout.ends_with(b"end")); + assert!(stdout[..BYTES as usize].iter().all(|byte| *byte == 0)); + assert_eq!(stderr, b"tail\n"); + + let sleeper = client.spawn(Command::new("sleep").arg("30")).await.unwrap(); + assert_eq!( + sleeper + .wait_timeout(Duration::from_millis(100)) + .await + .unwrap(), + None + ); + sleeper.signal(Signal::Terminate).await.unwrap(); + let status = sleeper.wait().await.unwrap(); + assert_eq!(status.signal(), Some(libc::SIGTERM), "{status}"); + // Asked again, the same exit (an older server's WAITs may each give it another time). + let same = |other: &yas_client::process::ExitStatus| { + (&other.kind, &other.reason, other.raw_code) + == (&status.kind, &status.reason, status.raw_code) + }; + let again = sleeper.wait().await.unwrap(); + assert!(same(&again), "{again:?} after {status:?}"); + let again = sleeper + .wait_timeout(Duration::from_millis(1)) + .await + .unwrap() + .expect("exited"); + assert!(same(&again), "{again:?} after {status:?}"); + + for _ in 0..20 { + let output = client + .spawn(&Command::new("true")) + .await + .unwrap() + .output() + .await + .unwrap(); + assert!(output.status.success()); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_spawned_process_reports_its_exit_without_a_wait() { + use yas_client::wire::schema::process as schema; + let server = tokio::time::timeout( + TIMEOUT, + HostedServer::start( + options() + .arg("--verbose") + .args(["--process-max-waits", "1"]), + ), + ) + .await + .expect("hosted server start timed out") + .expect("hosted server starts"); + let client = server.connect().await.unwrap(); + assert_ne!( + client.launcher_flags() & schema::SPAWN_REPORT_EXIT as u32, + 0 + ); + // With the only WAIT held, every exit below comes unasked. + let (sleeper, holder) = hold_the_only_wait(&client).await; + exits_come_with_all_their_output(&server, &client).await; + // A process attached to (not spawned) is still waited for: another session, its only + // WAIT held too, attaches to this one's sleeper and is refused a second WAIT. + let other = server.connect().await.unwrap(); + let (other_sleeper, other_holder) = hold_the_only_wait(&other).await; + let attached = other.attach(sleeper.handle(), false).await.unwrap(); + let refused = attached + .wait_timeout(Duration::from_millis(1)) + .await + .unwrap_err(); + assert_eq!( + refused.status(), + Some(yas_client::wire::core::Status::ResourceExhausted), + "{refused}" + ); + for (which, (sleeper, holder)) in [(sleeper, holder), (other_sleeper, other_holder)] + .into_iter() + .enumerate() + { + sleeper.kill().await.unwrap(); + // Killed by YAS at this client's request, as a WAIT would answer it. + let status = sleeper.wait().await.unwrap(); + assert_eq!( + (&status.kind, &status.reason), + ( + &yas_client::process::ExitKind::Killed, + &yas_client::process::ExitReason::Client + ), + "sleeper {which}: {status:?}" + ); + tokio::time::timeout(TIMEOUT, holder) + .await + .unwrap() + .unwrap(); + } +} + +/// Against a server from before SPAWN_REPORT_EXIT, the same commands are +/// waited for with WAIT. Run with +/// `YAS_OLD_SERVER=/path/to/yas cargo test -p yas-cli --test client_host -- --ignored`, with a +/// binary that paces process output (its long output must not reset the stream). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs YAS_OLD_SERVER: a yas binary from before SPAWN_REPORT_EXIT"] +async fn a_server_from_before_report_exit_is_waited_for() { + use yas_client::wire::schema::process as schema; + let binary = std::env::var_os("YAS_OLD_SERVER").expect("YAS_OLD_SERVER names a yas binary"); + let server = tokio::time::timeout( + TIMEOUT, + HostedServer::start(options_for(binary).args(["--process-max-waits", "1"])), + ) + .await + .expect("hosted server start timed out") + .expect("hosted server starts"); + let client = server.connect().await.unwrap(); + assert_eq!( + client.launcher_flags() & schema::SPAWN_REPORT_EXIT as u32, + 0 + ); + // Its exits take a WAIT: with the only one held, waiting is refused. + let (sleeper, holder) = hold_the_only_wait(&client).await; + let process = client + .spawn(Command::new("sh").args(["-c", "exit 5"])) + .await + .unwrap(); + let refused = process.wait().await.unwrap_err(); + assert_eq!( + refused.status(), + Some(yas_client::wire::core::Status::ResourceExhausted), + "{refused}" + ); + sleeper.kill().await.unwrap(); + tokio::time::timeout(TIMEOUT, holder) + .await + .unwrap() + .unwrap(); + assert_eq!(process.wait().await.unwrap().code(), Some(5)); + exits_come_with_all_their_output(&server, &client).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_commands_background_children_die_with_it() { let server = start().await; @@ -920,7 +1136,7 @@ async fn a_command_can_leave_its_background_running_with_a_null_stdin() { let client = server.connect().await.unwrap(); assert_eq!( client.launcher_flags(), - (schema::SPAWN_LEAVE_RESIDUE | schema::SPAWN_STDIN_NULL) as u32 + (schema::SPAWN_LEAVE_RESIDUE | schema::SPAWN_STDIN_NULL | schema::SPAWN_REPORT_EXIT) as u32 ); // The background `sleep` holds stdout: the exit comes after the grace, and // the sleep keeps running. diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index 76f1ff28f..ec5e7f2e0 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -51,6 +51,8 @@ pub(crate) enum Route { Transfer(u32), /// State events for one `(family, subscription_id)`. State(u16, u32), + /// Process EXIT events for one process handle (a SPAWN with REPORT_EXIT). + ProcessExit(u64), } /// Registered by a call: run by the reader on an OK `Result`, before any later @@ -633,6 +635,18 @@ fn dispatch(shared: &Shared, frame: Frame) -> Result<()> { } return Ok(()); } + if frame.header.family == family::PROCESS + && frame.header.kind == yas_wire::process::event_kind::EXIT + { + if let Some(handle) = yas_wire::process::ExitReport::handle_of(&frame.payload) { + shared + .router + .lock() + .unwrap() + .deliver(Route::ProcessExit(handle), frame); + } + return Ok(()); + } if frame.header.kind == 0 && family_has_state(frame.header.family) { if let Some(subscription) = state_subscription_id(&frame.payload) { shared diff --git a/crates/client/src/process.rs b/crates/client/src/process.rs index 855b2c675..41f9c7f9c 100644 --- a/crates/client/src/process.rs +++ b/crates/client/src/process.rs @@ -94,8 +94,8 @@ use yas_wire::{ family, process::{ self as wire, Attach, Control, ControlAction, ControlResult, Cwd, EnvEntry, - EnvironmentKind, ExitRecord, ProcessRecord, RemovedProcess, Spawn, StreamBundle, Wait, - request_kind, + EnvironmentKind, ExitRecord, ExitReport, ProcessRecord, RemovedProcess, Spawn, + StreamBundle, Wait, request_kind, }, schema::process as schema, state::{Phase, RecordKind, Watch as StateWatch}, @@ -485,6 +485,49 @@ pub struct Process { stdout_offset: u64, stderr_offset: u64, merged_stderr: bool, + /// Where the server sends its exit, for a process spawned with REPORT_EXIT. + reported: Option, +} + +/// The exit a server reports unasked (an EXIT event), received once and kept. +struct ReportedExit { + frames: tokio::sync::Mutex, + status: tokio::sync::OnceCell, +} + +impl std::fmt::Debug for ReportedExit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReportedExit") + .field("status", &self.status.get()) + .finish_non_exhaustive() + } +} + +impl ReportedExit { + async fn wait(&self, client: &Client) -> Result { + self.status + .get_or_try_init(|| async { + use yas_wire::Decode; + let frame = self.frames.lock().await.recv().await.ok_or_else(|| { + client + .closed_reason() + .unwrap_or_else(|| Error::protocol("Process EXIT route closed")) + })?; + Ok(ExitStatus::from_wire( + ExitReport::decode(&frame.payload)?.exit, + )) + }) + .await + .cloned() + } +} + +impl Drop for Process { + fn drop(&mut self) { + if self.reported.is_some() { + self.client.release(Route::ProcessExit(self.handle)); + } + } } impl Process { @@ -536,8 +579,13 @@ impl Process { self.stderr.take() } - /// Wait for the process to exit. + /// Wait for the process to exit. A process this session spawned from a server that + /// reports exits (SPAWN_REPORT_EXIT, in [`Client::launcher_flags`]) takes no request: + /// the server sends the exit as it happens. Otherwise this asks with WAIT. pub async fn wait(&self) -> Result { + if let Some(reported) = &self.reported { + return reported.wait(&self.client).await; + } self.client .wait_process(self.handle, None) .await? @@ -546,6 +594,12 @@ impl Process { /// Wait at most `timeout`; `None` if it is still running. pub async fn wait_timeout(&self, timeout: Duration) -> Result> { + if let Some(reported) = &self.reported { + return match tokio::time::timeout(timeout, reported.wait(&self.client)).await { + Ok(status) => status.map(Some), + Err(_) => Ok(None), + }; + } self.client.wait_process(self.handle, Some(timeout)).await } @@ -765,13 +819,18 @@ fn change_from_record(record: &yas_wire::state::Record) -> Result Hook { - Box::new(|prefix: &ResultPrefix| { +/// The routes a SPAWN or ATTACH Result announces: its streams, and with `report_exit` the +/// process's EXIT events. +fn bundle_hook(report_exit: bool) -> Hook { + Box::new(move |prefix: &ResultPrefix| { use yas_wire::Decode; let Ok(bundle) = StreamBundle::decode(&prefix.body) else { return Vec::new(); }; let mut routes = vec![Route::Transfer(bundle.stdout.transfer_id)]; + if report_exit { + routes.push(Route::ProcessExit(bundle.process_handle)); + } if let Some(stdin) = &bundle.stdin { routes.push(Route::Transfer(stdin.transfer_id)); } @@ -787,10 +846,15 @@ impl Client { /// happens to it and its children. pub async fn spawn(&self, command: &Command) -> Result { let stdin_null = self.launcher_flags() & schema::SPAWN_STDIN_NULL as u32 != 0; + // The server sends the exit unasked: waiting for it takes no round trip. + let report_exit = self.launcher_flags() & schema::SPAWN_REPORT_EXIT as u32 != 0; let window = command .window .unwrap_or_else(|| self.default_process_window()); - let spawn = command.to_wire(stdin_null, window)?; + let mut spawn = command.to_wire(stdin_null, window)?; + if report_exit { + spawn.flags |= schema::SPAWN_REPORT_EXIT as u16; + } // Servers from before the extended limits refuse more than 256 // entries as undecodable; say why instead. if let Some(limits) = self.process_limits() @@ -808,7 +872,7 @@ impl Client { request_kind::SPAWN, spawn.encode()?, Some(DEFAULT_REQUEST_TIMEOUT), - Some(bundle_hook()), + Some(bundle_hook(report_exit)), ) .await?; let mut process = self.process_from_reply(reply, window)?; @@ -847,7 +911,7 @@ impl Client { request_kind::ATTACH, attach.encode()?, Some(DEFAULT_REQUEST_TIMEOUT), - Some(bundle_hook()), + Some(bundle_hook(false)), ) .await?; self.process_from_reply(reply, window) @@ -893,6 +957,12 @@ impl Client { } None => None, }; + let reported = reply + .take(Route::ProcessExit(bundle.process_handle)) + .map(|frames| ReportedExit { + frames: tokio::sync::Mutex::new(frames), + status: tokio::sync::OnceCell::new(), + }); Ok(Process { client: self.clone(), handle: bundle.process_handle, @@ -902,6 +972,7 @@ impl Client { stdout_offset: bundle.stdout_lifetime_offset, stderr_offset: bundle.stderr_lifetime_offset, merged_stderr: bundle.merged_stderr, + reported, }) } @@ -1035,7 +1106,9 @@ impl Client { .and_then(|limits| wire::Limits::from_extensions(&limits).ok()) } - /// The opt-in SPAWN flags this server honours (`SPAWN_LEAVE_RESIDUE`, `SPAWN_STDIN_NULL`); + /// The opt-in SPAWN flags this server honours (`SPAWN_LEAVE_RESIDUE`, `SPAWN_STDIN_NULL`, + /// `SPAWN_REPORT_EXIT`, which [`Client::spawn`] sets itself so that [`Process::wait`] takes + /// no round trip); /// 0 for servers that predate them. [`Stdin::Null`] uses the null device where offered. pub fn launcher_flags(&self) -> u32 { self.process_limits() diff --git a/crates/server/src/yas.rs b/crates/server/src/yas.rs index 178306f9e..0a2eb6fc9 100644 --- a/crates/server/src/yas.rs +++ b/crates/server/src/yas.rs @@ -3011,6 +3011,16 @@ struct ProcessAttachment { stdout_transfer: u32, stderr_transfer: Option, exited: bool, + /// SPAWN_REPORT_EXIT: the exit goes to this session unasked, whatever becomes of the + /// attachment. + report_exit: Option, +} + +/// A SPAWN_REPORT_EXIT process's exit report: its handle, and whether its EXIT was sent. +#[derive(Clone)] +struct ExitReport { + process_handle: u64, + sent: Arc, } struct ProcessInputTransfer { @@ -3047,6 +3057,8 @@ enum ProcessOperationOutcome { fingerprint: [u8; 32], stdout_receive_credit: u64, stderr_receive_credit: u64, + /// SPAWN_REPORT_EXIT: send the exit to this session as an EXIT event. + report_exit: bool, outcome: Result, }, Attach { @@ -18211,6 +18223,7 @@ impl Session { let operation_id = request.operation_id; let stdout_receive_credit = request.stdout_receive_credit; let stderr_receive_credit = request.stderr_receive_credit; + let report_exit = request.flags & yas_wire::schema::process::SPAWN_REPORT_EXIT as u16 != 0; let session = self.process.as_ref().ok_or(())?.session.clone(); let internal = self.internal.clone(); let cancellation = self.cancellation.clone(); @@ -18228,6 +18241,7 @@ impl Session { fingerprint, stdout_receive_credit, stderr_receive_credit, + report_exit, outcome, }, _slot: None, @@ -25385,6 +25399,7 @@ impl Session { fingerprint, stdout_receive_credit, stderr_receive_credit, + report_exit, outcome, } => { if kind != yas_process_wire::request_kind::SPAWN { @@ -25442,6 +25457,7 @@ impl Session { } }; let body = bundle.encode().map_err(|_| ())?; + let report_exit = report_exit.then_some(bundle.process_handle); self.record_process_replay( operation_id, kind, @@ -25465,7 +25481,7 @@ impl Session { self.remove_process_attachment(attachment_id, true); return Err(()); } - self.activate_process_attachment(attachment_id, events)?; + self.activate_process_attachment(attachment_id, events, report_exit)?; Ok(()) } ProcessOperationOutcome::Attach { @@ -25526,7 +25542,7 @@ impl Session { self.remove_process_attachment(attachment_id, true); return Err(()); } - self.activate_process_attachment(attachment_id, events)?; + self.activate_process_attachment(attachment_id, events, None)?; Ok(()) } ProcessOperationOutcome::Control { @@ -25783,6 +25799,7 @@ impl Session { stdout_transfer, stderr_transfer, exited: false, + report_exit: None, }, ); Ok(( @@ -25801,10 +25818,13 @@ impl Session { )) } + /// Start forwarding an installed attachment's streams; with `report_exit` (the handle of a + /// process spawned with SPAWN_REPORT_EXIT), its exit is sent as an EXIT event too. fn activate_process_attachment( &mut self, attachment_id: u32, events: super::yas_process::AttachmentEvents, + report_exit: Option, ) -> Result<(), ()> { let attachment = self .process @@ -25820,6 +25840,10 @@ impl Session { .stderr_transfer .map(|transfer_id| self.outbound.get(&transfer_id).cloned().ok_or(())) .transpose()?; + let report_exit = report_exit.map(|process_handle| ExitReport { + process_handle, + sent: Arc::new(AtomicBool::new(false)), + }); let task = spawn_process_attachment( attachment_id, attachment.control.clone(), @@ -25828,16 +25852,19 @@ impl Session { attachment.stdout_transfer, stdout_flow, attachment.stderr_transfer.zip(stderr_flow), + report_exit.clone(), self.out.clone(), self.internal.clone(), self.cancellation.clone(), attachment.cancellation.clone(), ); - self.process + let attachment = self + .process .as_mut() .and_then(|process| process.attachments.get_mut(&attachment_id)) - .ok_or(())? - .task = Some(task); + .ok_or(())?; + attachment.task = Some(task); + attachment.report_exit = report_exit; Ok(()) } @@ -26083,9 +26110,39 @@ impl Session { self.outbound_sensitive.remove(&transfer_id); } } - if detach { + // A SPAWN_REPORT_EXIT process whose attachment goes before its exit (its streams were + // dropped, or its output route failed) still reports it: the server waits for it. + let report = attachment + .report_exit + .filter(|report| !report.sent.load(Ordering::Acquire)) + .zip(self.process.as_ref().map(|process| process.session.clone())); + if detach || report.is_some() { + let out = self.out.clone(); + let connection = self.cancellation.clone(); + let control = attachment.control; tokio::spawn(async move { - let _ = attachment.control.detach().await; + if detach { + let _ = control.detach().await; + } else { + drop(control); + } + let Some((report, session)) = report else { + return; + }; + let wait = yas_process_wire::Wait { + process_handle: report.process_handle, + timeout_ns: 0, + extensions: Extensions::default(), + }; + let exit = tokio::select! { + exit = session.wait(&wait) => exit, + _ = connection.cancelled() => return, + }; + if let Ok(exit) = exit + && !report.sent.swap(true, Ordering::AcqRel) + { + send_exit_report(&out, report.process_handle, exit, &connection).await; + } }); } } @@ -31003,6 +31060,29 @@ struct ProcessOutputChunk { data: Vec, } +/// A SPAWN_REPORT_EXIT process's exit, as the EXIT event its session gets unasked. +async fn send_exit_report( + out: &FrameSender, + process_handle: u64, + exit: super::yas_process::ExitInfo, + connection: &ConnectionCancellation, +) { + let report = yas_process_wire::ExitReport { + process_handle, + exit: exit.into_record(monotonic_ns()), + extensions: Extensions::default(), + }; + let _ = send_event_with_sensitivity( + out, + family::PROCESS, + yas_process_wire::event_kind::EXIT, + &report, + connection, + true, + ) + .await; +} + #[allow(clippy::too_many_arguments)] fn spawn_process_attachment( attachment_id: u32, @@ -31012,12 +31092,14 @@ fn spawn_process_attachment( stdout_transfer: u32, stdout_flow: Arc, stderr: Option<(u32, Arc)>, + report_exit: Option, out: FrameSender, internal: mpsc::Sender, connection: ConnectionCancellation, cancellation: ConnectionCancellation, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { + let exit_out = out.clone(); let (stdout_tx, stdout_rx) = mpsc::channel(PROCESS_STREAM_QUEUE); let stdout_task = tokio::spawn(run_process_output( attachment_id, @@ -31113,8 +31195,15 @@ fn spawn_process_attachment( return; } } - Some(super::yas_process::Event::Exit(_)) => { + Some(super::yas_process::Event::Exit(exit)) => { exited = true; + // As a WAIT would answer it, at once: the streams go on with what + // the process wrote before, at the pace of their credit. + if let Some(report) = &report_exit + && !report.sent.swap(true, Ordering::AcqRel) + { + send_exit_report(&exit_out, report.process_handle, exit, &connection).await; + } let _ = internal .send(Internal::ProcessExited { attachment_id }) .await; @@ -51805,7 +51894,7 @@ mod tests { yas_process_wire::Limits { max_mutation_replays: MAX_PROCESS_OPERATION_REPLAYS as u32, launcher_flags: if cfg!(unix) { - yas_wire::schema::process::SPAWN_LAUNCHER_FLAGS as u32 + yas_wire::schema::process::SPAWN_LAUNCHER_FLAGS_EXTENDED as u32 } else { yas_wire::schema::process::SPAWN_STDIN_NULL as u32 }, @@ -52170,6 +52259,142 @@ mod tests { timeout(TEST_TIMEOUT, server_task).await.unwrap().unwrap(); } + #[cfg(unix)] + #[tokio::test(flavor = "multi_thread")] + async fn process_spawn_with_report_exit_is_sent_its_exit_without_a_wait() { + use std::os::unix::ffi::OsStrExt; + + let sh = std::env::split_paths(&std::env::var_os("PATH").expect("test PATH is set")) + .map(|directory| directory.join("sh")) + .find(|path| path.is_file()) + .expect("sh is on PATH") + .as_os_str() + .as_bytes() + .to_vec(); + let state = super::super::tests::process_transport::test_state( + super::super::process::Server::new(false, true), + ); + let (mut client, codec, hello, server_task) = + start_registered_session(state, &[family::TRANSFER, family::PROCESS]).await; + let descriptor = hello + .families + .iter() + .find(|descriptor| descriptor.family_id == family::PROCESS) + .expect("Process negotiated"); + let limits = yas_process_wire::Limits::from_extensions(&descriptor.limits).unwrap(); + assert_ne!( + limits.launcher_flags & yas_wire::schema::process::SPAWN_REPORT_EXIT as u32, + 0 + ); + + let spawn = yas_process_wire::Spawn { + operation_id: [0x72; 16], + flags: (yas_wire::schema::process::SPAWN_REPORT_EXIT + | yas_wire::schema::process::SPAWN_STDIN_NULL) as u16, + environment_kind: yas_process_wire::EnvironmentKind::Empty, + cwd: yas_process_wire::Cwd::ServerDefault, + argv: vec![ + sh, + b"-c".to_vec(), + b"printf out; printf err >&2; exit 3".to_vec(), + ], + env: Vec::new(), + stdout_receive_credit: 1024 * 1024, + stderr_receive_credit: 1024 * 1024, + extensions: Extensions::default(), + }; + write_request( + &mut client, + &codec, + family::PROCESS, + yas_process_wire::request_kind::SPAWN, + 11, + &spawn, + ) + .await; + let (spawned, _) = next_process_result_collecting_state( + &mut client, + &codec, + yas_process_wire::request_kind::SPAWN, + 11, + ) + .await; + assert_eq!(spawned.status, Status::Ok); + let streams = yas_process_wire::StreamBundle::decode(&spawned.body).unwrap(); + assert!(streams.stdin.is_none()); + let stderr = streams.stderr.as_ref().expect("separate stderr Transfer"); + + // No WAIT is sent: the exit comes on its own, next to the streams. + let mut stdout = Vec::new(); + let mut stderr_bytes = Vec::new(); + let mut stdout_closed = false; + let mut stderr_closed = false; + let mut exit = None; + while !stdout_closed || !stderr_closed || exit.is_none() { + let frame = next_frame(&mut client, &codec).await; + match (frame.header.family, frame.header.kind) { + (family::PROCESS, yas_process_wire::event_kind::EXIT) => { + assert_eq!( + frame.header, + FrameHeader { + sensitive: true, + ..FrameHeader::event( + family::PROCESS, + yas_process_wire::event_kind::EXIT, + ) + } + ); + assert!(exit.is_none(), "a second EXIT"); + let report = yas_process_wire::ExitReport::decode(&frame.payload).unwrap(); + assert_eq!(report.process_handle, streams.process_handle); + assert_eq!( + yas_process_wire::ExitReport::handle_of(&frame.payload), + Some(streams.process_handle) + ); + exit = Some(report.exit); + } + (family::TRANSFER, yas_wire::schema::transfer::event::BYTE_DATA) => { + let data = ByteData::decode(&frame.payload).unwrap(); + let target = if data.transfer_id == streams.stdout.transfer_id { + &mut stdout + } else if data.transfer_id == stderr.transfer_id { + &mut stderr_bytes + } else { + panic!("bytes for unknown Process Transfer") + }; + assert_eq!(data.offset, target.len() as u64); + target.extend_from_slice(&data.data); + } + (family::TRANSFER, yas_wire::schema::transfer::event::CLOSE) => { + let close = Close::decode(&frame.payload).unwrap(); + assert_eq!(close.status, Status::Ok.code()); + if close.transfer_id == streams.stdout.transfer_id { + assert_eq!(close.final_data_bytes, stdout.len() as u64); + stdout_closed = true; + } else if close.transfer_id == stderr.transfer_id { + assert_eq!(close.final_data_bytes, stderr_bytes.len() as u64); + stderr_closed = true; + } else { + panic!("CLOSE for unknown Process Transfer") + } + } + _ => panic!( + "unexpected native frame from a reported Process: {:?}", + frame.header + ), + } + } + let exit = exit.unwrap(); + assert_eq!(exit.kind, yas_process_wire::ExitKind::Code); + assert_eq!(exit.code, 3); + assert_ne!(exit.exited_server_ns, 0); + assert_eq!(stdout, b"out"); + assert_eq!(stderr_bytes, b"err"); + + drop(client); + timeout(TEST_TIMEOUT, server_task).await.unwrap().unwrap(); + } + #[cfg(unix)] #[tokio::test(flavor = "multi_thread")] async fn terminal_query_transfer_and_cancel_are_native_and_correlated() { diff --git a/crates/server/src/yas_process.rs b/crates/server/src/yas_process.rs index 9c7637cc3..68d60e888 100644 --- a/crates/server/src/yas_process.rs +++ b/crates/server/src/yas_process.rs @@ -293,9 +293,10 @@ impl Runtime { } pub(crate) fn limits(&self) -> wire::Limits { - // LEAVE_RESIDUE works with Unix process groups and Windows jobs alike. + // LEAVE_RESIDUE works with Unix process groups and Windows jobs alike; REPORT_EXIT is + // the YAS connection's own. wire::Limits { - launcher_flags: schema::process::SPAWN_LAUNCHER_FLAGS as u32, + launcher_flags: schema::process::SPAWN_LAUNCHER_FLAGS_EXTENDED as u32, ..self.server.maxima().limits() } } @@ -381,7 +382,8 @@ impl Session { resolved_cwd: Option>, ) -> Result { let cwd = resolve_cwd(&request.cwd, resolved_cwd)?; - let flags = u8::try_from(request.flags) + // REPORT_EXIT asks the YAS connection for an EXIT event; the process is the same. + let flags = u8::try_from(request.flags & !(schema::process::SPAWN_REPORT_EXIT as u16)) .map_err(|_| Error::Invalid("Process SPAWN flags do not fit v1".to_owned()))?; let process_id = self.allocate_process_id()?; let (route, events) = self.install_route(process_id, false)?; diff --git a/crates/yas/src/generated.rs b/crates/yas/src/generated.rs index a3b9caf40..668600b12 100644 --- a/crates/yas/src/generated.rs +++ b/crates/yas/src/generated.rs @@ -3791,6 +3791,7 @@ pub const WAIT: u16 = 0x0005; pub mod event { pub const STATE: u16 = 0x0000; pub const STATE_ACK: u16 = 0x0001; +pub const EXIT: u16 = 0x0002; } pub const SPAWN_MERGE_STDERR: u64 = 1; pub const SPAWN_DETACHABLE: u64 = 2; @@ -3798,6 +3799,8 @@ pub const SPAWN_LEAVE_RESIDUE: u64 = 4; pub const SPAWN_STDIN_NULL: u64 = 8; pub const SPAWN_FLAGS: u64 = 3; pub const SPAWN_LAUNCHER_FLAGS: u64 = 12; +pub const SPAWN_REPORT_EXIT: u64 = 16; +pub const SPAWN_LAUNCHER_FLAGS_EXTENDED: u64 = 28; pub const ENV_EMPTY: u64 = 0; pub const ENV_SESSION: u64 = 1; pub const CWD_SERVER_DEFAULT: u64 = 0; @@ -3886,6 +3889,7 @@ pub const LIMIT_MAX_DETACHED_RETENTION_NS: u64 = 9; pub const MAX_MUTATION_REPLAYS: u64 = 65536; pub const LIMIT_MAX_MUTATION_REPLAYS: u64 = 10; pub const LIMIT_LAUNCHER_FLAGS: u64 = 11; +pub const LIMIT_LAUNCHER_FLAGS_EXTENDED: u64 = 19; pub static OPERATIONS: &[super::OperationMetadata] = &[ super::OperationMetadata { name: "WATCH", class: 1, kind: 0, direction: 0, sensitive: 1, compression: 0, datagram: 0, layout: "StateWatch; ResultPrefix + StateWatchResult" }, super::OperationMetadata { name: "UNWATCH", class: 1, kind: 1, direction: 0, sensitive: 0, compression: 0, datagram: 0, layout: "subscription_id:u32; ResultPrefix" }, @@ -3895,11 +3899,13 @@ super::OperationMetadata { name: "CONTROL", class: 1, kind: 4, direction: 0, sen super::OperationMetadata { name: "WAIT", class: 1, kind: 5, direction: 0, sensitive: 1, compression: 0, datagram: 0, layout: "process_handle:u64,timeout_ns:u64,Extensions; ResultPrefix + ExitRecord" }, super::OperationMetadata { name: "STATE", class: 0, kind: 0, direction: 1, sensitive: 1, compression: 0, datagram: 0, layout: "StateEvent" }, super::OperationMetadata { name: "STATE_ACK", class: 0, kind: 1, direction: 0, sensitive: 0, compression: 0, datagram: 0, layout: "StateAck" }, +super::OperationMetadata { name: "EXIT", class: 0, kind: 2, direction: 1, sensitive: 1, compression: 0, datagram: 0, layout: "ExitReport" }, ]; pub static TYPES: &[super::TypeMetadata] = &[ super::TypeMetadata { name: "cwd", layout: "kind:u8,reserved:[u8;3]=0; SERVER_DEFAULT empty, PATH path:bytes_u32, TERMINAL terminal_handle:u64, FS root_handle:u64,component_count:u16,repeated component:bytes_u16" }, super::TypeMetadata { name: "process_record", layout: "process_handle:u64,lifecycle:u8,stream_state:u8,flags:u16,native_pid:u64,owner_session:[u8;16],argv0:bytes_u32,stdin_received:u64,stdout_produced:u64,stderr_produced:u64,retention_deadline_server_ns:u64,exit_present:u8,reserved:[u8;7]=0,optional exit:bytes_u32 containing ExitRecord,Extensions" }, super::TypeMetadata { name: "remove_record", layout: "process_handle:u64" }, +super::TypeMetadata { name: "exit_report", layout: "process_handle:u64,exit:bytes_u32 containing ExitRecord,Extensions" }, super::TypeMetadata { name: "exit_record", layout: "kind:u8,reason:u8,reserved:u16=0,code:i32,exited_server_ns:u64,detail:bytes_u32" }, super::TypeMetadata { name: "stream_bundle", layout: "process_handle:u64,flags:u16,reserved:u16=0,stdout_lifetime_offset:u64,stderr_lifetime_offset:u64,optional stdin/stdout/stderr descriptor:bytes_u32 containing sensitive BYTE TransferDescriptor,Extensions" }, super::TypeMetadata { name: "state_entity_body", layout: "ADD/REPLACE complete ProcessRecord; REMOVE process_handle:u64" }, @@ -3924,6 +3930,7 @@ super::LimitMetadata { name: "MAX_STREAM_BUFFER_BYTES_EXTENDED", tag: 15, value_ super::LimitMetadata { name: "MAX_ENVC_EXTENDED", tag: 16, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 16384 }, super::LimitMetadata { name: "MAX_PENDING_WAITS", tag: 17, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 65536 }, super::LimitMetadata { name: "MAX_PENDING_OPERATIONS", tag: 18, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 16384 }, +super::LimitMetadata { name: "LAUNCHER_FLAGS_EXTENDED", tag: 19, value_type: super::LimitValueType::U32, required: false, hard_min: 0, hard_max: 28 }, ]; pub static CONSTANTS: &[super::ConstantMetadata] = &[ super::ConstantMetadata { name: "SPAWN_MERGE_STDERR", value: 1 }, @@ -3932,6 +3939,8 @@ super::ConstantMetadata { name: "SPAWN_LEAVE_RESIDUE", value: 4 }, super::ConstantMetadata { name: "SPAWN_STDIN_NULL", value: 8 }, super::ConstantMetadata { name: "SPAWN_FLAGS", value: 3 }, super::ConstantMetadata { name: "SPAWN_LAUNCHER_FLAGS", value: 12 }, +super::ConstantMetadata { name: "SPAWN_REPORT_EXIT", value: 16 }, +super::ConstantMetadata { name: "SPAWN_LAUNCHER_FLAGS_EXTENDED", value: 28 }, super::ConstantMetadata { name: "ENV_EMPTY", value: 0 }, super::ConstantMetadata { name: "ENV_SESSION", value: 1 }, super::ConstantMetadata { name: "CWD_SERVER_DEFAULT", value: 0 }, @@ -4020,6 +4029,7 @@ super::ConstantMetadata { name: "LIMIT_MAX_DETACHED_RETENTION_NS", value: 9 }, super::ConstantMetadata { name: "MAX_MUTATION_REPLAYS", value: 65536 }, super::ConstantMetadata { name: "LIMIT_MAX_MUTATION_REPLAYS", value: 10 }, super::ConstantMetadata { name: "LIMIT_LAUNCHER_FLAGS", value: 11 }, +super::ConstantMetadata { name: "LIMIT_LAUNCHER_FLAGS_EXTENDED", value: 19 }, ]; } pub mod net { @@ -5033,6 +5043,7 @@ GoldenVector { name: "yas.process.request.control.header", hex: "400004000901000 GoldenVector { name: "yas.process.request.wait.header", hex: "400005000901000000" }, GoldenVector { name: "yas.process.event.state.header", hex: "4000000008" }, GoldenVector { name: "yas.process.event.state_ack.header", hex: "4000010000" }, +GoldenVector { name: "yas.process.event.exit.header", hex: "4000020008" }, GoldenVector { name: "yas.net.request.open.header", hex: "410000000901000000" }, GoldenVector { name: "yas.net.request.close.header", hex: "410001000901000000" }, GoldenVector { name: "yas.net.event.datagram.header", hex: "4100000008" }, diff --git a/crates/yas/src/process.rs b/crates/yas/src/process.rs index 5a70d2cf1..22efd4c49 100644 --- a/crates/yas/src/process.rs +++ b/crates/yas/src/process.rs @@ -198,8 +198,8 @@ pub struct Spawn { impl Spawn { fn validate(&self) -> Result<()> { validate_operation_id(&self.operation_id)?; - let known = - crate::schema::process::SPAWN_FLAGS | crate::schema::process::SPAWN_LAUNCHER_FLAGS; + let known = crate::schema::process::SPAWN_FLAGS + | crate::schema::process::SPAWN_LAUNCHER_FLAGS_EXTENDED; if self.flags & !(known as u16) != 0 { return Err(Error::Invalid("Process spawn flags")); } @@ -914,6 +914,47 @@ impl Decode for RemovedProcess { } } +/// EXIT: a process's final exit, sent to the session that spawned it with +/// `SPAWN_REPORT_EXIT`, once the exit is final (as WAIT would answer it). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExitReport { + pub process_handle: u64, + pub exit: ExitRecord, + pub extensions: Extensions, +} + +impl Encode for ExitReport { + fn encode_to(&self, out: &mut Vec) -> Result<()> { + validate_handle(self.process_handle, "Process handle")?; + put_u64(out, self.process_handle); + put_bytes_u32(out, &self.exit.encode()?)?; + self.extensions.encode_tail(out) + } +} + +impl Decode for ExitReport { + fn decode(input: &[u8]) -> Result { + let mut decoder = Decoder::new(input); + let process_handle = decoder.u64()?; + let exit = ExitRecord::decode(decoder.len_bytes_u32()?)?; + let value = Self { + process_handle, + exit, + extensions: decoder.extensions()?, + }; + decoder.finish()?; + validate_handle(value.process_handle, "Process handle")?; + Ok(value) + } +} + +impl ExitReport { + /// The process handle of an EXIT event's payload, without decoding the rest. + pub fn handle_of(payload: &[u8]) -> Option { + Some(u64::from_le_bytes(payload.get(..8)?.try_into().ok()?)) + } +} + /// Process family maxima, as a server selects them in HELLO. /// /// The first ten fields are the family's original limits (tags 1–10). A @@ -937,8 +978,10 @@ pub struct Limits { pub max_stream_buffer_bytes: u64, pub max_detached_retention_ns: u64, pub max_mutation_replays: u32, - /// SPAWN flags of `SPAWN_LAUNCHER_FLAGS` the server honours (LEAVE_RESIDUE, STDIN_NULL); - /// 0 from servers that predate them. + /// SPAWN flags of `SPAWN_LAUNCHER_FLAGS_EXTENDED` the server honours (LEAVE_RESIDUE, + /// STDIN_NULL, REPORT_EXIT); 0 from servers that predate them. LAUNCHER_FLAGS (tag 11) + /// carries those of `SPAWN_LAUNCHER_FLAGS`, all older clients accept; + /// LAUNCHER_FLAGS_EXTENDED (tag 19) carries all of them, when there are more. pub launcher_flags: u32, /// Pending `WAIT`s one session may hold. pub max_pending_waits: u32, @@ -962,7 +1005,7 @@ impl Limits { max_mutation_replays: crate::schema::process::MAX_MUTATION_REPLAYS as u32, max_pending_waits: crate::schema::process::MAX_PENDING_WAITS as u32, max_pending_operations: crate::schema::process::MAX_PENDING_OPERATIONS as u32, - launcher_flags: crate::schema::process::SPAWN_LAUNCHER_FLAGS as u32, + launcher_flags: crate::schema::process::SPAWN_LAUNCHER_FLAGS_EXTENDED as u32, }; /// The original hard maxima: what an unconfigured server enforces, and @@ -1096,9 +1139,17 @@ impl Limits { self.max_pending_operations, )); } - if self.launcher_flags != 0 { + let legacy_launcher_flags = + self.launcher_flags & crate::schema::process::SPAWN_LAUNCHER_FLAGS as u32; + if legacy_launcher_flags != 0 { extensions.push(limit_u32( crate::schema::process::LIMIT_LAUNCHER_FLAGS, + legacy_launcher_flags, + )); + } + if self.launcher_flags != legacy_launcher_flags { + extensions.push(limit_u32( + crate::schema::process::LIMIT_LAUNCHER_FLAGS_EXTENDED, self.launcher_flags, )); } @@ -1126,6 +1177,7 @@ impl Limits { crate::schema::process::LIMIT_MAX_ENVC_EXTENDED as u16, crate::schema::process::LIMIT_MAX_PENDING_WAITS as u16, crate::schema::process::LIMIT_MAX_PENDING_OPERATIONS as u16, + crate::schema::process::LIMIT_LAUNCHER_FLAGS_EXTENDED as u16, ]; reject_unknown_required(extensions, &known)?; let legacy = Self::DEFAULT; @@ -1145,6 +1197,14 @@ impl Limits { extensions, crate::schema::process::LIMIT_MAX_STREAM_BUFFER_BYTES, )?; + let legacy_launcher_flags = + if extensions.0.iter().any(|extension| { + extension.tag == crate::schema::process::LIMIT_LAUNCHER_FLAGS as u16 + }) { + read_limit_u32(extensions, crate::schema::process::LIMIT_LAUNCHER_FLAGS)? + } else { + 0 + }; let value = Self { max_argc: read_limit_u32(extensions, crate::schema::process::LIMIT_MAX_ARGC)?, max_arg_bytes: read_limit_u32(extensions, crate::schema::process::LIMIT_MAX_ARG_BYTES)?, @@ -1175,13 +1235,11 @@ impl Limits { extensions, crate::schema::process::LIMIT_MAX_MUTATION_REPLAYS, )?, - launcher_flags: if extensions.0.iter().any(|extension| { - extension.tag == crate::schema::process::LIMIT_LAUNCHER_FLAGS as u16 - }) { - read_limit_u32(extensions, crate::schema::process::LIMIT_LAUNCHER_FLAGS)? - } else { - 0 - }, + launcher_flags: read_optional_limit_u32( + extensions, + crate::schema::process::LIMIT_LAUNCHER_FLAGS_EXTENDED, + )? + .unwrap_or(legacy_launcher_flags), max_pending_waits: u32_or( crate::schema::process::LIMIT_MAX_PENDING_WAITS, legacy.max_pending_waits, @@ -1204,6 +1262,11 @@ impl Limits { || value.max_processes < max_processes || value.max_pending_spawns < max_pending_spawns || value.max_stream_buffer_bytes < max_stream_buffer_bytes + // Tag 11 carries the v1 launcher flags and tag 19 all of them: the + // extended set keeps those the legacy tag promised, and no others of theirs. + || legacy_launcher_flags & !(crate::schema::process::SPAWN_LAUNCHER_FLAGS as u32) != 0 + || value.launcher_flags & crate::schema::process::SPAWN_LAUNCHER_FLAGS as u32 + != legacy_launcher_flags { return Err(Error::Invalid("Process family limit")); } @@ -1754,6 +1817,136 @@ mod tests { ); } + #[test] + fn report_exit_is_advertised_in_a_tag_older_clients_ignore() { + use crate::schema::process as p; + let v1 = p::SPAWN_LAUNCHER_FLAGS as u32; + let all = p::SPAWN_LAUNCHER_FLAGS_EXTENDED as u32; + assert_eq!(all & !v1, p::SPAWN_REPORT_EXIT as u32); + + // Only the v1 flags: tag 11 alone, as before REPORT_EXIT. + let before = Limits { + launcher_flags: v1, + ..Limits::DEFAULT + }; + let extensions = before.to_extensions().unwrap(); + assert_eq!(limit_value(&extensions, p::LIMIT_LAUNCHER_FLAGS), Some(12)); + assert_eq!( + limit_value(&extensions, p::LIMIT_LAUNCHER_FLAGS_EXTENDED), + None + ); + assert_eq!(Limits::from_extensions(&extensions).unwrap(), before); + + // With REPORT_EXIT: tag 11 keeps within the maximum clients from before it + // validate, and the optional tag 19 carries every flag. + let extended = Limits { + launcher_flags: all, + ..Limits::DEFAULT + }; + let extensions = extended.to_extensions().unwrap(); + assert_eq!( + limit_value(&extensions, p::LIMIT_LAUNCHER_FLAGS), + Some(p::SPAWN_LAUNCHER_FLAGS) + ); + assert_eq!( + limit_value(&extensions, p::LIMIT_LAUNCHER_FLAGS_EXTENDED), + Some(p::SPAWN_LAUNCHER_FLAGS_EXTENDED) + ); + assert!(extensions.0.iter().all(|extension| !extension.required)); + assert_eq!(Limits::from_extensions(&extensions).unwrap(), extended); + // A client that does not know tag 19 reads the v1 flags: no REPORT_EXIT. + let without_19 = Extensions( + extensions + .0 + .iter() + .filter(|e| u64::from(e.tag) != p::LIMIT_LAUNCHER_FLAGS_EXTENDED) + .cloned() + .collect(), + ); + assert_eq!(Limits::from_extensions(&without_19).unwrap(), before); + + let with = |tag_11: Option, tag_19: Option| { + let mut extensions = Limits::DEFAULT.to_extensions().unwrap(); + for (tag, value) in [ + (p::LIMIT_LAUNCHER_FLAGS, tag_11), + (p::LIMIT_LAUNCHER_FLAGS_EXTENDED, tag_19), + ] { + if let Some(value) = value { + extensions.0.push(limit_u32(tag, value)); + } + } + Limits::from_extensions(&extensions) + }; + assert_eq!( + with(None, Some(p::SPAWN_REPORT_EXIT as u32)) + .unwrap() + .launcher_flags, + p::SPAWN_REPORT_EXIT as u32 + ); + // REPORT_EXIT has no place in tag 11, which older clients bound to 12. + assert!(with(Some(all), None).is_err()); + assert!(with(Some(all), Some(all)).is_err()); + // Tag 19 keeps the v1 flags tag 11 promised, and adds none of theirs. + assert!(with(Some(v1), Some(p::SPAWN_REPORT_EXIT as u32)).is_err()); + assert!(with(None, Some(all)).is_err()); + assert!(with(Some(v1), Some(all << 1)).is_err()); + } + + #[test] + fn exit_report_round_trips_and_names_its_process_cheaply() { + let report = ExitReport { + process_handle: 0x0102_0304_0506_0708, + exit: ExitRecord { + kind: ExitKind::Code, + reason: crate::schema::process::EXIT_REASON_UNKNOWN as u8, + code: 3, + exited_server_ns: 9, + detail: b"exited".to_vec(), + }, + extensions: Extensions::default(), + }; + every_truncation(&report); + let bytes = report.encode().unwrap(); + assert_eq!(ExitReport::handle_of(&bytes), Some(report.process_handle)); + assert_eq!(ExitReport::handle_of(&bytes[..7]), None); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(ExitReport::decode(&trailing).is_err()); + assert!( + ExitReport { + process_handle: 0, + ..report.clone() + } + .encode() + .is_err() + ); + let mut zero = bytes; + zero[..8].fill(0); + assert!(ExitReport::decode(&zero).is_err()); + } + + #[test] + fn spawn_accepts_report_exit_among_its_flags() { + use crate::schema::process as p; + let spawn = |flags: u64| Spawn { + operation_id: [1; 16], + flags: flags as u16, + environment_kind: EnvironmentKind::Session, + cwd: Cwd::ServerDefault, + argv: vec![b"true".to_vec()], + env: Vec::new(), + stdout_receive_credit: 65_536, + stderr_receive_credit: 65_536, + extensions: Extensions::default(), + }; + every_truncation(&spawn(p::SPAWN_REPORT_EXIT | p::SPAWN_STDIN_NULL)); + assert!( + spawn(p::SPAWN_LAUNCHER_FLAGS_EXTENDED << 1) + .encode() + .is_err() + ); + } + #[test] fn invalid_environment_and_transfer_policy_fail() { let mut spawn = Spawn { diff --git a/docs/design/processes.md b/docs/design/processes.md index c0ad004c9..c3e7c5ea7 100644 --- a/docs/design/processes.md +++ b/docs/design/processes.md @@ -91,7 +91,13 @@ observe output, while at most one attachment owns stdin. `CONTROL` provides typed signal, terminate, kill, and detach actions under nonzero operation IDs. `WAIT` returns the final portable exit record or -`TIMEOUT`. Closing stdin half-closes the child stream. An ordinary child belongs +`TIMEOUT`. A client that sets `SPAWN_REPORT_EXIT`, where the server offers it +(family-limit tag 19), is sent that record as an `EXIT` event once the exit is +final, so a command's exit costs no round trip of its own: SPAWN's Result, its +output and its exit all travel from one request. yas-client sets it whenever +offered, and `Process::wait` then takes no request and no pending-`WAIT` slot; +against older servers it sends `WAIT` as before. Closing stdin half-closes the +child stream. An ordinary child belongs to its spawning session and is terminated when it disappears; a detachable child remains discoverable until its retained final record expires. diff --git a/docs/design/yas.md b/docs/design/yas.md index 7f7cbd1eb..18f69b5f1 100644 --- a/docs/design/yas.md +++ b/docs/design/yas.md @@ -3242,7 +3242,7 @@ of defining separate data and ACK messages. | Class | Kinds | | ------- | -------------------------------------------- | | Request | WATCH, UNWATCH, SPAWN, ATTACH, CONTROL, WAIT | -| Event | STATE, STATE_ACK | +| Event | STATE, STATE_ACK, EXIT | SPAWN executes exact argv and environment bytes without an implicit shell. It accepts explicit cwd, inherited terminal cwd, FS root/path, session environment, @@ -3268,6 +3268,27 @@ TERMINATE's escalation SIGKILLs members left after the kill grace (terminates the job on Windows) and stops waiting for their streams. The exit's detail says `residual process group left running` when members held the streams. +`REPORT_EXIT` (16) sends the spawning session the exit as it becomes final, +without a WAIT: one EXIT Event (`0x0002`, sensitive), `[process_handle: u64, +exit: bytes_u32 containing ExitRecord, Extensions]`, the record a WAIT would +return at that moment. The streams go on with what the process wrote before its +exit, at the pace of their credit, so the event can arrive before their last +bytes and their CLOSE. The report outlives the streams: when the spawning +session's attachment goes before the exit (it dropped its streams, or their +route failed), the server waits for the exit itself and still sends the EXIT, +once. It takes none of the session's pending WAITs, and +nothing changes for other sessions: they, and sessions that ATTACH, still WAIT. +A SPAWN retried under its operation ID shares the original attachment, whose +exit is reported once. + +Servers advertise the opt-in flags they honour in two optional family limits: +tag 11 `LAUNCHER_FLAGS` carries those of v1 (`LEAVE_RESIDUE`, `STDIN_NULL`), +at most 12, which is all clients from before `REPORT_EXIT` accept; tag 19 +`LAUNCHER_FLAGS_EXTENDED` carries every flag the server honours, when that is +more. Tag 19 names each flag tag 11 does and adds none of v1's. A client sets +`REPORT_EXIT` only when tag 19 offers it, and otherwise WAITs, so either side +may be older. + Catalog records contain argv0, native PID for diagnostics, lifecycle, owner session, detachable flag, stream offsets, exit record, and retention deadline. An ordinary process is owned by its spawning session and terminated when that @@ -3282,7 +3303,8 @@ pipe is read at most a window ahead); an attachment of another session that fall a window behind has its Transfers reset `RESOURCE_EXHAUSTED`, and nothing else changes. CONTROL provides portable signal, terminate, kill, and detach actions with operation IDs. Closing the stdin Transfer half-closes -stdin. WAIT returns the final exit record or TIMEOUT. +stdin. WAIT returns the final exit record or TIMEOUT; a process spawned with +`REPORT_EXIT` needs none, its EXIT Event carries the same record. The canonical v1 payloads are generated from `protocol/yas/families/process.toml`. SPAWN carries `[operation_id, flags, diff --git a/js/core/src/yas/generated.ts b/js/core/src/yas/generated.ts index 1fe937672..e5966364e 100644 --- a/js/core/src/yas/generated.ts +++ b/js/core/src/yas/generated.ts @@ -1649,12 +1649,15 @@ export const YAS_PROCESS_CONTROL = 4 as const; export const YAS_PROCESS_WAIT = 5 as const; export const YAS_PROCESS_STATE = 0 as const; export const YAS_PROCESS_STATE_ACK = 1 as const; +export const YAS_PROCESS_EXIT = 2 as const; export const YAS_PROCESS_SPAWN_MERGE_STDERR = 1 as const; export const YAS_PROCESS_SPAWN_DETACHABLE = 2 as const; export const YAS_PROCESS_SPAWN_LEAVE_RESIDUE = 4 as const; export const YAS_PROCESS_SPAWN_STDIN_NULL = 8 as const; export const YAS_PROCESS_SPAWN_FLAGS = 3 as const; export const YAS_PROCESS_SPAWN_LAUNCHER_FLAGS = 12 as const; +export const YAS_PROCESS_SPAWN_REPORT_EXIT = 16 as const; +export const YAS_PROCESS_SPAWN_LAUNCHER_FLAGS_EXTENDED = 28 as const; export const YAS_PROCESS_ENV_EMPTY = 0 as const; export const YAS_PROCESS_ENV_SESSION = 1 as const; export const YAS_PROCESS_CWD_SERVER_DEFAULT = 0 as const; @@ -1743,6 +1746,7 @@ export const YAS_PROCESS_LIMIT_MAX_DETACHED_RETENTION_NS = 9 as const; export const YAS_PROCESS_MAX_MUTATION_REPLAYS = 65536 as const; export const YAS_PROCESS_LIMIT_MAX_MUTATION_REPLAYS = 10 as const; export const YAS_PROCESS_LIMIT_LAUNCHER_FLAGS = 11 as const; +export const YAS_PROCESS_LIMIT_LAUNCHER_FLAGS_EXTENDED = 19 as const; export const YAS_FAMILY_NET = 65 as const; export const YAS_NET_VERSION = 1 as const; export const YAS_NET_OPEN = 0 as const; @@ -2251,6 +2255,7 @@ export const YAS_FAMILY_LIMIT_POLICIES: Readonly "64/2/5": [1, 0, 0], "64/0/0": [1, 0, 0], "64/0/1": [0, 0, 0], + "64/0/2": [1, 0, 0], "65/1/0": [1, 0, 0], "65/2/0": [1, 0, 0], "65/1/1": [1, 0, 0], @@ -2876,6 +2882,7 @@ export const YAS_OPERATION_DIRECTION_MASKS: Readonly> = { "64/1/5": 1, "64/0/0": 2, "64/0/1": 1, + "64/0/2": 2, "65/1/0": 1, "65/1/1": 1, "65/0/0": 3, @@ -11733,6 +11740,14 @@ export const YAS_SCHEMA = { "required": false, "hard_min": 1, "hard_max": 16384 + }, + { + "name": "LAUNCHER_FLAGS_EXTENDED", + "tag": 19, + "type": "u32", + "required": false, + "hard_min": 0, + "hard_max": 28 } ], "requests": [ @@ -11809,6 +11824,15 @@ export const YAS_SCHEMA = { "compression": "allowed", "datagram": "forbidden", "layout": "StateAck" + }, + { + "name": "EXIT", + "kind": 2, + "direction": "server_to_client", + "sensitive": "required", + "compression": "allowed", + "datagram": "forbidden", + "layout": "ExitReport" } ], "types": [ @@ -11824,6 +11848,10 @@ export const YAS_SCHEMA = { "name": "remove_record", "layout": "process_handle:u64" }, + { + "name": "exit_report", + "layout": "process_handle:u64,exit:bytes_u32 containing ExitRecord,Extensions" + }, { "name": "exit_record", "layout": "kind:u8,reason:u8,reserved:u16=0,code:i32,exited_server_ns:u64,detail:bytes_u32" @@ -11866,6 +11894,14 @@ export const YAS_SCHEMA = { "name": "SPAWN_LAUNCHER_FLAGS", "value": 12 }, + { + "name": "SPAWN_REPORT_EXIT", + "value": 16 + }, + { + "name": "SPAWN_LAUNCHER_FLAGS_EXTENDED", + "value": 28 + }, { "name": "ENV_EMPTY", "value": 0 @@ -12217,6 +12253,10 @@ export const YAS_SCHEMA = { { "name": "LIMIT_LAUNCHER_FLAGS", "value": 11 + }, + { + "name": "LIMIT_LAUNCHER_FLAGS_EXTENDED", + "value": 19 } ] }, @@ -15393,6 +15433,10 @@ export const YAS_GOLDEN_VECTORS = { "name": "yas.process.event.state_ack.header", "hex": "4000010000" }, + { + "name": "yas.process.event.exit.header", + "hex": "4000020008" + }, { "name": "yas.net.request.open.header", "hex": "410000000901000000" diff --git a/protocol/yas/families/process.toml b/protocol/yas/families/process.toml index 76f0f40db..6a56be040 100644 --- a/protocol/yas/families/process.toml +++ b/protocol/yas/families/process.toml @@ -20,6 +20,7 @@ limits = [ { name = "MAX_ENVC_EXTENDED", tag = 16, type = "u32", required = false, hard_min = 1, hard_max = 16384 }, { name = "MAX_PENDING_WAITS", tag = 17, type = "u32", required = false, hard_min = 1, hard_max = 65536 }, { name = "MAX_PENDING_OPERATIONS", tag = 18, type = "u32", required = false, hard_min = 1, hard_max = 16384 }, + { name = "LAUNCHER_FLAGS_EXTENDED", tag = 19, type = "u32", required = false, hard_min = 0, hard_max = 28 }, ] [[constant]] @@ -40,6 +41,15 @@ value = 3 [[constant]] name = "SPAWN_LAUNCHER_FLAGS" value = 12 +# The spawning session is sent an EXIT event once the exit is final, so it +# learns it without a WAIT. Advertised by LAUNCHER_FLAGS_EXTENDED (tag 19), +# which old clients ignore: LAUNCHER_FLAGS (tag 11) keeps its v1 maximum. +[[constant]] +name = "SPAWN_REPORT_EXIT" +value = 16 +[[constant]] +name = "SPAWN_LAUNCHER_FLAGS_EXTENDED" +value = 28 [[constant]] name = "ENV_EMPTY" @@ -325,6 +335,9 @@ value = 10 [[constant]] name = "LIMIT_LAUNCHER_FLAGS" value = 11 +[[constant]] +name = "LIMIT_LAUNCHER_FLAGS_EXTENDED" +value = 19 [[request]] name = "WATCH" @@ -398,6 +411,15 @@ compression = "allowed" datagram = "forbidden" layout = "StateAck" +[[event]] +name = "EXIT" +kind = 0x0002 +direction = "server_to_client" +sensitive = "required" +compression = "allowed" +datagram = "forbidden" +layout = "ExitReport" + [[type]] name = "cwd" layout = "kind:u8,reserved:[u8;3]=0; SERVER_DEFAULT empty, PATH path:bytes_u32, TERMINAL terminal_handle:u64, FS root_handle:u64,component_count:u16,repeated component:bytes_u16" @@ -410,6 +432,10 @@ layout = "process_handle:u64,lifecycle:u8,stream_state:u8,flags:u16,native_pid:u name = "remove_record" layout = "process_handle:u64" +[[type]] +name = "exit_report" +layout = "process_handle:u64,exit:bytes_u32 containing ExitRecord,Extensions" + [[type]] name = "exit_record" layout = "kind:u8,reason:u8,reserved:u16=0,code:i32,exited_server_ns:u64,detail:bytes_u32" diff --git a/protocol/yas/inspection.json b/protocol/yas/inspection.json index f4abc74d9..dd2d55655 100644 --- a/protocol/yas/inspection.json +++ b/protocol/yas/inspection.json @@ -5599,6 +5599,23 @@ "name": "STATE_ACK", "sensitive": "allowed" }, + { + "class": "event", + "class_id": 0, + "compression": "allowed", + "correlated": false, + "datagram": "forbidden", + "direction": "server_to_client", + "family": 64, + "family_name": "yas.process", + "family_version": 1, + "header_bytes": 5, + "key": "64/0/2", + "kind": 2, + "layout": "ExitReport", + "name": "EXIT", + "sensitive": "required" + }, { "class": "request", "class_id": 1, diff --git a/protocol/yas/schema.json b/protocol/yas/schema.json index 1cea9b7ba..d6d58672c 100644 --- a/protocol/yas/schema.json +++ b/protocol/yas/schema.json @@ -8817,6 +8817,14 @@ "required": false, "hard_min": 1, "hard_max": 16384 + }, + { + "name": "LAUNCHER_FLAGS_EXTENDED", + "tag": 19, + "type": "u32", + "required": false, + "hard_min": 0, + "hard_max": 28 } ], "requests": [ @@ -8893,6 +8901,15 @@ "compression": "allowed", "datagram": "forbidden", "layout": "StateAck" + }, + { + "name": "EXIT", + "kind": 2, + "direction": "server_to_client", + "sensitive": "required", + "compression": "allowed", + "datagram": "forbidden", + "layout": "ExitReport" } ], "types": [ @@ -8908,6 +8925,10 @@ "name": "remove_record", "layout": "process_handle:u64" }, + { + "name": "exit_report", + "layout": "process_handle:u64,exit:bytes_u32 containing ExitRecord,Extensions" + }, { "name": "exit_record", "layout": "kind:u8,reason:u8,reserved:u16=0,code:i32,exited_server_ns:u64,detail:bytes_u32" @@ -8950,6 +8971,14 @@ "name": "SPAWN_LAUNCHER_FLAGS", "value": 12 }, + { + "name": "SPAWN_REPORT_EXIT", + "value": 16 + }, + { + "name": "SPAWN_LAUNCHER_FLAGS_EXTENDED", + "value": 28 + }, { "name": "ENV_EMPTY", "value": 0 @@ -9301,6 +9330,10 @@ { "name": "LIMIT_LAUNCHER_FLAGS", "value": 11 + }, + { + "name": "LIMIT_LAUNCHER_FLAGS_EXTENDED", + "value": 19 } ] }, diff --git a/protocol/yas/vectors.json b/protocol/yas/vectors.json index 27f1e0fbc..a6ebb8422 100644 --- a/protocol/yas/vectors.json +++ b/protocol/yas/vectors.json @@ -753,6 +753,10 @@ "name": "yas.process.event.state_ack.header", "hex": "4000010000" }, + { + "name": "yas.process.event.exit.header", + "hex": "4000020008" + }, { "name": "yas.net.request.open.header", "hex": "410000000901000000" diff --git a/protocol/yas/wire.md b/protocol/yas/wire.md index 1d2f075fe..be0244cf2 100644 --- a/protocol/yas/wire.md +++ b/protocol/yas/wire.md @@ -893,6 +893,7 @@ Every Request kind has a correlated Result with the same family and kind. | ---: | --- | --- | --- | --- | --- | --- | | `0x0000` | `STATE` | `server_to_client` | `required` | `allowed` | `forbidden` | StateEvent | | `0x0001` | `STATE_ACK` | `client_to_server` | `allowed` | `allowed` | `forbidden` | StateAck | +| `0x0002` | `EXIT` | `server_to_client` | `required` | `allowed` | `forbidden` | ExitReport | ### Limits @@ -916,6 +917,7 @@ Every Request kind has a correlated Result with the same family and kind. | 16 | `MAX_ENVC_EXTENDED` | 4 | false | 1 | 16384 | | 17 | `MAX_PENDING_WAITS` | 4 | false | 1 | 65536 | | 18 | `MAX_PENDING_OPERATIONS` | 4 | false | 1 | 16384 | +| 19 | `LAUNCHER_FLAGS_EXTENDED` | 4 | false | 0 | 28 | ### Shared types @@ -924,6 +926,7 @@ Every Request kind has a correlated Result with the same family and kind. | `cwd` | kind:u8,reserved:[u8;3]=0; SERVER_DEFAULT empty, PATH path:bytes_u32, TERMINAL terminal_handle:u64, FS root_handle:u64,component_count:u16,repeated component:bytes_u16 | | `process_record` | process_handle:u64,lifecycle:u8,stream_state:u8,flags:u16,native_pid:u64,owner_session:[u8;16],argv0:bytes_u32,stdin_received:u64,stdout_produced:u64,stderr_produced:u64,retention_deadline_server_ns:u64,exit_present:u8,reserved:[u8;7]=0,optional exit:bytes_u32 containing ExitRecord,Extensions | | `remove_record` | process_handle:u64 | +| `exit_report` | process_handle:u64,exit:bytes_u32 containing ExitRecord,Extensions | | `exit_record` | kind:u8,reason:u8,reserved:u16=0,code:i32,exited_server_ns:u64,detail:bytes_u32 | | `stream_bundle` | process_handle:u64,flags:u16,reserved:u16=0,stdout_lifetime_offset:u64,stderr_lifetime_offset:u64,optional stdin/stdout/stderr descriptor:bytes_u32 containing sensitive BYTE TransferDescriptor,Extensions | | `state_entity_body` | ADD/REPLACE complete ProcessRecord; REMOVE process_handle:u64 | From f9a3673b6d7ee2713be89b20d2f3a937a4c8abad Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 05:54:38 +0000 Subject: [PATCH 09/10] Report an exit only from the attachment that saw it; WAIT once that goes Review of #86: the server's fallback wait, for a REPORT_EXIT process whose attachment went before its exit, bound the process again (a detached one's slot was taken back and ATTACH answered CONFLICT), hung Process::wait when that wait failed, and an attachment aborted mid-send lost the EXIT. The fallback is gone: the attachment reports the exit, from a task of its own so removing the attachment meanwhile cannot cut the EXIT short. When the attachment goes first (a Transfer RESET on any of its streams, stdin included, sent or received, or a DETACH) no EXIT comes, and yas-client knows: its router marks the process's report lost, and Process::wait and wait_timeout then WAIT, after taking an EXIT that already arrived. --- crates/cli/tests/client_host.rs | 98 +++++++++++++++++++++++++++++ crates/client/src/client.rs | 100 +++++++++++++++++++++++++++--- crates/client/src/process.rs | 105 +++++++++++++++++++++++--------- crates/server/src/yas.rs | 74 +++++----------------- docs/design/processes.md | 6 +- docs/design/yas.md | 9 +-- 6 files changed, 291 insertions(+), 101 deletions(-) diff --git a/crates/cli/tests/client_host.rs b/crates/cli/tests/client_host.rs index 034cd17d0..0fbf0f5d7 100644 --- a/crates/cli/tests/client_host.rs +++ b/crates/cli/tests/client_host.rs @@ -528,6 +528,104 @@ async fn a_dropped_output_stream_leaves_its_process_and_session_working() { still_runs_commands(&client).await; } +/// A spawned process whose attachment goes before its exit (it was detached, or its streams +/// were dropped) is asked for its exit with WAIT, which its attachment would have reported. +/// Nothing holds it to the session meanwhile: a detached one gives back its slot, and either +/// can be attached to again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_process_whose_attachment_went_is_waited_for_and_held_by_nothing() { + use yas_client::wire::schema::process as schema; + let server = start().await; + let client = server.connect().await.unwrap(); + assert_ne!( + client.launcher_flags() & schema::SPAWN_REPORT_EXIT as u32, + 0 + ); + let per_session = client.process_limits().unwrap().max_processes_per_session as usize; + // Detached, each gives back its slot: one more than the session holds still spawns. + let mut detached = Vec::new(); + for _ in 0..per_session { + let process = client + .spawn(Command::new("sleep").arg("30").detachable(true)) + .await + .unwrap(); + within("a detach", process.detach()).await.unwrap(); + detached.push(process); + } + let one_more = within( + "a spawn beside the detached", + client.spawn(Command::new("sh").args(["-c", "exit 6"])), + ) + .await + .unwrap(); + assert_eq!( + within("its exit", one_more.wait()).await.unwrap().code(), + Some(6) + ); + // A detached one can be attached to again, and its exit comes (by WAIT). + let again = within("an attach", client.attach(detached[0].handle(), false)) + .await + .unwrap(); + detached[0].kill().await.unwrap(); + let status = within("a detached one's exit", detached[0].wait()) + .await + .unwrap(); + // (Each answer stamps the time it is given.) + let attached = within("its exit, attached", again.wait()).await.unwrap(); + assert_eq!( + (&status.kind, &status.reason, status.raw_code), + (&attached.kind, &attached.reason, attached.raw_code) + ); + for process in &detached[1..] { + process.kill().await.unwrap(); + within("a detached one's exit", process.wait()) + .await + .unwrap(); + } + // An ordinary one whose streams were dropped: the same. + let mut sleeper = client.spawn(Command::new("sleep").arg("30")).await.unwrap(); + drop(sleeper.take_stdout()); + drop(sleeper.take_stderr()); + let again = within("an attach", client.attach(sleeper.handle(), false)) + .await + .unwrap_or_else(|error| panic!("{error}\n{}", server_log(&server))); + assert_eq!( + within( + "a short wait", + sleeper.wait_timeout(Duration::from_millis(200)) + ) + .await + .unwrap(), + None + ); + sleeper.kill().await.unwrap(); + let status = within("its exit", sleeper.wait()).await.unwrap(); + // (Each answer stamps the time it is given.) + let attached = within("its exit, attached", again.wait()).await.unwrap(); + assert_eq!( + (&status.kind, &status.reason, status.raw_code), + (&attached.kind, &attached.reason, attached.raw_code) + ); + // One whose stdin was aborted: its attachment goes with it. + let mut reader = client + .spawn( + Command::new("sh") + .args(["-c", "sleep 0.3; exit 3"]) + .stdin(Stdin::Piped), + ) + .await + .unwrap(); + reader.take_stdin().unwrap().abort(); + assert_eq!( + within("its exit, stdin aborted", reader.wait()) + .await + .unwrap() + .code(), + Some(3) + ); + still_runs_commands(&client).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_watcher_that_falls_behind_is_dropped_alone() { let server = start().await; diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index ec5e7f2e0..4f9be8f8c 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -20,7 +20,7 @@ //! terminates every ordinary process it spawned; see [`crate::process`]. use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; @@ -65,6 +65,8 @@ pub(crate) type FrameReceiver = mpsc::UnboundedReceiver; pub(crate) struct Reply { pub(crate) prefix: ResultPrefix, routes: Vec<(Route, FrameReceiver)>, + /// With a [`Route::ProcessExit`]: told if that process's attachment goes first. + report_lost: Option>, } impl Reply { @@ -72,6 +74,41 @@ impl Reply { let index = self.routes.iter().position(|(key, _)| *key == route)?; Some(self.routes.swap_remove(index).1) } + + pub(crate) fn take_report_lost(&mut self) -> Option> { + self.report_lost.take() + } +} + +/// Marked when the attachment that would report a process's exit (SPAWN_REPORT_EXIT) goes +/// before it: a Transfer RESET, sent or received, on any of its streams, or a DETACH. The +/// server then sends no EXIT, and the exit is asked for with WAIT. +#[derive(Debug, Default)] +pub(crate) struct ReportLost { + lost: AtomicBool, + notify: tokio::sync::Notify, +} + +impl ReportLost { + pub(crate) fn mark(&self) { + self.lost.store(true, Ordering::Release); + self.notify.notify_waiters(); + } + + pub(crate) fn is_lost(&self) -> bool { + self.lost.load(Ordering::Acquire) + } + + /// Once marked. + pub(crate) async fn lost(&self) { + loop { + let notified = self.notify.notified(); + if self.is_lost() { + return; + } + notified.await; + } + } } struct Pending { @@ -88,6 +125,8 @@ struct Router { orphan_bytes: usize, released: HashSet, released_order: VecDeque, + /// The open transfers of processes that report their exit: a RESET on one marks it. + report_watch: HashMap>, closed: Option, } @@ -108,6 +147,9 @@ impl Router { fn release(&mut self, route: Route) { self.routes.remove(&route); + if let Route::Transfer(transfer_id) = route { + self.report_watch.remove(&transfer_id); + } if let Some(frames) = self.orphans.remove(&route) { for frame in frames { self.orphan_bytes = self.orphan_bytes.saturating_sub(frame.payload.len()); @@ -124,6 +166,16 @@ impl Router { } } + /// A transfer closed, or reset (either way): a reset one's process, if it reports its + /// exit, will not. + fn transfer_ended(&mut self, transfer_id: u32, reset: bool) { + if let Some(lost) = self.report_watch.remove(&transfer_id) + && reset + { + lost.mark(); + } + } + fn deliver(&mut self, route: Route, frame: Frame) { if let Some(sender) = self.routes.get(&route) { if sender.send(frame).is_err() { @@ -163,6 +215,7 @@ impl Router { // Dropping the senders ends every stream and subscription; they then // report the session error. self.routes.clear(); + self.report_watch.clear(); self.orphans.clear(); self.orphan_order.clear(); self.orphan_bytes = 0; @@ -414,6 +467,16 @@ impl Client { if let Some(error) = self.closed_reason() { return Err(error); } + if frame.header.kind == yas_wire::transfer::kind::RESET + && let Some(transfer_id) = transfer_event_id(&frame) + { + self.inner + .shared + .router + .lock() + .unwrap() + .transfer_ended(transfer_id, true); + } self.inner .outbound .send(frame) @@ -619,7 +682,26 @@ fn dispatch(shared: &Shared, frame: Frame) -> Result<()> { routes.push((route, receiver)); } } - if pending.reply.send(Ok(Reply { prefix, routes })).is_err() { + // A process that reports its exit: any of its transfers reset (its attachment + // went) means no EXIT comes. + let report_lost = routes + .iter() + .any(|(route, _)| matches!(route, Route::ProcessExit(_))) + .then(|| { + let lost = Arc::new(ReportLost::default()); + for (route, _) in &routes { + if let Route::Transfer(transfer_id) = route { + router.report_watch.insert(*transfer_id, lost.clone()); + } + } + lost + }); + let reply = Reply { + prefix, + routes, + report_lost, + }; + if pending.reply.send(Ok(reply)).is_err() { // The caller went away between sending and now. } Ok(()) @@ -627,11 +709,15 @@ fn dispatch(shared: &Shared, frame: Frame) -> Result<()> { Class::Event => { if frame.header.family == family::TRANSFER { if let Some(transfer_id) = transfer_event_id(&frame) { - shared - .router - .lock() - .unwrap() - .deliver(Route::Transfer(transfer_id), frame); + let mut router = shared.router.lock().unwrap(); + let kind = frame.header.kind; + if matches!( + kind, + yas_wire::transfer::kind::CLOSE | yas_wire::transfer::kind::RESET + ) { + router.transfer_ended(transfer_id, kind == yas_wire::transfer::kind::RESET); + } + router.deliver(Route::Transfer(transfer_id), frame); } return Ok(()); } diff --git a/crates/client/src/process.rs b/crates/client/src/process.rs index 41f9c7f9c..bca92bee4 100644 --- a/crates/client/src/process.rs +++ b/crates/client/src/process.rs @@ -86,6 +86,7 @@ //! never more than 1 MiB nor less than 16 KiB. use std::ffi::OsStr; +use std::sync::Arc; use std::time::Duration; use yas_wire::{ @@ -489,10 +490,13 @@ pub struct Process { reported: Option, } -/// The exit a server reports unasked (an EXIT event), received once and kept. +/// The exit a server reports unasked (an EXIT event), received once and kept. Its attachment +/// reports it: once that goes before the exit (a stream dropped or reset, the process +/// detached), the exit is asked for with WAIT. struct ReportedExit { frames: tokio::sync::Mutex, - status: tokio::sync::OnceCell, + status: std::sync::OnceLock, + lost: Arc, } impl std::fmt::Debug for ReportedExit { @@ -504,21 +508,57 @@ impl std::fmt::Debug for ReportedExit { } impl ReportedExit { - async fn wait(&self, client: &Client) -> Result { - self.status - .get_or_try_init(|| async { - use yas_wire::Decode; - let frame = self.frames.lock().await.recv().await.ok_or_else(|| { - client - .closed_reason() - .unwrap_or_else(|| Error::protocol("Process EXIT route closed")) - })?; - Ok(ExitStatus::from_wire( - ExitReport::decode(&frame.payload)?.exit, - )) - }) - .await - .cloned() + /// The exit, waiting at most `timeout`: None if it is still running then. + async fn wait( + &self, + client: &Client, + handle: u64, + timeout: Option, + ) -> Result> { + use yas_wire::Decode; + let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); + let until_deadline = async { + match deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } + }; + tokio::pin!(until_deadline); + if let Some(status) = self.status.get() { + return Ok(Some(status.clone())); + } + let mut frames = tokio::select! { + frames = self.frames.lock() => frames, + () = &mut until_deadline => return Ok(None), + }; + if let Some(status) = self.status.get() { + return Ok(Some(status.clone())); + } + let frame = tokio::select! { + biased; + frame = frames.recv() => frame, + // Its attachment went: an EXIT it sent first may still be here. + () = self.lost.lost() => frames.try_recv().ok(), + () = &mut until_deadline => return Ok(None), + }; + let status = match frame { + Some(frame) => ExitStatus::from_wire(ExitReport::decode(&frame.payload)?.exit), + None if self.lost.is_lost() => { + let left = deadline.map(|deadline| { + deadline.saturating_duration_since(tokio::time::Instant::now()) + }); + match client.wait_process(handle, left).await? { + Some(status) => status, + None => return Ok(None), + } + } + None => { + return Err(client + .closed_reason() + .unwrap_or_else(|| Error::protocol("Process EXIT route closed"))); + } + }; + Ok(Some(self.status.get_or_init(|| status).clone())) } } @@ -581,24 +621,23 @@ impl Process { /// Wait for the process to exit. A process this session spawned from a server that /// reports exits (SPAWN_REPORT_EXIT, in [`Client::launcher_flags`]) takes no request: - /// the server sends the exit as it happens. Otherwise this asks with WAIT. + /// the server sends the exit as it happens. Otherwise, or once the attachment that would + /// report it went first (a stream dropped or reset before its end, a detach), this asks + /// with WAIT. pub async fn wait(&self) -> Result { - if let Some(reported) = &self.reported { - return reported.wait(&self.client).await; - } - self.client - .wait_process(self.handle, None) - .await? - .ok_or_else(|| Error::protocol("Process WAIT without timeout timed out")) + let status = match &self.reported { + Some(reported) => reported.wait(&self.client, self.handle, None).await?, + None => self.client.wait_process(self.handle, None).await?, + }; + status.ok_or_else(|| Error::protocol("Process WAIT without timeout timed out")) } /// Wait at most `timeout`; `None` if it is still running. pub async fn wait_timeout(&self, timeout: Duration) -> Result> { if let Some(reported) = &self.reported { - return match tokio::time::timeout(timeout, reported.wait(&self.client)).await { - Ok(status) => status.map(Some), - Err(_) => Ok(None), - }; + return reported + .wait(&self.client, self.handle, Some(timeout)) + .await; } self.client.wait_process(self.handle, Some(timeout)).await } @@ -627,6 +666,10 @@ impl Process { /// Make a detachable process independent of this session's attachment. pub async fn detach(&self) -> Result<()> { + // Its attachment goes, and would have reported the exit. + if let Some(reported) = &self.reported { + reported.lost.mark(); + } self.client .control_process(self.handle, ControlAction::Detach, 0) .await @@ -957,11 +1000,13 @@ impl Client { } None => None, }; + let lost = reply.take_report_lost().unwrap_or_default(); let reported = reply .take(Route::ProcessExit(bundle.process_handle)) .map(|frames| ReportedExit { frames: tokio::sync::Mutex::new(frames), - status: tokio::sync::OnceCell::new(), + status: std::sync::OnceLock::new(), + lost, }); Ok(Process { client: self.clone(), diff --git a/crates/server/src/yas.rs b/crates/server/src/yas.rs index 0a2eb6fc9..c88984cd2 100644 --- a/crates/server/src/yas.rs +++ b/crates/server/src/yas.rs @@ -3011,16 +3011,6 @@ struct ProcessAttachment { stdout_transfer: u32, stderr_transfer: Option, exited: bool, - /// SPAWN_REPORT_EXIT: the exit goes to this session unasked, whatever becomes of the - /// attachment. - report_exit: Option, -} - -/// A SPAWN_REPORT_EXIT process's exit report: its handle, and whether its EXIT was sent. -#[derive(Clone)] -struct ExitReport { - process_handle: u64, - sent: Arc, } struct ProcessInputTransfer { @@ -25799,7 +25789,6 @@ impl Session { stdout_transfer, stderr_transfer, exited: false, - report_exit: None, }, ); Ok(( @@ -25840,10 +25829,6 @@ impl Session { .stderr_transfer .map(|transfer_id| self.outbound.get(&transfer_id).cloned().ok_or(())) .transpose()?; - let report_exit = report_exit.map(|process_handle| ExitReport { - process_handle, - sent: Arc::new(AtomicBool::new(false)), - }); let task = spawn_process_attachment( attachment_id, attachment.control.clone(), @@ -25852,19 +25837,17 @@ impl Session { attachment.stdout_transfer, stdout_flow, attachment.stderr_transfer.zip(stderr_flow), - report_exit.clone(), + report_exit, self.out.clone(), self.internal.clone(), self.cancellation.clone(), attachment.cancellation.clone(), ); - let attachment = self - .process + self.process .as_mut() .and_then(|process| process.attachments.get_mut(&attachment_id)) - .ok_or(())?; - attachment.task = Some(task); - attachment.report_exit = report_exit; + .ok_or(())? + .task = Some(task); Ok(()) } @@ -26110,39 +26093,11 @@ impl Session { self.outbound_sensitive.remove(&transfer_id); } } - // A SPAWN_REPORT_EXIT process whose attachment goes before its exit (its streams were - // dropped, or its output route failed) still reports it: the server waits for it. - let report = attachment - .report_exit - .filter(|report| !report.sent.load(Ordering::Acquire)) - .zip(self.process.as_ref().map(|process| process.session.clone())); - if detach || report.is_some() { - let out = self.out.clone(); - let connection = self.cancellation.clone(); - let control = attachment.control; + // A SPAWN_REPORT_EXIT process whose attachment goes before its exit reports none: the + // client knows (it dropped a stream, detached, or saw its streams reset) and WAITs. + if detach { tokio::spawn(async move { - if detach { - let _ = control.detach().await; - } else { - drop(control); - } - let Some((report, session)) = report else { - return; - }; - let wait = yas_process_wire::Wait { - process_handle: report.process_handle, - timeout_ns: 0, - extensions: Extensions::default(), - }; - let exit = tokio::select! { - exit = session.wait(&wait) => exit, - _ = connection.cancelled() => return, - }; - if let Ok(exit) = exit - && !report.sent.swap(true, Ordering::AcqRel) - { - send_exit_report(&out, report.process_handle, exit, &connection).await; - } + let _ = attachment.control.detach().await; }); } } @@ -31092,7 +31047,7 @@ fn spawn_process_attachment( stdout_transfer: u32, stdout_flow: Arc, stderr: Option<(u32, Arc)>, - report_exit: Option, + report_exit: Option, out: FrameSender, internal: mpsc::Sender, connection: ConnectionCancellation, @@ -31199,10 +31154,13 @@ fn spawn_process_attachment( exited = true; // As a WAIT would answer it, at once: the streams go on with what // the process wrote before, at the pace of their credit. - if let Some(report) = &report_exit - && !report.sent.swap(true, Ordering::AcqRel) - { - send_exit_report(&exit_out, report.process_handle, exit, &connection).await; + // In a task of its own: removing the attachment meanwhile aborts this + // one, and the exit is already this attachment's to report. + if let Some(process_handle) = report_exit { + let (out, connection) = (exit_out.clone(), connection.clone()); + tokio::spawn(async move { + send_exit_report(&out, process_handle, exit, &connection).await; + }); } let _ = internal .send(Internal::ProcessExited { attachment_id }) diff --git a/docs/design/processes.md b/docs/design/processes.md index c3e7c5ea7..133f4c4d0 100644 --- a/docs/design/processes.md +++ b/docs/design/processes.md @@ -95,8 +95,10 @@ nonzero operation IDs. `WAIT` returns the final portable exit record or (family-limit tag 19), is sent that record as an `EXIT` event once the exit is final, so a command's exit costs no round trip of its own: SPAWN's Result, its output and its exit all travel from one request. yas-client sets it whenever -offered, and `Process::wait` then takes no request and no pending-`WAIT` slot; -against older servers it sends `WAIT` as before. Closing stdin half-closes the +offered, and `Process::wait` then takes no request and no pending-`WAIT` slot +unless the attachment that reports the exit went first (a stream reset or +dropped before its end, a detach), when it sends `WAIT`, as it does against +older servers. Closing stdin half-closes the child stream. An ordinary child belongs to its spawning session and is terminated when it disappears; a detachable child remains discoverable until its retained final record expires. diff --git a/docs/design/yas.md b/docs/design/yas.md index 18f69b5f1..10099b9a2 100644 --- a/docs/design/yas.md +++ b/docs/design/yas.md @@ -3273,10 +3273,11 @@ without a WAIT: one EXIT Event (`0x0002`, sensitive), `[process_handle: u64, exit: bytes_u32 containing ExitRecord, Extensions]`, the record a WAIT would return at that moment. The streams go on with what the process wrote before its exit, at the pace of their credit, so the event can arrive before their last -bytes and their CLOSE. The report outlives the streams: when the spawning -session's attachment goes before the exit (it dropped its streams, or their -route failed), the server waits for the exit itself and still sends the EXIT, -once. It takes none of the session's pending WAITs, and +bytes and their CLOSE. The spawning session's attachment sends it: when that +attachment goes before the exit (a Transfer RESET on any of its streams, stdin +included, from either side, or a DETACH), no EXIT is sent and the client WAITs +for the exit instead, as it would without the flag. yas-client does so on its +own. The report takes none of the session's pending WAITs, and nothing changes for other sessions: they, and sessions that ATTACH, still WAIT. A SPAWN retried under its operation ID shares the original attachment, whose exit is reported once. From 29cd072962e25f307ad9ca234f3051e56e15857f Mon Sep 17 00:00:00 2001 From: Pierre Carrier Date: Wed, 30 Sep 2026 06:38:27 +0000 Subject: [PATCH 10/10] Make tag 19 a set of SPAWN flags whose unknown bits clients ignore LAUNCHER_FLAGS_EXTENDED's hard maximum was 28, the flags it carries today: a later server adding one would have failed the HELLO of every client from this PR, as tag 11's maximum of 12 would have failed older clients for REPORT_EXIT. Its hard maximum is now 65535 (any u16 of SPAWN flags) and Limits::from_extensions keeps only the flags it knows, so the next flag needs no new tag. --- crates/yas/src/generated.rs | 2 +- crates/yas/src/process.rs | 16 ++++++++++++++++ docs/design/yas.md | 7 ++++--- js/core/src/yas/generated.ts | 4 ++-- protocol/yas/families/process.toml | 3 ++- protocol/yas/schema.json | 2 +- protocol/yas/wire.md | 2 +- 7 files changed, 27 insertions(+), 9 deletions(-) diff --git a/crates/yas/src/generated.rs b/crates/yas/src/generated.rs index 668600b12..3b7ecee5e 100644 --- a/crates/yas/src/generated.rs +++ b/crates/yas/src/generated.rs @@ -3930,7 +3930,7 @@ super::LimitMetadata { name: "MAX_STREAM_BUFFER_BYTES_EXTENDED", tag: 15, value_ super::LimitMetadata { name: "MAX_ENVC_EXTENDED", tag: 16, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 16384 }, super::LimitMetadata { name: "MAX_PENDING_WAITS", tag: 17, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 65536 }, super::LimitMetadata { name: "MAX_PENDING_OPERATIONS", tag: 18, value_type: super::LimitValueType::U32, required: false, hard_min: 1, hard_max: 16384 }, -super::LimitMetadata { name: "LAUNCHER_FLAGS_EXTENDED", tag: 19, value_type: super::LimitValueType::U32, required: false, hard_min: 0, hard_max: 28 }, +super::LimitMetadata { name: "LAUNCHER_FLAGS_EXTENDED", tag: 19, value_type: super::LimitValueType::U32, required: false, hard_min: 0, hard_max: 65535 }, ]; pub static CONSTANTS: &[super::ConstantMetadata] = &[ super::ConstantMetadata { name: "SPAWN_MERGE_STDERR", value: 1 }, diff --git a/crates/yas/src/process.rs b/crates/yas/src/process.rs index 22efd4c49..e0dd35876 100644 --- a/crates/yas/src/process.rs +++ b/crates/yas/src/process.rs @@ -1235,10 +1235,13 @@ impl Limits { extensions, crate::schema::process::LIMIT_MAX_MUTATION_REPLAYS, )?, + // A flag this side does not know is one it never sets: ignored, so a later flag + // needs no new tag. launcher_flags: read_optional_limit_u32( extensions, crate::schema::process::LIMIT_LAUNCHER_FLAGS_EXTENDED, )? + .map(|flags| flags & crate::schema::process::SPAWN_LAUNCHER_FLAGS_EXTENDED as u32) .unwrap_or(legacy_launcher_flags), max_pending_waits: u32_or( crate::schema::process::LIMIT_MAX_PENDING_WAITS, @@ -1890,6 +1893,19 @@ mod tests { assert!(with(Some(v1), Some(p::SPAWN_REPORT_EXIT as u32)).is_err()); assert!(with(None, Some(all)).is_err()); assert!(with(Some(v1), Some(all << 1)).is_err()); + // Tag 19 is a set of SPAWN flags: those this side does not know (a later server's) are + // ignored, and any u16 of them passes the family's limit bounds. + assert_eq!( + with(Some(v1), Some(all | 1 << 15)).unwrap().launcher_flags, + all + ); + let tag_19 = crate::schema::family_metadata(crate::family::PROCESS, 1) + .unwrap() + .limits + .iter() + .find(|limit| u64::from(limit.tag) == p::LIMIT_LAUNCHER_FLAGS_EXTENDED) + .unwrap(); + assert_eq!(tag_19.hard_max, u64::from(u16::MAX)); } #[test] diff --git a/docs/design/yas.md b/docs/design/yas.md index 10099b9a2..a1f84edef 100644 --- a/docs/design/yas.md +++ b/docs/design/yas.md @@ -3286,9 +3286,10 @@ Servers advertise the opt-in flags they honour in two optional family limits: tag 11 `LAUNCHER_FLAGS` carries those of v1 (`LEAVE_RESIDUE`, `STDIN_NULL`), at most 12, which is all clients from before `REPORT_EXIT` accept; tag 19 `LAUNCHER_FLAGS_EXTENDED` carries every flag the server honours, when that is -more. Tag 19 names each flag tag 11 does and adds none of v1's. A client sets -`REPORT_EXIT` only when tag 19 offers it, and otherwise WAITs, so either side -may be older. +more. Tag 19 names each flag tag 11 does and adds none of v1's. It is a set of +SPAWN flags, any u16: a client ignores the flags it does not know, so later +flags need no new tag. A client sets `REPORT_EXIT` only when tag 19 offers it, +and otherwise WAITs, so either side may be older. Catalog records contain argv0, native PID for diagnostics, lifecycle, owner session, detachable flag, stream offsets, exit record, and retention deadline. diff --git a/js/core/src/yas/generated.ts b/js/core/src/yas/generated.ts index e5966364e..181bd3609 100644 --- a/js/core/src/yas/generated.ts +++ b/js/core/src/yas/generated.ts @@ -2255,7 +2255,7 @@ export const YAS_FAMILY_LIMIT_POLICIES: Readonly