diff --git a/Cargo.lock b/Cargo.lock index 0bd97ebf01..24fdb44e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3797,6 +3797,7 @@ version = "0.76.0-rc7" dependencies = [ "anyhow", "chrono", + "clap", "dirs", "hex", "iroh", diff --git a/crates/mesh-llm-cli/src/parser/commands.rs b/crates/mesh-llm-cli/src/parser/commands.rs index 9efaa36db4..1842542fe0 100644 --- a/crates/mesh-llm-cli/src/parser/commands.rs +++ b/crates/mesh-llm-cli/src/parser/commands.rs @@ -1453,6 +1453,8 @@ mod tests { ("--spec-draft-models", "/models/draft.gguf"), ("--spec-draft-max-tokens", "8"), ("--spec-draft-min-tokens", "2"), + ("--spec-draft-acceptance-threshold", "0.8"), + ("--spec-draft-split-probability", "0.5"), ("--spec-ngram-min", "2"), ("--spec-ngram-max", "4"), ] { diff --git a/crates/mesh-llm-commands/Cargo.toml b/crates/mesh-llm-commands/Cargo.toml index a1a511a80d..6bcdbe0133 100644 --- a/crates/mesh-llm-commands/Cargo.toml +++ b/crates/mesh-llm-commands/Cargo.toml @@ -51,6 +51,7 @@ url = "2" zeroize = { version = "1", features = ["derive"] } [dev-dependencies] +clap.workspace = true rand = "0.10" serial_test = "3" tempfile = "3" diff --git a/crates/mesh-llm-commands/src/operational_logging.rs b/crates/mesh-llm-commands/src/operational_logging.rs index 67e36498e5..e70ae72ae0 100644 --- a/crates/mesh-llm-commands/src/operational_logging.rs +++ b/crates/mesh-llm-commands/src/operational_logging.rs @@ -1,21 +1,20 @@ //! Bounded CLI command dispatch events. -//! -//! This boundary starts after `clap` has parsed a [`Command`] and finishes at -//! the top-level command dispatcher. It deliberately does not inspect command -//! arguments or error text. Parse failures happen before a `Command` exists -//! and remain outside this boundary. + +mod command_summary; use anyhow::Result; +use command_summary::command_summary; use mesh_llm_cli::Command; #[cfg(test)] use mesh_llm_events::OutputEvent; -use mesh_llm_events::{CliCommandFamily, CliCommandOutcome, emit_cli_command_event}; +use mesh_llm_events::{ + CliCommandFamily, CliCommandOutcome, CliCommandSummary, emit_cli_command_event, +}; use std::fmt; use std::sync::{Arc, LazyLock, RwLock}; /// Marker a handler can retain in its error chain when it explicitly rejects a -/// parsed command. Other handler or dispatcher errors are classified as -/// failures without reading their details. +/// parsed command. #[derive(Debug)] pub struct CommandDispatchRejected; @@ -27,35 +26,25 @@ impl fmt::Display for CommandDispatchRejected { impl std::error::Error for CommandDispatchRejected {} -/// Fail-open durable-audit bridge invoked after each command boundary -/// emission. The caller installs it only after logging runtime -/// initialization, so no durable record is produced before initialization. -/// The bridge receives only the static command family and outcome; it never -/// sees command arguments, argv, or error text. -pub type CliOperationalAuditBridge = Arc; +/// Fail-open durable-audit bridge invoked after each command boundary emission. +pub type CliOperationalAuditBridge = + Arc) + Send + Sync>; static CLI_OPERATIONAL_AUDIT_BRIDGE: LazyLock>> = LazyLock::new(|| RwLock::new(None)); -/// Install the durable command audit bridge. Callers invoke this only after -/// the logging runtime has been initialized so command behavior is unchanged -/// before initialization. Replacing the bridge is allowed for tests and -/// embedded hosts. pub fn install_cli_operational_audit_bridge(bridge: CliOperationalAuditBridge) { *CLI_OPERATIONAL_AUDIT_BRIDGE .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(bridge); } -/// Remove the command audit bridge. Used by tests and by callers that must -/// stop persisting command outcomes. pub fn clear_cli_operational_audit_bridge() { *CLI_OPERATIONAL_AUDIT_BRIDGE .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = None; } -/// Return the installed bridge, if any. pub fn cli_operational_audit_bridge() -> Option { CLI_OPERATIONAL_AUDIT_BRIDGE .read() @@ -63,42 +52,71 @@ pub fn cli_operational_audit_bridge() -> Option { .clone() } -/// Process-local lifecycle emitter for one parsed command dispatch. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct CommandDispatchBoundary { family: CliCommandFamily, + summary: Option, } impl CommandDispatchBoundary { - /// Start the command boundary after parsing succeeds. pub fn start(command: &Command) -> Self { - Self::start_family(command_family(command)) + Self::start_with_family(command_family(command), command) + } + + pub fn start_with_family(family: CliCommandFamily, command: &Command) -> Self { + Self::start_with_summary(family, command_summary(command)) + } + + pub fn start_with_cli(cli: &mesh_llm_cli::Cli, family: CliCommandFamily) -> Self { + let Some(command) = cli.command.as_ref() else { + return Self::start_family(family); + }; + let mut summary = command_summary(command).map(|value| value.as_str().to_owned()); + if let Some(summary_value) = summary.as_mut() { + for (present, marker) in [ + (!cli.join.is_empty(), " --join [REDACTED]"), + (!cli.relay.is_empty(), " --root-relay [REDACTED]"), + (!cli.relay_auth.is_empty(), " --relay-auth [REDACTED]"), + ] { + if present && !summary_value.contains(marker) { + summary_value.push_str(marker); + } + } + } + Self::start_with_summary( + family, + summary.and_then(|value| CliCommandSummary::sanitize(&value)), + ) + } + + fn start_with_summary(family: CliCommandFamily, summary: Option) -> Self { + let boundary = Self { family, summary }; + boundary.emit(CliCommandOutcome::Started); + boundary } - /// Start a command boundary whose bounded family was resolved by the - /// dispatcher (for example, external plugin versus truly unknown). pub fn start_family(family: CliCommandFamily) -> Self { - let boundary = Self { family }; + let boundary = Self { + family, + summary: None, + }; boundary.emit(CliCommandOutcome::Started); boundary } - /// Emit exactly one terminal result for the command dispatcher outcome. pub fn finish(self, result: &Result<()>) { self.emit(command_outcome(result)); } - fn emit(self, outcome: CliCommandOutcome) { - // Command results must retain their existing return and output behavior - // even if the process-local presentation sink is unavailable. + fn emit(&self, outcome: CliCommandOutcome) { let _ = emit_cli_command_event(self.family, outcome); if let Some(bridge) = cli_operational_audit_bridge() { - bridge(self.family, outcome); + bridge(self.family, outcome, self.summary.clone()); } } #[cfg(test)] - fn event(self, outcome: CliCommandOutcome) -> OutputEvent { + fn event(&self, outcome: CliCommandOutcome) -> OutputEvent { OutputEvent::CliCommandLifecycle { family: self.family, outcome, @@ -106,14 +124,14 @@ impl CommandDispatchBoundary { } } -/// Emit a process lifecycle outcome that occurs before a parsed [`Command`] -/// exists, while retaining the same presentation and durable-audit bridge. pub fn emit_cli_process_event(family: CliCommandFamily, outcome: CliCommandOutcome) { - let boundary = CommandDispatchBoundary { family }; + let boundary = CommandDispatchBoundary { + family, + summary: None, + }; boundary.emit(outcome); } -/// Map a parsed command to a stable, argument-free event family. pub fn command_family(command: &Command) -> CliCommandFamily { match command { Command::Models { .. } | Command::Download { .. } | Command::ModelPrepare { .. } => { @@ -156,6 +174,7 @@ fn command_outcome(result: &Result<()>) -> CliCommandOutcome { mod tests { use super::*; use anyhow::{Error, anyhow}; + use clap::Parser; use mesh_llm_events::{clear_output_sink, set_cli_command_event_verbose, set_output_sink}; use std::io; @@ -211,14 +230,12 @@ mod tests { impl Drop for CommandEventVerboseResetGuard { fn drop(&mut self) { - mesh_llm_events::set_cli_command_event_verbose(false); + set_cli_command_event_verbose(false); } } fn lifecycle_events(command: &Command, result: &Result<()>) -> [OutputEvent; 2] { - let boundary = CommandDispatchBoundary { - family: command_family(command), - }; + let boundary = CommandDispatchBoundary::start(command); [ boundary.event(CliCommandOutcome::Started), boundary.event(command_outcome(result)), @@ -226,42 +243,33 @@ mod tests { } #[test] + #[serial_test::serial] fn command_dispatch_orders_started_before_completed_without_command_arguments() { let command = Command::Load { name: "private-model.gguf?token=private-token".to_string(), port: 41731, }; - let result = Ok(()); - - let events = lifecycle_events(&command, &result); - + let events = lifecycle_events(&command, &Ok(())); assert_eq!( - events, - [ - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }, - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }, - ] + events[0].clone(), + OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Started + } + ); + assert_eq!( + events[1].clone(), + OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Completed + } ); let serialized = format!("{events:?}"); - for raw_value in [ - "private-model.gguf?token=private-token", - "41731", - "private-token", - ] { - assert!( - !serialized.contains(raw_value), - "command metadata must not enter lifecycle events" - ); - } + assert!(!serialized.contains("private-token")); } #[test] + #[serial_test::serial] fn explicit_handler_rejection_maps_to_rejected_without_error_detail() { let command = Command::Discover { name: Some("private-mesh".to_string()), @@ -273,31 +281,19 @@ mod tests { }; let result: Result<()> = Err(Error::new(CommandDispatchRejected) .context("private command rejection detail with private-token")); - let events = lifecycle_events(&command, &result); - assert_eq!( - events[1], + events[1].clone(), OutputEvent::CliCommandLifecycle { family: CliCommandFamily::Discovery, - outcome: CliCommandOutcome::Rejected, + outcome: CliCommandOutcome::Rejected } ); - let serialized = format!("{events:?}"); - for raw_value in [ - "private-mesh", - "private-region", - "wss://relay.private.example/?token=private-token", - "private command rejection detail with private-token", - ] { - assert!( - !serialized.contains(raw_value), - "rejection detail must not enter lifecycle events" - ); - } + assert!(!format!("{events:?}").contains("private-token")); } #[test] + #[serial_test::serial] fn unmarked_dispatch_failure_maps_to_failed_without_error_detail() { let command = Command::Download { name: Some("private/model?token=private-token".to_string()), @@ -306,18 +302,15 @@ mod tests { let result: Result<()> = Err(anyhow!( "download failed for https://private.example/model?token=private-token" )); - let events = lifecycle_events(&command, &result); - assert_eq!( - events[1], + events[1].clone(), OutputEvent::CliCommandLifecycle { family: CliCommandFamily::Models, - outcome: CliCommandOutcome::Failed, + outcome: CliCommandOutcome::Failed } ); - let serialized = format!("{events:?}"); - assert!(!serialized.contains("https://private.example/model?token=private-token")); + assert!(!format!("{events:?}").contains("private-token")); } #[test] @@ -331,42 +324,25 @@ mod tests { let recording = Arc::new(RecordingBridge::default()); let _bridge_guard = BridgeResetGuard; let bridge_recording = recording.clone(); - install_cli_operational_audit_bridge(Arc::new(move |family, outcome| { + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, _summary| { bridge_recording .calls .lock() .expect("recording bridge mutex poisoned") .push((family, outcome)); })); - - let command = Command::Load { + let boundary = CommandDispatchBoundary::start(&Command::Load { name: "model.gguf".to_string(), port: 9337, - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Ok(()); - boundary.finish(&result); - - assert_eq!( - sink.take_events(), - vec![ - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }, - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }, - ] - ); + }); + boundary.finish(&Ok(())); + assert_eq!(sink.take_events().len(), 2); assert_eq!( recording.take_calls(), vec![ (CliCommandFamily::Runtime, CliCommandOutcome::Started), - (CliCommandFamily::Runtime, CliCommandOutcome::Completed), - ], - "installed bridge fires exactly once per outcome" + (CliCommandFamily::Runtime, CliCommandOutcome::Completed) + ] ); } @@ -374,18 +350,12 @@ mod tests { #[serial_test::serial] fn command_dispatch_without_bridge_stays_fail_open() { clear_cli_operational_audit_bridge(); - let command = Command::Load { + let boundary = CommandDispatchBoundary::start(&Command::Load { name: "private-model.gguf".to_string(), port: 41731, - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Err(anyhow!("private dispatch failure")); - boundary.finish(&result); - - assert!( - cli_operational_audit_bridge().is_none(), - "uninstalled bridge must remain absent after dispatch" - ); + }); + boundary.finish(&Err(anyhow!("private dispatch failure"))); + assert!(cli_operational_audit_bridge().is_none()); } #[test] @@ -394,46 +364,156 @@ mod tests { let recording = Arc::new(RecordingBridge::default()); let _bridge_guard = BridgeResetGuard; let bridge_recording = recording.clone(); - install_cli_operational_audit_bridge(Arc::new(move |family, outcome| { + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, _summary| { bridge_recording .calls .lock() .expect("recording bridge mutex poisoned") .push((family, outcome)); })); - - let command = Command::Discover { + let boundary = CommandDispatchBoundary::start(&Command::Discover { name: Some("private-mesh".to_string()), model: None, min_vram: None, region: Some("private-region".to_string()), auto: false, relay: vec!["wss://relay.private.example/?token=private-token".to_string()], - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Err(Error::new(CommandDispatchRejected) - .context("private command rejection detail with private-token")); - boundary.finish(&result); - - let calls = recording.take_calls(); + }); + boundary.finish(&Err(Error::new(CommandDispatchRejected) + .context("private command rejection detail with private-token"))); assert_eq!( - calls, + recording.take_calls(), vec![ (CliCommandFamily::Discovery, CliCommandOutcome::Started), - (CliCommandFamily::Discovery, CliCommandOutcome::Rejected), + (CliCommandFamily::Discovery, CliCommandOutcome::Rejected) ] ); - let serialized = format!("{calls:?}"); - for raw_value in [ - "private-mesh", - "private-region", - "wss://relay.private.example/?token=private-token", - "private command rejection detail with private-token", - ] { - assert!( - !serialized.contains(raw_value), - "command metadata must not cross the audit bridge" - ); - } + } + + #[test] + #[serial_test::serial] + fn dispatch_summary_is_shared_by_started_and_terminal_bridge_records() { + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let expected = CliCommandSummary::sanitize("mesh-llm load --port 9337 name [REDACTED]") + .expect("bounded summary"); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |_, _, summary| { + calls.lock().expect("bridge mutex").push(summary) + })); + let boundary = CommandDispatchBoundary::start(&Command::Load { + name: "model.gguf".to_owned(), + port: 9337, + }); + boundary.finish(&Ok(())); + assert_eq!( + recording.lock().expect("bridge mutex").as_slice(), + &[Some(expected.clone()), Some(expected)] + ); + } + + #[test] + #[serial_test::serial] + fn start_with_cli_redacts_global_credentials_in_both_bridge_records() { + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay", + "wss://relay.example/?token=private-relay-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "load", + "private/model", + ]); + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, summary| { + calls + .lock() + .expect("bridge mutex") + .push((family, outcome, summary)); + })); + + let boundary = CommandDispatchBoundary::start_with_cli(&cli, CliCommandFamily::Runtime); + boundary.finish(&Ok(())); + + let calls = recording.lock().expect("bridge mutex"); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, CliCommandFamily::Runtime); + assert_eq!(calls[1].0, CliCommandFamily::Runtime); + assert_eq!(calls[0].1, CliCommandOutcome::Started); + assert_eq!(calls[1].1, CliCommandOutcome::Completed); + assert_eq!(calls[0].2, calls[1].2); + let summary = calls[0].2.as_ref().expect("CLI summary").as_str(); + assert!(summary.contains("--join [REDACTED]")); + assert!(summary.contains("--root-relay [REDACTED]")); + assert!(summary.contains("--relay-auth [REDACTED]")); + assert!(CliCommandSummary::sanitize(summary).is_some()); + assert!(!summary.contains("private-invite-token")); + assert!(!summary.contains("private-relay-token")); + assert!(!summary.contains("private-token")); + } + + #[test] + #[serial_test::serial] + fn start_with_cli_does_not_fabricate_global_markers_without_base_summary() { + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "benchmark", + "tune", + "--model", + "private-model", + "--json", + "--ctx-sizes", + "1", + "--batch-sizes", + "1", + "--ubatch-sizes", + "1", + "--mmap-values", + "auto", + "--mlock-values", + "enabled", + "--flash-attention", + "on", + "--speculative-types", + "auto", + "--spec-draft-models", + "private-draft.gguf", + "--spec-draft-max-tokens", + "1", + "--spec-draft-min-tokens", + "1", + "--spec-ngram-min", + "1", + "--spec-ngram-max", + "1", + "--apply", + "--launch-args", + "--debug-telemetry", + ]); + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, summary| { + calls + .lock() + .expect("bridge mutex") + .push((family, outcome, summary)); + })); + + let boundary = CommandDispatchBoundary::start_with_cli(&cli, CliCommandFamily::Benchmark); + boundary.finish(&Ok(())); + + let calls = recording.lock().expect("bridge mutex"); + assert_eq!(calls.len(), 2); + assert!(calls[0].2.is_none()); + assert!(calls[1].2.is_none()); } } diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary.rs new file mode 100644 index 0000000000..799720e1e9 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary.rs @@ -0,0 +1,97 @@ +mod administration; +mod auth; +mod benchmark; +mod dispatch; +mod models; +mod runtime; + +use mesh_llm_cli::Command; +use mesh_llm_events::CliCommandSummary; + +const DEFAULT_LOCAL_PORT: u16 = 3131; +const DEFAULT_AGENT_PORT: u16 = 9337; + +#[derive(Default)] +struct SummaryAssembly { + command: String, + flags: Vec, + values: Vec, + redacted: Vec<&'static str>, +} + +struct ModelPrepareSummary<'a> { + source_repo: &'a Option, + quant: &'a Option, + target: &'a Option, + model_id: &'a Option, + flavor: &'a str, + timeout: &'a str, + mesh_llm_ref: &'a str, + dry_run: bool, + confirm: bool, + follow: bool, + json: bool, + status: &'a Option, + logs: &'a Option, + cancel: &'a Option, + list: bool, + update_script: bool, +} + +impl SummaryAssembly { + fn new(command: &str) -> Self { + Self { + command: command.to_owned(), + ..Self::default() + } + } + + fn flag(&mut self, name: &'static str, present: bool) { + if present { + let marker = format!("--{name}"); + if !self.flags.contains(&marker) { + self.flags.push(marker); + } + } + } + + fn redact(&mut self, name: &'static str, present: bool) { + if present { + self.redacted.push(name); + } + } + + fn port(&mut self, port: u16, default: u16) { + if port != default { + self.values.push(format!("--port {port}")); + } + } + + fn finish(self) -> Option { + let mut value = self.command; + for part in self.flags.into_iter().chain(self.values) { + value.push(' '); + value.push_str(&part); + } + for name in self.redacted { + value.push(' '); + value.push_str(name); + value.push_str(" [REDACTED]"); + } + CliCommandSummary::sanitize(&value) + } +} + +pub(super) fn command_summary(command: &Command) -> Option { + let mut assembly = SummaryAssembly::new("mesh-llm"); + dispatch::format_command(command, &mut assembly); + assembly.finish() +} + +#[cfg(test)] +#[path = "command_summary_tests.rs"] +mod command_summary_tests; + +#[cfg(test)] +#[path = "command_summary_context_tests.rs"] +mod command_summary_context_tests; diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs new file mode 100644 index 0000000000..9d973cafa7 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs @@ -0,0 +1,102 @@ +use super::{DEFAULT_LOCAL_PORT, SummaryAssembly}; + +pub(super) fn format_gpus( + json: bool, + command: Option<&mesh_llm_cli::GpuCommand>, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" gpus"); + let child_json = match command { + None => false, + Some(mesh_llm_cli::GpuCommand::Detect { json }) => { + assembly.command.push_str(" detect"); + *json + } + Some(mesh_llm_cli::GpuCommand::RunBenchmark { backend }) => { + assembly.command.push_str(" run-benchmark --backend "); + assembly + .command + .push_str(&format!("{backend:?}").to_lowercase()); + false + } + }; + assembly.flag("json", json || child_json); +} + +pub(super) fn format_config(command: &mesh_llm_cli::ConfigCommand, assembly: &mut SummaryAssembly) { + let mesh_llm_cli::ConfigCommand::Validate { config_path, json } = command; + assembly.command.push_str(" config validate"); + assembly.redact("--config-path", config_path.is_some()); + assembly.flag("json", *json); +} + +pub(super) fn format_doctor( + command: Option<&mesh_llm_cli::DoctorCommand>, + json: bool, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" doctor"); + let mut child_json = false; + if let Some(mesh_llm_cli::DoctorCommand::Split { + model_ref, + port, + json: split_json, + output_dir, + }) = command + { + assembly.command.push_str(" split"); + assembly.redact("--model-ref", !model_ref.is_empty()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + child_json = *split_json; + assembly.redact("--output-dir", output_dir.is_some()); + } + assembly.flag("json", json || child_json); +} + +pub(super) fn format_skills(command: &mesh_llm_cli::SkillCommand, assembly: &mut SummaryAssembly) { + let mesh_llm_cli::SkillCommand::Install { + agent, + all, + dry_run, + force, + } = command; + assembly.command.push_str(" skills install"); + assembly.redact("--agent", !agent.is_empty()); + assembly.flag("all", *all); + assembly.flag("dry-run", *dry_run); + assembly.flag("force", *force); +} + +pub(super) fn format_plugin(command: &mesh_llm_cli::PluginCommand, assembly: &mut SummaryAssembly) { + use mesh_llm_cli::PluginCommand; + match command { + PluginCommand::Install { + reference, + archive, + name, + version, + } => { + assembly.command.push_str(" plugins install"); + assembly.redact("reference", reference.is_some()); + assembly.redact("--archive", archive.is_some()); + assembly.redact("--name", name.is_some()); + assembly.redact("--version", version.is_some()); + } + PluginCommand::Update { .. } => format_plugin_name("update", assembly), + PluginCommand::Enable { .. } => format_plugin_name("enable", assembly), + PluginCommand::Disable { .. } => format_plugin_name("disable", assembly), + PluginCommand::Delete { .. } => format_plugin_name("delete", assembly), + PluginCommand::Info { .. } => format_plugin_name("info", assembly), + PluginCommand::Search { query } => { + assembly.command.push_str(" plugins search"); + assembly.redact("query", query.is_some()); + } + PluginCommand::List => assembly.command.push_str(" plugins list"), + } +} + +fn format_plugin_name(name: &str, assembly: &mut SummaryAssembly) { + assembly.command.push_str(" plugins "); + assembly.command.push_str(name); + assembly.redact("name", true); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs new file mode 100644 index 0000000000..8df09d0417 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs @@ -0,0 +1,146 @@ +use super::SummaryAssembly; + +pub(super) fn format_auth(command: &mesh_llm_cli::AuthCommand, assembly: &mut SummaryAssembly) { + use mesh_llm_cli::{AuthCommand, TrustCommand}; + match command { + AuthCommand::Init { + owner_key, + force, + no_passphrase, + keychain, + } => { + assembly.command.push_str(" auth init"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.flag("force", *force); + assembly.flag("no-passphrase", *no_passphrase); + assembly.flag("keychain", *keychain); + } + AuthCommand::Status { + owner_key, + node_key, + node_ownership, + trust_store, + } => { + assembly.command.push_str(" auth status"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--node-ownership", node_ownership.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::SignNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + } + | AuthCommand::RenewNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + } => { + assembly + .command + .push_str(if matches!(command, AuthCommand::SignNode { .. }) { + " auth sign-node" + } else { + " auth renew-node" + }); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--out", out.is_some()); + assembly.redact("--hostname-hint", hostname_hint.is_some()); + assembly.redact("--node-label", node_label.is_some()); + assembly.redact("--expires-in-hours", *expires_in_hours != 168); + } + AuthCommand::VerifyNode { + file, + node_id, + trust_store, + trust_policy, + } => { + assembly.command.push_str(" auth verify-node"); + assembly.redact("--file", file.is_some()); + assembly.redact("--node-id", node_id.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + assembly.redact("--verify-trust-policy", trust_policy.is_some()); + } + AuthCommand::RotateNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + revoke_current, + reason, + trust_store, + } => { + assembly.command.push_str(" auth rotate-node"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--out", out.is_some()); + assembly.redact("--hostname-hint", hostname_hint.is_some()); + assembly.redact("--node-label", node_label.is_some()); + assembly.redact("--expires-in-hours", *expires_in_hours != 168); + assembly.flag("revoke-current", *revoke_current); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RevokeOwner { + reason, + trust_store, + .. + } => { + assembly.command.push_str(" auth revoke-owner"); + assembly.redact("owner_id", true); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RevokeNode { + cert_id, + node_id, + reason, + trust_store, + } => { + assembly.command.push_str(" auth revoke-node"); + assembly.redact("--cert-id", cert_id.is_some()); + assembly.redact("--node-id", node_id.is_some()); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RotateOwner { + owner_key, + no_passphrase, + force, + } => { + assembly.command.push_str(" auth rotate-owner"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.flag("no-passphrase", *no_passphrase); + assembly.flag("force", *force); + } + AuthCommand::Trust { command } => match command { + TrustCommand::Add { + label, trust_store, .. + } => { + assembly.command.push_str(" auth trust add"); + assembly.redact("owner_id", true); + assembly.redact("--label", label.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + TrustCommand::Remove { trust_store, .. } => { + assembly.command.push_str(" auth trust remove"); + assembly.redact("owner_id", true); + assembly.redact("--trust-store", trust_store.is_some()); + } + TrustCommand::List { trust_store } => { + assembly.command.push_str(" auth trust list"); + assembly.redact("--trust-store", trust_store.is_some()); + } + }, + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs new file mode 100644 index 0000000000..8471100bd5 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs @@ -0,0 +1,70 @@ +use super::SummaryAssembly; + +pub(super) fn format_benchmark( + command: &mesh_llm_cli::benchmark::BenchmarkCommand, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::benchmark::BenchmarkCommand; + match command { + BenchmarkCommand::Tune(tune) => { + assembly.command.push_str(" benchmark tune"); + assembly.redact("--model", tune.model.is_some()); + assembly.redact("--models", !tune.models.is_empty()); + assembly.flag("json", tune.json); + assembly.redact("--ctx-sizes", !tune.ctx_sizes.is_empty()); + assembly.redact("--batch-sizes", !tune.batch_sizes.is_empty()); + assembly.redact("--ubatch-sizes", !tune.ubatch_sizes.is_empty()); + assembly.redact("--mmap-values", !tune.mmap_values.is_empty()); + assembly.redact("--mlock-values", !tune.mlock_values.is_empty()); + assembly.redact("--flash-attention", !tune.flash_attention.is_empty()); + assembly.redact("--speculative-types", !tune.speculative_types.is_empty()); + assembly.flag("no-speculative-tune", tune.no_speculative_tune); + assembly.redact("--spec-draft-models", !tune.spec_draft_models.is_empty()); + assembly.redact( + "--spec-draft-max-tokens", + !tune.spec_draft_max_tokens.is_empty(), + ); + assembly.redact( + "--spec-draft-min-tokens", + !tune.spec_draft_min_tokens.is_empty(), + ); + assembly.redact("--spec-ngram-min", !tune.spec_ngram_min.is_empty()); + assembly.redact("--spec-ngram-max", !tune.spec_ngram_max.is_empty()); + assembly.redact( + "--spec-draft-acceptance-threshold", + !tune.spec_draft_acceptance_threshold.is_empty(), + ); + assembly.redact( + "--spec-draft-split-probability", + !tune.spec_draft_split_probability.is_empty(), + ); + assembly.flag("apply", tune.apply); + assembly.flag("replace-existing", tune.replace_existing); + assembly.flag("launch-args", tune.launch_args); + assembly.redact( + "--throughput-tolerance-pct", + tune.throughput_tolerance_pct != 10.0, + ); + assembly.redact("--max-tokens", tune.max_tokens != 128); + assembly.redact("--startup-timeout-secs", tune.startup_timeout_secs != 600); + assembly.redact("--request-timeout-secs", tune.request_timeout_secs != 600); + assembly.flag("debug-telemetry", tune.debug_telemetry); + assembly.redact( + "--prompt", + tune.prompt != "Write a concise paragraph about distributed GPU inference.", + ); + } + BenchmarkCommand::ImportPrompts { + source: _, + limit, + max_tokens, + output: _, + } => { + assembly.command.push_str(" benchmark import-prompts"); + assembly.redact("--source", true); + assembly.redact("--limit", *limit != 20); + assembly.redact("--max-tokens", max_tokens.is_some()); + assembly.redact("--output", true); + } + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs new file mode 100644 index 0000000000..e931654fa4 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs @@ -0,0 +1,170 @@ +use mesh_llm_cli::Command; + +use super::{ + DEFAULT_AGENT_PORT, DEFAULT_LOCAL_PORT, ModelPrepareSummary, SummaryAssembly, administration, + auth, benchmark, models, runtime, +}; + +pub(super) fn format_command(command: &Command, assembly: &mut SummaryAssembly) { + match command { + Command::Models { command } => models::format_models(command, assembly), + Command::Runtime { command } => runtime::format_runtime(command.as_ref(), assembly), + Command::Plugin { command } => administration::format_plugin(command, assembly), + Command::Auth { command } => auth::format_auth(command, assembly), + Command::Benchmark { command } => benchmark::format_benchmark(command, assembly), + Command::Config { command } => administration::format_config(command, assembly), + Command::Doctor { command, json } => { + administration::format_doctor(command.as_ref(), *json, assembly); + } + Command::Skills { command } => administration::format_skills(command, assembly), + Command::Download { name, draft } => { + assembly.command.push_str(" download"); + assembly.redact("name", name.is_some()); + assembly.flag("draft", *draft); + } + Command::Update { + version, + flavor, + detect_flavor, + } => { + assembly.command.push_str(" update"); + assembly.redact("--version", version.is_some()); + assembly.redact("--flavor", flavor.is_some()); + assembly.flag("detect-flavor", *detect_flavor); + } + Command::Gpus { json, command } => { + administration::format_gpus(*json, command.as_ref(), assembly); + } + Command::Setup { + yes, + no_interactive, + service, + no_service, + skip_runtime, + verbose, + } => { + assembly.command.push_str(" setup"); + assembly.flag("yes", *yes); + assembly.flag("no-interactive", *no_interactive); + assembly.flag("service", *service); + assembly.flag("no-service", *no_service); + assembly.flag("skip-runtime", *skip_runtime); + assembly.flag("verbose", *verbose); + } + Command::Uninstall { + dry_run, + yes, + keep_cache, + keep_service_files, + purge_config, + keep_config, + binary_path, + json, + verbose, + } => { + assembly.command.push_str(" uninstall"); + assembly.flag("dry-run", *dry_run); + assembly.flag("yes", *yes); + assembly.flag("keep-cache", *keep_cache); + assembly.flag("keep-service-files", *keep_service_files); + assembly.flag("purge-config", *purge_config); + assembly.flag("keep-config", *keep_config); + assembly.redact("--binary-path", binary_path.is_some()); + assembly.flag("json", *json); + assembly.flag("verbose", *verbose); + } + Command::Load { port, .. } => runtime::format_local_model("load", *port, assembly), + Command::Unload { port, .. } => runtime::format_local_model("unload", *port, assembly), + Command::Status { port } => { + assembly.command.push_str(" status"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + } + Command::Discover { + name, + model, + min_vram, + region, + auto, + relay, + } => { + assembly.command.push_str(" discover"); + assembly.redact("--name", name.is_some()); + assembly.redact("--model", model.is_some()); + assembly.redact("--min-vram", min_vram.is_some()); + assembly.redact("--region", region.is_some()); + assembly.flag("auto", *auto); + assembly.redact("--relay", !relay.is_empty()); + } + Command::RotateKey => assembly.command.push_str(" rotate-key"), + Command::Goose { model, port } => format_agent("goose", model, *port, assembly), + Command::Claude { model, port } => format_agent("claude", model, *port, assembly), + Command::Pi { model, host, write } => format_client("pi", model, host, *write, assembly), + Command::Opencode { model, host, write } => { + format_client("opencode", model, host, *write, assembly); + } + Command::Stop => assembly.command.push_str(" stop"), + Command::ModelPrepare { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + dry_run, + confirm, + follow, + json, + status, + logs, + cancel, + list, + update_script, + } => models::format_model_prepare( + ModelPrepareSummary { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + dry_run: *dry_run, + confirm: *confirm, + follow: *follow, + json: *json, + status, + logs, + cancel, + list: *list, + update_script: *update_script, + }, + assembly, + ), + Command::ExternalPlugin(args) => { + assembly.command.push_str(" external-plugin"); + assembly.redact("argv", !args.is_empty()); + } + } +} + +fn format_agent(name: &str, model: &Option, port: u16, assembly: &mut SummaryAssembly) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.redact("--model", model.is_some()); + assembly.port(port, DEFAULT_AGENT_PORT); +} + +fn format_client( + name: &str, + model: &Option, + host: &str, + write: bool, + assembly: &mut SummaryAssembly, +) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.redact("--model", model.is_some()); + assembly.redact("--host", host != "127.0.0.1:9337"); + assembly.flag("write", write); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs new file mode 100644 index 0000000000..5f3da370f5 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs @@ -0,0 +1,169 @@ +use super::{ModelPrepareSummary, SummaryAssembly}; + +pub(super) fn format_models( + command: &mesh_llm_cli::models::ModelsCommand, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::models::ModelsCommand; + match command { + ModelsCommand::Package { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + experimental, + dry_run, + confirm, + follow, + status, + logs, + cancel, + list, + update_script, + json, + } => { + assembly.command.push_str(" models package"); + assembly.redact("source_repo", source_repo.is_some()); + assembly.redact("--quant", quant.is_some()); + assembly.redact("--target", target.is_some()); + assembly.redact("--model-id", model_id.is_some()); + assembly.redact("--flavor", flavor != "auto"); + assembly.redact("--timeout", timeout != "1h"); + assembly.redact("--mesh-llm-ref", mesh_llm_ref != "main"); + assembly.flag("experimental", *experimental); + assembly.flag("dry-run", *dry_run); + assembly.flag("confirm", *confirm); + assembly.flag("follow", *follow); + assembly.redact("--status", status.is_some()); + assembly.redact("--logs", logs.is_some()); + assembly.redact("--cancel", cancel.is_some()); + assembly.flag("list", *list); + assembly.flag("update-script", *update_script); + assembly.flag("json", *json); + } + ModelsCommand::Recommended { json } => format_model_json("recommended", *json, assembly), + ModelsCommand::Installed { json } => format_model_json("installed", *json, assembly), + ModelsCommand::Cleanup { + unused_since, + yes, + json, + } => { + assembly.command.push_str(" models cleanup"); + assembly.redact("--unused-since", unused_since.is_some()); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + ModelsCommand::Prune { yes, json } => { + assembly.command.push_str(" models prune"); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + ModelsCommand::Certify { + model, + report_out, + json, + package_only, + api_base, + prompt, + max_tokens, + } => { + assembly.command.push_str(" models certify"); + assembly.redact("model", !model.is_empty()); + assembly.redact("--report-out", report_out.is_some()); + assembly.flag("json", *json); + assembly.flag("package-only", *package_only); + assembly.redact("--api-base", api_base.is_some()); + assembly.redact("--prompt", prompt != "Say ok."); + assembly.redact("--max-tokens", *max_tokens != 2); + } + ModelsCommand::List { json } => format_model_json("list", *json, assembly), + ModelsCommand::Search { + query, + gguf, + mlx, + catalog, + limit, + sort, + json, + } => { + assembly.command.push_str(" models search"); + assembly.redact("query", !query.is_empty()); + assembly.flag("gguf", *gguf); + assembly.flag("mlx", *mlx); + assembly.flag("catalog", *catalog); + assembly.redact("--limit", *limit != 20); + assembly.redact( + "--sort", + *sort != mesh_llm_cli::models::ModelSearchSort::Trending, + ); + assembly.flag("json", *json); + } + ModelsCommand::Show { model, json } => { + assembly.command.push_str(" models show"); + assembly.redact("model", !model.is_empty()); + assembly.flag("json", *json); + } + ModelsCommand::Download { + model, + draft, + direct, + json, + } => { + assembly.command.push_str(" models download"); + assembly.redact("model", !model.is_empty()); + assembly.flag("draft", *draft); + assembly.flag("direct", *direct); + assembly.flag("json", *json); + } + ModelsCommand::Updates { + repo, + all, + check, + json, + } => { + assembly.command.push_str(" models updates"); + assembly.redact("repo", repo.is_some()); + assembly.flag("all", *all); + assembly.flag("check", *check); + assembly.flag("json", *json); + } + ModelsCommand::Delete { model, yes, json } => { + assembly.command.push_str(" models delete"); + assembly.redact("model", !model.is_empty()); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + } +} + +fn format_model_json(name: &str, json: bool, assembly: &mut SummaryAssembly) { + assembly.command.push_str(" models "); + assembly.command.push_str(name); + assembly.flag("json", json); +} + +pub(super) fn format_model_prepare( + options: ModelPrepareSummary<'_>, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" model-prepare"); + assembly.redact("source_repo", options.source_repo.is_some()); + assembly.redact("--quant", options.quant.is_some()); + assembly.redact("--target", options.target.is_some()); + assembly.redact("--model-id", options.model_id.is_some()); + assembly.redact("--flavor", options.flavor != "auto"); + assembly.redact("--timeout", options.timeout != "1h"); + assembly.redact("--mesh-llm-ref", options.mesh_llm_ref != "main"); + assembly.flag("dry-run", options.dry_run); + assembly.flag("confirm", options.confirm); + assembly.flag("follow", options.follow); + assembly.flag("json", options.json); + assembly.redact("--status", options.status.is_some()); + assembly.redact("--logs", options.logs.is_some()); + assembly.redact("--cancel", options.cancel.is_some()); + assembly.flag("list", options.list); + assembly.flag("update-script", options.update_script); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs new file mode 100644 index 0000000000..12636b6bd2 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs @@ -0,0 +1,148 @@ +use super::{DEFAULT_LOCAL_PORT, SummaryAssembly}; + +pub(super) fn format_local_model(name: &str, port: u16, assembly: &mut SummaryAssembly) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.port(port, DEFAULT_LOCAL_PORT); + assembly.redact("name", true); +} + +pub(super) fn format_runtime( + command: Option<&mesh_llm_cli::runtime::RuntimeCommand>, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::runtime::RuntimeCommand; + assembly.command.push_str(" runtime"); + let Some(command) = command else { return }; + match command { + RuntimeCommand::Status { port } => { + assembly.command.push_str(" status"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + } + RuntimeCommand::Load { port, .. } => format_local_model("load", *port, assembly), + RuntimeCommand::Unload { port, .. } => format_local_model("unload", *port, assembly), + RuntimeCommand::Guardrails { mode, port, json } => { + assembly.command.push_str(" guardrails --mode "); + assembly + .command + .push_str(&format!("{mode:?}").to_lowercase()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::Bootstrap { port, json } => { + assembly.command.push_str(" bootstrap"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::List { + available, + installed, + manifest, + bundle_dirs, + cache_dir, + json, + } => { + assembly.command.push_str(" list"); + assembly.flag("available", *available); + assembly.flag("installed", *installed); + assembly.redact("--manifest", manifest.is_some()); + assembly.redact("--bundle-dir", !bundle_dirs.is_empty()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Install { + runtime, + manifest, + bundle_dirs, + cache_dir, + json, + } => { + assembly.command.push_str(" install"); + assembly.redact("runtime_ref", runtime.is_some()); + assembly.redact("--manifest", manifest.is_some()); + assembly.redact("--bundle-dir", !bundle_dirs.is_empty()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Remove { + mesh_version, + cache_dir, + json, + .. + } => { + assembly.command.push_str(" remove"); + assembly.redact("native_runtime_id", true); + assembly.redact("--mesh-version", mesh_version.is_some()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Prune { + active_only, + mesh_version, + cache_dir, + json, + } => { + assembly.command.push_str(" prune"); + assembly.flag("active-only", *active_only); + assembly.redact("--mesh-version", mesh_version.is_some()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::GetConfig { port, json, .. } + | RuntimeCommand::ScanRefresh { port, json, .. } + | RuntimeCommand::RefreshInventory { port, json, .. } => { + assembly.command.push_str(" remote"); + assembly.redact("--endpoint", true); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::LoadModel { + profile, + port, + json, + .. + } + | RuntimeCommand::EnsureModel { + profile, + port, + json, + .. + } => { + assembly.command.push_str(" remote-model"); + assembly.redact("--endpoint", true); + assembly.redact("--model", true); + assembly.redact("--profile", profile.is_some()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::UnloadModel { + model, + instance_id, + port, + json, + .. + } + | RuntimeCommand::DrainModel { + model, + instance_id, + port, + json, + .. + } => { + assembly.command.push_str(" remote-model"); + assembly.redact("--endpoint", true); + assembly.redact("--model", model.is_some()); + assembly.redact("--instance-id", instance_id.is_some()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::ApplyConfig { port, json, .. } => { + assembly.command.push_str(" apply-config"); + assembly.redact("--endpoint", true); + assembly.redact("--expected-revision", true); + assembly.redact("--config", true); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs new file mode 100644 index 0000000000..770f6c41b4 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs @@ -0,0 +1,257 @@ +use super::command_summary; +use clap::Parser; +use mesh_llm_events::CliCommandSummary; + +fn assert_parsed_summaries(cases: &[(&[&str], &str)]) { + for (args, marker) in cases { + let cli = mesh_llm_cli::Cli::parse_from(*args); + let command = cli.command.as_ref().expect("parsed command"); + let summary = command_summary(command) + .map(|summary| summary.as_str().to_owned()) + .unwrap_or_else(|| panic!("producer summary for {args:?}")); + assert!(summary.contains(marker), "missing {marker} in {summary}"); + assert!( + CliCommandSummary::sanitize(&summary).is_some(), + "rejected: {summary}" + ); + } +} + +#[test] +fn parsed_commands_emit_local_raw_options() { + assert_parsed_summaries(&[ + (&["mesh-llm", "status", "--port", "41731"], "--port 41731"), + ( + &["mesh-llm", "load", "private/model", "--port", "41731"], + "--port 41731", + ), + ( + &["mesh-llm", "unload", "private/model", "--port", "41731"], + "--port 41731", + ), + (&["mesh-llm", "goose", "--port", "41731"], "--port 41731"), + (&["mesh-llm", "claude", "--port", "41731"], "--port 41731"), + ( + &[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "doctor", + "--json", + "split", + "--model-ref", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &["mesh-llm", "gpus", "run-benchmark", "--backend", "cuda"], + "--backend cuda", + ), + ( + &[ + "mesh-llm", + "gpus", + "--json", + "run-benchmark", + "--backend", + "cuda", + ], + "--backend cuda", + ), + ]); +} + +#[test] +fn parsed_commands_emit_runtime_raw_options() { + assert_parsed_summaries(&[ + ( + &[ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--port", + "41731", + ], + "--mode metrics", + ), + ( + &[ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--json", + "--port", + "41731", + ], + "--mode metrics", + ), + ( + &["mesh-llm", "runtime", "status", "--port", "41731"], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "unload", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &["mesh-llm", "runtime", "bootstrap", "--port", "41731"], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "bootstrap", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ]); +} + +#[test] +fn parsed_commands_emit_remote_raw_options() { + assert_parsed_summaries(&[ + ( + &[ + "mesh-llm", + "runtime", + "get-config", + "--endpoint", + "private/endpoint", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "get-config", + "--endpoint", + "private/endpoint", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load-model", + "--endpoint", + "private/endpoint", + "--model", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load-model", + "--endpoint", + "private/endpoint", + "--model", + "private/model", + "--profile", + "profile-a", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "unload-model", + "--endpoint", + "private/endpoint", + "--instance-id", + "instance-a", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "apply-config", + "--endpoint", + "private/endpoint", + "--expected-revision", + "7", + "--config", + "/private/config.toml", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "apply-config", + "--endpoint", + "private/endpoint", + "--expected-revision", + "7", + "--config", + "/private/config.toml", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ]); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs new file mode 100644 index 0000000000..2483fb44c0 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs @@ -0,0 +1,319 @@ +use super::{SummaryAssembly, command_summary}; +use clap::Parser; +use mesh_llm_events::CliCommandSummary; + +fn parsed_summary(args: &[&str]) -> String { + let cli = mesh_llm_cli::Cli::parse_from(args); + let Some(command) = cli.command.as_ref() else { + return String::new(); + }; + match command_summary(command) { + Some(summary) => summary.as_str().to_owned(), + None => String::new(), + } +} + +#[test] +fn representative_command_summaries_survive_strict_sanitization() { + let commands: &[&[&str]] = &[ + &["mesh-llm", "setup"], + &["mesh-llm", "uninstall", "--yes"], + &["mesh-llm", "gpus", "detect"], + &["mesh-llm", "discover", "--auto"], + &["mesh-llm", "goose"], + &["mesh-llm", "plugins", "list"], + &["mesh-llm", "config", "validate"], + &[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + ], + &["mesh-llm", "models", "recommended"], + &["mesh-llm", "runtime", "bootstrap"], + &[ + "mesh-llm", + "benchmark", + "import-prompts", + "--source", + "mt-bench", + "--output", + "private.jsonl", + ], + &["mesh-llm", "model-prepare", "--list"], + &["mesh-llm", "auth", "status"], + ]; + + for args in commands { + let summary = parsed_summary(args); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer emitted unsanitized summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); + } +} + +#[test] +fn over_limit_producer_summary_is_omitted_without_fallback() { + let mut assembly = SummaryAssembly::new("mesh-llm"); + for _ in 0..32 { + assembly.redact("name", true); + } + + assert!(assembly.finish().is_none()); +} + +#[test] +fn uninstall_boolean_flags_survive_producer_sanitization() { + let summaries = [ + parsed_summary(&[ + "mesh-llm", + "uninstall", + "--dry-run", + "--yes", + "--keep-cache", + "--keep-service-files", + "--keep-config", + "--json", + "--verbose", + ]), + parsed_summary(&["mesh-llm", "uninstall", "--purge-config"]), + ]; + + for summary in summaries { + assert!(!summary.is_empty()); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); + } +} + +#[test] +fn command_summary_retains_static_prefix_nested_names_and_redacts_plugin_values() { + let summary = parsed_summary(&[ + "mesh-llm", + "plugins", + "install", + "https://user:secret@example.test/plugin?token=query-secret", + ]); + assert_eq!(summary, "mesh-llm plugins install reference [REDACTED]"); + assert!(!summary.contains("secret")); + assert!(!summary.contains("query-secret")); +} + +#[test] +fn command_summary_covers_plugin_config_and_doctor_values_without_defaults() { + assert_eq!( + parsed_summary(&["mesh-llm", "plugins", "update", "private-plugin"]), + "mesh-llm plugins update name [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "config", + "validate", + "--config-path", + "/private/config.toml" + ]), + "mesh-llm config validate --config-path [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + "--output-dir", + "/private/report" + ]), + "mesh-llm doctor split --model-ref [REDACTED] --output-dir [REDACTED]" + ); +} + +#[test] +fn command_summary_covers_auth_trust_and_nested_command_families() { + let auth = parsed_summary(&[ + "mesh-llm", + "auth", + "rotate-node", + "--owner-key", + "/private/owner.json", + "--node-key", + "/private/node.key", + "--reason", + "private reason", + "--trust-store", + "/private/trust.json", + ]); + for marker in [ + "mesh-llm auth rotate-node", + "--owner-key [REDACTED]", + "--node-key [REDACTED]", + "--reason [REDACTED]", + "--trust-store [REDACTED]", + ] { + assert!(auth.contains(marker)); + } + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "auth", + "trust", + "add", + "private-owner-id", + "--trust-store", + "/private/trust.json" + ]), + "mesh-llm auth trust add owner_id [REDACTED] --trust-store [REDACTED]" + ); + assert!( + parsed_summary(&["mesh-llm", "models", "show", "private/model"]) + .contains("mesh-llm models show model [REDACTED]") + ); + assert!( + parsed_summary(&["mesh-llm", "runtime", "load", "private/model"]) + .contains("mesh-llm runtime load name [REDACTED]") + ); + assert!( + parsed_summary(&["mesh-llm", "benchmark", "tune", "--model", "private/model"]) + .contains("mesh-llm benchmark tune --model [REDACTED]") + ); +} + +#[test] +fn command_summary_retains_explicit_runtime_install_without_runtime_value() { + let summary = parsed_summary(&["mesh-llm", "runtime", "install", "private-runtime-value"]); + + assert_eq!(summary, "mesh-llm runtime install runtime_ref [REDACTED]"); + assert!(!summary.contains("private-runtime-value")); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); +} + +#[test] +fn command_summary_omits_default_ports_and_retains_explicit_safe_port() { + assert_eq!( + parsed_summary(&["mesh-llm", "load", "private/model"]), + "mesh-llm load name [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "load", "private/model", "--port", "41731"]), + "mesh-llm load --port 41731 name [REDACTED]" + ); +} + +#[test] +fn command_summary_deduplicates_parent_and_child_json_flags() { + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "doctor", + "--json", + "split", + "--model-ref", + "private/model", + "--json", + ]), + "mesh-llm doctor split --json --model-ref [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "gpus", "--json", "detect", "--json"]), + "mesh-llm gpus detect --json" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "gpus", + "--json", + "run-benchmark", + "--backend", + "cuda", + ]), + "mesh-llm gpus run-benchmark --backend cuda --json" + ); +} + +#[test] +fn summary_assembly_deduplicates_repeated_flags() { + let mut assembly = SummaryAssembly::new("mesh-llm models list"); + assembly.flag("json", true); + assembly.flag("json", true); + + assert_eq!( + assembly.finish().map(|summary| summary.as_str().to_owned()), + Some("mesh-llm models list --json".to_owned()) + ); +} + +#[test] +fn command_summary_redacts_non_default_model_search_sort() { + let summary = parsed_summary(&[ + "mesh-llm", + "models", + "search", + "private-query", + "--sort", + "downloads", + ]); + assert_eq!( + summary, + "mesh-llm models search query [REDACTED] --sort [REDACTED]" + ); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); +} + +#[test] +fn command_summary_redacts_spec_draft_tune_values() { + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "benchmark", + "tune", + "--spec-draft-acceptance-threshold", + "0.8125,0.9375", + ]), + "mesh-llm benchmark tune --spec-draft-acceptance-threshold [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "benchmark", + "tune", + "--spec-draft-split-probability", + "0.125,0.375", + ]), + "mesh-llm benchmark tune --spec-draft-split-probability [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "benchmark", "tune"]), + "mesh-llm benchmark tune" + ); +} + +#[test] +fn command_summary_redacts_external_argv_without_global_cli_context() { + assert_eq!( + parsed_summary(&["mesh-llm", "private-plugin", "--api-key", "private-key"]), + "mesh-llm external-plugin argv [REDACTED]" + ); + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "load", + "private/model", + ]); + let Some(command) = cli.command.as_ref() else { + return; + }; + let summary = command_summary(command) + .map(|value| value.as_str().to_owned()) + .unwrap_or_default(); + assert_eq!(summary, "mesh-llm load name [REDACTED]"); +} diff --git a/crates/mesh-llm-events/src/audit.rs b/crates/mesh-llm-events/src/audit.rs index f59f83dc83..95fd76948b 100644 --- a/crates/mesh-llm-events/src/audit.rs +++ b/crates/mesh-llm-events/src/audit.rs @@ -16,6 +16,7 @@ use uuid::Uuid; mod sanitization; use sanitization::redact_secrets; +pub use sanitization::{SanitizedAuditDetailJson, SanitizedAuditScalar}; /// Audit log format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, clap::ValueEnum)] diff --git a/crates/mesh-llm-events/src/audit/sanitization.rs b/crates/mesh-llm-events/src/audit/sanitization.rs index 06193cafb3..51625be869 100644 --- a/crates/mesh-llm-events/src/audit/sanitization.rs +++ b/crates/mesh-llm-events/src/audit/sanitization.rs @@ -1,7 +1,27 @@ //! Audit-event privacy and size-boundary enforcement. use super::AuditEvent; -use serde_json::Value; + +mod detail; + +pub use detail::SanitizedAuditDetailJson; + +/// Free-form audit scalar that has passed the shared privacy and size policy. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SanitizedAuditScalar(String); + +impl SanitizedAuditScalar { + pub fn sanitize(value: &str) -> Self { + Self(sanitize_audit_text(value, true)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[cfg(test)] +mod tests; const SECRET_PATTERNS: &[&str] = &[ "token", @@ -19,91 +39,34 @@ const SECRET_PATTERNS: &[&str] = &[ "certificate", ]; -const MAX_AUDIT_TEXT_LEN: usize = 1024; -const MAX_AUDIT_METADATA_NODES: usize = 256; -const MAX_AUDIT_METADATA_DEPTH: usize = 8; +pub(super) const MAX_AUDIT_TEXT_LEN: usize = 1024; +pub(super) const MAX_AUDIT_DETAIL_RAW_BYTES: usize = 256 * 1024; +pub(super) const MAX_AUDIT_METADATA_NODES: usize = 256; +pub(super) const MAX_AUDIT_METADATA_DEPTH: usize = 8; +const AUDIT_TRUNCATION_SUFFIX: &str = "... [TRUNCATED]"; /// Redact sensitive fields and bound untrusted audit metadata before emission. pub(super) fn redact_secrets(event: AuditEvent) -> AuditEvent { let mut redacted = event; - - // Bound the complete metadata tree as well as each individual string. This - // keeps one hostile event from bypassing rotation with a wide/deep value. - let metadata = std::mem::take(&mut redacted.metadata); - let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; - for (key, mut value) in metadata { - if remaining_nodes == 0 { - break; - } - remaining_nodes -= 1; - redact_json_value(Some(&key), &mut value, 0, &mut remaining_nodes); - redacted.metadata.insert(key, value); - } - + redacted.metadata = detail::sanitize_metadata(std::mem::take(&mut redacted.metadata)); redacted.action = redact_audit_text(&redacted.action); redacted.resource = redacted.resource.as_deref().map(redact_audit_text); redacted.actor = redacted.actor.as_deref().map(redact_audit_text); redacted.error = redacted.error.as_deref().map(redact_audit_text); - redacted } -fn redact_json_value( - key: Option<&str>, - value: &mut Value, - depth: usize, - remaining_nodes: &mut usize, -) { - if let Value::String(token) = value - && key.is_some_and(is_invite_token_key) - { - *token = sanitize_audit_text(token, false); - return; +pub(super) fn sanitize_audit_key(key: &str) -> String { + if is_invite_token_key(key) { + return redact_audit_text(key); } - if key.is_some_and(is_sensitive_key) { - *value = Value::String("[REDACTED]".to_string()); - return; - } - if depth >= MAX_AUDIT_METADATA_DEPTH && matches!(value, Value::Object(_) | Value::Array(_)) { - *value = Value::String("[TRUNCATED]".to_string()); - return; - } - match value { - Value::Object(object) => { - let original = std::mem::take(object); - for (nested_key, mut nested_value) in original { - if *remaining_nodes == 0 { - break; - } - *remaining_nodes -= 1; - redact_json_value( - Some(&nested_key), - &mut nested_value, - depth + 1, - remaining_nodes, - ); - object.insert(nested_key, nested_value); - } - } - Value::Array(values) => { - let original = std::mem::take(values); - for mut nested_value in original { - if *remaining_nodes == 0 { - break; - } - *remaining_nodes -= 1; - redact_json_value(None, &mut nested_value, depth + 1, remaining_nodes); - values.push(nested_value); - } - } - Value::String(string) => { - *string = sanitize_audit_text(string, true); - } - _ => {} + if is_sensitive_key(key) { + return "[REDACTED]".to_owned(); } + redact_audit_text(key) } -fn is_invite_token_key(key: &str) -> bool { +pub(super) fn is_invite_token_key(key: &str) -> bool { let compact = key .chars() .filter(|character| character.is_ascii_alphanumeric()) @@ -112,7 +75,7 @@ fn is_invite_token_key(key: &str) -> bool { compact.ends_with("invitetoken") } -fn is_sensitive_key(key: &str) -> bool { +pub(super) fn is_sensitive_key(key: &str) -> bool { let key = key.to_lowercase(); SECRET_PATTERNS.iter().any(|pattern| key.contains(pattern)) } @@ -121,7 +84,7 @@ fn redact_audit_text(value: &str) -> String { sanitize_audit_text(value, true) } -fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { +pub(super) fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { let mut sanitized = value.to_string(); for variable in ["HOME", "USERPROFILE"] { if let Ok(home) = std::env::var(variable) @@ -130,14 +93,15 @@ fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { sanitized = sanitized.replace(&home, "~"); } } + sanitized.retain(|character| !character.is_control()); if redact_secret_values && contains_secret_value(&sanitized) { - return "[REDACTED]".to_string(); + return "[REDACTED]".to_owned(); } - if sanitized.chars().count() > MAX_AUDIT_TEXT_LEN { - let prefix: String = sanitized.chars().take(MAX_AUDIT_TEXT_LEN).collect(); - return format!("{prefix}... [TRUNCATED]"); + let prefix_len = MAX_AUDIT_TEXT_LEN - AUDIT_TRUNCATION_SUFFIX.chars().count(); + let prefix = sanitized.chars().take(prefix_len).collect::(); + return format!("{prefix}{AUDIT_TRUNCATION_SUFFIX}"); } sanitized } @@ -179,146 +143,3 @@ fn contains_marker_at_token_boundary(value: &str, marker: &str) -> bool { .is_none_or(|character| !character.is_ascii_alphanumeric() && character != '_') }) } - -#[cfg(test)] -mod tests { - use super::*; - use crate::audit::{AuditCategory, AuditOutcome}; - - fn event() -> AuditEvent { - AuditEvent::new(AuditCategory::System, "event", AuditOutcome::Success) - } - - #[test] - fn nested_metadata_is_recursively_redacted() { - let event = event().with_metadata( - "changes", - serde_json::json!({ - "old_value": {"api_token": "secret-one"}, - "new_value": [{"PASSWORD": "secret-two"}, "Bearer secret-three"] - }), - ); - let serialized = redact_secrets(event).to_json_line().unwrap(); - assert!(!serialized.contains("secret-one")); - assert!(!serialized.contains("secret-two")); - assert!(!serialized.contains("secret-three")); - } - - #[test] - fn invite_token_metadata_is_preserved() { - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!("mesh-llm-invite-token-abc123"), - )); - - assert_eq!( - redacted.metadata["invite_token"], - "mesh-llm-invite-token-abc123" - ); - } - - #[test] - fn invite_token_metadata_is_path_sanitized_and_bounded() { - let home = ["HOME", "USERPROFILE"] - .into_iter() - .find_map(|variable| std::env::var(variable).ok()) - .filter(|home| !home.is_empty()) - .expect("HOME or USERPROFILE should be available to the test"); - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!(format!("{home}/{}", "x".repeat(MAX_AUDIT_TEXT_LEN + 1))), - )); - let token = redacted.metadata["invite_token"] - .as_str() - .expect("invite token should remain a string"); - - assert!(!token.contains(&home)); - assert!(token.starts_with("~/")); - assert!(token.ends_with("... [TRUNCATED]")); - assert!(token.chars().count() <= MAX_AUDIT_TEXT_LEN + "... [TRUNCATED]".len()); - } - - #[test] - fn metadata_strings_are_recursively_secret_path_and_length_safe() { - let home = ["HOME", "USERPROFILE"] - .into_iter() - .find_map(|variable| std::env::var(variable).ok()) - .filter(|home| !home.is_empty()) - .expect("HOME or USERPROFILE should be available to the test"); - let redacted = redact_secrets( - event() - .with_metadata("refresh", serde_json::json!("refresh_token=secret-value")) - .with_metadata( - "nested", - serde_json::json!({ - "note": format!("{} {}", home, "x".repeat(MAX_AUDIT_TEXT_LEN + 1)), - "credential": "private_key=secret-key" - }), - ), - ); - - assert_eq!(redacted.metadata["refresh"], "[REDACTED]"); - assert_eq!(redacted.metadata["nested"]["credential"], "[REDACTED]"); - let note = redacted.metadata["nested"]["note"] - .as_str() - .expect("note should remain a string"); - assert!(!note.contains(&home)); - assert!(note.ends_with("... [TRUNCATED]")); - assert!(note.chars().count() <= MAX_AUDIT_TEXT_LEN + "... [TRUNCATED]".len()); - } - - #[test] - fn metadata_collections_are_bounded_before_serialization() { - let values = (0..1_000) - .map(|index| serde_json::json!({"index": index, "note": "x".repeat(2_048)})) - .collect::>(); - let redacted = redact_secrets(event().with_metadata("items", Value::Array(values))); - let items = redacted.metadata["items"] - .as_array() - .expect("items should remain an array"); - - assert!(items.len() <= MAX_AUDIT_METADATA_NODES); - assert!(redacted.to_json_line().unwrap().len() < 300_000); - } - - #[test] - fn invite_token_exception_does_not_bypass_nested_sanitization() { - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!({"password": "must-not-survive"}), - )); - - assert_eq!(redacted.metadata["invite_token"], "[REDACTED]"); - } - - #[test] - fn free_form_audit_fields_are_redacted_and_bounded() { - let event = AuditEvent::new( - AuditCategory::System, - "restart failed: Bearer audit-secret".to_string(), - AuditOutcome::Failure, - ) - .with_resource("model sk-live-example") - .with_actor("ghp_exampletoken") - .with_error("password=example-password") - .with_metadata("note", serde_json::json!("Basic YWxpY2U6cGFzcw==")); - let redacted = redact_secrets(event); - - assert_eq!(redacted.action, "[REDACTED]"); - assert_eq!(redacted.resource.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.actor.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.error.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.metadata["note"], "[REDACTED]"); - - let long_action = AuditEvent::new( - AuditCategory::System, - "x".repeat(MAX_AUDIT_TEXT_LEN + 1), - AuditOutcome::Success, - ); - assert!( - redact_secrets(long_action) - .action - .ends_with("... [TRUNCATED]") - ); - } -} diff --git a/crates/mesh-llm-events/src/audit/sanitization/detail.rs b/crates/mesh-llm-events/src/audit/sanitization/detail.rs new file mode 100644 index 0000000000..e08a2b49ce --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/detail.rs @@ -0,0 +1,152 @@ +use super::{ + MAX_AUDIT_DETAIL_RAW_BYTES, MAX_AUDIT_METADATA_DEPTH, MAX_AUDIT_METADATA_NODES, + MAX_AUDIT_TEXT_LEN, is_invite_token_key, is_sensitive_key, sanitize_audit_key, + sanitize_audit_text, +}; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, BTreeSet}; + +/// JSON audit detail that has passed the shared privacy and size policy. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SanitizedAuditDetailJson(String); + +impl SanitizedAuditDetailJson { + /// Bound raw input, sanitize valid JSON recursively, and replace malformed input completely. + pub fn sanitize(raw: &str) -> Self { + if raw.len() > MAX_AUDIT_DETAIL_RAW_BYTES { + return Self(Value::String("[TRUNCATED]".to_owned()).to_string()); + } + let Ok(mut value) = serde_json::from_str(raw) else { + return Self(Value::String("[REDACTED]".to_owned()).to_string()); + }; + sanitize_root(&mut value); + Self(value.to_string()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +pub(super) fn sanitize_metadata(metadata: BTreeMap) -> BTreeMap { + let mut object = metadata.into_iter().collect::>(); + let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; + sanitize_object(&mut object, 0, &mut remaining_nodes); + object.into_iter().collect() +} + +fn sanitize_root(value: &mut Value) { + let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; + sanitize_json_value(None, value, 0, &mut remaining_nodes); +} + +fn sanitize_json_value( + key: Option<&str>, + value: &mut Value, + depth: usize, + remaining_nodes: &mut usize, +) { + if let Value::String(token) = value + && key.is_some_and(is_invite_token_key) + { + *token = sanitize_audit_text(token, false); + return; + } + if key.is_some_and(is_sensitive_key) { + *value = Value::String("[REDACTED]".to_owned()); + return; + } + if depth >= MAX_AUDIT_METADATA_DEPTH && matches!(value, Value::Object(_) | Value::Array(_)) { + *value = Value::String("[TRUNCATED]".to_owned()); + return; + } + match value { + Value::Object(object) => sanitize_object(object, depth, remaining_nodes), + Value::Array(values) => sanitize_array(values, depth, remaining_nodes), + Value::String(string) => *string = sanitize_audit_text(string, true), + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + +fn sanitize_object(object: &mut Map, depth: usize, remaining_nodes: &mut usize) { + let mut original = std::mem::take(object).into_iter().collect::>(); + original.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + let mut entries = Vec::new(); + for (original_key, mut nested_value) in original { + if *remaining_nodes == 0 { + break; + } + *remaining_nodes -= 1; + sanitize_json_value( + Some(&original_key), + &mut nested_value, + depth + 1, + remaining_nodes, + ); + let canonical_key = sanitize_audit_key(&original_key); + entries.push((original_key, canonical_key, nested_value)); + } + insert_collision_safe(object, entries); +} + +fn sanitize_array(values: &mut Vec, depth: usize, remaining_nodes: &mut usize) { + let original = std::mem::take(values); + for mut nested_value in original { + if *remaining_nodes == 0 { + break; + } + *remaining_nodes -= 1; + sanitize_json_value(None, &mut nested_value, depth + 1, remaining_nodes); + values.push(nested_value); + } +} + +fn insert_collision_safe(object: &mut Map, entries: Vec<(String, String, Value)>) { + let reserved_keys = entries + .iter() + .filter(|(original, canonical, _)| original == canonical) + .map(|(_, canonical, _)| canonical.clone()) + .collect::>(); + let mut used_keys = BTreeSet::new(); + let mut collision_indices = BTreeMap::new(); + for (original_key, canonical_key, value) in entries { + let owns_canonical = original_key == canonical_key; + let final_key = if (owns_canonical || !reserved_keys.contains(&canonical_key)) + && used_keys.insert(canonical_key.clone()) + { + canonical_key + } else { + next_collision_key( + &canonical_key, + &reserved_keys, + &mut used_keys, + &mut collision_indices, + ) + }; + object.insert(final_key, value); + } +} + +fn next_collision_key( + canonical_key: &str, + reserved_keys: &BTreeSet, + used_keys: &mut BTreeSet, + collision_indices: &mut BTreeMap, +) -> String { + let next_index = collision_indices + .entry(canonical_key.to_owned()) + .or_insert(2); + loop { + let suffix = format!("#{next_index}"); + let prefix_len = MAX_AUDIT_TEXT_LEN.saturating_sub(suffix.chars().count()); + let prefix = canonical_key.chars().take(prefix_len).collect::(); + let candidate = format!("{prefix}{suffix}"); + *next_index += 1; + if !reserved_keys.contains(&candidate) && used_keys.insert(candidate.clone()) { + return candidate; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs b/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs new file mode 100644 index 0000000000..1b9a036e09 --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs @@ -0,0 +1,171 @@ +use super::*; + +fn parse_detail(raw: &str) -> Value { + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON") +} + +fn home_directory() -> String { + ["HOME", "USERPROFILE"] + .into_iter() + .find_map(|variable| std::env::var(variable).ok()) + .filter(|home| !home.is_empty()) + .expect("HOME or USERPROFILE should be available to the test") +} + +#[test] +fn audit_detail_sanitization_preserves_safe_json_structure() { + let raw = r#"{"safe":{"outcome":"completed","count":3},"nested":{"password":"detail-secret"}}"#; + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + let value: Value = + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON"); + assert_eq!(value["safe"]["outcome"], "completed"); + assert_eq!(value["safe"]["count"], 3); + assert!(!sanitized.as_str().contains("detail-secret")); +} + +#[test] +fn audit_detail_sanitization_replaces_malformed_sensitive_input() { + let raw = r#"{"note":"Bearer malformed-credential""#; + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + assert_eq!(sanitized.as_str(), r#""[REDACTED]""#); + assert!(!sanitized.as_str().contains("malformed-credential")); +} + +#[test] +fn audit_detail_sanitization_truncates_oversized_raw_input_before_parsing() { + let raw = serde_json::json!({ + "sentinel": "OVERSIZED-RAW-SENTINEL", + "padding": "x".repeat(256 * 1024), + }) + .to_string(); + let sanitized = SanitizedAuditDetailJson::sanitize(&raw); + assert_eq!(sanitized.as_str(), r#""[TRUNCATED]""#); + assert!(!sanitized.as_str().contains("OVERSIZED-RAW-SENTINEL")); +} + +#[test] +fn audit_detail_sanitization_bounds_object_keys() { + let oversized_key = "k".repeat(MAX_AUDIT_TEXT_LEN + 1); + let raw = serde_json::json!({ oversized_key: "safe-value" }).to_string(); + let value = parse_detail(&raw); + let object = value + .as_object() + .expect("sanitized detail remains an object"); + let (key, stored_value) = object.iter().next().expect("bounded entry remains present"); + assert!(key.chars().count() <= MAX_AUDIT_TEXT_LEN); + assert_eq!(stored_value, "safe-value"); +} + +#[test] +fn audit_detail_sanitization_removes_private_and_credential_key_text() { + let home = home_directory(); + let raw = serde_json::json!({ + format!("{home}/private/audit\npath"): "safe-path-value", + "Bearer object-key-credential": "sensitive-field-value", + }) + .to_string(); + let value = parse_detail(&raw); + let keys = value + .as_object() + .expect("sanitized detail remains an object") + .keys() + .cloned() + .collect::>(); + assert!(keys.iter().all(|key| !key.contains(&home))); + assert!(keys.iter().all(|key| !key.contains("Bearer"))); + assert!( + keys.iter() + .all(|key| !key.contains("object-key-credential")) + ); + assert!(keys.iter().all(|key| !key.contains('\n'))); +} + +#[test] +fn audit_detail_sanitization_preserves_values_at_deterministic_collision_keys() { + let home = home_directory(); + let private_key = format!("{home}/collision"); + let canonical_key = "~/collision".to_string(); + let raw = serde_json::json!({ + private_key: "private-path-value", + canonical_key: "canonical-value", + }) + .to_string(); + let value = parse_detail(&raw); + assert_eq!(value["~/collision"], "canonical-value"); + assert_eq!(value["~/collision#2"], "private-path-value"); +} + +#[test] +fn sensitive_key_contract_redacts_nested_bare_key_names() { + // Given: nested detail containing every bare sensitive-key spelling. + let raw = serde_json::json!({ + "nested": { + "password": "password-value", + "credential": "credential-value", + "api_key": "api-key-value", + "private_key": "private-key-value", + "authorization": "authorization-value", + } + }) + .to_string(); + + // When: the durable detail sanitizer processes the document. + let sanitized = SanitizedAuditDetailJson::sanitize(&raw); + + // Then: neither sensitive names nor their values persist. + for sensitive in [ + "password", + "credential", + "api_key", + "private_key", + "authorization", + ] { + assert!(!sanitized.as_str().contains(sensitive)); + } + let value: Value = + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON"); + assert_eq!(value["nested"].as_object().map(Map::len), Some(5)); +} + +#[test] +fn sensitive_key_contract_preserves_invite_token_exception() { + // Given: a string value under the invite-token exception key. + let raw = r#"{"invite_token":"mesh-llm-invite-token-abc123"}"#; + + // When: the durable detail sanitizer processes the document. + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + + // Then: the key and bounded string value remain available. + assert_eq!(sanitized.as_str(), raw); +} + +#[test] +fn sensitive_key_contract_reserves_canonical_collision_chain() { + // Given: canonical keys that reserve the first two collision positions. + let raw = r#"{"[REDACTED]":"canonical","[REDACTED]#2":"reserved","authorization":"auth-value","password":"password-value"}"#; + + // When: sensitive key names normalize in sorted original-key order. + let value = parse_detail(raw); + + // Then: every entry remains and later collisions use unreserved suffixes. + assert_eq!(value["[REDACTED]"], "canonical"); + assert_eq!(value["[REDACTED]#2"], "reserved"); + assert_eq!(value["[REDACTED]#3"], "[REDACTED]"); + assert_eq!(value["[REDACTED]#4"], "[REDACTED]"); + assert_eq!(value.as_object().map(Map::len), Some(4)); +} + +#[test] +fn sensitive_key_contract_is_byte_idempotent() { + // Given: detail with sensitive-key and canonical-key collisions. + let raw = + r#"{"[REDACTED]":"canonical","credential":"credential-value","password":"password-value"}"#; + + // When: the sanitizer processes its own output. + let first = SanitizedAuditDetailJson::sanitize(raw); + let second = SanitizedAuditDetailJson::sanitize(first.as_str()); + + // Then: the serialized bytes do not change. + assert_eq!(second.as_str(), first.as_str()); +} diff --git a/crates/mesh-llm-events/src/audit/sanitization/tests.rs b/crates/mesh-llm-events/src/audit/sanitization/tests.rs new file mode 100644 index 0000000000..29a01b7d90 --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/tests.rs @@ -0,0 +1,178 @@ +use super::*; +use crate::audit::{AuditCategory, AuditOutcome}; +use serde_json::Value; +use std::collections::BTreeMap; + +fn event() -> AuditEvent { + AuditEvent::new(AuditCategory::System, "event", AuditOutcome::Success) +} + +fn home_directory() -> String { + ["HOME", "USERPROFILE"] + .into_iter() + .find_map(|variable| std::env::var(variable).ok()) + .filter(|home| !home.is_empty()) + .expect("HOME or USERPROFILE should be available to the test") +} + +#[test] +fn audit_scalar_sanitizes_private_paths_controls_credentials_and_bounds() { + let home = home_directory(); + let private_path = SanitizedAuditScalar::sanitize(&format!("{home}/private\naudit")); + let credential = SanitizedAuditScalar::sanitize("Bearer audit-secret"); + let oversized = SanitizedAuditScalar::sanitize(&"x".repeat(MAX_AUDIT_TEXT_LEN + 1)); + + assert_eq!(private_path.as_str(), "~/privateaudit"); + assert_eq!(credential.as_str(), "[REDACTED]"); + assert!(oversized.as_str().ends_with("... [TRUNCATED]")); + assert!(oversized.as_str().chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn audit_scalar_preserves_safe_and_fixed_values() { + for value in ["runtime", "runtime_ready", "[REDACTED]", "[TRUNCATED]"] { + assert_eq!(SanitizedAuditScalar::sanitize(value).as_str(), value); + } +} + +#[test] +fn nested_metadata_is_recursively_redacted() { + let event = event().with_metadata( + "changes", + serde_json::json!({ + "old_value": {"api_token": "secret-one"}, + "new_value": [{"PASSWORD": "secret-two"}, "Bearer secret-three"] + }), + ); + let serialized = redact_secrets(event).to_json_line().unwrap(); + assert!(!serialized.contains("secret-one")); + assert!(!serialized.contains("secret-two")); + assert!(!serialized.contains("secret-three")); +} + +#[test] +fn invite_token_metadata_is_preserved() { + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!("mesh-llm-invite-token-abc123"), + )); + assert_eq!( + redacted.metadata["invite_token"], + "mesh-llm-invite-token-abc123" + ); +} + +#[test] +fn invite_token_metadata_is_path_sanitized_and_bounded() { + let home = home_directory(); + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!(format!("{home}/{}", "x".repeat(MAX_AUDIT_TEXT_LEN + 1))), + )); + let token = redacted.metadata["invite_token"] + .as_str() + .expect("invite token should remain a string"); + assert!(!token.contains(&home)); + assert!(token.starts_with("~/")); + assert!(token.ends_with("... [TRUNCATED]")); + assert!(token.chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn metadata_strings_are_recursively_secret_path_and_length_safe() { + let home = home_directory(); + let redacted = redact_secrets( + event() + .with_metadata("refresh", serde_json::json!("refresh_token=secret-value")) + .with_metadata( + "nested", + serde_json::json!({ + "note": format!("{} {}", home, "x".repeat(MAX_AUDIT_TEXT_LEN + 1)), + "credential": "private_key=secret-key" + }), + ), + ); + assert_eq!(redacted.metadata["refresh"], "[REDACTED]"); + assert_eq!(redacted.metadata["nested"]["[REDACTED]"], "[REDACTED]"); + let note = redacted.metadata["nested"]["note"] + .as_str() + .expect("note should remain a string"); + assert!(!note.contains(&home)); + assert!(note.ends_with("... [TRUNCATED]")); + assert!(note.chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn metadata_collections_are_bounded_before_serialization() { + let values = (0..1_000) + .map(|index| serde_json::json!({"index": index, "note": "x".repeat(2_048)})) + .collect::>(); + let redacted = redact_secrets(event().with_metadata("items", Value::Array(values))); + let items = redacted.metadata["items"] + .as_array() + .expect("items should remain an array"); + assert!(items.len() <= MAX_AUDIT_METADATA_NODES); + assert!(redacted.to_json_line().unwrap().len() < 300_000); +} + +#[test] +fn invite_token_exception_does_not_bypass_nested_sanitization() { + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!({"password": "must-not-survive"}), + )); + assert_eq!(redacted.metadata["invite_token"], "[REDACTED]"); +} + +#[test] +fn free_form_audit_fields_are_redacted_and_bounded() { + let event = AuditEvent::new( + AuditCategory::System, + "restart failed: Bearer audit-secret".to_string(), + AuditOutcome::Failure, + ) + .with_resource("model sk-live-example") + .with_actor("ghp_exampletoken") + .with_error("password=example-password") + .with_metadata("note", serde_json::json!("Basic YWxpY2U6cGFzcw==")); + let redacted = redact_secrets(event); + assert_eq!(redacted.action, "[REDACTED]"); + assert_eq!(redacted.resource.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.actor.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.error.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.metadata["note"], "[REDACTED]"); + + let long_action = AuditEvent::new( + AuditCategory::System, + "x".repeat(MAX_AUDIT_TEXT_LEN + 1), + AuditOutcome::Success, + ); + assert!( + redact_secrets(long_action) + .action + .ends_with("... [TRUNCATED]") + ); +} + +#[test] +fn sensitive_key_contract_matches_top_level_audit_metadata() { + // Given: equivalent detail JSON and top-level audit metadata. + let raw = r#"{"[REDACTED]":"canonical","credential":"credential-value","invite_token":"mesh-llm-invite-token-abc123","password":"password-value"}"#; + let metadata: BTreeMap = + serde_json::from_str(raw).expect("fixture is a JSON object"); + let mut audit_event = event(); + audit_event.metadata = metadata; + + // When: both audit sanitization paths process the same keys. + let detail = SanitizedAuditDetailJson::sanitize(raw); + let redacted_event = redact_secrets(audit_event); + + // Then: their normalized key and value behavior is identical. + let detail_value: Value = + serde_json::from_str(detail.as_str()).expect("sanitized detail is valid JSON"); + let event_value = + serde_json::to_value(redacted_event.metadata).expect("sanitized metadata is serializable"); + assert_eq!(event_value, detail_value); + assert!(!event_value.to_string().contains("credential")); + assert!(!event_value.to_string().contains("password")); +} diff --git a/crates/mesh-llm-events/src/command_lifecycle.rs b/crates/mesh-llm-events/src/command_lifecycle.rs index 395a80bb00..a9d4da525d 100644 --- a/crates/mesh-llm-events/src/command_lifecycle.rs +++ b/crates/mesh-llm-events/src/command_lifecycle.rs @@ -9,6 +9,9 @@ use super::{LogFormat, OutputEvent, OutputLevel, OutputSink, output_sink}; use std::io::{self, Write}; use std::sync::atomic::{AtomicBool, Ordering}; +#[path = "command_summary_grammar.rs"] +mod command_summary_grammar; + /// Stable family for a parsed command. The enum intentionally groups commands /// by responsibility instead of carrying user-supplied subcommand names or /// arguments. @@ -30,6 +33,44 @@ pub enum CliCommandFamily { Unknown, } +/// A grammar-validated summary of a parsed CLI command. +/// +/// External callers must use [`CliCommandSummary::sanitize`] to construct a +/// summary. +/// +/// ```compile_fail +/// use mesh_llm_events::CliCommandSummary; +/// +/// let _summary = CliCommandSummary::new("mesh-llm status"); +/// ``` +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CliCommandSummary(String); + +impl CliCommandSummary { + fn new(value: &str) -> Option { + let token_count = value.split_whitespace().count(); + (!value.is_empty() + && token_count > 0 + && token_count <= 32 + && value.chars().count() <= 256 + && !value.chars().any(char::is_control)) + .then(|| Self(value.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn sanitize(value: &str) -> Option { + let summary = Self::new(value)?; + is_safe_summary(&summary.0).then_some(summary) + } +} + +fn is_safe_summary(value: &str) -> bool { + command_summary_grammar::is_safe_summary(value) +} + impl CliCommandFamily { pub const fn as_str(self) -> &'static str { match self { @@ -164,7 +205,10 @@ fn write_cli_command_event_to_stderr( stderr: &mut W, ) -> io::Result<()> { let OutputEvent::CliCommandLifecycle { family, outcome } = event else { - unreachable!("CLI command event adapter received a different event variant"); + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "CLI command event adapter received a different event variant", + )); }; writeln!( @@ -178,161 +222,5 @@ fn write_cli_command_event_to_stderr( } #[cfg(test)] -mod tests { - use super::*; - use crate::{clear_output_sink, set_output_sink}; - use std::sync::{Arc, Mutex}; - - struct RecordingSink { - mode: LogFormat, - events: Mutex>, - } - - impl RecordingSink { - fn new(mode: LogFormat) -> Self { - Self { - mode, - events: Mutex::new(Vec::new()), - } - } - } - - impl OutputSink for RecordingSink { - fn emit_event(&self, event: OutputEvent) -> io::Result<()> { - self.events.lock().expect("recording sink lock").push(event); - Ok(()) - } - - fn mode(&self) -> LogFormat { - self.mode - } - } - - struct VerboseResetGuard; - - impl Drop for VerboseResetGuard { - fn drop(&mut self) { - set_cli_command_event_verbose(false); - } - } - - struct OutputSinkResetGuard; - - impl Drop for OutputSinkResetGuard { - fn drop(&mut self) { - clear_output_sink(); - } - } - - #[test] - fn public_emit_is_silent_unless_verbose_enabled() { - let sink = Arc::new(RecordingSink::new(LogFormat::Pretty)); - let _sink_guard = OutputSinkResetGuard; - let _verbose_guard = VerboseResetGuard; - set_output_sink(sink.clone()); - - set_cli_command_event_verbose(false); - emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Started) - .expect("silent emission must succeed"); - assert!( - sink.events.lock().expect("recording sink lock").is_empty(), - "without --debug the command event must not be presented" - ); - - set_cli_command_event_verbose(true); - emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Completed) - .expect("verbose emission must succeed"); - assert_eq!( - *sink.events.lock().expect("recording sink lock"), - vec![OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }], - "with --debug the command event must reach the pretty sink" - ); - } - - #[test] - fn pretty_sink_receives_typed_command_event() { - let sink = RecordingSink::new(LogFormat::Pretty); - let mut stderr = Vec::new(); - let event = OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }; - - emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) - .expect("pretty sink should receive command event"); - - assert_eq!( - *sink.events.lock().expect("recording sink lock"), - vec![event] - ); - assert!(stderr.is_empty()); - } - - #[test] - fn json_sink_keeps_command_event_off_stdout() { - let sink = RecordingSink::new(LogFormat::Json); - let mut stderr = Vec::new(); - let event = OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Models, - outcome: CliCommandOutcome::Completed, - }; - - emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) - .expect("stderr fallback should write command event"); - - assert!(sink.events.lock().expect("recording sink lock").is_empty()); - assert_eq!( - String::from_utf8(stderr).expect("stderr must be utf-8"), - "mesh-llm command event: family=models code=cli_command_completed outcome=completed\n" - ); - } - - #[test] - fn command_event_vocabulary_is_bounded_and_static() { - let families = [ - CliCommandFamily::Agent, - CliCommandFamily::Benchmark, - CliCommandFamily::Configuration, - CliCommandFamily::Diagnostics, - CliCommandFamily::Discovery, - CliCommandFamily::Hardware, - CliCommandFamily::Identity, - CliCommandFamily::Installation, - CliCommandFamily::Models, - CliCommandFamily::Plugin, - CliCommandFamily::Process, - CliCommandFamily::Runtime, - CliCommandFamily::Skills, - CliCommandFamily::Unknown, - ]; - let outcomes = [ - CliCommandOutcome::Started, - CliCommandOutcome::Completed, - CliCommandOutcome::Failed, - CliCommandOutcome::Rejected, - CliCommandOutcome::ParseFailed, - ]; - - for family in families { - assert!(family.as_str().len() <= 24); - assert!( - family - .as_str() - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte == b'_') - ); - } - for outcome in outcomes { - assert!(outcome.code().len() <= 48); - assert!( - outcome - .code() - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte == b'_') - ); - } - } -} +#[path = "command_lifecycle/tests.rs"] +mod tests; diff --git a/crates/mesh-llm-events/src/command_lifecycle/tests.rs b/crates/mesh-llm-events/src/command_lifecycle/tests.rs new file mode 100644 index 0000000000..bb7562b763 --- /dev/null +++ b/crates/mesh-llm-events/src/command_lifecycle/tests.rs @@ -0,0 +1,359 @@ +use super::*; +use crate::{clear_output_sink, set_output_sink}; +use std::sync::{Arc, Mutex}; + +struct RecordingSink { + mode: LogFormat, + events: Mutex>, +} + +impl RecordingSink { + fn new(mode: LogFormat) -> Self { + Self { + mode, + events: Mutex::new(Vec::new()), + } + } +} + +impl OutputSink for RecordingSink { + fn emit_event(&self, event: OutputEvent) -> io::Result<()> { + self.events.lock().expect("recording sink lock").push(event); + Ok(()) + } + + fn mode(&self) -> LogFormat { + self.mode + } +} + +struct VerboseResetGuard; + +impl Drop for VerboseResetGuard { + fn drop(&mut self) { + set_cli_command_event_verbose(false); + } +} + +struct OutputSinkResetGuard; + +impl Drop for OutputSinkResetGuard { + fn drop(&mut self) { + clear_output_sink(); + } +} + +#[test] +fn public_emit_is_silent_unless_verbose_enabled() { + let sink = Arc::new(RecordingSink::new(LogFormat::Pretty)); + let _sink_guard = OutputSinkResetGuard; + let _verbose_guard = VerboseResetGuard; + set_output_sink(sink.clone()); + + set_cli_command_event_verbose(false); + emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Started) + .expect("silent emission must succeed"); + assert!( + sink.events.lock().expect("recording sink lock").is_empty(), + "without --debug the command event must not be presented" + ); + + set_cli_command_event_verbose(true); + emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Completed) + .expect("verbose emission must succeed"); + assert_eq!( + *sink.events.lock().expect("recording sink lock"), + vec![OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Completed, + }], + "with --debug the command event must reach the pretty sink" + ); +} + +#[test] +fn pretty_sink_receives_typed_command_event() { + let sink = RecordingSink::new(LogFormat::Pretty); + let mut stderr = Vec::new(); + let event = OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Started, + }; + + emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) + .expect("pretty sink should receive command event"); + + assert_eq!( + *sink.events.lock().expect("recording sink lock"), + vec![event] + ); + assert!(stderr.is_empty()); +} + +#[test] +fn json_sink_keeps_command_event_off_stdout() { + let sink = RecordingSink::new(LogFormat::Json); + let mut stderr = Vec::new(); + let event = OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Models, + outcome: CliCommandOutcome::Completed, + }; + + emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) + .expect("stderr fallback should write command event"); + + assert!(sink.events.lock().expect("recording sink lock").is_empty()); + assert_eq!( + String::from_utf8(stderr).expect("stderr must be utf-8"), + "mesh-llm command event: family=models code=cli_command_completed outcome=completed\n" + ); +} + +#[test] +fn command_event_vocabulary_is_bounded_and_static() { + let families = [ + CliCommandFamily::Agent, + CliCommandFamily::Benchmark, + CliCommandFamily::Configuration, + CliCommandFamily::Diagnostics, + CliCommandFamily::Discovery, + CliCommandFamily::Hardware, + CliCommandFamily::Identity, + CliCommandFamily::Installation, + CliCommandFamily::Models, + CliCommandFamily::Plugin, + CliCommandFamily::Process, + CliCommandFamily::Runtime, + CliCommandFamily::Skills, + CliCommandFamily::Unknown, + ]; + let outcomes = [ + CliCommandOutcome::Started, + CliCommandOutcome::Completed, + CliCommandOutcome::Failed, + CliCommandOutcome::Rejected, + CliCommandOutcome::ParseFailed, + ]; + + for family in families { + assert!(family.as_str().len() <= 24); + assert!( + family + .as_str() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } + for outcome in outcomes { + assert!(outcome.code().len() <= 48); + assert!( + outcome + .code() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } +} + +#[test] +fn command_summary_rejects_control_text_and_unbounded_tokens() { + assert!(CliCommandSummary::new(" ").is_none()); + assert!(CliCommandSummary::new("runtime\nload").is_none()); + assert!(CliCommandSummary::new(&"token ".repeat(33)).is_none()); + assert!(CliCommandSummary::new(&"x".repeat(257)).is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_arbitrary_values_and_preserves_safe_shape() { + assert!(CliCommandSummary::sanitize("mesh-llm load model-private").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load --port 41731 name [REDACTED]").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm load name [REDACTED]").is_some()); + assert!(CliCommandSummary::sanitize(&format!("mesh-llm {}", "load ".repeat(32))).is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_unknown_options_and_global_numeric_values() { + assert!(CliCommandSummary::sanitize("mesh-llm --api-key 123456").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load 123456 name [REDACTED]").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load --port 41731 name [REDACTED]").is_some()); + assert!( + CliCommandSummary::sanitize("mesh-llm runtime install runtime_ref [REDACTED]").is_some() + ); +} + +#[test] +fn command_summary_sanitizer_rejects_non_canonical_whole_command_shapes() { + let summaries = [ + " ", + "mesh-llm models list --json --json", + "mesh-llm models --json list", + "mesh-llm load name [REDACTED] name [REDACTED]", + "mesh-llm gpus run-benchmark --backend cuda --json --json", + "mesh-llm load --port 41731 --port 41732", + "mesh-llm load --port 41731 name [REDACTED] --json", + "mesh-llm load name [REDACTED] status", + "mesh-llm models nonsense", + "mesh-llm load --json name [REDACTED]", + "mesh-llm runtime status name [REDACTED]", + "mesh-llm models list name [REDACTED] --json", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "non-canonical summary was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_non_canonical_ascii_whitespace() { + for summary in [ + " mesh-llm models list", + "mesh-llm models list ", + "mesh-llm models list", + "mesh-llm\tmodels list", + "mesh-llm models\nlist", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "non-canonical whitespace was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_conflicting_boolean_pairs() { + for summary in [ + "mesh-llm setup --service --no-service", + "mesh-llm setup --no-service --service", + "mesh-llm uninstall --purge-config --keep-config", + "mesh-llm uninstall --keep-config --purge-config", + "mesh-llm auth init --no-passphrase --keychain", + "mesh-llm auth init --keychain --no-passphrase", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "conflicting flags were accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_accepts_global_relay_redaction_only() { + assert!( + CliCommandSummary::sanitize("mesh-llm load name [REDACTED] --root-relay [REDACTED]") + .is_some() + ); + for summary in [ + "mesh-llm load name [REDACTED] --relay private-relay", + "mesh-llm load name [REDACTED] --root-relay [REDACTED] value", + "mesh-llm load name [REDACTED] --relay-auth private-token", + "mesh-llm load --root-relay [REDACTED] name [REDACTED]", + "mesh-llm load name [REDACTED] --relay-auth [REDACTED] --root-relay [REDACTED]", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "malformed global relay marker was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_contextualizes_enum_values() { + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --backend cuda").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime guardrails --mode metrics").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --backend rocm").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime guardrails --mode strict").is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_raw_options_in_wrong_command_contexts() { + assert!(CliCommandSummary::sanitize("mesh-llm load --mode enforce").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime status --backend cuda").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --port 1234").is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_deep_static_prefix_without_panicking() { + let result = std::panic::catch_unwind(|| { + CliCommandSummary::sanitize( + "mesh-llm load unload status discover rotate-key setup --port 1234", + ) + }); + + assert!(result.is_ok()); + assert!(result.is_ok_and(|summary| summary.is_none())); +} + +#[test] +fn command_summary_sanitizer_rejects_inserted_prefix_tokens_before_raw_options() { + let summaries = [ + "mesh-llm gpus --draft run-benchmark --backend cuda", + "mesh-llm gpus run-benchmark model [REDACTED] --backend cuda", + "mesh-llm gpus run-benchmark detect --backend cuda", + "mesh-llm runtime guardrails status --mode metrics", + "mesh-llm gpus --json run-benchmark --backend cuda", + "mesh-llm doctor --json split --port 1234", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "malformed producer prefix was accepted: {summary}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_impossible_boolean_and_raw_option_orders() { + let summaries = [ + "mesh-llm runtime guardrails --mode metrics --port 41731 --json", + "mesh-llm runtime bootstrap --port 41731 --json", + "mesh-llm runtime remote --port 41731 --json --endpoint [REDACTED]", + "mesh-llm runtime remote --endpoint [REDACTED] --json --port 41731", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "impossible producer ordering was accepted: {summary}" + ); + } +} + +#[test] +fn command_summary_sanitizer_accepts_every_producer_raw_option_context() { + let summaries = [ + "mesh-llm status --port 41731", + "mesh-llm load --port 41731 name [REDACTED]", + "mesh-llm unload --port 41731 name [REDACTED]", + "mesh-llm goose --port 41731 --model [REDACTED]", + "mesh-llm claude --port 41731 --model [REDACTED]", + "mesh-llm doctor split --port 41731 --model-ref [REDACTED]", + "mesh-llm doctor split --json --port 41731 --model-ref [REDACTED]", + "mesh-llm doctor split --json --port 41731 --model-ref [REDACTED]", + "mesh-llm gpus run-benchmark --backend cuda", + "mesh-llm gpus run-benchmark --backend cuda --json", + "mesh-llm runtime status --port 41731", + "mesh-llm runtime load --port 41731 name [REDACTED]", + "mesh-llm runtime unload --port 41731 name [REDACTED]", + "mesh-llm runtime guardrails --mode metrics --port 41731", + "mesh-llm runtime guardrails --mode metrics --json --port 41731", + "mesh-llm runtime bootstrap --port 41731", + "mesh-llm runtime bootstrap --json --port 41731", + "mesh-llm runtime remote --port 41731 --endpoint [REDACTED]", + "mesh-llm runtime remote --json --port 41731 --endpoint [REDACTED]", + "mesh-llm runtime remote-model --port 41731 --endpoint [REDACTED] --model [REDACTED]", + "mesh-llm runtime remote-model --json --port 41731 --endpoint [REDACTED] --model [REDACTED]", + "mesh-llm runtime apply-config --port 41731 --endpoint [REDACTED] --expected-revision [REDACTED] --config [REDACTED]", + "mesh-llm runtime apply-config --json --port 41731 --endpoint [REDACTED] --expected-revision [REDACTED] --config [REDACTED]", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_some(), + "producer-reachable summary was rejected: {summary}" + ); + } +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar.rs b/crates/mesh-llm-events/src/command_summary_grammar.rs new file mode 100644 index 0000000000..c45b9c0f74 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar.rs @@ -0,0 +1,28 @@ +#[path = "command_summary_grammar/descriptors.rs"] +mod descriptors; +#[path = "command_summary_grammar/raw_options.rs"] +mod raw_options; +#[path = "command_summary_grammar/validation.rs"] +mod validation; +#[path = "command_summary_grammar/vocabulary.rs"] +mod vocabulary; + +use descriptors::DESCRIPTOR_GROUPS; +use validation::validate_descriptor; + +pub(super) fn is_safe_summary(value: &str) -> bool { + let tokens = value.split(' ').collect::>(); + !tokens.is_empty() + && tokens + .iter() + .all(|token| !token.is_empty() && !token.chars().any(char::is_whitespace)) + && DESCRIPTOR_GROUPS + .iter() + .flat_map(|group| group.iter()) + .copied() + .any(|descriptor| validate_descriptor(&tokens, descriptor)) +} + +#[cfg(test)] +#[path = "command_summary_grammar/tests.rs"] +mod tests; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs new file mode 100644 index 0000000000..fbf079db91 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs @@ -0,0 +1,72 @@ +#[path = "descriptors/auth.rs"] +mod auth; +#[path = "descriptors/models.rs"] +mod models; +#[path = "descriptors/plugins_benchmark.rs"] +mod plugins_benchmark; +#[path = "descriptors/runtime.rs"] +mod runtime; +#[path = "descriptors/top_level.rs"] +mod top_level; + +#[derive(Clone, Copy)] +pub(super) enum RawKind { + Backend, + Mode, + None, +} + +#[derive(Clone, Copy)] +pub(super) struct Descriptor { + pub(super) path: &'static [&'static str], + pub(super) booleans: &'static [&'static str], + pub(super) redacted: &'static [&'static str], + pub(super) conflicts: &'static [&'static [&'static str]], + pub(super) has_port: bool, + pub(super) raw: RawKind, +} + +const NONE: &[&str] = &[]; +const JSON: &[&str] = &["--json"]; +const YES_JSON: &[&str] = &["--yes", "--json"]; +const REDACTED_NAME: &[&str] = &["name"]; +const NO_CONFLICTS: &[&[&str]] = &[]; + +pub(super) const GLOBAL_REDACTED: &[&str] = &["--join", "--root-relay", "--relay-auth"]; + +const fn descriptor( + path: &'static [&'static str], + booleans: &'static [&'static str], + redacted: &'static [&'static str], + has_port: bool, + raw: RawKind, +) -> Descriptor { + descriptor_with_conflicts(path, booleans, redacted, NO_CONFLICTS, has_port, raw) +} + +const fn descriptor_with_conflicts( + path: &'static [&'static str], + booleans: &'static [&'static str], + redacted: &'static [&'static str], + conflicts: &'static [&'static [&'static str]], + has_port: bool, + raw: RawKind, +) -> Descriptor { + Descriptor { + path, + booleans, + redacted, + conflicts, + has_port, + raw, + } +} + +pub(super) const DESCRIPTOR_GROUPS: &[&[Descriptor]] = &[ + top_level::DESCRIPTORS, + plugins_benchmark::PLUGIN_DESCRIPTORS, + models::DESCRIPTORS, + plugins_benchmark::BENCHMARK_DESCRIPTORS, + runtime::DESCRIPTORS, + auth::DESCRIPTORS, +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs new file mode 100644 index 0000000000..8758c3f408 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs @@ -0,0 +1,125 @@ +use super::{Descriptor, NONE, RawKind, descriptor, descriptor_with_conflicts}; + +const AUTH_INIT_FLAGS: &[&str] = &["--force", "--no-passphrase", "--keychain"]; +const AUTH_INIT_CONFLICTS: &[&[&str]] = &[&["--no-passphrase", "--keychain"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "auth", "init"], + AUTH_INIT_FLAGS, + &["--owner-key"], + AUTH_INIT_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "status"], + NONE, + &[ + "--owner-key", + "--node-key", + "--node-ownership", + "--trust-store", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "sign-node"], + NONE, + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "renew-node"], + NONE, + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "verify-node"], + NONE, + &[ + "--file", + "--node-id", + "--trust-store", + "--verify-trust-policy", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "rotate-node"], + &["--revoke-current"], + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + "--reason", + "--trust-store", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "revoke-owner"], + NONE, + &["owner_id", "--reason", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "revoke-node"], + NONE, + &["--cert-id", "--node-id", "--reason", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "rotate-owner"], + &["--no-passphrase", "--force"], + &["--owner-key"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "add"], + NONE, + &["owner_id", "--label", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "remove"], + NONE, + &["owner_id", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "list"], + NONE, + &["--trust-store"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs new file mode 100644 index 0000000000..04b5e3f288 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs @@ -0,0 +1,119 @@ +use super::{Descriptor, JSON, NONE, RawKind, YES_JSON, descriptor, descriptor_with_conflicts}; + +const MODEL_PACKAGE_FLAGS: &[&str] = &[ + "--experimental", + "--dry-run", + "--confirm", + "--follow", + "--list", + "--update-script", + "--json", +]; +const SEARCH_FLAGS: &[&str] = &["--gguf", "--mlx", "--catalog", "--json"]; +const MODEL_CERTIFY_FLAGS: &[&str] = &["--json", "--package-only"]; +const MODEL_SEARCH_CONFLICTS: &[&[&str]] = &[&["--gguf", "--mlx"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor( + &["mesh-llm", "models", "package"], + MODEL_PACKAGE_FLAGS, + &[ + "source_repo", + "--quant", + "--target", + "--model-id", + "--flavor", + "--timeout", + "--mesh-llm-ref", + "--status", + "--logs", + "--cancel", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "recommended"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "installed"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "cleanup"], + YES_JSON, + &["--unused-since"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "prune"], + YES_JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "certify"], + MODEL_CERTIFY_FLAGS, + &[ + "model", + "--report-out", + "--api-base", + "--prompt", + "--max-tokens", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "list"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "models", "search"], + SEARCH_FLAGS, + &["query", "--limit", "--sort"], + MODEL_SEARCH_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "show"], + JSON, + &["model"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "download"], + &["--draft", "--direct", "--json"], + &["model"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "updates"], + &["--all", "--check", "--json"], + &["repo"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "delete"], + YES_JSON, + &["model"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs new file mode 100644 index 0000000000..70502d6f0b --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs @@ -0,0 +1,122 @@ +use super::{Descriptor, NONE, REDACTED_NAME, RawKind, descriptor, descriptor_with_conflicts}; + +const PLUGIN_INSTALL_CONFLICTS: &[&[&str]] = &[&["reference", "--archive"]]; +const TUNE_FLAGS: &[&str] = &[ + "--json", + "--no-speculative-tune", + "--apply", + "--replace-existing", + "--launch-args", + "--debug-telemetry", +]; +const TUNE_CONFLICTS: &[&[&str]] = &[ + &["--model", "--models"], + &["--no-speculative-tune", "--speculative-types"], + &["--no-speculative-tune", "--spec-draft-models"], + &["--no-speculative-tune", "--spec-draft-max-tokens"], + &["--no-speculative-tune", "--spec-draft-min-tokens"], + &["--no-speculative-tune", "--spec-draft-acceptance-threshold"], + &["--no-speculative-tune", "--spec-draft-split-probability"], + &["--no-speculative-tune", "--spec-ngram-min"], + &["--no-speculative-tune", "--spec-ngram-max"], +]; + +pub(super) const PLUGIN_DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "plugins", "install"], + NONE, + &["reference", "--archive", "--name", "--version"], + PLUGIN_INSTALL_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "update"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "enable"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "disable"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "delete"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "info"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "search"], + NONE, + &["query"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "list"], + NONE, + NONE, + false, + RawKind::None, + ), +]; + +pub(super) const BENCHMARK_DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "benchmark", "tune"], + TUNE_FLAGS, + &[ + "--model", + "--models", + "--ctx-sizes", + "--batch-sizes", + "--ubatch-sizes", + "--mmap-values", + "--mlock-values", + "--flash-attention", + "--speculative-types", + "--spec-draft-models", + "--spec-draft-max-tokens", + "--spec-draft-min-tokens", + "--spec-ngram-min", + "--spec-ngram-max", + "--spec-draft-acceptance-threshold", + "--spec-draft-split-probability", + "--throughput-tolerance-pct", + "--max-tokens", + "--startup-timeout-secs", + "--request-timeout-secs", + "--prompt", + ], + TUNE_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "benchmark", "import-prompts"], + NONE, + &["--source", "--limit", "--max-tokens", "--output"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs new file mode 100644 index 0000000000..c28a5c8ce5 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs @@ -0,0 +1,96 @@ +use super::{Descriptor, JSON, NONE, RawKind, descriptor, descriptor_with_conflicts}; + +const RUNTIME_LIST_FLAGS: &[&str] = &["--available", "--installed", "--json"]; +const RUNTIME_PRUNE_FLAGS: &[&str] = &["--active-only", "--json"]; +const RUNTIME_LIST_CONFLICTS: &[&[&str]] = &[&["--available", "--installed"]]; +const REMOTE_MODEL_CONFLICTS: &[&[&str]] = &[&["--model", "--instance-id"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor(&["mesh-llm", "runtime"], NONE, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "runtime", "status"], + NONE, + NONE, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "load"], + NONE, + &["name"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "unload"], + NONE, + &["name"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "guardrails"], + JSON, + NONE, + true, + RawKind::Mode, + ), + descriptor( + &["mesh-llm", "runtime", "bootstrap"], + JSON, + NONE, + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "runtime", "list"], + RUNTIME_LIST_FLAGS, + &["--manifest", "--bundle-dir", "--cache-dir"], + RUNTIME_LIST_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "install"], + JSON, + &["runtime_ref", "--manifest", "--bundle-dir", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "remove"], + JSON, + &["native_runtime_id", "--mesh-version", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "prune"], + RUNTIME_PRUNE_FLAGS, + &["--mesh-version", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "remote"], + JSON, + &["--endpoint"], + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "runtime", "remote-model"], + JSON, + &["--endpoint", "--model", "--profile", "--instance-id"], + REMOTE_MODEL_CONFLICTS, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "apply-config"], + JSON, + &["--endpoint", "--expected-revision", "--config"], + true, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs new file mode 100644 index 0000000000..991b64d1b5 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs @@ -0,0 +1,189 @@ +use super::{ + Descriptor, JSON, NONE, REDACTED_NAME, RawKind, descriptor, descriptor_with_conflicts, +}; + +const SETUP_FLAGS: &[&str] = &[ + "--yes", + "--no-interactive", + "--service", + "--no-service", + "--skip-runtime", + "--verbose", +]; +const UNINSTALL_FLAGS: &[&str] = &[ + "--dry-run", + "--yes", + "--keep-cache", + "--keep-service-files", + "--purge-config", + "--keep-config", + "--json", + "--verbose", +]; +const MODEL_PREPARE_FLAGS: &[&str] = &[ + "--dry-run", + "--confirm", + "--follow", + "--json", + "--list", + "--update-script", +]; +const SETUP_CONFLICTS: &[&[&str]] = &[&["--service", "--no-service"]]; +const UNINSTALL_CONFLICTS: &[&[&str]] = &[&["--purge-config", "--keep-config"]]; +const UPDATE_CONFLICTS: &[&[&str]] = &[&["--flavor", "--detect-flavor"]]; +const SKILLS_INSTALL_CONFLICTS: &[&[&str]] = &[&["--agent", "--all"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "setup"], + SETUP_FLAGS, + NONE, + SETUP_CONFLICTS, + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "uninstall"], + UNINSTALL_FLAGS, + &["--binary-path"], + UNINSTALL_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "download"], + &["--draft"], + &["name"], + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "update"], + &["--detect-flavor"], + &["--version", "--flavor"], + UPDATE_CONFLICTS, + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "status"], NONE, NONE, true, RawKind::None), + descriptor( + &["mesh-llm", "load"], + NONE, + REDACTED_NAME, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "unload"], + NONE, + REDACTED_NAME, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "discover"], + &["--auto"], + &["--name", "--model", "--min-vram", "--region", "--relay"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "rotate-key"], + NONE, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "goose"], + NONE, + &["--model"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "claude"], + NONE, + &["--model"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "pi"], + &["--write"], + &["--model", "--host"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "opencode"], + &["--write"], + &["--model", "--host"], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "stop"], NONE, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "external-plugin"], + NONE, + &["argv"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "model-prepare"], + MODEL_PREPARE_FLAGS, + &[ + "source_repo", + "--quant", + "--target", + "--model-id", + "--flavor", + "--timeout", + "--mesh-llm-ref", + "--status", + "--logs", + "--cancel", + ], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "gpus"], JSON, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "gpus", "detect"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "gpus", "run-benchmark"], + JSON, + NONE, + false, + RawKind::Backend, + ), + descriptor( + &["mesh-llm", "config", "validate"], + JSON, + &["--config-path"], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "doctor"], JSON, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "doctor", "split"], + JSON, + &["--model-ref", "--output-dir"], + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "skills", "install"], + &["--all", "--dry-run", "--force"], + &["--agent"], + SKILLS_INSTALL_CONFLICTS, + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs b/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs new file mode 100644 index 0000000000..090946f488 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs @@ -0,0 +1,111 @@ +use super::vocabulary::is_static_summary_token; + +#[derive(Clone, Copy)] +struct CommandPath<'a> { + tokens: [Option<&'a str>; 4], + len: usize, + valid: bool, +} + +impl CommandPath<'_> { + fn matches(self, expected: &[&str]) -> bool { + self.valid + && self.len == expected.len() + && expected + .iter() + .enumerate() + .all(|(index, token)| self.tokens[index] == Some(*token)) + } +} + +pub(super) fn raw_option_is_allowed(tokens: &[&str], index: usize, option: &str) -> bool { + let prefix = &tokens[..index]; + match option { + "--backend" => matches_backend_prefix(prefix), + "--mode" => matches_mode_prefix(prefix), + "--port" => matches_port_prefix(prefix), + _ => false, + } +} + +fn command_path<'a>(tokens: &'a [&'a str], end: usize) -> CommandPath<'a> { + let mut path = CommandPath { + tokens: [None; 4], + len: 0, + valid: true, + }; + for token in &tokens[..end] { + if !is_static_summary_token(token) { + continue; + } + if path.len >= path.tokens.len() { + path.valid = false; + return path; + } + path.tokens[path.len] = Some(*token); + path.len += 1; + } + path +} + +fn matches_backend_prefix(prefix: &[&str]) -> bool { + let path = command_path(prefix, prefix.len()); + path.matches(&["mesh-llm", "gpus", "run-benchmark"]) + && prefix == ["mesh-llm", "gpus", "run-benchmark"] +} + +fn matches_mode_prefix(prefix: &[&str]) -> bool { + prefix == ["mesh-llm", "runtime", "guardrails"] +} + +fn matches_port_prefix(prefix: &[&str]) -> bool { + matches!( + prefix, + ["mesh-llm", "status"] + | ["mesh-llm", "load"] + | ["mesh-llm", "unload"] + | ["mesh-llm", "goose"] + | ["mesh-llm", "claude"] + | ["mesh-llm", "doctor", "split"] + | ["mesh-llm", "doctor", "split", "--json"] + | ["mesh-llm", "doctor", "split", "--json", "--json"] + | ["mesh-llm", "runtime", "status"] + | ["mesh-llm", "runtime", "load"] + | ["mesh-llm", "runtime", "unload"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "disabled"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "metrics"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "enforce"] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "disabled", + "--json", + ] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--json", + ] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "enforce", + "--json", + ] + | ["mesh-llm", "runtime", "bootstrap"] + | ["mesh-llm", "runtime", "bootstrap", "--json"] + | ["mesh-llm", "runtime", "remote"] + | ["mesh-llm", "runtime", "remote", "--json"] + | ["mesh-llm", "runtime", "remote-model"] + | ["mesh-llm", "runtime", "remote-model", "--json"] + | ["mesh-llm", "runtime", "apply-config"] + | ["mesh-llm", "runtime", "apply-config", "--json"] + ) +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/tests.rs b/crates/mesh-llm-events/src/command_summary_grammar/tests.rs new file mode 100644 index 0000000000..4be1eb992b --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/tests.rs @@ -0,0 +1,107 @@ +use super::descriptors::{DESCRIPTOR_GROUPS, RawKind}; +use super::is_safe_summary; +use super::raw_options::raw_option_is_allowed; +use super::vocabulary::{is_boolean_option, is_redacted_marker, is_static_summary_token}; + +#[test] +fn vocabulary_classifies_static_boolean_and_redacted_tokens() { + assert!(is_static_summary_token("mesh-llm")); + assert!(is_static_summary_token("run-benchmark")); + assert!(!is_static_summary_token("private-model")); + + assert!(is_boolean_option("--json")); + assert!(is_boolean_option("--no-speculative-tune")); + assert!(!is_boolean_option("--backend")); + + assert!(is_redacted_marker("model")); + assert!(is_redacted_marker("--relay-auth")); + assert!(!is_redacted_marker("[REDACTED]")); +} + +#[test] +fn raw_options_are_limited_to_candidate_producer_contexts() { + let backend = ["mesh-llm", "gpus", "run-benchmark", "--backend", "cuda"]; + let mode = ["mesh-llm", "runtime", "guardrails", "--mode", "metrics"]; + let port = ["mesh-llm", "doctor", "split", "--json", "--port", "41731"]; + + assert!(raw_option_is_allowed(&backend, 3, "--backend")); + assert!(raw_option_is_allowed(&mode, 3, "--mode")); + assert!(raw_option_is_allowed(&port, 4, "--port")); + assert!(!raw_option_is_allowed(&backend, 3, "--mode")); +} + +#[test] +fn benchmark_tune_rejects_each_speculative_option_when_tuning_is_disabled() { + let speculative_options = [ + "--speculative-types", + "--spec-draft-models", + "--spec-draft-max-tokens", + "--spec-draft-min-tokens", + "--spec-draft-acceptance-threshold", + "--spec-draft-split-probability", + "--spec-ngram-min", + "--spec-ngram-max", + ]; + + for option in speculative_options { + let summary = format!("mesh-llm benchmark tune --no-speculative-tune {option} [REDACTED]"); + assert!(!is_safe_summary(&summary), "accepted conflict: {option}"); + } +} + +#[test] +fn ports_accept_only_ascii_decimal_u16_values() { + for port in ["0", "1", "65535"] { + let summary = format!("mesh-llm status --port {port}"); + assert!(is_safe_summary(&summary), "rejected valid port: {port}"); + } + + for port in ["+1", "-1", "65536", "1.0", "١"] { + let summary = format!("mesh-llm status --port {port}"); + assert!(!is_safe_summary(&summary), "accepted invalid port: {port}"); + } +} + +#[test] +fn every_descriptor_accepts_its_complete_canonical_shape() { + for descriptor in DESCRIPTOR_GROUPS.iter().flat_map(|group| group.iter()) { + let mut prefix = descriptor.path.to_vec(); + match descriptor.raw { + RawKind::Backend => prefix.extend_from_slice(&["--backend", "cuda"]), + RawKind::Mode => prefix.extend_from_slice(&["--mode", "metrics"]), + RawKind::None => {} + } + let mut tokens = prefix.clone(); + tokens.extend_from_slice(descriptor.booleans); + if descriptor.has_port { + tokens.extend_from_slice(&["--port", "41731"]); + } + for marker in descriptor.redacted { + tokens.extend_from_slice(&[*marker, "[REDACTED]"]); + } + if descriptor + .conflicts + .iter() + .any(|pair| pair.iter().all(|flag| tokens.contains(flag))) + { + assert!(!is_safe_summary(&tokens.join(" "))); + continue; + } + let summary = tokens.join(" "); + if tokens.len() <= 32 && summary.chars().count() <= 256 { + assert!(is_safe_summary(&summary), "descriptor rejected: {summary}"); + } else { + for marker in descriptor.booleans.iter().chain(descriptor.redacted) { + let mut single = prefix.clone(); + single.push(*marker); + if descriptor.redacted.contains(marker) { + single.push("[REDACTED]"); + } + assert!( + is_safe_summary(&single.join(" ")), + "descriptor marker rejected: {marker}" + ); + } + } + } +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/validation.rs b/crates/mesh-llm-events/src/command_summary_grammar/validation.rs new file mode 100644 index 0000000000..08aa7578e2 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/validation.rs @@ -0,0 +1,123 @@ +use super::descriptors::{Descriptor, GLOBAL_REDACTED, RawKind}; +use super::raw_options::raw_option_is_allowed; +use super::vocabulary::{is_boolean_option, is_redacted_marker, is_static_summary_token}; + +#[derive(Clone, Copy)] +enum Phase { + Booleans, + Port, + Redacted, +} + +fn descriptor_matches(tokens: &[&str], descriptor: Descriptor) -> bool { + descriptor.path.len() <= tokens.len() + && descriptor + .path + .iter() + .enumerate() + .all(|(index, token)| is_static_summary_token(token) && tokens[index] == *token) +} + +fn raw_value_is_valid(raw: RawKind, tokens: &[&str], index: usize) -> Option { + let (option, valid_values): (&str, &[&str]) = match raw { + RawKind::Backend => ("--backend", &["metal", "cuda", "hip", "intel"]), + RawKind::Mode => ("--mode", &["disabled", "metrics", "enforce"]), + RawKind::None => return Some(index), + }; + (tokens.get(index) == Some(&option) + && tokens + .get(index + 1) + .is_some_and(|value| valid_values.contains(value))) + .then_some(index + 2) +} + +pub(super) fn validate_descriptor(tokens: &[&str], descriptor: Descriptor) -> bool { + if !descriptor_matches(tokens, descriptor) { + return false; + } + let Some(mut index) = raw_value_is_valid(descriptor.raw, tokens, descriptor.path.len()) else { + return false; + }; + let mut phase = Phase::Booleans; + let mut seen_booleans = Vec::new(); + let mut seen_redacted = Vec::new(); + let mut seen_tokens = Vec::new(); + let mut port_seen = false; + let mut global_phase = false; + let mut last_global_rank = 0; + + while index < tokens.len() { + let Some(token) = tokens.get(index) else { + return false; + }; + if is_boolean_option(token) { + if !matches!(phase, Phase::Booleans) + || !descriptor.booleans.contains(token) + || seen_booleans.contains(token) + { + return false; + } + seen_booleans.push(*token); + seen_tokens.push(*token); + index += 1; + continue; + } + if *token == "--port" { + if matches!(phase, Phase::Redacted) + || port_seen + || !descriptor.has_port + || !raw_option_is_allowed(tokens, index, token) + { + return false; + } + let Some(value) = tokens.get(index + 1) else { + return false; + }; + if value.is_empty() + || !value.bytes().all(|byte| byte.is_ascii_digit()) + || value.parse::().is_err() + { + return false; + } + port_seen = true; + phase = Phase::Port; + index += 2; + continue; + } + if is_redacted_marker(token) { + let is_global = GLOBAL_REDACTED.contains(token); + if global_phase && !is_global { + return false; + } + if is_global { + let global_rank = match *token { + "--join" => 1, + "--root-relay" => 2, + "--relay-auth" => 3, + _ => 0, + }; + if global_rank <= last_global_rank { + return false; + } + global_phase = true; + last_global_rank = global_rank; + } + if (!descriptor.redacted.contains(token) && !GLOBAL_REDACTED.contains(token)) + || seen_redacted.contains(token) + || tokens.get(index + 1) != Some(&"[REDACTED]") + { + return false; + } + seen_redacted.push(*token); + seen_tokens.push(*token); + phase = Phase::Redacted; + index += 2; + continue; + } + return false; + } + !descriptor + .conflicts + .iter() + .any(|pair| pair.iter().all(|flag| seen_tokens.contains(flag))) +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs b/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs new file mode 100644 index 0000000000..c3e048c7ce --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs @@ -0,0 +1,205 @@ +pub(super) fn is_static_summary_token(token: &str) -> bool { + matches!( + token, + "mesh-llm" + | "load" + | "unload" + | "status" + | "discover" + | "rotate-key" + | "goose" + | "claude" + | "pi" + | "opencode" + | "stop" + | "external-plugin" + | "setup" + | "uninstall" + | "gpus" + | "detect" + | "run-benchmark" + | "config" + | "validate" + | "doctor" + | "split" + | "skills" + | "install" + | "plugins" + | "update" + | "enable" + | "disable" + | "delete" + | "info" + | "search" + | "list" + | "models" + | "package" + | "recommended" + | "installed" + | "cleanup" + | "prune" + | "certify" + | "show" + | "download" + | "updates" + | "benchmark" + | "tune" + | "import-prompts" + | "model-prepare" + | "runtime" + | "guardrails" + | "bootstrap" + | "remove" + | "remote" + | "remote-model" + | "apply-config" + | "auth" + | "init" + | "sign-node" + | "renew-node" + | "verify-node" + | "rotate-node" + | "revoke-owner" + | "revoke-node" + | "rotate-owner" + | "trust" + | "add" + ) +} + +pub(super) fn is_boolean_option(token: &str) -> bool { + matches!( + token, + "--draft" + | "--detect-flavor" + | "--yes" + | "--no-interactive" + | "--service" + | "--no-service" + | "--skip-runtime" + | "--verbose" + | "--dry-run" + | "--keep-cache" + | "--keep-config" + | "--keep-service-files" + | "--json" + | "--auto" + | "--write" + | "--all" + | "--force" + | "--experimental" + | "--confirm" + | "--follow" + | "--list" + | "--update-script" + | "--gguf" + | "--mlx" + | "--catalog" + | "--direct" + | "--check" + | "--no-speculative-tune" + | "--apply" + | "--replace-existing" + | "--launch-args" + | "--debug-telemetry" + | "--available" + | "--installed" + | "--active-only" + | "--keychain" + | "--no-passphrase" + | "--revoke-current" + | "--package-only" + | "--purge-config" + ) +} + +pub(super) fn is_redacted_marker(token: &str) -> bool { + matches!( + token, + "argv" + | "model" + | "name" + | "native_runtime_id" + | "owner_id" + | "query" + | "reference" + | "repo" + | "runtime_ref" + | "source_repo" + | "--version" + | "--flavor" + | "--binary-path" + | "--name" + | "--model" + | "--min-vram" + | "--region" + | "--relay" + | "--root-relay" + | "--join" + | "--relay-auth" + | "--sort" + | "--host" + | "--config-path" + | "--model-ref" + | "--output-dir" + | "--agent" + | "--archive" + | "--quant" + | "--target" + | "--model-id" + | "--timeout" + | "--mesh-llm-ref" + | "--status" + | "--logs" + | "--cancel" + | "--unused-since" + | "--report-out" + | "--api-base" + | "--prompt" + | "--max-tokens" + | "--limit" + | "--models" + | "--ctx-sizes" + | "--batch-sizes" + | "--ubatch-sizes" + | "--mmap-values" + | "--mlock-values" + | "--flash-attention" + | "--speculative-types" + | "--spec-draft-models" + | "--spec-draft-max-tokens" + | "--spec-draft-min-tokens" + | "--spec-ngram-min" + | "--spec-ngram-max" + | "--spec-draft-acceptance-threshold" + | "--spec-draft-split-probability" + | "--throughput-tolerance-pct" + | "--startup-timeout-secs" + | "--request-timeout-secs" + | "--source" + | "--output" + | "--manifest" + | "--bundle-dir" + | "--cache-dir" + | "--mesh-version" + | "--endpoint" + | "--profile" + | "--instance-id" + | "--expected-revision" + | "--config" + | "--owner-key" + | "--node-key" + | "--node-ownership" + | "--trust-store" + | "--out" + | "--hostname-hint" + | "--node-label" + | "--expires-in-hours" + | "--file" + | "--node-id" + | "--verify-trust-policy" + | "--reason" + | "--cert-id" + | "--label" + ) +} diff --git a/crates/mesh-llm-events/src/lib.rs b/crates/mesh-llm-events/src/lib.rs index 0d05646684..ad373ac962 100644 --- a/crates/mesh-llm-events/src/lib.rs +++ b/crates/mesh-llm-events/src/lib.rs @@ -14,7 +14,8 @@ pub mod terminal_progress; mod command_lifecycle; pub use audit::*; pub use command_lifecycle::{ - CliCommandFamily, CliCommandOutcome, emit_cli_command_event, set_cli_command_event_verbose, + CliCommandFamily, CliCommandOutcome, CliCommandSummary, emit_cli_command_event, + set_cli_command_event_verbose, }; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] diff --git a/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs b/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs index f4a9f35612..e1af365038 100644 --- a/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs @@ -41,36 +41,43 @@ impl ManagementLifecycleContext { } pub(super) fn request_id_from_raw(raw: &[u8]) -> RequestId { - let value = crate::api::access::request_header(raw, "x-request-id") + let mut headers = [httparse::EMPTY_HEADER; 64]; + let mut request = httparse::Request::new(&mut headers); + let Ok(httparse::Status::Complete(_)) = request.parse(raw) else { + return RequestId::default(); + }; + let mut request_id_headers = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")); + let Some(header) = request_id_headers.next() else { + return RequestId::default(); + }; + if request_id_headers.next().is_some() { + return RequestId::default(); + } + std::str::from_utf8(header.value) .ok() - .flatten() - .and_then(openai_frontend::parse_request_id); - value.unwrap_or_default() + .and_then(openai_frontend::parse_request_id) + .unwrap_or_default() } pub(super) fn eligible_management_route(method: &str, path: &str) -> bool { - !(is_read_only_observation_method(method) - || path == "/models" - || path.starts_with("/v1/") - || path == "/api/chat" - || path == "/api/responses" - || path == "/api/objects" - || path.starts_with("/api/objects/") - || path == "/mesh/hook" - || path == "/api/logs" - || path.starts_with("/api/logs/")) + is_mutation_method(method) + && path != "/models" + && !path.starts_with("/v1/") + && path != "/api/chat" + && path != "/api/responses" + && path != "/api/objects" + && !path.starts_with("/api/objects/") + && path != "/mesh/hook" + && path != "/api/logs" + && !path.starts_with("/api/logs/") && (path.starts_with("/api/") || path == "/mcp") } -/// Read-only console observation must never become workload ledger traffic. -/// -/// These endpoints are frequently polled or held open by the local console. -/// Recording them as ordinary management work makes request counts describe -/// the observer rather than the workload it is observing. This applies to all -/// safe observation methods, including future console endpoints. Mutations -/// continue through the management lifecycle and remain auditable. -fn is_read_only_observation_method(method: &str) -> bool { - matches!(method, "GET" | "HEAD" | "OPTIONS") +fn is_mutation_method(method: &str) -> bool { + matches!(method, "POST" | "PUT" | "PATCH" | "DELETE") } pub(super) fn method_route_label(method: &str, path: &str) -> &'static str { @@ -190,6 +197,20 @@ mod tests { } } + #[test] + fn duplicate_request_ids_are_replaced_instead_of_selecting_the_first() { + let first = "00000000-0000-4000-8000-000000000051"; + let second = "00000000-0000-4000-8000-000000000052"; + let raw = format!( + "POST /api/runtime/config/validate HTTP/1.1\r\nHost: localhost\r\nx-request-id: {first}\r\nX-Request-Id: {second}\r\nContent-Length: 0\r\n\r\n" + ); + + let request_id = request_id_from_raw(raw.as_bytes()); + + assert_ne!(request_id.as_uuid().to_string(), first); + assert_ne!(request_id.as_uuid().to_string(), second); + } + #[tokio::test] async fn trusted_local_and_server_failures_use_bounded_terminal_states() { let (_, rejected) = lifecycle(RequestId::new()); @@ -234,32 +255,45 @@ mod tests { } #[test] - fn logs_openai_and_all_read_only_management_routes_are_excluded() { - for path in [ - "/api/logs/requests", - "/api/logs/events", - "/api/status", - "/api/events", - "/api/runtime/events", + fn management_lifecycle_routes_are_mutation_only() { + let mutation_methods = ["POST", "PUT", "PATCH", "DELETE"]; + let observation_methods = ["GET", "HEAD", "OPTIONS", "TRACE", "CONNECT"]; + let management_paths = ["/api/runtime/control", "/api/plugins/install", "/mcp"]; + let excluded_paths = [ "/v1/chat/completions", "/models", "/api/chat", "/api/responses", "/api/objects", + "/api/objects/item", + "/api/logs", + "/api/logs/requests", "/mesh/hook", - ] { - assert!( - !eligible_management_route("GET", path), - "{path} must stay excluded" - ); + ]; + + for method in mutation_methods { + for path in management_paths { + assert!( + eligible_management_route(method, path), + "{method} {path} must be eligible" + ); + } + } + for method in observation_methods { + for path in management_paths { + assert!( + !eligible_management_route(method, path), + "{method} {path} must be excluded" + ); + } + } + for method in mutation_methods.into_iter().chain(observation_methods) { + for path in excluded_paths { + assert!( + !eligible_management_route(method, path), + "{method} {path} must stay excluded" + ); + } } - assert!(!eligible_management_route("GET", "/api/models")); - assert!(!eligible_management_route("HEAD", "/api/plugins")); - assert!(!eligible_management_route( - "OPTIONS", - "/api/runtime/control" - )); - assert!(eligible_management_route("POST", "/api/runtime/control")); - assert!(eligible_management_route("POST", "/api/status")); } } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs index 529ad8a0a0..bec384ca9d 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs @@ -236,7 +236,9 @@ fn prepare_terminal_delete( }); } match facade.request(&request_id)? { - Some(record) if record.outcome == "active" => return Err(LogsError::ActiveRequest), + Some(record) if record.request.outcome == "active" => { + return Err(LogsError::ActiveRequest); + } Some(_) => {} None => return Err(LogsError::NotFound), } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs index 1b3b5db1d9..4d5321d774 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs @@ -2,12 +2,14 @@ use base64::Engine; use mesh_llm_events::logging::envelope::CanonicalEnvelope; use mesh_llm_events::logging::events::{LifecycleEvent, TokenUsage}; use mesh_llm_log_store::{ - ArtifactContent, ArtifactRecord, AuditEntryRow, AuditEntrySeverity, EventRecord, ProxyRecord, - RequestRecord, + ArtifactContent, ArtifactRecord, AuditEntryDetail, AuditEntrySeverity, EventRecord, + ProxyRecord, RequestRecordWithCaller, }; use serde::Serialize; use std::collections::BTreeMap; +use mesh_llm_events::CliCommandSummary; + use super::{LogsError, event_kind}; use crate::logging::RequestSummaryEntry; @@ -30,37 +32,75 @@ pub(crate) struct RequestDto { provider: Option, engine: Option, status_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_path_type: Option, source: &'static str, } impl RequestDto { - pub(super) fn durable(record: RequestRecord) -> Self { + pub(super) fn durable(record: RequestRecordWithCaller) -> Self { + let request = record.request; Self { - request_id: record.request_id, - outcome: record.outcome, - created_at: record.created_at, - terminal_at: record.terminal_at, - route: record.route.as_deref().map(safe_metadata), - model: record.model.as_deref().map(safe_metadata), - provider: record.provider.as_deref().map(safe_metadata), - engine: record.engine.as_deref().map(safe_metadata), - status_code: record.status_code, + request_id: request.request_id, + outcome: request.outcome, + created_at: request.created_at, + terminal_at: request.terminal_at, + route: request.route.as_deref().map(safe_metadata), + model: request.model.as_deref().map(safe_metadata), + provider: request.provider.as_deref().map(safe_metadata), + engine: request.engine.as_deref().map(safe_metadata), + status_code: request.status_code, + caller_endpoint_id: record.caller_endpoint_id.as_deref().map(safe_metadata), + caller_addr: record.caller_addr.as_deref().map(safe_metadata), + caller_path_type: record.caller_path_type.as_deref().map(safe_metadata), source: "durable", } } - pub(super) fn active(entry: RequestSummaryEntry, metadata: Option) -> Self { + pub(super) fn active( + entry: RequestSummaryEntry, + metadata: Option, + ) -> Self { let summary_metadata = entry.metadata.clone(); let metadata = metadata.map(|record| { ( - record.route.as_deref().map(safe_metadata), - record.model.as_deref().map(safe_metadata), - record.provider.as_deref().map(safe_metadata), - record.engine.as_deref().map(safe_metadata), - record.status_code, + record.request.route.as_deref().map(safe_metadata), + record.request.model.as_deref().map(safe_metadata), + record.request.provider.as_deref().map(safe_metadata), + record.request.engine.as_deref().map(safe_metadata), + record.request.status_code, + record.caller_endpoint_id.as_deref().map(safe_metadata), + record.caller_addr.as_deref().map(safe_metadata), + record.caller_path_type.as_deref().map(safe_metadata), ) }); - let (route, model, provider, engine, status_code) = metadata.unwrap_or_default(); + let ( + route, + model, + provider, + engine, + status_code, + caller_endpoint_id, + caller_addr, + caller_path_type, + ) = metadata.unwrap_or_default(); + let active_caller = ( + summary_metadata.caller_endpoint_id().map(safe_metadata), + summary_metadata.caller_addr().map(safe_metadata), + summary_metadata.caller_path_type().map(str::to_owned), + ); + let caller = if active_caller.0.is_some() + || active_caller.1.is_some() + || active_caller.2.is_some() + { + active_caller + } else { + (caller_endpoint_id, caller_addr, caller_path_type) + }; Self { request_id: entry.request_id, outcome: entry.state, @@ -71,6 +111,9 @@ impl RequestDto { provider: summary_metadata.provider().map(safe_metadata).or(provider), engine: summary_metadata.engine().map(safe_metadata).or(engine), status_code, + caller_endpoint_id: caller.0, + caller_addr: caller.1, + caller_path_type: caller.2, source: "active", } } @@ -314,6 +357,10 @@ pub(crate) struct AuditDto { #[serde(skip_serializing_if = "Option::is_none")] subject_id: Option, #[serde(skip_serializing_if = "Option::is_none")] + remote_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] operation_id: Option, #[serde(skip_serializing_if = "Option::is_none")] request_id: Option, @@ -322,13 +369,16 @@ pub(crate) struct AuditDto { #[serde(skip_serializing_if = "Option::is_none")] outcome: Option, #[serde(skip_serializing_if = "Option::is_none")] + command_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] duration_ms: Option, #[serde(skip_serializing_if = "BTreeMap::is_empty")] numeric_summaries: BTreeMap, } -impl From for AuditDto { - fn from(row: AuditEntryRow) -> Self { +impl From for AuditDto { + fn from(detail: AuditEntryDetail) -> Self { + let row = detail.entry; Self { sequence: row.sequence, entry_id: row.entry_id, @@ -343,10 +393,16 @@ impl From for AuditDto { context_version: row.context_version, subject_kind: row.subject_kind, subject_id: row.subject_id.as_deref().map(safe_metadata), + remote_addr: detail.remote_addr, + path_type: detail.path_type, operation_id: row.operation_id.as_deref().map(safe_metadata), request_id: row.correlation_request_id.as_deref().map(safe_metadata), reason_code: row.reason_code, outcome: row.outcome, + command_summary: detail + .command_summary + .and_then(|summary| CliCommandSummary::sanitize(&summary)) + .map(|summary| summary.as_str().to_owned()), duration_ms: row.duration_ms, numeric_summaries: row.numeric_summaries, } @@ -426,7 +482,7 @@ mod usage_tests { } } -fn safe_metadata(value: &str) -> String { +pub(super) fn safe_metadata(value: &str) -> String { let trimmed = value.trim(); let path_shaped = trimmed.starts_with('/') || trimmed.starts_with("~/") @@ -445,6 +501,43 @@ mod tests { envelope::CanonicalEnvelope, identifiers::{EventId, RequestId}, }; + use mesh_llm_log_store::{LogStore, RealClock, RequestRecord}; + use std::sync::Arc; + + fn detailed_request( + request: RequestRecord, + caller: (Option<&str>, Option<&str>, Option<&str>), + ) -> RequestRecordWithCaller { + let root = tempfile::tempdir().expect("temporary detailed request store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open request store"); + let insert_result = match caller { + (None, None, None) => store.upsert_summary_metadata( + &request.request_id, + request.model.as_deref(), + request.route.as_deref(), + request.provider.as_deref(), + request.engine.as_deref(), + &request.created_at, + ), + (caller_endpoint_id, caller_addr, caller_path_type) => store + .upsert_summary_metadata_with_caller( + &request.request_id, + request.model.as_deref(), + request.route.as_deref(), + request.provider.as_deref(), + request.engine.as_deref(), + caller_endpoint_id, + caller_addr, + caller_path_type, + &request.created_at, + ), + }; + insert_result.expect("insert detailed request"); + store + .query_request_with_caller(&request.request_id) + .expect("query detailed request") + .expect("detailed request") + } #[test] fn event_dto_projects_authoritative_usage_and_terminal_status() { @@ -531,20 +624,27 @@ mod tests { #[test] fn metadata_and_proxy_dtos_do_not_leak_paths_or_credentials() { - let dto = RequestDto::durable(RequestRecord { - request_id: uuid::Uuid::new_v4().to_string(), - outcome: "completed".into(), - created_at: "2026-01-01T00:00:00Z".into(), - terminal_at: None, - route: Some("/private/secret".into()), - model: Some("Bearer secret-token".into()), - provider: None, - engine: None, - status_code: None, - }); + let dto = RequestDto::durable(detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "completed".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: Some("/private/secret".into()), + model: Some("Bearer secret-token".into()), + provider: None, + engine: None, + status_code: None, + }, + (None, None, None), + )); let json = serde_json::to_string(&dto).expect("serialize dto"); assert!(!json.contains("secret-token")); assert!(!json.contains("/private/secret")); + let json: serde_json::Value = serde_json::from_str(&json).expect("request DTO JSON"); + assert!(json.get("callerEndpointId").is_none()); + assert!(json.get("callerAddr").is_none()); + assert!(json.get("callerPathType").is_none()); let proxy = ProxyDto::from(ProxyRecord { attempt_id: uuid::Uuid::new_v4().to_string(), @@ -563,6 +663,130 @@ mod tests { assert!(!json.contains("token=secret")); } + #[test] + fn active_and_durable_request_dtos_expose_local_http_socket_addresses() { + for caller_addr in [ + "127.0.0.1:40123", + "192.0.2.10:40123", + "[2001:db8::10]:40123", + ] { + let metadata = crate::logging::RequestSummaryMetadata::default().with_caller_identity( + None, + Some(caller_addr), + Some(crate::logging::CallerPathType::LocalHttp), + ); + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata, + }, + None, + ); + let durable = RequestDto::durable(detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "completed".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: Some("2026-01-01T00:00:01Z".into()), + route: None, + model: None, + provider: None, + engine: None, + status_code: Some(200), + }, + (None, Some(caller_addr), Some("local_http")), + )); + + for json in [ + serde_json::to_value(active).expect("active DTO"), + serde_json::to_value(durable).expect("durable DTO"), + ] { + assert_eq!(json["callerAddr"], caller_addr); + assert_eq!(json["callerPathType"], "local_http"); + assert!(json.get("callerEndpointId").is_none()); + } + } + } + + #[test] + fn active_caller_tuple_is_selected_atomically_before_durable_fallback() { + let endpoint_id = "abababababababababababababababababababababababababababababababab"; + let active_metadata = crate::logging::RequestSummaryMetadata::default() + .with_caller_identity(Some(endpoint_id), None, None); + let durable_local = detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: None, + model: None, + provider: None, + engine: None, + status_code: None, + }, + (None, Some("127.0.0.1:40123"), Some("local_http")), + ); + + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata: active_metadata, + }, + Some(durable_local), + ); + let json = serde_json::to_value(active).expect("active DTO"); + + assert_eq!(json["callerEndpointId"], endpoint_id); + assert!(json.get("callerAddr").is_none()); + assert!(json.get("callerPathType").is_none()); + } + + #[test] + fn active_request_without_caller_uses_complete_durable_caller_tuple() { + let endpoint_id = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac"; + let durable = detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: None, + model: None, + provider: None, + engine: None, + status_code: None, + }, + ( + Some(endpoint_id), + Some("192.0.2.44:11204"), + Some("remote_quic_http"), + ), + ); + + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata: crate::logging::RequestSummaryMetadata::default(), + }, + Some(durable), + ); + let json = serde_json::to_value(active).expect("active DTO"); + + assert_eq!(json["callerEndpointId"], endpoint_id); + assert_eq!(json["callerAddr"], "192.0.2.44:11204"); + assert_eq!(json["callerPathType"], "remote_quic_http"); + } + #[test] fn intentionally_omitted_artifact_is_unavailable_not_missing() { let record = ArtifactRecord { diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs index 3d7afa2441..c351018c37 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs @@ -123,7 +123,7 @@ impl LogsError { Self::SchemaIncompatible { .. } => ( 503, "logging_schema_incompatible", - "the local log database schema is incompatible with this MeshLLM version", + "the local log database schema is unknown or incompatible", ), }; let details = match self { @@ -223,12 +223,12 @@ mod tests { fn schema_incompatibility_preserves_only_the_version_pair() { assert_eq!( LogsError::from(mesh_llm_log_store::LogStoreError::SchemaIncompatible { - found: 14, - supported: 11, + found: 2, + supported: 1, }), LogsError::SchemaIncompatible { - found: 14, - supported: 11, + found: 2, + supported: 1, } ); } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs index c1dc724846..606cd7d69b 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs @@ -1,15 +1,15 @@ use mesh_llm_events::logging::envelope::CanonicalEnvelope; use mesh_llm_events::logging::events::LifecycleEvent; use mesh_llm_events::logging::replay::ReplayChannel; -use mesh_llm_log_store::{AuditEntryRow, AuditEntrySeverity}; use serde::Serialize; -use std::collections::BTreeMap; use super::super::event_kind; use super::query::{AuditCursor, Cursor}; -use crate::logging::OperationalAuditContext; +use crate::logging::ReplayRecord; use crate::logging::RequestSummaryEventSnapshots; -use crate::logging::{AuditReplayRecord, ReplayRecord}; + +mod audit_entry; +pub(super) use audit_entry::{audit_entry_frame, durable_audit_entry_frame}; pub(in crate::api::routes::logs) const MAX_FRAME_BYTES: usize = 16 * 1024; @@ -56,6 +56,12 @@ struct PublicRequest { model: Option, provider: Option, engine: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_path_type: Option, status_code: Option, source: &'static str, } @@ -150,6 +156,9 @@ fn request_projection( model: metadata.model().map(str::to_owned), provider: metadata.provider().map(str::to_owned), engine: metadata.engine().map(str::to_owned), + caller_endpoint_id: metadata.caller_endpoint_id().map(str::to_owned), + caller_addr: metadata.caller_addr().map(str::to_owned), + caller_path_type: metadata.caller_path_type().map(str::to_owned), status_code: lifecycle_status_code(&envelope.event), source: "active", }) @@ -222,164 +231,6 @@ fn frame(event: &str, id: &str, data: &T) -> Result { (frame.len() <= MAX_FRAME_BYTES).then_some(frame).ok_or(()) } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct AuditEntryData { - entry_id: String, - occurred_at: String, - source: String, - code: String, - #[serde(skip_serializing_if = "Option::is_none")] - severity: Option, - #[serde(skip_serializing_if = "Option::is_none")] - context_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - subject_kind: Option, - #[serde(skip_serializing_if = "Option::is_none")] - subject_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - operation_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - request_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - reason_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - outcome: Option, - #[serde(skip_serializing_if = "Option::is_none")] - duration_ms: Option, - #[serde(skip_serializing_if = "BTreeMap::is_empty")] - numeric_summaries: BTreeMap, - sequence: u64, -} - -/// Audit entry frame: privacy-safe projection of an audit replay record. -/// Never contains `canonical_envelope` or arbitrary `detail_json`. -pub(super) fn audit_entry_frame(record: &AuditReplayRecord) -> Result { - let payload: serde_json::Value = serde_json::from_str(&record.entry.payload).map_err(|_| ())?; - let entry_id = payload - .get("entry_id") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let occurred_at = payload - .get("occurred_at") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let source = payload - .get("source") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let code = payload - .get("code") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let severity = payload - .get("severity") - .and_then(|v| v.as_str()) - .map(str::to_owned); - let context_version = payload - .get("context_version") - .and_then(serde_json::Value::as_u64) - .and_then(|value| u8::try_from(value).ok()) - .filter(|value| *value == 1); - let context_string = |key: &str| { - context_version.and_then(|_| { - payload - .get(key) - .and_then(serde_json::Value::as_str) - .filter(|value| !value.is_empty() && value.chars().count() <= 256) - .map(str::to_owned) - }) - }; - let context_code = |key: &str| { - context_version.and_then(|_| { - payload - .get(key) - .and_then(serde_json::Value::as_str) - .filter(|value| OperationalAuditContext::valid_static_code(value)) - .map(str::to_owned) - }) - }; - let subject_kind = context_version.and_then(|_| { - payload - .get("subject_kind") - .and_then(serde_json::Value::as_str) - .and_then(crate::logging::OperationalAuditSubjectKind::parse) - .map(|kind| kind.as_str().to_owned()) - }); - let numeric_summaries = context_version.map_or_else(BTreeMap::new, |_| { - payload - .get("numeric_summaries") - .and_then(serde_json::Value::as_object) - .into_iter() - .flatten() - .filter(|(key, _)| OperationalAuditContext::valid_static_code(key)) - .filter_map(|(key, value)| value.as_u64().map(|value| (key.clone(), value))) - .take(8) - .collect() - }); - - let data = AuditEntryData { - entry_id, - occurred_at, - source, - code, - severity, - context_version, - subject_kind, - subject_id: context_string("subject_id"), - operation_id: context_string("operation_id"), - request_id: context_string("request_id"), - reason_code: context_code("reason_code"), - outcome: context_code("outcome"), - duration_ms: context_version.and_then(|_| { - payload - .get("duration_ms") - .and_then(serde_json::Value::as_u64) - }), - numeric_summaries, - sequence: record.sequence, - }; - frame( - "audit_entry", - &AuditCursor(record.sequence).event_id(), - &data, - ) -} - -/// Render a row read back from the durable store. This is the production audit -/// reconciliation path and deliberately uses the database sequence as the SSE -/// cursor so entries written by another process share the same ordering. -pub(super) fn durable_audit_entry_frame(record: AuditEntryRow) -> Result { - let sequence = u64::try_from(record.sequence).map_err(|_| ())?; - let severity = record.severity.map(|severity| match severity { - AuditEntrySeverity::Info => "info".to_string(), - AuditEntrySeverity::Warning => "warning".to_string(), - AuditEntrySeverity::Error => "error".to_string(), - }); - let data = AuditEntryData { - entry_id: record.entry_id, - occurred_at: record.occurred_at, - source: record.source, - code: record.code, - severity, - context_version: record.context_version, - subject_kind: record.subject_kind, - subject_id: record.subject_id, - operation_id: record.operation_id, - request_id: record.correlation_request_id, - reason_code: record.reason_code, - outcome: record.outcome, - duration_ms: record.duration_ms, - numeric_summaries: record.numeric_summaries, - sequence, - }; - frame("audit_entry", &AuditCursor(sequence).event_id(), &data) -} - /// Audit gap frame: points to `/api/logs/audit` for recovery. pub(super) fn audit_gap_frame( from_sequence: u64, @@ -393,109 +244,6 @@ pub(super) fn audit_gap_frame( #[cfg(test)] mod tests { use super::*; - use crate::logging::BusEntry; - - fn audit_record(sequence: u64) -> AuditReplayRecord { - AuditReplayRecord { - entry: BusEntry { - payload: serde_json::json!({ - "kind": "audit", - "entry_id": "test-entry-id", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "runtime", - "code": "startup_complete", - "severity": "info", - }) - .to_string(), - channel_hint: 2, - }, - sequence, - cursor: sequence, - } - } - - #[test] - fn audit_entry_frame_shape_and_fields() { - let record = audit_record(7); - let frame = audit_entry_frame(&record).expect("audit entry frame"); - - assert!(frame.contains("event: audit_entry")); - assert!(frame.contains("id: a1:7")); - assert!(frame.contains("\"entryId\":\"test-entry-id\"")); - assert!(frame.contains("\"occurredAt\":\"2026-01-01T00:00:00Z\"")); - assert!(frame.contains("\"source\":\"runtime\"")); - assert!(frame.contains("\"code\":\"startup_complete\"")); - assert!(frame.contains("\"severity\":\"info\"")); - assert!(frame.contains("\"sequence\":7")); - assert!(!frame.contains("canonical_envelope")); - assert!(!frame.contains("detail_json")); - assert!(frame.len() <= MAX_FRAME_BYTES); - } - - #[test] - fn audit_entry_frame_omits_severity_when_none() { - let mut record = audit_record(3); - record.entry.payload = serde_json::json!({ - "kind": "audit", - "entry_id": "id-3", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "cli", - "code": "command_executed", - }) - .to_string(); - let frame = audit_entry_frame(&record).expect("audit entry without severity"); - assert!(!frame.contains("severity")); - } - - #[test] - fn audit_entry_frame_filters_invalid_typed_context_fields() { - let mut record = audit_record(4); - record.entry.payload = serde_json::json!({ - "kind": "audit", - "entry_id": "id-4", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "runtime", - "code": "startup_complete", - "context_version": 1, - "subject_kind": "not_a_subject", - "reason_code": "NOT VALID", - "outcome": "completed", - "numeric_summaries": { - "bad-key": 0, - "metric_0": 0, - "metric_1": 1, - "metric_2": 2, - "metric_3": 3, - "metric_4": 4, - "metric_5": 5, - "metric_6": 6, - "metric_7": 7, - "metric_8": 8 - } - }) - .to_string(); - let frame = audit_entry_frame(&record).expect("audit context frame"); - let data = frame - .lines() - .find_map(|line| line.strip_prefix("data: ")) - .and_then(|line| serde_json::from_str::(line).ok()) - .expect("audit frame data"); - - assert!(data.get("subjectKind").is_none()); - assert!(data.get("reasonCode").is_none()); - assert_eq!(data["outcome"], "completed"); - assert_eq!(data["numericSummaries"].as_object().unwrap().len(), 8); - assert!(data["numericSummaries"].get("metric_7").is_some()); - assert!(data["numericSummaries"].get("metric_8").is_none()); - assert!(data["numericSummaries"].get("bad-key").is_none()); - } - - #[test] - fn audit_entry_frame_rejects_malformed_payload() { - let mut record = audit_record(1); - record.entry.payload = "not-json".to_string(); - assert!(audit_entry_frame(&record).is_err()); - } #[test] fn audit_gap_frame_carries_audit_endpoint() { diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs new file mode 100644 index 0000000000..8e88d14a84 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs @@ -0,0 +1,233 @@ +use mesh_llm_events::CliCommandSummary; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntrySeverity}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::net::SocketAddr; + +use super::super::query::AuditCursor; +use crate::api::routes::logs::dto::safe_metadata; +use crate::logging::{AuditReplayRecord, OperationalAuditContext}; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct AuditEntryData { + entry_id: String, + occurred_at: String, + source: String, + code: String, + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + context_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + subject_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + subject_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + remote_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path_type: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + operation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reason_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + outcome: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + numeric_summaries: BTreeMap, + sequence: u64, +} + +#[derive(Default)] +struct AuditPathProjection { + remote_addr: Option, + path_type: Option<&'static str>, +} + +fn audit_path_projection( + context_version: Option, + path_type: Option<&str>, + remote_addr: Option<&str>, +) -> AuditPathProjection { + if context_version != Some(1) { + return AuditPathProjection::default(); + } + match path_type { + Some("direct") => AuditPathProjection { + remote_addr: remote_addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + path_type: Some("direct"), + }, + Some("relay") => AuditPathProjection { + remote_addr: None, + path_type: Some("relay"), + }, + Some(_) | None => AuditPathProjection::default(), + } +} + +/// Audit entry frame: privacy-safe projection of an audit replay record. +/// Never contains `canonical_envelope` or arbitrary `detail_json`. +pub(in super::super) fn audit_entry_frame(record: &AuditReplayRecord) -> Result { + let payload: serde_json::Value = serde_json::from_str(&record.entry.payload).map_err(|_| ())?; + let entry_id = payload + .get("entry_id") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let occurred_at = payload + .get("occurred_at") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let source = payload + .get("source") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let code = payload + .get("code") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let severity = payload + .get("severity") + .and_then(|v| v.as_str()) + .map(str::to_owned); + let context_version = payload + .get("context_version") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .filter(|value| *value == 1); + let context_string = |key: &str| { + context_version.and_then(|_| { + payload + .get(key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty() && value.chars().count() <= 256) + .map(str::to_owned) + }) + }; + let context_code = |key: &str| { + context_version.and_then(|_| { + payload + .get(key) + .and_then(serde_json::Value::as_str) + .filter(|value| OperationalAuditContext::valid_static_code(value)) + .map(str::to_owned) + }) + }; + let subject_kind = context_version.and_then(|_| { + payload + .get("subject_kind") + .and_then(serde_json::Value::as_str) + .and_then(crate::logging::OperationalAuditSubjectKind::parse) + .map(|kind| kind.as_str().to_owned()) + }); + let numeric_summaries = context_version.map_or_else(BTreeMap::new, |_| { + payload + .get("numeric_summaries") + .and_then(serde_json::Value::as_object) + .into_iter() + .flatten() + .filter(|(key, _)| OperationalAuditContext::valid_static_code(key)) + .filter_map(|(key, value)| value.as_u64().map(|value| (key.clone(), value))) + .take(8) + .collect() + }); + let path = audit_path_projection( + context_version, + payload.get("path_type").and_then(serde_json::Value::as_str), + payload + .get("remote_addr") + .and_then(serde_json::Value::as_str), + ); + + let data = AuditEntryData { + entry_id, + occurred_at, + source, + code, + severity, + context_version, + subject_kind, + subject_id: context_string("subject_id"), + remote_addr: path.remote_addr, + path_type: path.path_type, + operation_id: context_string("operation_id"), + request_id: context_string("request_id"), + reason_code: context_code("reason_code"), + outcome: context_code("outcome"), + command_summary: context_version.and_then(|_| { + payload + .get("command_summary") + .and_then(serde_json::Value::as_str) + .and_then(CliCommandSummary::sanitize) + .map(|summary| summary.as_str().to_owned()) + }), + duration_ms: context_version.and_then(|_| { + payload + .get("duration_ms") + .and_then(serde_json::Value::as_u64) + }), + numeric_summaries, + sequence: record.sequence, + }; + super::frame( + "audit_entry", + &AuditCursor(record.sequence).event_id(), + &data, + ) +} + +/// Render a row read back from the durable store. This is the production audit +/// reconciliation path and deliberately uses the database sequence as the SSE +/// cursor so entries written by another process share the same ordering. +pub(in super::super) fn durable_audit_entry_frame(detail: AuditEntryDetail) -> Result { + let record = detail.entry; + let sequence = u64::try_from(record.sequence).map_err(|_| ())?; + let path = audit_path_projection( + record.context_version, + detail.path_type.as_deref(), + detail.remote_addr.as_deref(), + ); + let severity = record.severity.map(|severity| match severity { + AuditEntrySeverity::Info => "info".to_string(), + AuditEntrySeverity::Warning => "warning".to_string(), + AuditEntrySeverity::Error => "error".to_string(), + }); + let data = AuditEntryData { + entry_id: record.entry_id, + occurred_at: record.occurred_at, + source: record.source, + code: record.code, + severity, + context_version: record.context_version, + subject_kind: record.subject_kind, + subject_id: record.subject_id.as_deref().map(safe_metadata), + remote_addr: path.remote_addr, + path_type: path.path_type, + operation_id: record.operation_id.as_deref().map(safe_metadata), + request_id: record.correlation_request_id.as_deref().map(safe_metadata), + reason_code: record.reason_code, + outcome: record.outcome, + command_summary: detail + .command_summary + .and_then(|summary| CliCommandSummary::sanitize(&summary)) + .map(|summary| summary.as_str().to_owned()), + duration_ms: record.duration_ms, + numeric_summaries: record.numeric_summaries, + sequence, + }; + super::frame("audit_entry", &AuditCursor(sequence).event_id(), &data) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs new file mode 100644 index 0000000000..d2abbcf110 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs @@ -0,0 +1,344 @@ +use super::*; +use crate::logging::{AuditReplayRecord, BusEntry}; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntryFilters, LogStore, RealClock}; +use std::sync::Arc; + +use super::super::MAX_FRAME_BYTES; + +fn audit_record(sequence: u64) -> AuditReplayRecord { + AuditReplayRecord { + entry: BusEntry { + payload: serde_json::json!({ + "kind": "audit", + "entry_id": "test-entry-id", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "runtime", + "code": "startup_complete", + "severity": "info", + }) + .to_string(), + channel_hint: 2, + }, + sequence, + cursor: sequence, + } +} + +fn durable_audit_detail(entry_id: &str, detail: serde_json::Value) -> AuditEntryDetail { + let root = tempfile::tempdir().expect("temporary durable audit store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit store"); + store + .insert_audit_entry( + entry_id, + None, + "2026-01-01T00:00:00Z", + "cli", + "command_completed", + Some(&detail.to_string()), + ) + .expect("insert durable audit entry"); + store + .list_audit_entry_details_after_sequence(0, 1, AuditEntryFilters::default()) + .expect("query durable audit entry") + .into_iter() + .next() + .expect("durable audit entry") +} + +fn frame_data(frame: &str) -> serde_json::Value { + frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .and_then(|line| serde_json::from_str::(line).ok()) + .expect("audit frame data") +} + +#[test] +fn audit_entry_frame_shape_and_fields() { + let record = audit_record(7); + let frame = audit_entry_frame(&record).expect("audit entry frame"); + + assert!(frame.contains("event: audit_entry")); + assert!(frame.contains("id: a1:7")); + assert!(frame.contains("\"entryId\":\"test-entry-id\"")); + assert!(frame.contains("\"occurredAt\":\"2026-01-01T00:00:00Z\"")); + assert!(frame.contains("\"source\":\"runtime\"")); + assert!(frame.contains("\"code\":\"startup_complete\"")); + assert!(frame.contains("\"severity\":\"info\"")); + assert!(frame.contains("\"sequence\":7")); + assert!(!frame.contains("canonical_envelope")); + assert!(!frame.contains("detail_json")); + assert!(frame.len() <= MAX_FRAME_BYTES); +} + +#[test] +fn audit_entry_frame_omits_severity_when_none() { + let mut record = audit_record(3); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-3", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_executed", + }) + .to_string(); + let frame = audit_entry_frame(&record).expect("audit entry without severity"); + assert!(!frame.contains("severity")); +} + +#[test] +fn audit_entry_frame_filters_invalid_typed_context_fields() { + let mut record = audit_record(4); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-4", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "runtime", + "code": "startup_complete", + "context_version": 1, + "subject_kind": "not_a_subject", + "reason_code": "NOT VALID", + "outcome": "completed", + "numeric_summaries": { + "bad-key": 0, + "metric_0": 0, + "metric_1": 1, + "metric_2": 2, + "metric_3": 3, + "metric_4": 4, + "metric_5": 5, + "metric_6": 6, + "metric_7": 7, + "metric_8": 8 + } + }) + .to_string(); + let frame = audit_entry_frame(&record).expect("audit context frame"); + let data = frame_data(&frame); + + assert!(data.get("subjectKind").is_none()); + assert!(data.get("reasonCode").is_none()); + assert_eq!(data["outcome"], "completed"); + assert_eq!(data["numericSummaries"].as_object().unwrap().len(), 8); + assert!(data["numericSummaries"].get("metric_7").is_some()); + assert!(data["numericSummaries"].get("metric_8").is_none()); + assert!(data["numericSummaries"].get("bad-key").is_none()); +} + +#[test] +fn live_audit_path_projection_is_privacy_filtered() { + let cases = [ + ( + 1, + "direct", + Some("[2001:0db8:0:0:0:0:0:1]:443"), + Some("direct"), + Some("[2001:db8::1]:443"), + ), + ( + 1, + "relay", + Some("private-relay.example:443"), + Some("relay"), + None, + ), + ( + 1, + "direct", + Some("private-hostname:443"), + Some("direct"), + None, + ), + (1, "unknown", Some("192.0.2.1:443"), None, None), + (2, "direct", Some("192.0.2.1:443"), None, None), + ]; + + for (context_version, path_type, remote_addr, expected_path, expected_addr) in cases { + let mut record = audit_record(20); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "mesh-path", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "mesh", + "code": "peer_path_observed", + "context_version": context_version, + "path_type": path_type, + "remote_addr": remote_addr, + "canonical_envelope": "must-not-leak", + "detail_json": "must-not-leak" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit path frame"); + let data = frame_data(&frame); + + assert_eq!( + data.get("pathType").and_then(|value| value.as_str()), + expected_path + ); + assert_eq!( + data.get("remoteAddr").and_then(|value| value.as_str()), + expected_addr + ); + assert!(data.get("canonical_envelope").is_none()); + assert!(data.get("detail_json").is_none()); + } +} + +#[test] +fn audit_entry_frame_rejects_malformed_payload() { + let mut record = audit_record(1); + record.entry.payload = "not-json".to_string(); + assert!(audit_entry_frame(&record).is_err()); +} + +#[test] +fn live_audit_frame_drops_malformed_command_summary() { + let mut record = audit_record(8); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-8", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm gpus --draft run-benchmark --backend cuda" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("private-model-name")); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_drops_duplicate_command_summary_flags() { + let mut record = audit_record(13); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-13", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm models list --json --json" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_drops_deep_malformed_command_summary() { + let mut record = audit_record(11); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-11", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm load unload status discover rotate-key setup --port 1234" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_preserves_valid_command_summary() { + let mut record = audit_record(10); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-10", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(frame.contains( + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + )); +} + +#[test] +fn durable_audit_frame_drops_malformed_command_summary() { + let record = durable_audit_detail( + "id-9", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm gpus --draft run-benchmark --backend cuda", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(!frame.contains("private-model-name")); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn durable_audit_frame_drops_deep_malformed_command_summary() { + let record = durable_audit_detail( + "id-12", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm load unload status discover rotate-key setup --port 1234", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn durable_audit_frame_preserves_valid_command_summary() { + let record = durable_audit_detail( + "id-11", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(frame.contains( + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + )); +} + +#[test] +fn durable_audit_frame_redacts_unsafe_rest_parity_metadata() { + let record = durable_audit_detail( + "id-14", + serde_json::json!({ + "context_version": 1, + "subject_id": "https://alice:subject-secret@example.test/model?api_key=subject-query", + "operation_id": "/Users/alice/private-operation", + "request_id": "request-7?token=request-secret", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + let data = frame_data(&frame); + + assert_eq!(data["subjectId"], "[REDACTED]"); + assert_eq!(data["operationId"], "[REDACTED]"); + assert_eq!(data["requestId"], "[REDACTED]"); + + let serialized = data.to_string(); + for unsafe_value in [ + "alice", + "subject-secret", + "subject-query", + "/Users/alice/private-operation", + "request-secret", + ] { + assert!(!serialized.contains(unsafe_value)); + } +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs index 5345c22771..69c68559cf 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs @@ -118,11 +118,11 @@ impl ReplaySession { pub(super) fn durable_audit_frames( &mut self, - records: Vec, + records: Vec, ) -> Vec { let mut frames = Vec::with_capacity(records.len()); for record in records { - let Ok(sequence) = u64::try_from(record.sequence) else { + let Ok(sequence) = u64::try_from(record.entry.sequence) else { continue; }; if sequence <= self.audit_cursor.sequence() { @@ -512,6 +512,10 @@ impl ConnectionQueue { } impl ConnectionReceiver { + pub(super) fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + pub(super) async fn recv(&mut self) -> Option { if self.cancelled.load(Ordering::Acquire) { return None; diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs index 69b768bdea..628b8a46f5 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs @@ -4,12 +4,16 @@ use mesh_llm_events::logging::events::LifecycleEvent; use mesh_llm_events::logging::identifiers::{EventId, RequestId}; use mesh_llm_events::logging::replay::ReplaySequence; +use super::super::protocol::MAX_FRAME_BYTES; use super::*; use crate::logging::{ - LoggingService, ReplayBus, RequestSummaryEntry, RequestSummaryEventSnapshots, + CallerPathType, LoggingService, ReplayBus, RequestSummaryEntry, RequestSummaryEventSnapshots, RequestSummaryMetadata, TerminalOutcome, }; +mod audit; +mod queue; + fn summary(created_at: &str, state: &str, metadata: RequestSummaryMetadata) -> RequestSummaryEntry { RequestSummaryEntry { request_id: "test-request-summary".into(), @@ -601,295 +605,88 @@ fn terminal_replay_uses_enriched_summary_metadata_for_filters() { } #[test] -fn heartbeat_is_an_sse_comment() { - assert_eq!(super::super::heartbeat_frame(), ": keepalive\n\n"); -} - -#[tokio::test] -async fn queue_bounds_slow_consumers_and_cancellation() { - let (queue, mut receiver) = ConnectionQueue::new(1); - queue.try_send("first".into()).expect("first fits"); - assert_eq!( - queue.try_send("second".into()), - Err(QueueError::SlowConsumer) - ); - assert_eq!(receiver.recv().await.as_deref(), Some("first")); - queue.cancel(); - assert_eq!( - queue.try_send("after-cancel".into()), - Err(QueueError::Cancelled) - ); - assert!(receiver.recv().await.is_none()); -} - -#[tokio::test] -async fn queue_times_out_a_slow_socket_writer_without_unbounded_growth() { - let (queue, _receiver) = ConnectionQueue::new(1); - queue.try_send("first".into()).unwrap(); - assert_eq!( - queue - .send_with_timeout("second".into(), std::time::Duration::from_millis(5)) - .await, - Err(QueueError::SlowConsumer) - ); -} - -fn audit_subscription(cursor: AuditCursor) -> Subscription { - Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(AuditSelection { - cursor, - source: None, - severity: None, - }), - } -} - -fn push_audit(bus: &ReplayBus, payload: &str) { - bus.push_audit_replay(payload.to_string(), 2); -} - -fn audit_payload( - entry_id: &str, - occurred_at: &str, - source: &str, - code: &str, - severity: Option<&str>, -) -> String { - let mut obj = serde_json::json!({ - "kind": "audit", - "entry_id": entry_id, - "occurred_at": occurred_at, - "source": source, - "code": code, - }); - if let Some(s) = severity { - obj.as_object_mut() - .unwrap() - .insert("severity".into(), serde_json::json!(s)); - } - obj.to_string() -} - -#[test] -fn audit_replay_emits_ordered_frames_with_a1_ids() { +fn lifecycle_replay_projects_canonical_caller_attribution() { let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload( - "id-1", - "2026-01-01T00:00:01Z", - "runtime", - "startup", - Some("info"), - ), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "mesh", "peer_joined", None), - ); - push_audit( - &bus, - &audit_payload( - "id-3", - "2026-01-01T00:00:03Z", - "cli", - "command", - Some("warning"), + let cases = [ + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.71:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.71:11204"), + Some("remote_quic_http"), ), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); - assert_eq!(frames.len(), 3); - assert!(frames[0].contains("event: audit_entry")); - assert!(frames[0].contains("id: a1:1")); - assert!(frames[1].contains("id: a1:2")); - assert!(frames[2].contains("id: a1:3")); - assert!(frames[0].contains("\"entryId\":\"id-1\"")); - assert!(frames[1].contains("\"entryId\":\"id-2\"")); - assert!(frames[2].contains("\"entryId\":\"id-3\"")); -} - -#[test] -fn audit_live_updates_are_incremental_and_advance_filtered_records() { - let bus = ReplayBus::new(4); - let mut selected = audit_subscription(AuditCursor(0)); - selected.audit.as_mut().unwrap().source = Some("runtime".into()); - let mut session = ReplaySession::new(selected); - let mut updates = bus.subscribe_updates(); - - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "ignored", None), - ); - let filtered = updates.try_recv().expect("filtered audit update"); - assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); - assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); - - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "selected", None), - ); - let update = updates.try_recv().expect("selected audit update"); - let frames = session.next_update_frames(&bus, &update, None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id: a1:2")); - assert!(frames[0].contains("\"entryId\":\"id-2\"")); -} - -#[test] -fn audit_replay_dedupes_records_at_or_below_cursor() { - let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), - ); - push_audit( - &bus, - &audit_payload("id-3", "2026-01-01T00:00:03Z", "runtime", "c", None), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(2)), None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id: a1:3")); - assert!(frames[0].contains("id-3")); -} - -#[test] -fn audit_replay_filters_by_source() { - let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "a", None), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), - ); - push_audit( - &bus, - &audit_payload("id-3", "2026-01-01T00:00:03Z", "mesh", "c", None), - ); - - let sel = AuditSelection { - cursor: AuditCursor(0), - source: Some("mesh".to_owned()), - severity: None, - }; - let sub = Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(sel.clone()), - }; - let frames = replay_frames(&bus, &sub, None); - assert_eq!(frames.len(), 2); - assert!(frames[0].contains("id-1")); - assert!(frames[1].contains("id-3")); - assert!(!frames[0].contains("id-2")); -} - -#[test] -fn audit_replay_filters_by_severity() { - let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", Some("info")), - ); - push_audit( - &bus, - &audit_payload( - "id-2", - "2026-01-01T00:00:02Z", - "runtime", - "b", - Some("warning"), + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"), + Some("192.0.2.72:11204"), + Some(CallerPathType::Relay), + ), + Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"), + None, + Some("relay"), ), - ); - push_audit( - &bus, - &audit_payload( - "id-3", - "2026-01-01T00:00:03Z", - "runtime", - "c", - Some("error"), + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"), + None, + None, + ), + Some("fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"), + None, + None, ), - ); - - let sel = AuditSelection { - cursor: AuditCursor(0), - source: None, - severity: Some("warning".to_owned()), - }; - let sub = Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(sel), - }; - let frames = replay_frames(&bus, &sub, None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id-2")); -} - -#[test] -fn audit_gap_emits_recovery_endpoint() { - let bus = ReplayBus::new(1); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), - ); - - let frames = replay_frames( - &bus, - &audit_subscription(AuditCursor(0)), - Some("a1:2".to_owned()), - ); - assert!(frames[0].contains("event: replay_gap")); - assert!(frames[0].contains("/api/logs/audit")); - assert!(frames[0].contains("id: a1:1")); - assert!(frames[1].contains("event: audit_entry")); - assert!(frames[1].contains("id: a1:2")); -} - -#[test] -fn audit_frames_never_contain_lifecycle_fields() { - let bus = ReplayBus::new(3); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); - assert_eq!(frames.len(), 1); - assert!(!frames[0].contains("canonical_envelope")); - assert!(!frames[0].contains("detail_json")); - assert!(!frames[0].contains("requestId")); + (RequestSummaryMetadata::default(), None, None, None), + ]; + + for (sequence, (metadata, endpoint_id, caller_addr, path_type)) in cases.into_iter().enumerate() + { + entry_with_metadata( + &bus, + ReplayChannel::Requests, + sequence as u64 + 1, + RequestId::new(), + metadata, + ); + let frame = replay_frames( + &bus, + &subscription(vec![ReplayChannel::Requests], Cursor::default()), + None, + ) + .pop() + .expect("caller attribution replay frame"); + let data = frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .and_then(|line| serde_json::from_str::(line).ok()) + .expect("lifecycle replay data"); + let request = &data["request"]; + + assert_eq!( + request + .get("callerEndpointId") + .and_then(|value| value.as_str()), + endpoint_id + ); + assert!(endpoint_id.is_some() || request.get("callerEndpointId").is_none()); + assert_eq!( + request.get("callerAddr").and_then(|value| value.as_str()), + caller_addr + ); + assert!(caller_addr.is_some() || request.get("callerAddr").is_none()); + assert_eq!( + request + .get("callerPathType") + .and_then(|value| value.as_str()), + path_type + ); + assert!(path_type.is_some() || request.get("callerPathType").is_none()); + } } #[test] -fn lifecycle_gap_regression_still_uses_requests_endpoint() { - let bus = ReplayBus::new(1); - entry(&bus, ReplayChannel::Requests, 1, RequestId::new()); - entry(&bus, ReplayChannel::Requests, 2, RequestId::new()); - - let frames = replay_frames( - &bus, - &subscription(vec![ReplayChannel::Requests], Cursor::default()), - Some("opaque-rest-cursor".into()), - ); - assert!(frames[0].contains("event: replay_gap")); - assert!(frames[0].contains("/api/logs/requests")); - assert!(!frames[0].contains("/api/logs/audit")); +fn heartbeat_is_an_sse_comment() { + assert_eq!(super::super::heartbeat_frame(), ": keepalive\n\n"); } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs new file mode 100644 index 0000000000..15488ee7c7 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs @@ -0,0 +1,378 @@ +use super::*; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntryFilters, LogStore, RealClock}; + +fn durable_path_details() -> Vec { + let root = tempfile::tempdir().expect("temporary durable audit store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit store"); + for (entry_id, subject_id, path_type, remote_addr) in [ + ( + "durable-direct", + "peer-direct", + "direct", + "[2001:0db8:0:0:0:0:0:1]:443", + ), + ( + "durable-relay", + "peer-relay", + "relay", + "private-relay.example:443", + ), + ] { + let detail = serde_json::json!({ + "context_version": 1, + "subject_kind": "mesh_peer", + "subject_id": subject_id, + "path_type": path_type, + "remote_addr": remote_addr, + }); + store + .insert_audit_entry( + entry_id, + None, + "2026-01-01T00:00:01Z", + "mesh", + "peer_path_observed", + Some(&detail.to_string()), + ) + .expect("insert durable audit entry"); + } + store + .list_audit_entry_details_after_sequence(0, 2, AuditEntryFilters::default()) + .expect("query durable audit entries") +} + +fn audit_subscription(cursor: AuditCursor) -> Subscription { + Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(AuditSelection { + cursor, + source: None, + severity: None, + }), + } +} + +fn push_audit(bus: &ReplayBus, payload: &str) { + bus.push_audit_replay(payload.to_string(), 2); +} + +fn audit_payload( + entry_id: &str, + occurred_at: &str, + source: &str, + code: &str, + severity: Option<&str>, +) -> String { + let mut obj = serde_json::json!({ + "kind": "audit", + "entry_id": entry_id, + "occurred_at": occurred_at, + "source": source, + "code": code, + }); + if let Some(s) = severity { + obj.as_object_mut() + .unwrap() + .insert("severity".into(), serde_json::json!(s)); + } + obj.to_string() +} + +#[test] +fn audit_replay_emits_ordered_frames_with_a1_ids() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload( + "id-1", + "2026-01-01T00:00:01Z", + "runtime", + "startup", + Some("info"), + ), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "mesh", "peer_joined", None), + ); + push_audit( + &bus, + &audit_payload( + "id-3", + "2026-01-01T00:00:03Z", + "cli", + "command", + Some("warning"), + ), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); + assert_eq!(frames.len(), 3); + assert!(frames[0].contains("event: audit_entry")); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[2].contains("id: a1:3")); + assert!(frames[0].contains("\"entryId\":\"id-1\"")); + assert!(frames[1].contains("\"entryId\":\"id-2\"")); + assert!(frames[2].contains("\"entryId\":\"id-3\"")); +} + +#[test] +fn audit_replay_session_projects_direct_and_relay_paths_without_raw_fields() { + let bus = ReplayBus::new(4); + for (entry_id, path_type, remote_addr) in [ + ("direct-entry", "direct", "192.0.2.44:11204"), + ("relay-entry", "relay", "private-relay.example:443"), + ] { + push_audit( + &bus, + &serde_json::json!({ + "kind": "audit", + "entry_id": entry_id, + "occurred_at": "2026-01-01T00:00:01Z", + "source": "mesh", + "code": "peer_path_observed", + "context_version": 1, + "path_type": path_type, + "remote_addr": remote_addr, + "canonical_envelope": "must-not-leak", + "detail_json": "must-not-leak" + }) + .to_string(), + ); + } + let mut session = ReplaySession::new(audit_subscription(AuditCursor(0))); + + let frames = session.next_frames(&bus, None); + + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[0].contains("\"pathType\":\"direct\"")); + assert!(frames[0].contains("\"remoteAddr\":\"192.0.2.44:11204\"")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[1].contains("\"pathType\":\"relay\"")); + assert!(!frames[1].contains("remoteAddr")); + assert!(!frames[1].contains("private-relay.example")); + assert!( + frames + .iter() + .all(|frame| !frame.contains("canonical_envelope")) + ); + assert!(frames.iter().all(|frame| !frame.contains("detail_json"))); + assert!(frames.iter().all(|frame| !frame.contains("must-not-leak"))); + assert!(frames.iter().all(|frame| frame.len() <= MAX_FRAME_BYTES)); +} + +#[test] +fn durable_audit_reconciliation_projects_paths_and_advances_cursor() { + let records = durable_path_details(); + let direct = records[0].clone(); + let relay = records[1].clone(); + let mut session = ReplaySession::new(audit_subscription(AuditCursor(0))); + + let frames = session.durable_audit_frames(vec![direct.clone(), relay.clone()]); + + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[0].contains("\"pathType\":\"direct\"")); + assert!(frames[0].contains("\"remoteAddr\":\"[2001:db8::1]:443\"")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[1].contains("\"pathType\":\"relay\"")); + assert!(!frames[1].contains("remoteAddr")); + assert!(!frames[1].contains("private-relay.example")); + assert!( + frames + .iter() + .all(|frame| !frame.contains("canonical_envelope")) + ); + assert!(frames.iter().all(|frame| !frame.contains("detail_json"))); + assert!(session.durable_audit_frames(vec![direct, relay]).is_empty()); + assert_eq!( + session.durable_audit_query().map(|(cursor, _)| cursor), + Some(2) + ); +} + +#[test] +fn audit_live_updates_are_incremental_and_advance_filtered_records() { + let bus = ReplayBus::new(4); + let mut selected = audit_subscription(AuditCursor(0)); + selected.audit.as_mut().unwrap().source = Some("runtime".into()); + let mut session = ReplaySession::new(selected); + let mut updates = bus.subscribe_updates(); + + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "ignored", None), + ); + let filtered = updates.try_recv().expect("filtered audit update"); + assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); + assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); + + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "selected", None), + ); + let update = updates.try_recv().expect("selected audit update"); + let frames = session.next_update_frames(&bus, &update, None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id: a1:2")); + assert!(frames[0].contains("\"entryId\":\"id-2\"")); +} + +#[test] +fn audit_replay_dedupes_records_at_or_below_cursor() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + push_audit( + &bus, + &audit_payload("id-3", "2026-01-01T00:00:03Z", "runtime", "c", None), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(2)), None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id: a1:3")); + assert!(frames[0].contains("id-3")); +} + +#[test] +fn audit_replay_filters_by_source() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + push_audit( + &bus, + &audit_payload("id-3", "2026-01-01T00:00:03Z", "mesh", "c", None), + ); + + let sel = AuditSelection { + cursor: AuditCursor(0), + source: Some("mesh".to_owned()), + severity: None, + }; + let sub = Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(sel.clone()), + }; + let frames = replay_frames(&bus, &sub, None); + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id-1")); + assert!(frames[1].contains("id-3")); + assert!(!frames[0].contains("id-2")); +} + +#[test] +fn audit_replay_filters_by_severity() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", Some("info")), + ); + push_audit( + &bus, + &audit_payload( + "id-2", + "2026-01-01T00:00:02Z", + "runtime", + "b", + Some("warning"), + ), + ); + push_audit( + &bus, + &audit_payload( + "id-3", + "2026-01-01T00:00:03Z", + "runtime", + "c", + Some("error"), + ), + ); + + let sel = AuditSelection { + cursor: AuditCursor(0), + source: None, + severity: Some("warning".to_owned()), + }; + let sub = Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(sel), + }; + let frames = replay_frames(&bus, &sub, None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id-2")); +} + +#[test] +fn audit_gap_emits_recovery_endpoint() { + let bus = ReplayBus::new(1); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + + let frames = replay_frames( + &bus, + &audit_subscription(AuditCursor(0)), + Some("a1:2".to_owned()), + ); + assert!(frames[0].contains("event: replay_gap")); + assert!(frames[0].contains("/api/logs/audit")); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[1].contains("event: audit_entry")); + assert!(frames[1].contains("id: a1:2")); +} + +#[test] +fn audit_frames_never_contain_lifecycle_fields() { + let bus = ReplayBus::new(3); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); + assert_eq!(frames.len(), 1); + assert!(!frames[0].contains("canonical_envelope")); + assert!(!frames[0].contains("detail_json")); + assert!(!frames[0].contains("requestId")); +} + +#[test] +fn lifecycle_gap_regression_still_uses_requests_endpoint() { + let bus = ReplayBus::new(1); + entry(&bus, ReplayChannel::Requests, 1, RequestId::new()); + entry(&bus, ReplayChannel::Requests, 2, RequestId::new()); + + let frames = replay_frames( + &bus, + &subscription(vec![ReplayChannel::Requests], Cursor::default()), + Some("opaque-rest-cursor".into()), + ); + assert!(frames[0].contains("event: replay_gap")); + assert!(frames[0].contains("/api/logs/requests")); + assert!(!frames[0].contains("/api/logs/audit")); +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs new file mode 100644 index 0000000000..2ba10e6ef0 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs @@ -0,0 +1,30 @@ +use super::*; + +#[tokio::test] +async fn queue_bounds_slow_consumers_and_cancellation() { + let (queue, mut receiver) = ConnectionQueue::new(1); + queue.try_send("first".into()).expect("first fits"); + assert_eq!( + queue.try_send("second".into()), + Err(QueueError::SlowConsumer) + ); + assert_eq!(receiver.recv().await.as_deref(), Some("first")); + queue.cancel(); + assert_eq!( + queue.try_send("after-cancel".into()), + Err(QueueError::Cancelled) + ); + assert!(receiver.recv().await.is_none()); +} + +#[tokio::test] +async fn queue_times_out_a_slow_socket_writer_without_unbounded_growth() { + let (queue, _receiver) = ConnectionQueue::new(1); + queue.try_send("first".into()).unwrap(); + assert_eq!( + queue + .send_with_timeout("second".into(), std::time::Duration::from_millis(5)) + .await, + Err(QueueError::SlowConsumer) + ); +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs index fb4140504f..9b14e4a5fd 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs @@ -59,14 +59,14 @@ async fn run( query_facade, subscription, recovery_cursor, - queue.clone(), + queue, updates, )); while let Some(frame) = receiver.recv().await { let write = tokio::time::timeout(WRITE_TIMEOUT, stream.write_all(frame.as_bytes())).await; if !matches!(write, Ok(Ok(()))) { - queue.cancel(); + receiver.cancel(); break; } } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs index 4c7c86cb8f..5e53d6e91c 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; -use mesh_llm_log_store::{ArtifactRecord, EventRecord, LogStoreError, RequestRecord}; +use mesh_llm_log_store::{ArtifactRecord, EventRecord, LogStoreError, RequestRecordWithCaller}; use serde::Serialize; use tokio::net::TcpStream; @@ -90,7 +90,7 @@ fn build_export( let request_ids = page .items .iter() - .map(|record| record.request_id.clone()) + .map(|record| record.request.request_id.clone()) .collect::>(); // Each item may consume at most the complete export row budget. Fetch one // extra child per owner so a partial child window remains detectable while @@ -122,7 +122,7 @@ fn build_export( set_resume_cursor(&mut export); break; } - let request_id = record.request_id.clone(); + let request_id = record.request.request_id.clone(); let built = export_item( ExportItemInput { record, @@ -219,7 +219,7 @@ fn set_resume_cursor(export: &mut ExportDto) { } struct ExportItemInput { - record: RequestRecord, + record: RequestRecordWithCaller, event_records: Vec, artifact_records: Vec, child_row_limit: usize, diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs index 2c683ef1fc..32e0e46237 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs @@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet}; pub(crate) use dto::{ArtifactDto, AuditDto, EventDto, PageDto, ProxyDto, RequestDto}; pub(crate) use error::LogsError; use mesh_llm_events::logging::events::LifecycleEvent; -use mesh_llm_log_store::{ArtifactRecord, LogStoreError, QuerySort, RequestRecord}; +use mesh_llm_log_store::{ArtifactRecord, LogStoreError, QuerySort, RequestRecordWithCaller}; use self::dto::artifact_state; use self::parse::SourceFilter; @@ -601,7 +601,7 @@ fn load_active_metadata( active: &[RequestSummaryEntry], parsed: &parse::RequestListQuery, cursor_is_active: bool, -) -> Result, LogsError> { +) -> Result, LogsError> { let request_ids = if parsed.source != Some(SourceFilter::Durable) { active .iter() @@ -622,14 +622,14 @@ fn load_active_metadata( Ok(facade .requests_by_ids(&request_ids)? .into_iter() - .map(|record| (record.request_id.clone(), record)) + .map(|record| (record.request.request_id.clone(), record)) .collect()) } fn validate_cursor_scope( facade: &LoggingQueryFacade, active: &[RequestSummaryEntry], - active_metadata: &HashMap, + active_metadata: &HashMap, parsed: &parse::RequestListQuery, cursor_is_active: bool, ) -> Result<(), LogsError> { @@ -673,7 +673,7 @@ fn collect_durable( active_ids: &HashSet, parsed: &parse::RequestListQuery, cursor_is_active: bool, -) -> Result, LogsError> { +) -> Result, LogsError> { let mut query = parsed.store.clone(); if cursor_is_active { // An active-only cursor is not a durable row, so passing it to SQLite @@ -688,7 +688,9 @@ fn collect_durable( for _ in 0..max_pages { let page = facade.requests(&query)?; for record in page.items { - if !active_ids.contains(&record.request_id) && durable_after_cursor(&record, parsed) { + if !active_ids.contains(&record.request.request_id) + && durable_after_cursor(&record, parsed) + { result.push(record); } } @@ -700,11 +702,17 @@ fn collect_durable( Ok(result) } -fn durable_after_cursor(record: &RequestRecord, parsed: &parse::RequestListQuery) -> bool { +fn durable_after_cursor( + record: &RequestRecordWithCaller, + parsed: &parse::RequestListQuery, +) -> bool { let Some((timestamp, request_id)) = &parsed.cursor_boundary else { return true; }; - let row = (record.created_at.as_str(), record.request_id.as_str()); + let row = ( + record.request.created_at.as_str(), + record.request.request_id.as_str(), + ); let cursor = (timestamp.as_str(), request_id.as_str()); match parsed.store.sort { QuerySort::Ascending => row > cursor, @@ -714,7 +722,7 @@ fn durable_after_cursor(record: &RequestRecord, parsed: &parse::RequestListQuery fn active_matches( entry: &RequestSummaryEntry, - metadata: Option<&RequestRecord>, + metadata: Option<&RequestRecordWithCaller>, parsed: &parse::RequestListQuery, ) -> bool { if !active_matches_filters(entry, metadata, parsed) { @@ -733,16 +741,27 @@ fn active_matches( fn active_matches_filters( entry: &RequestSummaryEntry, - metadata: Option<&RequestRecord>, + metadata: Option<&RequestRecordWithCaller>, parsed: &parse::RequestListQuery, ) -> bool { let query = &parsed.store; - if let Some(route) = &query.route - && entry - .metadata - .route() - .or_else(|| metadata.and_then(|record| record.route.as_deref())) - != Some(route.as_str()) + let route = entry + .metadata + .route() + .or_else(|| metadata.and_then(|record| record.request.route.as_deref())); + if let Some(selected_route) = &query.route + && route != Some(selected_route.as_str()) + { + return false; + } + if query + .exclude_route + .as_deref() + .is_some_and(|excluded| route == Some(excluded)) + || query + .exclude_route_prefix + .as_deref() + .is_some_and(|prefix| route.is_some_and(|route| route.starts_with(prefix))) { return false; } @@ -750,7 +769,7 @@ fn active_matches_filters( && entry .metadata .model() - .or_else(|| metadata.and_then(|record| record.model.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.model.as_deref())) != Some(model.as_str()) { return false; @@ -759,7 +778,7 @@ fn active_matches_filters( && entry .metadata .provider() - .or_else(|| metadata.and_then(|record| record.provider.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.provider.as_deref())) != Some(provider.as_str()) { return false; @@ -768,13 +787,13 @@ fn active_matches_filters( && entry .metadata .engine() - .or_else(|| metadata.and_then(|record| record.engine.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.engine.as_deref())) != Some(engine.as_str()) { return false; } if let Some(status_code) = query.status_code - && metadata.and_then(|record| record.status_code) != Some(i64::from(status_code)) + && metadata.and_then(|record| record.request.status_code) != Some(i64::from(status_code)) { return false; } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs index 8db6f0640e..f7611e3ad8 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs @@ -34,6 +34,8 @@ pub(super) fn request_query(path: &str) -> Result { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, @@ -49,6 +51,8 @@ pub(super) fn request_query(path: &str) -> Result { "from" => query.from = Some(timestamp(&value)?), "to" => query.to = Some(timestamp(&value)?), "route" => query.route = Some(filter(value)?), + "exclude_route" => query.exclude_route = Some(filter(value)?), + "exclude_route_prefix" => query.exclude_route_prefix = Some(filter(value)?), "model" => query.model = Some(filter(value)?), "provider" => query.provider = Some(filter(value)?), "engine" => query.engine = Some(filter(value)?), @@ -593,6 +597,31 @@ mod tests { ); } + #[test] + fn parses_and_validates_request_route_exclusions() { + // Given / When + let query = request_query( + "/api/logs/requests?exclude_route=models&exclude_route_prefix=management_", + ) + .expect("parse route exclusions"); + + // Then + assert_eq!(query.store.exclude_route.as_deref(), Some("models")); + assert_eq!( + query.store.exclude_route_prefix.as_deref(), + Some("management_") + ); + for path in [ + "/api/logs/requests?exclude_route=", + "/api/logs/requests?exclude_route_prefix=%00", + ] { + assert!(matches!( + request_query(path), + Err(LogsError::InvalidQuery(_)) + )); + } + } + #[test] fn export_requires_a_bounded_reason_and_durable_scope() { assert!(matches!( diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs index 877e3a980f..844bcbb947 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs @@ -6,6 +6,8 @@ use openai_frontend::{ use super::*; +mod audit_sanitization; + #[test] fn export_route_classifies_before_method_validation() { let get_route = classify_mutating_route(Some("/api/logs/requests/export"), "GET") @@ -138,6 +140,91 @@ async fn list_merges_active_and_durable_without_duplicate_ids() { drop(active_guard); } +#[test] +fn route_exclusions_apply_to_every_merged_active_and_durable_page() { + // Given + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + for (request_id, route, created_at) in [ + ("durable-visible", Some("responses"), "2026-08-03T00:00:01Z"), + ( + "durable-hidden-management", + Some("management_get_status"), + "2026-08-03T00:00:02Z", + ), + ( + "durable-hidden-models", + Some("models"), + "2026-08-03T00:00:03Z", + ), + ] { + store + .insert_summary( + request_id, None, route, None, None, created_at, None, None, None, + ) + .expect("seed durable summary"); + } + let active = [ + ( + "active-hidden-management", + Some("management_post"), + "2026-08-03T00:00:04Z", + ), + ("active-visible-null", None, "2026-08-03T00:00:05Z"), + ( + "active-visible-chat", + Some("chat_completions"), + "2026-08-03T00:00:06Z", + ), + ( + "active-hidden-models", + Some("models"), + "2026-08-03T00:00:07Z", + ), + ] + .map(|(request_id, route, created_at)| RequestSummaryEntry { + request_id: request_id.to_string(), + state: "active".to_string(), + created_at: created_at.to_string(), + terminal_at: None, + metadata: crate::logging::RequestSummaryMetadata::from_parts(route, None, None, None), + }) + .to_vec(); + let facade = state.query_facade().expect("query facade"); + let base_path = + "/api/logs/requests?limit=1&exclude_route=models&exclude_route_prefix=management_"; + + // When + let first = list_requests_blocking( + facade.clone(), + active.clone(), + parse::request_query(base_path).expect("parse first page"), + ) + .expect("list first page"); + let first_cursor = first.next_cursor.expect("first page cursor"); + let second = list_requests_blocking( + facade.clone(), + active.clone(), + parse::request_query(&format!("{base_path}&cursor={first_cursor}")) + .expect("parse second page"), + ) + .expect("list second page"); + let second_cursor = second.next_cursor.expect("second page cursor"); + let third = list_requests_blocking( + facade, + active, + parse::request_query(&format!("{base_path}&cursor={second_cursor}")) + .expect("parse third page"), + ) + .expect("list third page"); + + // Then + assert_eq!(first.items[0].request_id(), "active-visible-chat"); + assert_eq!(second.items[0].request_id(), "active-visible-null"); + assert_eq!(third.items[0].request_id(), "durable-visible"); + assert!(third.next_cursor.is_none()); +} + #[test] fn active_request_time_bounds_compare_instants_within_the_boundary_second() { let (_temp, state) = runtime(); @@ -241,6 +328,11 @@ async fn active_request_uses_registered_metadata_before_durable_persistence() { None, None, None, + ) + .with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.42:11204"), + Some(crate::logging::CallerPathType::RemoteQuicHttp), ), ); service.merge_request_metadata( @@ -277,10 +369,79 @@ async fn active_request_uses_registered_metadata_before_durable_persistence() { assert_eq!(json["items"][0]["model"], "acme/model"); assert_eq!(json["items"][0]["provider"], "mesh"); assert_eq!(json["items"][0]["engine"], "raw_ingress"); + assert_eq!( + json["items"][0]["callerEndpointId"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert_eq!(json["items"][0]["callerAddr"], "192.0.2.42:11204"); + assert_eq!(json["items"][0]["callerPathType"], "remote_quic_http"); assert_eq!(json["items"][0]["source"], "active"); drop(guard); } +#[tokio::test] +async fn active_and_durable_api_json_expose_endpoint_only_caller_without_path_fields() { + let (_temp, state) = runtime(); + let service = state.service_for_test().expect("logging service"); + let active_request_id = RequestId::new(); + let durable_request_id = RequestId::new(); + let active_endpoint_id = "ad".repeat(32); + let durable_endpoint_id = "ae".repeat(32); + let (active_guard, _) = service.register_request_with_metadata( + active_request_id, + crate::logging::RequestSummaryMetadata::from_parts( + Some("responses"), + Some("active-model"), + None, + None, + ) + .with_caller_identity(Some(&active_endpoint_id), None, None), + ); + let (durable_guard, _) = service.register_request_with_metadata( + durable_request_id, + crate::logging::RequestSummaryMetadata::from_parts( + Some("responses"), + Some("durable-model"), + None, + None, + ) + .with_caller_identity(Some(&durable_endpoint_id), None, None), + ); + service + .transition_terminal( + durable_request_id, + &durable_guard, + crate::logging::TerminalOutcome::Completed, + ) + .expect("durable request terminalizes"); + assert!(service.pump_sync().await > 0); + + let active_page = list_requests( + &state, + "/api/logs/requests?source=active&model=active-model", + ) + .await + .expect("active endpoint-only request"); + let durable_page = list_requests( + &state, + "/api/logs/requests?source=durable&model=durable-model&outcome=completed", + ) + .await + .expect("durable endpoint-only request"); + + for (page, endpoint_id) in [ + (active_page, active_endpoint_id.as_str()), + (durable_page, durable_endpoint_id.as_str()), + ] { + let json = serde_json::to_value(page).expect("request page JSON"); + assert_eq!(json["items"].as_array().expect("items").len(), 1); + assert_eq!(json["items"][0]["callerEndpointId"], endpoint_id); + assert!(json["items"][0].get("callerAddr").is_none()); + assert!(json["items"][0].get("callerPathType").is_none()); + } + drop(active_guard); +} + #[tokio::test] async fn active_request_listing_batches_durable_metadata_fallback() { let (_temp, state) = runtime(); @@ -345,6 +506,11 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica None, None, None, + ) + .with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + None, + Some(crate::logging::CallerPathType::Relay), ), ); service.merge_request_metadata( @@ -388,6 +554,12 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica assert_eq!(json["items"][0]["model"], "acme/model"); assert_eq!(json["items"][0]["provider"], "mesh"); assert_eq!(json["items"][0]["engine"], "raw_ingress"); + assert_eq!( + json["items"][0]["callerEndpointId"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert!(json["items"][0].get("callerAddr").is_none()); + assert_eq!(json["items"][0]["callerPathType"], "relay"); assert_eq!(json["items"][0]["source"], "durable"); let absent_detail = request_detail(&state, &absent.as_uuid().to_string()) @@ -398,6 +570,9 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica assert_eq!(absent_json["model"], serde_json::Value::Null); assert_eq!(absent_json["provider"], serde_json::Value::Null); assert_eq!(absent_json["engine"], serde_json::Value::Null); + assert!(absent_json.get("callerEndpointId").is_none()); + assert!(absent_json.get("callerAddr").is_none()); + assert!(absent_json.get("callerPathType").is_none()); } #[tokio::test] @@ -638,6 +813,150 @@ async fn audit_list_exposes_typed_context_without_arbitrary_detail() { assert!(!json.to_string().contains("SENTINEL-AUDIT-SECRET")); } +#[tokio::test] +async fn audit_list_omits_malformed_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000024", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm gpus --draft run-benchmark --backend cuda"}"#, + ), + ) + .expect("seed malformed command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list malformed command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); + assert!(!json.to_string().contains("private-model-name")); +} + +#[tokio::test] +async fn audit_list_omits_duplicate_command_summary_flags() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000027", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm models list --json --json"}"#), + ) + .expect("seed duplicate command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list duplicate command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); +} + +#[tokio::test] +async fn audit_list_omits_deep_malformed_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000026", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm load unload status discover rotate-key setup --port 1234"}"#, + ), + ) + .expect("seed deep malformed command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list deep malformed command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); +} + +#[tokio::test] +async fn audit_list_preserves_valid_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000025", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]"}"#), + ) + .expect("seed valid command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list valid command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert_eq!( + json["items"][0]["commandSummary"], + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + ); +} + +#[tokio::test] +async fn audit_list_exposes_mesh_peer_direct_path_and_omits_relay_address() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000023", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-direct","remote_addr":"192.168.1.44:11204","path_type":"direct"}"#, + ), + ) + .expect("seed direct peer audit"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000024", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-relay","remote_addr":"203.0.113.10:443","path_type":"relay"}"#, + ), + ) + .expect("seed relay peer audit"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list peer audits"); + let json = serde_json::to_value(page).expect("serialize page"); + let direct = &json["items"][0]; + let relay = &json["items"][1]; + + assert_eq!(direct["subjectKind"], "mesh_peer"); + assert_eq!(direct["subjectId"], "peer-direct"); + assert_eq!(direct["remoteAddr"], "192.168.1.44:11204"); + assert_eq!(direct["pathType"], "direct"); + assert_eq!(relay["pathType"], "relay"); + assert!(relay.get("remoteAddr").is_none()); +} + #[tokio::test] async fn audit_filters_by_inclusive_canonical_time_bounds_before_pagination() { let (_temp, state) = runtime(); diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs new file mode 100644 index 0000000000..8dc145dead --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs @@ -0,0 +1,43 @@ +use super::*; + +#[tokio::test] +async fn audit_list_sanitizes_raw_actor_and_action_before_rest_serialization() { + // Given: a raw SQLite row that bypasses the typed write sanitizer. + let (_temp, state) = runtime(); + let raw_actor = "Bearer rest-raw-actor-secret"; + let raw_action = format!("REST-RAW-ACTION-SENTINEL-{}", "x".repeat(1_100)); + state + .store() + .expect("store") + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + ( + "00000000-0000-4000-8000-000000000099", + "2026-08-22T12:00:00.000000000Z", + raw_actor, + raw_action.as_str(), + r#"{"severity":"error"}"#, + ), + ) + .expect("insert raw audit row"); + + // When: the REST route reads and serializes the durable row. + let page = list_audits(&state, "/api/logs/audit?limit=1") + .await + .expect("list raw audit row"); + let json = serde_json::to_value(page).expect("serialize raw audit page"); + let row = &json["items"][0]; + + // Then: the API exposes only canonicalized scalar values. + assert_eq!(row["source"], "[REDACTED]"); + assert_eq!(row["severity"], "error"); + let code = row["code"].as_str().expect("audit code string"); + assert_eq!(code.chars().count(), 1_024); + assert!(code.ends_with("... [TRUNCATED]")); + let wire = json.to_string(); + assert!(!wire.contains(raw_actor)); + assert!(!wire.contains(&raw_action)); +} diff --git a/crates/mesh-llm-host-runtime/src/api/server.rs b/crates/mesh-llm-host-runtime/src/api/server.rs index a69a5c0ec8..4380de271a 100644 --- a/crates/mesh-llm-host-runtime/src/api/server.rs +++ b/crates/mesh-llm-host-runtime/src/api/server.rs @@ -240,6 +240,7 @@ pub(crate) async fn handle_request(mut stream: TcpStream, state: &MeshApi) -> an state.register_management_request( request_id, super::management_lifecycle::method_route_label(method, path_only), + source_addr, ) }) { return super::management_lifecycle::scope(lifecycle, dispatch).await; diff --git a/crates/mesh-llm-host-runtime/src/api/status.rs b/crates/mesh-llm-host-runtime/src/api/status.rs index 55b5b9eea8..19b382d734 100644 --- a/crates/mesh-llm-host-runtime/src/api/status.rs +++ b/crates/mesh-llm-host-runtime/src/api/status.rs @@ -1638,8 +1638,8 @@ mod tests { LoggingStatusPayload { metadata_available: false, metadata_state: "schema_incompatible", - schema_version: Some(14), - supported_schema_version: Some(11), + schema_version: Some(2), + supported_schema_version: Some(1), capture_mode: "unavailable", artifact_capture_available: false, artifact_capture_ready: false, diff --git a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs index 9d5bef4ccc..aa2ccfc9cd 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs @@ -1,5 +1,7 @@ use super::*; +mod audit_sanitization; + #[tokio::test] async fn log_routes_reject_methods_and_invalid_paths_with_bounded_json_errors() { let requests = [ @@ -219,6 +221,45 @@ async fn logs_events_rejects_invalid_raw_requests_before_sse_headers() { } } +#[tokio::test] +#[serial] +async fn logs_events_audit_reconcile_failure_writes_one_terminal_frame_then_eof() { + let _temporary_directory = install_sse_logging().await; + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + b"GET /api/logs/events?audit=true&cursor=a1:18446744073709551615 HTTP/1.1\r\nHost: localhost\r\nAccept: text/event-stream\r\n\r\n", + ) + .await + .unwrap(); + + let mut response = read_until_contains( + &mut stream, + b"\"code\":\"audit_reconcile_failed\"", + Duration::from_secs(2), + ) + .await; + let mut remainder = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), stream.read_to_end(&mut remainder)) + .await + .expect("audit reconcile failure stream must reach EOF") + .unwrap(); + response.extend_from_slice(&remainder); + let response = String::from_utf8(response).unwrap(); + assert_eq!(response.matches("event: stream_error").count(), 1); + assert_eq!( + response + .matches("\"code\":\"audit_reconcile_failed\"") + .count(), + 1 + ); + + server.await.unwrap().unwrap(); + disable_sse_logging().await; +} + #[tokio::test] #[serial] async fn logs_events_stops_after_a_disconnected_tcp_client() { diff --git a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs new file mode 100644 index 0000000000..8255d09ff8 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs @@ -0,0 +1,66 @@ +use super::*; + +#[tokio::test] +#[serial] +async fn logs_events_audit_sanitizes_raw_sql_actor_and_action() { + // Given: an audit row inserted outside the typed repository with unsafe scalars. + let _temporary_directory = install_sse_logging().await; + let logging = crate::logging_runtime_state().unwrap(); + let raw_actor = "Bearer sse-raw-actor-secret"; + let oversized_marker = "SSE-RAW-OVERSIZED-ACTION"; + let raw_action = format!("{}{oversized_marker}", "x".repeat(1_100)); + logging + .store() + .expect("installed log store") + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + ( + "00000000-0000-4000-8000-0000000000aa", + "2026-08-22T12:00:00.000000000Z", + raw_actor, + raw_action.as_str(), + r#"{"severity":"warning"}"#, + ), + ) + .expect("insert raw audit row"); + let expected_code = mesh_llm_events::audit::SanitizedAuditScalar::sanitize(&raw_action); + let expected_code_json = format!( + "\"code\":{}", + serde_json::to_string(expected_code.as_str()).expect("serialize expected audit code") + ); + + // When: an audit SSE client reconciles durable rows through the management route. + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + b"GET /api/logs/events?audit=true&cursor=a1:0 HTTP/1.1\r\nHost: localhost\r\nAccept: text/event-stream\r\n\r\n", + ) + .await + .unwrap(); + let response = read_until_contains( + &mut stream, + expected_code_json.as_bytes(), + Duration::from_secs(2), + ) + .await; + let response = String::from_utf8(response).unwrap(); + + // Then: only the canonical scalar values cross the SSE boundary. + assert!(response.contains("HTTP/1.1 200 OK")); + assert!(response.contains("event: audit_entry")); + assert!(response.contains("\"source\":\"[REDACTED]\"")); + assert!(response.contains(&expected_code_json)); + assert!(!response.contains(raw_actor)); + assert!(!response.contains("sse-raw-actor-secret")); + assert!(!response.contains(&raw_action)); + assert!(!response.contains(oversized_marker)); + + drop(stream); + server.abort(); + disable_sse_logging().await; +} diff --git a/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs b/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs index 954ae34f71..d8f5414c13 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs @@ -62,13 +62,142 @@ async fn management_health_is_a_json_liveness_response() { .await; server.await.unwrap().unwrap(); - assert!(response.starts_with("HTTP/1.1 200 OK"), "response: {response}"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); assert!(response.contains("Content-Type: application/json")); assert_eq!(json_body(&response)["status"], "ok"); assert_eq!(json_body(&response)["mesh"]["status"], "standalone"); assert_eq!(json_body(&response)["serving"]["status"], "idle"); } +#[tokio::test] +#[serial] +async fn trusted_local_management_mutation_persists_the_tcp_caller_address() { + let root = tempfile::tempdir().expect("temporary logging root"); + let config = mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(root.path().to_path_buf()), + ..Default::default() + }; + crate::initialize_logging_foundation(&config).await; + + let state = build_test_mesh_api().await; + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind management listener"); + let address = listener.local_addr().expect("management listener address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept management caller"); + handle_request(stream, &state).await + }); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let body = r#""active""#; + let request = format!( + "PUT /api/runtime/activity/override HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + request_id.as_uuid(), + body.len(), + body, + ); + let mut client = TcpStream::connect(address) + .await + .expect("connect management caller"); + let caller_addr = client + .local_addr() + .expect("management caller address") + .to_string(); + client + .write_all(request.as_bytes()) + .await + .expect("write management mutation"); + client.shutdown().await.expect("finish management request"); + let mut response = Vec::new(); + client + .read_to_end(&mut response) + .await + .expect("read management response"); + server + .await + .expect("management task joins") + .expect("request"); + assert!( + String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK"), + "trusted-local mutation succeeds" + ); + + let logging = crate::logging_runtime_state().expect("installed logging runtime"); + let request_key = request_id.as_uuid().to_string(); + let active = logging + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_key) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); + logging.pump_persistence_for_test().await; + let durable = logging + .store() + .expect("metadata store") + .query_request_with_caller(&request_key) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); +} + +#[tokio::test] +#[serial] +async fn logs_and_read_only_management_routes_never_self_record() { + let root = tempfile::tempdir().expect("temporary logging root"); + let config = mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(root.path().to_path_buf()), + ..Default::default() + }; + crate::initialize_logging_foundation(&config).await; + let state = build_test_mesh_api().await; + + let mut request_ids = Vec::new(); + for path in ["/api/logs/requests", "/api/status"] { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let (address, server) = spawn_management_test_server(state.clone()).await; + let response = send_management_request( + address, + format!( + "GET {path} HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ), + ) + .await; + server + .await + .expect("management task joins") + .expect("request"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); + request_ids.push(request_id); + } + + let service = crate::logging_runtime_state() + .expect("installed logging runtime") + .service_for_test() + .expect("logging service"); + for request_id in request_ids { + assert!( + service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .is_none(), + "excluded route must not register a lifecycle" + ); + } +} + #[tokio::test] async fn management_health_remains_available_in_headless_mode() { let state = build_test_mesh_api().await; @@ -81,7 +210,10 @@ async fn management_health_remains_available_in_headless_mode() { .await; server.await.unwrap().unwrap(); - assert!(response.starts_with("HTTP/1.1 200 OK"), "response: {response}"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); assert_eq!(json_body(&response)["status"], "ok"); } @@ -113,14 +245,13 @@ async fn management_health_reports_client_worker_and_serving_modes() { .await; server.await.unwrap().unwrap(); assert_eq!(json_body(&worker_response)["mode"], "worker"); - assert_eq!( - json_body(&worker_response)["serving"]["status"], - "idle" - ); + assert_eq!(json_body(&worker_response)["serving"]["status"], "idle"); state.update(true, true).await; - node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }).await; - node.set_hosted_models(vec!["served-model".to_string()]).await; + node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }) + .await; + node.set_hosted_models(vec!["served-model".to_string()]) + .await; let (address, server) = spawn_management_test_server(state).await; let serving_response = send_management_request( address, @@ -323,7 +454,8 @@ async fn management_health_reports_cached_plugin_models_for_serving_host() { let plugin_manager = build_inference_endpoint_plugin_manager(&["plugin-model"]).await; let state = build_test_mesh_api_with_plugin_manager(3131, plugin_manager).await; let node = state.node().await; - node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }).await; + node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }) + .await; let (address, server) = spawn_management_test_server(state).await; let response = send_management_request( diff --git a/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs b/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs new file mode 100644 index 0000000000..8efc9ddf33 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs @@ -0,0 +1,113 @@ +use super::*; + +const CLIENT_REQUEST_ID: &str = "00000000-0000-4000-8000-000000000041"; +const SECOND_CLIENT_REQUEST_ID: &str = "00000000-0000-4000-8000-000000000042"; + +struct RequestIdObservation { + response_id: String, + lifecycle_id: String, + lifecycle_state: String, +} + +async fn observe_management_request_id(request_id_headers: &str) -> RequestIdObservation { + let temporary_directory = tempfile::tempdir().expect("temporary logging root"); + crate::initialize_logging_foundation(&mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(temporary_directory.path().join("logging")), + replay_capacity: 16, + ..Default::default() + }) + .await; + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let body = r#"{"toml":"version = 1\n","path":"x"}"#; + let response = send_management_request( + address, + format!( + "POST /api/runtime/config/validate HTTP/1.1\r\nHost: localhost\r\n{request_id_headers}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ), + ) + .await; + server + .await + .expect("management task joins") + .expect("management request succeeds"); + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + + let response_ids = response + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("x-request-id")) + .map(|(_, value)| value.trim().to_string()) + .collect::>(); + assert_eq!( + response_ids.len(), + 1, + "response must contain one request ID" + ); + let response_id = response_ids[0].clone(); + let summary = crate::logging_runtime_state() + .expect("installed logging runtime") + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&response_id) + .expect("response request ID identifies the lifecycle"); + let observation = RequestIdObservation { + response_id, + lifecycle_id: summary.request_id, + lifecycle_state: summary.state, + }; + + crate::initialize_logging_foundation(&mesh_llm_config::LoggingConfig { + enabled: false, + ..Default::default() + }) + .await; + observation +} + +#[tokio::test] +#[serial] +async fn single_valid_client_request_id_is_the_canonical_management_id() { + let observation = + observe_management_request_id(&format!("x-request-id: {CLIENT_REQUEST_ID}\r\n")).await; + + assert_eq!(observation.response_id, CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_id, CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_state, "completed"); +} + +#[tokio::test] +#[serial] +async fn duplicate_client_request_ids_generate_a_new_canonical_management_id() { + let observation = observe_management_request_id(&format!( + "x-request-id: {CLIENT_REQUEST_ID}\r\nX-Request-Id: {SECOND_CLIENT_REQUEST_ID}\r\n" + )) + .await; + + assert_ne!(observation.response_id, CLIENT_REQUEST_ID); + assert_ne!(observation.response_id, SECOND_CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} + +#[tokio::test] +#[serial] +async fn malformed_client_request_id_generates_a_canonical_management_id() { + let observation = observe_management_request_id("x-request-id: not-a-uuid\r\n").await; + + assert_ne!(observation.response_id, "not-a-uuid"); + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} + +#[tokio::test] +#[serial] +async fn missing_client_request_id_generates_a_canonical_management_id() { + let observation = observe_management_request_id("").await; + + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} diff --git a/crates/mesh-llm-host-runtime/src/api/tests/mod.rs b/crates/mesh-llm-host-runtime/src/api/tests/mod.rs index 2ace24238a..0932f42a88 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/mod.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/mod.rs @@ -50,6 +50,7 @@ impl Drop for HomeEnvGuard { mod apply_config_diagnostics; mod apply_config_validation_authority; mod logs_api_routes; +mod management_request_id; mod runtime_config; mod runtime_config_validation_authority; mod runtime_control_state; diff --git a/crates/mesh-llm-host-runtime/src/lib.rs b/crates/mesh-llm-host-runtime/src/lib.rs index 72236b8f91..558321d204 100644 --- a/crates/mesh-llm-host-runtime/src/lib.rs +++ b/crates/mesh-llm-host-runtime/src/lib.rs @@ -33,8 +33,8 @@ pub use crypto::{ verify_release_attestation, }; pub use logging::{ - LoggingRuntimeState, OperationalAuditContext, OperationalAuditRecord, OperationalAuditSeverity, - OperationalAuditSubjectKind, + LoggingRuntimeState, OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, OperationalAuditSubjectKind, }; pub use mesh::requirements::{ BootstrapStatus, DIRECT_NODE_ADMISSION_PROOF_MAX_CLOCK_SKEW_MS, DirectNodeAdmissionProof, diff --git a/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs index e6f8d4896f..653734dcd0 100644 --- a/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs @@ -1,15 +1,20 @@ //! Metadata-only lifecycle ownership for one management API request. -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, +use std::{ + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, }; use mesh_llm_events::logging::{ events::LifecycleEvent, identifiers::RequestId, replay::ReplayChannel, }; -use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome}; +use super::{ + CallerPathType, LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome, +}; pub(crate) struct ManagementRequestLifecycle { request_id: RequestId, @@ -24,6 +29,25 @@ impl ManagementRequestLifecycle { request_id: RequestId, method_route: &'static str, ) -> Self { + Self::register_with_caller_addr(service, request_id, method_route, None) + } + + pub(crate) fn register_with_source_addr( + service: Arc, + request_id: RequestId, + method_route: &'static str, + source_addr: SocketAddr, + ) -> Self { + Self::register_with_caller_addr(service, request_id, method_route, Some(source_addr)) + } + + fn register_with_caller_addr( + service: Arc, + request_id: RequestId, + method_route: &'static str, + source_addr: Option, + ) -> Self { + let caller_addr = source_addr.map(|addr| addr.to_string()); let metadata = RequestSummaryMetadata::from_parts( Some(method_route), None, @@ -31,7 +55,12 @@ impl ManagementRequestLifecycle { Some(method_route), ) .with_source(Some("direct_http")) - .with_method(Some(management_method_label(method_route))); + .with_method(Some(management_method_label(method_route))) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let (guard, _) = service.register_request_with_metadata(request_id, metadata.clone()); if let Ok(payload) = serde_json::to_string(&LifecycleEvent::RouteSelected { model: None, @@ -135,6 +164,32 @@ mod tests { assert!(summary.metadata.model().is_none()); assert_eq!(summary.metadata.provider(), Some("management_api")); assert_eq!(summary.metadata.engine(), Some("management_get_status")); + assert!(summary.metadata.caller_endpoint_id().is_none()); + assert!(summary.metadata.caller_addr().is_none()); + assert!(summary.metadata.caller_path_type().is_none()); + } + + #[test] + fn management_registration_captures_truthful_local_http_caller() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let request_id = RequestId::new(); + let source_addr = "[::1]:40123".parse().expect("IPv6 source address"); + let lifecycle = ManagementRequestLifecycle::register_with_source_addr( + Arc::clone(&service), + request_id, + "management_post", + source_addr, + ); + + lifecycle.finish_status(200); + + let summary = service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("terminal request summary"); + assert_eq!(summary.metadata.caller_addr(), Some("[::1]:40123")); + assert_eq!(summary.metadata.caller_path_type(), Some("local_http")); + assert!(summary.metadata.caller_endpoint_id().is_none()); } #[test] diff --git a/crates/mesh-llm-host-runtime/src/logging/mod.rs b/crates/mesh-llm-host-runtime/src/logging/mod.rs index 9f91933548..4ae3ee5101 100644 --- a/crates/mesh-llm-host-runtime/src/logging/mod.rs +++ b/crates/mesh-llm-host-runtime/src/logging/mod.rs @@ -51,12 +51,12 @@ pub(crate) use openai_lifecycle::{ }; pub use persistence::LogStoreSink; pub(crate) use raw_mesh_lifecycle::{ - ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRemoteSuppressionLease, - RawMeshRequestLifecycle, + ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRemoteAttributionLease, + RawMeshRemoteSuppressionLease, RawMeshRequestLifecycle, }; pub use registry::{ActiveRequestSnapshot, RegistryConfig, RequestRegistry, RequestSummaryEntry}; pub(crate) use registry::{RequestSummaryEventSnapshots, RequestSummarySnapshot}; -pub(crate) use request_metadata::RequestSummaryMetadata; +pub(crate) use request_metadata::{CallerPathType, RequestSummaryMetadata}; #[cfg(test)] pub use runtime_state::ArtifactCaptureRequest; pub(crate) use runtime_state::LoggingQueryFacade; @@ -66,8 +66,8 @@ pub use sequences::SequenceGenerators; pub(crate) use service::ArtifactUnavailableReason; pub use service::{ ArtifactCaptureEntry, ArtifactPersistenceStatus, Clock, LoggingService, - OperationalAuditContext, OperationalAuditRecord, OperationalAuditSeverity, - OperationalAuditSubjectKind, PersistSink, ServiceConfig, SystemClock, + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, OperationalAuditSubjectKind, PersistSink, ServiceConfig, SystemClock, }; pub(crate) use webhook_delivery::{ RandomWebhookJitter, ReqwestWebhookTransport, SystemWebhookWorkerClock, WebhookDeliveryWorker, diff --git a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs index 72fc1ec87d..e06f75d4b8 100644 --- a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs @@ -19,6 +19,7 @@ use openai_frontend::{ OpenAiUsage, }; +use super::raw_mesh_lifecycle::FrontendAdmissionDecision; use super::{ ArtifactUnavailableReason, LifecycleGuard, LoggingService, ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRequestLifecycle, RequestSummaryMetadata, @@ -434,28 +435,28 @@ impl OpenAiLifecycleLoggingAdapter { fn admit(&self, context: &OpenAiLifecycleContext) { let request_id = context.request_id; - // A raw mesh ingress request owns its lifecycle before it reaches an - // embedded frontend. Direct frontend loopback traffic never claims this - // registry and remains owned by this adapter. - if self.raw_mesh_owners.is_claimed(request_id) { - return; - } - let mut tracked = lock_recover(&self.tracked); - if tracked.requests.contains_key(&request_id) || !tracked.make_room() { - return; - } + self.raw_mesh_owners.admit_frontend(request_id, || { + let mut tracked = lock_recover(&self.tracked); + if tracked.requests.contains_key(&request_id) { + return FrontendAdmissionDecision::Rejected; + } + let Ok(evicted) = tracked.make_room() else { + return FrontendAdmissionDecision::Rejected; + }; - let (guard, _) = self.service.register_request_with_metadata( - request_id, - RequestSummaryMetadata::from_openai_frontend_route(context.route) - .with_source(Some("direct_http")) - .with_method(Some(openai_method_label(context.method))), - ); - tracked.requests.insert( - request_id, - TrackedRequest::Active(ActiveRequest::new(guard)), - ); - tracked.insertion_order.push_back(request_id); + let (guard, _) = self.service.register_request_with_metadata( + request_id, + RequestSummaryMetadata::from_openai_frontend_route(context.route) + .with_source(Some("direct_http")) + .with_method(Some(openai_method_label(context.method))), + ); + tracked.requests.insert( + request_id, + TrackedRequest::Active(ActiveRequest::new(guard)), + ); + tracked.insertion_order.push_back(request_id); + FrontendAdmissionDecision::Registered { evicted } + }); } fn backend_dispatched(&self, request_id: RequestId, operation: OpenAiBackendOperation) { @@ -653,6 +654,7 @@ impl OpenAiLifecycleLoggingAdapter { let _ = self .service .transition_terminal(request_id, &guard, outcome); + self.raw_mesh_owners.release_frontend(request_id); } #[cfg(test)] @@ -670,19 +672,20 @@ const fn openai_method_label(method: OpenAiRequestMethod) -> &'static str { } impl TrackedRequests { - fn make_room(&mut self) -> bool { - while self.requests.len() >= MAX_TRACKED_REQUESTS { - let Some(oldest) = self.insertion_order.pop_front() else { - return false; - }; - if matches!(self.requests.get(&oldest), Some(TrackedRequest::Terminal)) { - self.requests.remove(&oldest); - continue; - } + fn make_room(&mut self) -> Result, ()> { + if self.requests.len() < MAX_TRACKED_REQUESTS { + return Ok(None); + } + let Some(oldest) = self.insertion_order.pop_front() else { + return Err(()); + }; + if matches!(self.requests.get(&oldest), Some(TrackedRequest::Terminal)) { + self.requests.remove(&oldest); + Ok(Some(oldest)) + } else { self.insertion_order.push_front(oldest); - return false; + Err(()) } - true } fn is_active(&self, request_id: RequestId) -> bool { @@ -952,6 +955,41 @@ mod tests { assert_eq!(adapter.tracked_len(), 1); } + #[test] + fn terminalization_releases_frontend_owner_without_disturbing_raw_owner() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let adapter = OpenAiLifecycleLoggingAdapter::new(Arc::clone(&service), Arc::clone(&owners)); + let frontend_request_id = RequestId::new(); + let frontend_context = context(frontend_request_id); + let raw_request_id = RequestId::new(); + let raw = RawMeshRequestLifecycle::register( + Arc::clone(&service), + Arc::clone(&owners), + raw_request_id, + ) + .expect("raw request should own its lifecycle"); + adapter.observe(&OpenAiLifecycleEvent::Admitted { + context: frontend_context.clone(), + }); + assert!(owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + + adapter.observe(&OpenAiLifecycleEvent::NonStreamTerminal { + context: frontend_context.clone(), + result: OpenAiTerminalResult::Completed { status_code: 200 }, + }); + adapter.observe(&OpenAiLifecycleEvent::NonStreamTerminal { + context: frontend_context, + result: OpenAiTerminalResult::Completed { status_code: 200 }, + }); + + assert!(!owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + assert_eq!(adapter.tracked_len(), 1); + raw.terminal(TerminalOutcome::Completed); + } + #[test] fn backend_stream_and_usage_events_map_to_canonical_children_once() { let (service, adapter) = adapter(); @@ -1132,6 +1170,104 @@ mod tests { ); } + #[test] + fn markerless_authenticated_attribution_and_frontend_admission_choose_one_parent_in_both_orders() + { + for attribution_first in [true, false] { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let adapter = + OpenAiLifecycleLoggingAdapter::new(Arc::clone(&service), Arc::clone(&owners)); + let request_id = RequestId::new(); + let endpoint_id = "83".repeat(32); + let authenticated = RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.83:11204"), + Some(super::super::CallerPathType::RemoteQuicHttp), + ); + let request_metadata = RequestSummaryMetadata::from_parts( + Some("chat_completions"), + Some("model-a"), + Some("openai_frontend"), + Some("chat_completion"), + ) + .with_source(Some("direct_http")) + .with_method(Some("POST")); + + let attribution = if attribution_first { + super::super::RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + authenticated.clone(), + ) + .expect("authenticated attribution") + } else { + adapter.admit(&context(request_id)); + service.merge_request_metadata(request_id, request_metadata.clone()); + super::super::RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + authenticated, + ) + .expect("authenticated attribution") + }; + if attribution_first { + adapter.admit(&context(request_id)); + } + + let raw = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + request_metadata, + ); + match raw { + Some(raw) => raw.terminal(TerminalOutcome::Completed), + None => adapter.terminal(request_id, TerminalOutcome::Completed), + } + drop(attribution); + + let events = canonical_events(&service); + assert_eq!( + count_events(&events, |event| matches!( + event, + LifecycleEvent::Admitted { .. } + )), + 1, + "admission order: attribution_first={attribution_first}" + ); + assert_eq!( + count_events(&events, |event| matches!( + event, + LifecycleEvent::Completed { .. } + )), + 1, + "terminal order: attribution_first={attribution_first}" + ); + let summary = service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("terminal summary"); + assert_eq!(summary.metadata.route(), Some("chat_completions")); + assert_eq!(summary.metadata.model(), Some("model-a")); + assert_eq!(summary.metadata.provider(), Some("openai_frontend")); + assert_eq!(summary.metadata.engine(), Some("chat_completion")); + assert_eq!(summary.metadata.source(), Some("direct_http")); + assert_eq!(summary.metadata.method(), Some("POST")); + assert_eq!( + summary.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(summary.metadata.caller_addr(), Some("192.0.2.83:11204")); + assert_eq!( + summary.metadata.caller_path_type(), + Some("remote_quic_http") + ); + } + } + #[test] fn ingress_attachment_keeps_direct_frontend_on_one_parent() { let service = Arc::new(LoggingService::new_disabled(Default::default())); diff --git a/crates/mesh-llm-host-runtime/src/logging/persistence.rs b/crates/mesh-llm-host-runtime/src/logging/persistence.rs index ca79ed5f22..25da1c86cf 100644 --- a/crates/mesh-llm-host-runtime/src/logging/persistence.rs +++ b/crates/mesh-llm-host-runtime/src/logging/persistence.rs @@ -143,12 +143,15 @@ impl LogStoreSink { impl PersistSink for LogStoreSink { async fn persist_summary(&self, entry: RequestSummaryEntry) -> Result<(), String> { self.run_blocking(move |store| { - store.upsert_summary_metadata( + store.upsert_summary_metadata_with_caller( &entry.request_id, entry.metadata.model(), entry.metadata.route(), entry.metadata.provider(), entry.metadata.engine(), + entry.metadata.caller_endpoint_id(), + entry.metadata.caller_addr(), + entry.metadata.caller_path_type(), &entry.created_at, ) }) diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs index 32574d094b..bf200876ec 100644 --- a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs @@ -5,33 +5,52 @@ //! Direct loopback requests still belong to the frontend observer because they //! never claim this raw-ingress ownership. -use std::{ - collections::HashMap, - sync::{ - Arc, Mutex, - atomic::{AtomicU64, Ordering}, - }, -}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::sync::{Arc, Mutex}; -use mesh_llm_events::logging::{ - events::LifecycleEvent, - identifiers::{AttemptId, RequestId}, - proxy::ProxyRecord, - replay::ReplayChannel, -}; +use mesh_llm_events::logging::identifiers::RequestId; -use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome}; +#[cfg(test)] +use mesh_llm_events::logging::events::LifecycleEvent; + +use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata}; + +#[cfg(test)] +use super::TerminalOutcome; + +mod event_emission; +mod ownership; +mod proxy_attempts; +mod remote_attribution; -const MAX_RAW_MESH_LIFECYCLE_OWNERS: usize = 1_024; -const MAX_LOGGED_COMPLETION_TOKENS: u64 = u32::MAX as u64; +pub(crate) use proxy_attempts::{ProxyAttemptFinish, RawMeshProxyAttempt}; +pub(crate) use remote_attribution::{RawMeshRemoteAttributionLease, RawMeshRemoteSuppressionLease}; + +const MAX_TRACKED_REQUESTS: usize = 1_024; + +#[cfg(test)] +const MAX_RAW_MESH_LIFECYCLE_OWNERS: usize = MAX_TRACKED_REQUESTS; #[derive(Default)] pub(crate) struct RawMeshLifecycleOwners { - active: Mutex>, - remote_suppressions: Mutex>, + coordination: Mutex, next_token: AtomicU64, } +#[derive(Default)] +struct LifecycleCoordination { + owners: HashMap, + raw_owner_count: usize, + remote_attributions: HashMap, + remote_suppressions: HashMap, +} + +enum LifecycleOwnerEntry { + Raw(RawMeshLifecycleEntry), + Frontend, +} + struct RawMeshLifecycleEntry { guard: LifecycleGuard, token: u64, @@ -47,488 +66,28 @@ struct RemoteSuppressionEntry { leases: u32, } -pub(crate) struct RawMeshRequestLifecycle { - service: Arc, - owners: Arc, - request_id: RequestId, +struct RemoteAttributionEntry { token: u64, - guard: LifecycleGuard, + leases: u32, + metadata: RequestSummaryMetadata, } -/// One transport attempt that is already owned by a raw mesh request parent. -/// -/// The attempt identifier is created by the canonical lifecycle recorder and -/// is reused for durable proxy metadata; this type cannot create a second -/// terminal owner. -pub(crate) struct RawMeshProxyAttempt { - attempt_id: AttemptId, - started_at: String, +enum RemoteAttributionPlacement { + Applied, + Pending(u64), } -/// Sanitized terminal metadata for one durable proxy attempt record. -pub(crate) struct ProxyAttemptFinish { - pub(crate) target: &'static str, - pub(crate) provider: Option<&'static str>, - pub(crate) engine: Option<&'static str>, - pub(crate) status_code: Option, - pub(crate) lifecycle_error: Option<&'static str>, - pub(crate) error: Option<&'static str>, +pub(crate) enum FrontendAdmissionDecision { + Rejected, + Registered { evicted: Option }, } -/// A fail-open, process-local marker for a trusted remote tunnel request. -/// -/// Unlike a raw ingress lifecycle, this never registers a logging parent. It -/// only prevents the target's embedded frontend observer from registering a -/// duplicate parent while the authenticated tunnel relay is active. -pub(crate) struct RawMeshRemoteSuppressionLease { +pub(crate) struct RawMeshRequestLifecycle { + service: Arc, owners: Arc, request_id: RequestId, token: u64, -} - -impl RawMeshLifecycleOwners { - pub(crate) fn is_claimed(&self, request_id: RequestId) -> bool { - lock_recover(&self.active).contains_key(&request_id) - || lock_recover(&self.remote_suppressions).contains_key(&request_id) - } - - fn acquire_remote_suppression(&self, request_id: RequestId) -> Option { - let mut suppressions = lock_recover(&self.remote_suppressions); - if let Some(existing) = suppressions.get_mut(&request_id) { - existing.leases = existing.leases.saturating_add(1); - return Some(existing.token); - } - if suppressions.len() >= MAX_RAW_MESH_LIFECYCLE_OWNERS { - return None; - } - - let token = self.next_token.fetch_add(1, Ordering::Relaxed); - suppressions.insert(request_id, RemoteSuppressionEntry { token, leases: 1 }); - Some(token) - } - - fn release_remote_suppression(&self, request_id: RequestId, token: u64) { - let mut suppressions = lock_recover(&self.remote_suppressions); - let Some(entry) = suppressions.get_mut(&request_id) else { - return; - }; - if entry.token != token { - return; - } - if entry.leases > 1 { - entry.leases -= 1; - } else { - suppressions.remove(&request_id); - } - } - - fn claim( - &self, - service: &LoggingService, - request_id: RequestId, - metadata: RequestSummaryMetadata, - ) -> Option<(LifecycleGuard, u64)> { - let mut active = lock_recover(&self.active); - if let Some(existing) = active.get(&request_id) { - let claim = (existing.guard.clone(), existing.token); - drop(active); - service.merge_request_metadata(request_id, metadata); - return Some(claim); - } - if active.len() >= MAX_RAW_MESH_LIFECYCLE_OWNERS { - return None; - } - - let (guard, _) = service.register_request_with_metadata(request_id, metadata); - let token = self.next_token.fetch_add(1, Ordering::Relaxed); - active.insert( - request_id, - RawMeshLifecycleEntry { - guard: guard.clone(), - token, - route_selected: false, - stream_started: false, - first_token_recorded: false, - stream_completed: false, - stream_error: false, - }, - ); - Some((guard, token)) - } - - fn emit_route_selected( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - model: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.route_selected { - false - } else { - entry.route_selected = true; - true - } - }; - if !should_emit { - return; - } - - let metadata = RequestSummaryMetadata::from_parts(None, model, provider, engine); - service.merge_request_metadata(request_id, metadata.clone()); - let event = LifecycleEvent::RouteSelected { - model: metadata.model().map(str::to_owned), - provider: metadata.provider().map(str::to_owned), - engine: metadata.engine().map(str::to_owned), - }; - if let Ok(payload) = serde_json::to_string(&event) { - let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); - } - } - - fn emit_stream_started( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - model: Option<&str>, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.stream_started || entry.stream_completed { - false - } else { - entry.stream_started = true; - entry.first_token_recorded = false; - entry.stream_completed = false; - entry.stream_error = false; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamStarted { - model: bounded_route_metadata(model), - }, - ); - } - } - - fn emit_stream_chunk(&self, service: &LoggingService, request_id: RequestId, token: u64) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token - || !entry.stream_started - || entry.stream_completed - || entry.stream_error - || entry.first_token_recorded - { - false - } else { - entry.first_token_recorded = true; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamChunk { tokens: None }, - ); - } - } - - fn emit_stream_completed( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - usage: Option, - ) { - let usage = usage.and_then(|usage| { - mesh_llm_events::logging::events::TokenUsage::from_counts( - usage.prompt_tokens, - usage.completion_tokens, - usage.total_tokens, - ) - }); - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token - || !entry.stream_started - || entry.stream_completed - || entry.stream_error - { - false - } else { - entry.stream_completed = true; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamCompleted { - tokens: bounded_completion_tokens( - usage.and_then(|usage| usage.completion_tokens), - ), - usage, - }, - ); - } - } - - fn emit_stream_error( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - label: &'static str, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.stream_completed || entry.stream_error { - false - } else { - entry.stream_error = true; - entry.stream_started = false; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamError { - error: Some(label.to_owned()), - }, - ); - } - } - - fn release(&self, request_id: RequestId, token: u64) { - let mut active = lock_recover(&self.active); - if active - .get(&request_id) - .is_some_and(|entry| entry.token == token) - { - active.remove(&request_id); - } - } -} - -fn enqueue_stream_event(service: &LoggingService, request_id: RequestId, event: LifecycleEvent) { - if let Ok(payload) = serde_json::to_string(&event) { - let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); - } -} - -impl RawMeshRequestLifecycle { - pub(crate) fn request_id(&self) -> RequestId { - self.request_id - } - - pub(crate) fn register( - service: Arc, - owners: Arc, - request_id: RequestId, - ) -> Option { - Self::register_with_metadata( - service, - owners, - request_id, - RequestSummaryMetadata::default(), - ) - } - - /// Register a raw ingress parent with the metadata known before routing. - pub(crate) fn register_with_metadata( - service: Arc, - owners: Arc, - request_id: RequestId, - metadata: RequestSummaryMetadata, - ) -> Option { - let (guard, token) = owners.claim(&service, request_id, metadata)?; - Some(Self { - service, - owners, - request_id, - token, - guard, - }) - } - - pub(crate) fn route_selected(&self, model: Option<&str>) { - self.route_selected_with_metadata(model, Some("mesh"), Some("raw_ingress")); - } - - pub(crate) fn route_selected_with_metadata( - &self, - model: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - ) { - self.owners.emit_route_selected( - &self.service, - self.request_id, - self.token, - model, - provider, - engine, - ); - } - - pub(crate) fn stream_started(&self, model: Option<&str>) { - self.owners - .emit_stream_started(&self.service, self.request_id, self.token, model); - } - - /// Record the first produced stream chunk. The canonical event contract - /// deliberately keeps this metadata-only and represents the first-token - /// boundary as the first `stream_chunk` event. - pub(crate) fn stream_first_token(&self) { - self.owners - .emit_stream_chunk(&self.service, self.request_id, self.token); - } - - pub(crate) fn stream_chunk(&self) { - self.owners - .emit_stream_chunk(&self.service, self.request_id, self.token); - } - - pub(crate) fn stream_completed( - &self, - usage: Option, - ) { - self.owners - .emit_stream_completed(&self.service, self.request_id, self.token, usage); - } - - pub(crate) fn stream_error(&self, label: &'static str) { - self.owners - .emit_stream_error(&self.service, self.request_id, self.token, label); - } - - pub(crate) fn stream_cancelled(&self) { - self.stream_error("client_disconnected"); - } - - /// Start one bounded transport attempt beneath this raw mesh parent. - pub(crate) fn start_attempt(&self) -> AttemptId { - self.service.start_attempt(self.request_id, &self.guard) - } - - /// Start one lifecycle attempt and retain only the bounded metadata needed - /// to persist its transport result later. - pub(crate) fn start_proxy_attempt(&self) -> RawMeshProxyAttempt { - RawMeshProxyAttempt { - attempt_id: self.start_attempt(), - started_at: self.service.proxy_record_timestamp(), - } - } - - /// Complete one previously started raw mesh transport attempt. - pub(crate) fn complete_attempt(&self, attempt_id: AttemptId, status_code: u16) { - self.service - .complete_attempt(self.request_id, attempt_id, Some(status_code)); - } - - /// Fail one previously started raw mesh transport attempt with a bounded - /// static outcome label. The parent remains active for later targets. - pub(crate) fn fail_attempt(&self, attempt_id: AttemptId, label: &'static str) { - self.service - .fail_attempt(self.request_id, attempt_id, label.to_owned()); - } - - /// Finish the existing lifecycle attempt and enqueue one metadata-only - /// durable proxy record. Persistence is deliberately fail-open and this - /// method never terminalizes the parent request. - pub(crate) fn finish_proxy_attempt( - &self, - attempt: RawMeshProxyAttempt, - finish: ProxyAttemptFinish, - ) { - let ProxyAttemptFinish { - target, - provider, - engine, - status_code, - lifecycle_error, - error, - } = finish; - let completed_at = self.service.proxy_record_timestamp(); - let mut record = ProxyRecord::new( - attempt.attempt_id, - self.request_id, - target.to_owned(), - attempt.started_at, - ); - record.provider = provider.map(str::to_owned); - record.engine = engine.map(str::to_owned); - - if let Some(status_code) = status_code { - self.complete_attempt(attempt.attempt_id, status_code); - record.complete(status_code, completed_at); - record.error = error.map(str::to_owned); - } else { - let lifecycle_error = lifecycle_error.unwrap_or("retryable_unavailable"); - let error = error.unwrap_or("unavailable"); - self.fail_attempt(attempt.attempt_id, lifecycle_error); - record.fail(error.to_owned(), completed_at); - } - - let _ = self.service.enqueue_proxy_record(record); - } - - pub(crate) fn terminal(&self, outcome: TerminalOutcome) { - let _ = self - .service - .transition_terminal(self.request_id, &self.guard, outcome); - self.owners.release(self.request_id, self.token); - } -} - -impl RawMeshRemoteSuppressionLease { - pub(crate) fn acquire( - owners: Arc, - request_id: RequestId, - ) -> Option { - let token = owners.acquire_remote_suppression(request_id)?; - Some(Self { - owners, - request_id, - token, - }) - } -} - -impl Drop for RawMeshRemoteSuppressionLease { - fn drop(&mut self) { - self.owners - .release_remote_suppression(self.request_id, self.token); - } + guard: LifecycleGuard, } fn lock_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { @@ -537,28 +96,5 @@ fn lock_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -const MAX_ROUTE_METADATA_CHARS: usize = 64; - -fn bounded_route_metadata(value: Option<&str>) -> Option { - let value = value?; - let (value, _) = super::policy::apply_redaction(value); - let bounded: String = value - .chars() - .take(MAX_ROUTE_METADATA_CHARS) - .map(|character| { - if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { - character - } else { - '_' - } - }) - .collect(); - (!bounded.is_empty()).then_some(bounded) -} - -fn bounded_completion_tokens(tokens: Option) -> Option { - tokens.filter(|value| *value <= MAX_LOGGED_COMPLETION_TOKENS) -} - #[cfg(test)] mod tests; diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs new file mode 100644 index 0000000000..b4268cc4c2 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs @@ -0,0 +1,235 @@ +use mesh_llm_events::logging::{ + events::{LifecycleEvent, TokenUsage}, + identifiers::RequestId, + replay::ReplayChannel, +}; + +use super::{ + LifecycleOwnerEntry, LoggingService, RawMeshLifecycleOwners, RawMeshRequestLifecycle, + lock_recover, +}; + +const MAX_LOGGED_COMPLETION_TOKENS: u64 = u32::MAX as u64; +const MAX_ROUTE_METADATA_CHARS: usize = 64; + +impl RawMeshLifecycleOwners { + fn emit_stream_started( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + model: Option<&str>, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.stream_started || entry.stream_completed { + false + } else { + entry.stream_started = true; + entry.first_token_recorded = false; + entry.stream_completed = false; + entry.stream_error = false; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamStarted { + model: bounded_route_metadata(model), + }, + ); + } + } + + fn emit_stream_chunk(&self, service: &LoggingService, request_id: RequestId, token: u64) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token + || !entry.stream_started + || entry.stream_completed + || entry.stream_error + || entry.first_token_recorded + { + false + } else { + entry.first_token_recorded = true; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamChunk { tokens: None }, + ); + } + } + + fn emit_stream_completed( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + usage: Option, + ) { + let usage = usage.and_then(|usage| { + TokenUsage::from_counts( + usage.prompt_tokens, + usage.completion_tokens, + usage.total_tokens, + ) + }); + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token + || !entry.stream_started + || entry.stream_completed + || entry.stream_error + { + false + } else { + entry.stream_completed = true; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamCompleted { + tokens: bounded_completion_tokens( + usage.and_then(|usage| usage.completion_tokens), + ), + usage, + }, + ); + } + } + + fn emit_stream_error( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + label: &'static str, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.stream_completed || entry.stream_error { + false + } else { + entry.stream_error = true; + entry.stream_started = false; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamError { + error: Some(label.to_owned()), + }, + ); + } + } +} + +impl RawMeshRequestLifecycle { + pub(crate) fn route_selected(&self, model: Option<&str>) { + self.route_selected_with_metadata(model, Some("mesh"), Some("raw_ingress")); + } + + pub(crate) fn route_selected_with_metadata( + &self, + model: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + ) { + self.owners.emit_route_selected( + &self.service, + self.request_id, + self.token, + model, + provider, + engine, + ); + } + + pub(crate) fn stream_started(&self, model: Option<&str>) { + self.owners + .emit_stream_started(&self.service, self.request_id, self.token, model); + } + + /// Record the first produced stream chunk. The canonical event contract + /// deliberately keeps this metadata-only and represents the first-token + /// boundary as the first `stream_chunk` event. + pub(crate) fn stream_first_token(&self) { + self.owners + .emit_stream_chunk(&self.service, self.request_id, self.token); + } + + pub(crate) fn stream_chunk(&self) { + self.owners + .emit_stream_chunk(&self.service, self.request_id, self.token); + } + + pub(crate) fn stream_completed(&self, usage: Option) { + self.owners + .emit_stream_completed(&self.service, self.request_id, self.token, usage); + } + + pub(crate) fn stream_error(&self, label: &'static str) { + self.owners + .emit_stream_error(&self.service, self.request_id, self.token, label); + } + + pub(crate) fn stream_cancelled(&self) { + self.stream_error("client_disconnected"); + } +} + +fn enqueue_stream_event(service: &LoggingService, request_id: RequestId, event: LifecycleEvent) { + if let Ok(payload) = serde_json::to_string(&event) { + let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); + } +} + +fn bounded_route_metadata(value: Option<&str>) -> Option { + let value = value?; + let (value, _) = super::super::policy::apply_redaction(value); + let bounded: String = value + .chars() + .take(MAX_ROUTE_METADATA_CHARS) + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { + character + } else { + '_' + } + }) + .collect(); + (!bounded.is_empty()).then_some(bounded) +} + +fn bounded_completion_tokens(tokens: Option) -> Option { + tokens.filter(|value| *value <= MAX_LOGGED_COMPLETION_TOKENS) +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs new file mode 100644 index 0000000000..07573346e2 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs @@ -0,0 +1,208 @@ +use std::sync::{Arc, atomic::Ordering}; + +use mesh_llm_events::logging::{ + events::LifecycleEvent, identifiers::RequestId, replay::ReplayChannel, +}; + +use super::super::TerminalOutcome; + +use super::{ + FrontendAdmissionDecision, LifecycleGuard, LifecycleOwnerEntry, LoggingService, + MAX_TRACKED_REQUESTS, RawMeshLifecycleEntry, RawMeshLifecycleOwners, RawMeshRequestLifecycle, + RequestSummaryMetadata, lock_recover, +}; + +impl RawMeshLifecycleOwners { + pub(crate) fn is_claimed(&self, request_id: RequestId) -> bool { + let coordination = lock_recover(&self.coordination); + coordination.owners.contains_key(&request_id) + || coordination.remote_attributions.contains_key(&request_id) + || coordination.remote_suppressions.contains_key(&request_id) + } + + /// Coordinate frontend eligibility and registration with raw claims, + /// authenticated attribution, and suppression placement. The callback may + /// briefly lock the frontend tracker and service registry; callers must not + /// enter this method while holding either inner lock. + pub(crate) fn admit_frontend( + &self, + request_id: RequestId, + register: impl FnOnce() -> FrontendAdmissionDecision, + ) { + let mut coordination = lock_recover(&self.coordination); + if coordination.owners.contains_key(&request_id) + || coordination.remote_attributions.contains_key(&request_id) + || coordination.remote_suppressions.contains_key(&request_id) + { + return; + } + let FrontendAdmissionDecision::Registered { evicted } = register() else { + return; + }; + if let Some(evicted) = evicted + && matches!( + coordination.owners.get(&evicted), + Some(LifecycleOwnerEntry::Frontend) + ) + { + coordination.owners.remove(&evicted); + } + coordination + .owners + .insert(request_id, LifecycleOwnerEntry::Frontend); + } + + pub(crate) fn release_frontend(&self, request_id: RequestId) { + let mut coordination = lock_recover(&self.coordination); + if matches!( + coordination.owners.get(&request_id), + Some(LifecycleOwnerEntry::Frontend) + ) { + coordination.owners.remove(&request_id); + } + } + + fn claim( + &self, + service: &LoggingService, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option<(LifecycleGuard, u64)> { + let mut coordination = lock_recover(&self.coordination); + if let Some(existing) = coordination.owners.get(&request_id) { + return match existing { + LifecycleOwnerEntry::Raw(existing) => { + let claim = (existing.guard.clone(), existing.token); + service.merge_request_metadata(request_id, metadata); + Some(claim) + } + LifecycleOwnerEntry::Frontend => { + service.merge_request_metadata(request_id, metadata); + None + } + }; + } + if coordination.raw_owner_count >= MAX_TRACKED_REQUESTS { + return None; + } + + let mut metadata = metadata; + if let Some(attribution) = coordination.remote_attributions.remove(&request_id) { + metadata.merge_authenticated_remote_caller(attribution.metadata); + } + let (guard, _) = service.register_request_with_metadata(request_id, metadata); + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination.owners.insert( + request_id, + LifecycleOwnerEntry::Raw(RawMeshLifecycleEntry { + guard: guard.clone(), + token, + route_selected: false, + stream_started: false, + first_token_recorded: false, + stream_completed: false, + stream_error: false, + }), + ); + coordination.raw_owner_count += 1; + Some((guard, token)) + } + + pub(super) fn emit_route_selected( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + model: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.route_selected { + false + } else { + entry.route_selected = true; + true + } + }; + if !should_emit { + return; + } + + let metadata = RequestSummaryMetadata::from_parts(None, model, provider, engine); + service.merge_request_metadata(request_id, metadata.clone()); + let event = LifecycleEvent::RouteSelected { + model: metadata.model().map(str::to_owned), + provider: metadata.provider().map(str::to_owned), + engine: metadata.engine().map(str::to_owned), + }; + if let Ok(payload) = serde_json::to_string(&event) { + let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); + } + } + + fn release(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + if matches!( + coordination.owners.get(&request_id), + Some(LifecycleOwnerEntry::Raw(entry)) if entry.token == token + ) { + coordination.owners.remove(&request_id); + coordination.raw_owner_count -= 1; + } + } +} + +impl RawMeshRequestLifecycle { + pub(crate) fn request_id(&self) -> RequestId { + self.request_id + } + + pub(crate) fn register( + service: Arc, + owners: Arc, + request_id: RequestId, + ) -> Option { + Self::register_with_metadata( + service, + owners, + request_id, + RequestSummaryMetadata::default(), + ) + } + + /// Register a raw ingress parent with the metadata known before routing. + pub(crate) fn register_with_metadata( + service: Arc, + owners: Arc, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + let (guard, token) = owners.claim(&service, request_id, metadata)?; + Some(Self { + service, + owners, + request_id, + token, + guard, + }) + } + + pub(crate) fn terminal(&self, outcome: TerminalOutcome) { + let _ = self + .service + .transition_terminal(self.request_id, &self.guard, outcome); + self.owners.release(self.request_id, self.token); + } +} + +impl Drop for RawMeshRequestLifecycle { + fn drop(&mut self) { + self.owners.release(self.request_id, self.token); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs new file mode 100644 index 0000000000..0a45fbfd44 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs @@ -0,0 +1,93 @@ +use mesh_llm_events::logging::identifiers::AttemptId; +use mesh_llm_events::logging::proxy::ProxyRecord; + +use super::RawMeshRequestLifecycle; + +/// One transport attempt that is already owned by a raw mesh request parent. +/// +/// The attempt identifier is created by the canonical lifecycle recorder and +/// is reused for durable proxy metadata; this type cannot create a second +/// terminal owner. +pub(crate) struct RawMeshProxyAttempt { + attempt_id: AttemptId, + started_at: String, +} + +/// Sanitized terminal metadata for one durable proxy attempt record. +pub(crate) struct ProxyAttemptFinish { + pub(crate) target: &'static str, + pub(crate) provider: Option<&'static str>, + pub(crate) engine: Option<&'static str>, + pub(crate) status_code: Option, + pub(crate) lifecycle_error: Option<&'static str>, + pub(crate) error: Option<&'static str>, +} + +impl RawMeshRequestLifecycle { + /// Start one bounded transport attempt beneath this raw mesh parent. + pub(crate) fn start_attempt(&self) -> AttemptId { + self.service.start_attempt(self.request_id, &self.guard) + } + + /// Start one lifecycle attempt and retain only the bounded metadata needed + /// to persist its transport result later. + pub(crate) fn start_proxy_attempt(&self) -> RawMeshProxyAttempt { + RawMeshProxyAttempt { + attempt_id: self.start_attempt(), + started_at: self.service.proxy_record_timestamp(), + } + } + + /// Complete one previously started raw mesh transport attempt. + pub(crate) fn complete_attempt(&self, attempt_id: AttemptId, status_code: u16) { + self.service + .complete_attempt(self.request_id, attempt_id, Some(status_code)); + } + + /// Fail one previously started raw mesh transport attempt with a bounded + /// static outcome label. The parent remains active for later targets. + pub(crate) fn fail_attempt(&self, attempt_id: AttemptId, label: &'static str) { + self.service + .fail_attempt(self.request_id, attempt_id, label.to_owned()); + } + + /// Finish the existing lifecycle attempt and enqueue one metadata-only + /// durable proxy record. Persistence is deliberately fail-open and this + /// method never terminalizes the parent request. + pub(crate) fn finish_proxy_attempt( + &self, + attempt: RawMeshProxyAttempt, + finish: ProxyAttemptFinish, + ) { + let ProxyAttemptFinish { + target, + provider, + engine, + status_code, + lifecycle_error, + error, + } = finish; + let completed_at = self.service.proxy_record_timestamp(); + let mut record = ProxyRecord::new( + attempt.attempt_id, + self.request_id, + target.to_owned(), + attempt.started_at, + ); + record.provider = provider.map(str::to_owned); + record.engine = engine.map(str::to_owned); + + if let Some(status_code) = status_code { + self.complete_attempt(attempt.attempt_id, status_code); + record.complete(status_code, completed_at); + record.error = error.map(str::to_owned); + } else { + let lifecycle_error = lifecycle_error.unwrap_or("retryable_unavailable"); + let error = error.unwrap_or("unavailable"); + self.fail_attempt(attempt.attempt_id, lifecycle_error); + record.fail(error.to_owned(), completed_at); + } + + let _ = self.service.enqueue_proxy_record(record); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs new file mode 100644 index 0000000000..02e307cb64 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs @@ -0,0 +1,157 @@ +use std::sync::{Arc, atomic::Ordering}; + +use mesh_llm_events::logging::identifiers::RequestId; + +use super::{ + LoggingService, MAX_TRACKED_REQUESTS, RawMeshLifecycleOwners, RemoteAttributionEntry, + RemoteAttributionPlacement, RemoteSuppressionEntry, RequestSummaryMetadata, lock_recover, +}; + +/// A fail-open, process-local marker for a trusted remote tunnel request. +/// +/// Unlike a raw ingress lifecycle, this never registers a logging parent. It +/// only prevents the target's embedded frontend observer from registering a +/// duplicate parent while the authenticated tunnel relay is active. +pub(crate) struct RawMeshRemoteSuppressionLease { + owners: Arc, + request_id: RequestId, + token: u64, +} + +pub(crate) struct RawMeshRemoteAttributionLease { + owners: Arc, + request_id: RequestId, + token: Option, +} + +impl RawMeshLifecycleOwners { + fn acquire_remote_suppression(&self, request_id: RequestId) -> Option { + let mut coordination = lock_recover(&self.coordination); + if let Some(existing) = coordination.remote_suppressions.get_mut(&request_id) { + existing.leases = existing.leases.saturating_add(1); + return Some(existing.token); + } + if coordination.remote_suppressions.len() >= MAX_TRACKED_REQUESTS { + return None; + } + + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination + .remote_suppressions + .insert(request_id, RemoteSuppressionEntry { token, leases: 1 }); + Some(token) + } + + fn attribute_remote_caller( + &self, + service: &LoggingService, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + if !metadata.has_authenticated_remote_caller() { + return None; + } + let mut coordination = lock_recover(&self.coordination); + if coordination.owners.contains_key(&request_id) { + service.merge_authenticated_remote_caller(request_id, metadata); + return Some(RemoteAttributionPlacement::Applied); + } + + if let Some(existing) = coordination.remote_attributions.get_mut(&request_id) { + existing.leases = existing.leases.saturating_add(1); + return Some(RemoteAttributionPlacement::Pending(existing.token)); + } + if coordination.remote_attributions.len() >= MAX_TRACKED_REQUESTS { + return None; + } + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination.remote_attributions.insert( + request_id, + RemoteAttributionEntry { + token, + leases: 1, + metadata, + }, + ); + Some(RemoteAttributionPlacement::Pending(token)) + } + + fn release_remote_attribution(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + let Some(entry) = coordination.remote_attributions.get_mut(&request_id) else { + return; + }; + if entry.token != token { + return; + } + if entry.leases > 1 { + entry.leases -= 1; + } else { + coordination.remote_attributions.remove(&request_id); + } + } + + fn release_remote_suppression(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + let Some(entry) = coordination.remote_suppressions.get_mut(&request_id) else { + return; + }; + if entry.token != token { + return; + } + if entry.leases > 1 { + entry.leases -= 1; + } else { + coordination.remote_suppressions.remove(&request_id); + } + } +} + +impl RawMeshRemoteSuppressionLease { + pub(crate) fn acquire( + owners: Arc, + request_id: RequestId, + ) -> Option { + let token = owners.acquire_remote_suppression(request_id)?; + Some(Self { + owners, + request_id, + token, + }) + } +} + +impl RawMeshRemoteAttributionLease { + pub(crate) fn acquire( + service: &LoggingService, + owners: Arc, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + let placement = owners.attribute_remote_caller(service, request_id, metadata)?; + Some(Self { + owners, + request_id, + token: match placement { + RemoteAttributionPlacement::Applied => None, + RemoteAttributionPlacement::Pending(token) => Some(token), + }, + }) + } +} + +impl Drop for RawMeshRemoteAttributionLease { + fn drop(&mut self) { + if let Some(token) = self.token { + self.owners + .release_remote_attribution(self.request_id, token); + } + } +} + +impl Drop for RawMeshRemoteSuppressionLease { + fn drop(&mut self) { + self.owners + .release_remote_suppression(self.request_id, self.token); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs index 36112c7685..916082e529 100644 --- a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::logging::{OpenAiLifecycleAttachment, ServiceConfig}; +use crate::logging::{CallerPathType, OpenAiLifecycleAttachment, ServiceConfig}; fn recorded_events(service: &LoggingService) -> Vec { service @@ -15,6 +15,31 @@ fn recorded_events(service: &LoggingService) -> Vec { .collect() } +#[test] +fn frontend_release_is_idempotent_and_preserves_raw_ownership() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let raw_request_id = RequestId::new(); + let raw = RawMeshRequestLifecycle::register( + Arc::clone(&service), + Arc::clone(&owners), + raw_request_id, + ) + .expect("raw request should own its lifecycle"); + let frontend_request_id = RequestId::new(); + owners.admit_frontend(frontend_request_id, || { + FrontendAdmissionDecision::Registered { evicted: None } + }); + + owners.release_frontend(frontend_request_id); + owners.release_frontend(frontend_request_id); + owners.release_frontend(raw_request_id); + + assert!(!owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + raw.terminal(TerminalOutcome::Completed); +} + #[test] fn raw_mesh_lifecycle_orders_metadata_events_without_payloads() { let service = Arc::new(LoggingService::new_disabled(Default::default())); @@ -365,6 +390,45 @@ fn raw_mesh_owner_reuses_one_guard_and_terminalizes_once() { assert!(!owners.is_claimed(request_id)); } +#[test] +fn dropping_unterminalized_raw_mesh_lifecycle_releases_owner_slot() { + // Given + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let lifecycle = + RawMeshRequestLifecycle::register(service, Arc::clone(&owners), request_id).unwrap(); + + // When + drop(lifecycle); + + // Then + assert!(!owners.is_claimed(request_id)); +} + +#[test] +fn dropping_stale_duplicate_does_not_release_replacement_owner() { + // Given + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let first = + RawMeshRequestLifecycle::register(Arc::clone(&service), Arc::clone(&owners), request_id) + .unwrap(); + let stale_duplicate = + RawMeshRequestLifecycle::register(Arc::clone(&service), Arc::clone(&owners), request_id) + .unwrap(); + first.terminal(TerminalOutcome::Completed); + let _replacement = + RawMeshRequestLifecycle::register(service, Arc::clone(&owners), request_id).unwrap(); + + // When + drop(stale_duplicate); + + // Then + assert!(owners.is_claimed(request_id)); +} + #[test] fn claimed_plan_failure_terminalizes_without_route_selected_event() { let service = Arc::new(LoggingService::new_disabled(Default::default())); @@ -483,3 +547,140 @@ fn remote_suppression_cap_fails_open_without_registering_parents() { drop(leases); } + +#[test] +fn dropping_first_pending_remote_attribution_releases_token_zero() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let endpoint_id = "81".repeat(32); + let attribution = RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.81:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ) + .expect("first pending attribution"); + + drop(attribution); + let lifecycle = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ) + .expect("local lifecycle"); + + let summary = service + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active summary"); + assert_eq!(summary.metadata.caller_endpoint_id(), None); + assert_eq!(summary.metadata.caller_addr(), Some("127.0.0.1:40123")); + assert_eq!(summary.metadata.caller_path_type(), Some("local_http")); + lifecycle.terminal(TerminalOutcome::Completed); +} + +#[test] +fn remote_attribution_rejects_incomplete_authenticated_tuple() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + + assert!( + RawMeshRemoteAttributionLease::acquire( + &service, + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("192.0.2.82:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ) + .is_none() + ); +} + +#[test] +fn pending_endpoint_only_attribution_survives_provisional_local_registration() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let endpoint_id = "83".repeat(32); + let attribution = RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None), + ) + .expect("pending endpoint-only attribution"); + + let lifecycle = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ) + .expect("local lifecycle"); + + let summary = service + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active summary"); + assert_eq!( + summary.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(summary.metadata.caller_addr(), None); + assert_eq!(summary.metadata.caller_path_type(), None); + lifecycle.terminal(TerminalOutcome::Completed); + drop(attribution); +} + +#[test] +fn remote_attribution_rejects_invalid_endpoint_only_variants() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let endpoint_id = "84".repeat(32); + + for metadata in [ + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.84:11204"), + None, + ), + RequestSummaryMetadata::default().with_caller_identity( + Some("not-an-authenticated-endpoint"), + None, + None, + ), + RequestSummaryMetadata::default().with_caller_identity(None, None, None), + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ] { + assert!( + RawMeshRemoteAttributionLease::acquire( + &service, + Arc::new(RawMeshLifecycleOwners::default()), + RequestId::new(), + metadata, + ) + .is_none() + ); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/registry.rs b/crates/mesh-llm-host-runtime/src/logging/registry.rs index 617de073ee..d63c088374 100644 --- a/crates/mesh-llm-host-runtime/src/logging/registry.rs +++ b/crates/mesh-llm-host-runtime/src/logging/registry.rs @@ -321,6 +321,29 @@ impl RequestRegistry { }) } + pub(crate) fn merge_authenticated_remote_caller( + &self, + request_id: &str, + metadata: RequestSummaryMetadata, + ) -> Option { + let mut active = self.active.lock().expect("registry mutex poisoned"); + if let Some(entry) = active.get_mut(request_id) { + return entry + .metadata + .merge_authenticated_remote_caller(metadata) + .then(|| entry.clone()); + } + drop(active); + + let mut recent = self.recent.lock().expect("registry mutex poisoned"); + recent.get_mut(request_id).and_then(|entry| { + entry + .metadata + .merge_authenticated_remote_caller(metadata) + .then(|| entry.clone()) + }) + } + /// Atomically move an active request to recent while retaining metadata /// merged before its terminal transition. pub(crate) fn terminalize( diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs index 0a40581616..d150ad1ec2 100644 --- a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs @@ -9,6 +9,10 @@ use openai_frontend::OpenAiFrontendRoute; use super::policy::{RedactMode, apply_redaction}; +mod caller_identity; + +pub(crate) use caller_identity::{CallerPathType, authenticated_endpoint_id}; + const MAX_REQUEST_METADATA_CHARS: usize = 64; /// The small, privacy-safe metadata projection shared by active summaries, @@ -27,6 +31,12 @@ pub(crate) struct RequestSummaryMetadata { source: Option, #[serde(default, skip_serializing_if = "Option::is_none")] method: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_path_type: Option, } impl RequestSummaryMetadata { @@ -44,6 +54,9 @@ impl RequestSummaryMetadata { engine: bounded_metadata(engine), source: None, method: None, + caller_endpoint_id: None, + caller_addr: None, + caller_path_type: None, } } @@ -114,18 +127,33 @@ impl RequestSummaryMetadata { && self.engine.is_none() && self.source.is_none() && self.method.is_none() + && self.caller_endpoint_id.is_none() + && self.caller_addr.is_none() + && self.caller_path_type.is_none() } /// Preserve the first truthful value for each field. A later source can /// fill a missing classification but cannot overwrite an earlier one. pub(crate) fn merge_missing(&mut self, update: Self) -> bool { + let Self { + route, + model, + provider, + engine, + source, + method, + caller_endpoint_id, + caller_addr, + caller_path_type, + } = update; let mut changed = false; - changed |= merge_field(&mut self.route, update.route); - changed |= merge_field(&mut self.model, update.model); - changed |= merge_field(&mut self.provider, update.provider); - changed |= merge_field(&mut self.engine, update.engine); - changed |= merge_field(&mut self.source, update.source); - changed |= merge_field(&mut self.method, update.method); + changed |= merge_field(&mut self.route, route); + changed |= merge_field(&mut self.model, model); + changed |= merge_field(&mut self.provider, provider); + changed |= merge_field(&mut self.engine, engine); + changed |= merge_field(&mut self.source, source); + changed |= merge_field(&mut self.method, method); + changed |= self.merge_missing_caller(caller_endpoint_id, caller_addr, caller_path_type); changed } } diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs new file mode 100644 index 0000000000..810495bb03 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs @@ -0,0 +1,126 @@ +use std::net::SocketAddr; + +use super::RequestSummaryMetadata; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CallerPathType { + LocalHttp, + RemoteQuicHttp, + Relay, +} + +impl CallerPathType { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::LocalHttp => "local_http", + Self::RemoteQuicHttp => "remote_quic_http", + Self::Relay => "relay", + } + } +} + +impl RequestSummaryMetadata { + pub(crate) fn with_caller_identity( + mut self, + endpoint_id: Option<&str>, + addr: Option<&str>, + path_type: Option, + ) -> Self { + let endpoint_id = endpoint_id + .filter(|value| authenticated_endpoint_id(value)) + .map(str::to_owned); + let (caller_endpoint_id, caller_addr, caller_path_type) = match path_type { + Some(CallerPathType::LocalHttp) if endpoint_id.is_none() => { + match bounded_caller_addr(addr) { + Some(addr) => (None, Some(addr), path_type), + None => (None, None, None), + } + } + Some(CallerPathType::RemoteQuicHttp) if endpoint_id.is_some() => { + (endpoint_id, bounded_caller_addr(addr), path_type) + } + Some(CallerPathType::Relay) if endpoint_id.is_some() => (endpoint_id, None, path_type), + None if endpoint_id.is_some() && addr.is_none() => (endpoint_id, None, None), + Some( + CallerPathType::LocalHttp | CallerPathType::RemoteQuicHttp | CallerPathType::Relay, + ) + | None => (None, None, None), + }; + self.caller_endpoint_id = caller_endpoint_id; + self.caller_addr = caller_addr; + self.caller_path_type = caller_path_type; + self + } + + pub(crate) fn caller_endpoint_id(&self) -> Option<&str> { + self.caller_endpoint_id.as_deref() + } + + pub(crate) fn caller_addr(&self) -> Option<&str> { + self.caller_addr.as_deref() + } + + pub(crate) fn caller_path_type(&self) -> Option<&'static str> { + self.caller_path_type.map(CallerPathType::as_str) + } + + pub(crate) fn merge_authenticated_remote_caller(&mut self, update: Self) -> bool { + if !update.has_authenticated_remote_caller() || self.has_authenticated_remote_caller() { + return false; + } + + let changed = self.caller_endpoint_id != update.caller_endpoint_id + || self.caller_addr != update.caller_addr + || self.caller_path_type != update.caller_path_type; + self.caller_endpoint_id = update.caller_endpoint_id; + self.caller_addr = update.caller_addr; + self.caller_path_type = update.caller_path_type; + changed + } + + pub(super) fn merge_missing_caller( + &mut self, + caller_endpoint_id: Option, + caller_addr: Option, + caller_path_type: Option, + ) -> bool { + let current_is_empty = self.caller_endpoint_id.is_none() + && self.caller_addr.is_none() + && self.caller_path_type.is_none(); + let update_is_empty = + caller_endpoint_id.is_none() && caller_addr.is_none() && caller_path_type.is_none(); + if !current_is_empty || update_is_empty { + return false; + } + self.caller_endpoint_id = caller_endpoint_id; + self.caller_addr = caller_addr; + self.caller_path_type = caller_path_type; + true + } + + pub(crate) fn has_authenticated_remote_caller(&self) -> bool { + self.caller_endpoint_id + .as_deref() + .is_some_and(authenticated_endpoint_id) + && match self.caller_path_type { + Some(CallerPathType::LocalHttp) => false, + Some(CallerPathType::RemoteQuicHttp) => true, + Some(CallerPathType::Relay) | None => self.caller_addr.is_none(), + } + } +} + +fn bounded_caller_addr(value: Option<&str>) -> Option { + value? + .parse::() + .ok() + .map(|addr| addr.to_string()) +} + +pub(crate) fn authenticated_endpoint_id(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs new file mode 100644 index 0000000000..3834a539bd --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs @@ -0,0 +1,92 @@ +use super::*; + +#[test] +fn local_http_caller_identity_keeps_ipv4_and_ipv6_socket_addresses() { + for caller_addr in [ + "127.0.0.1:40123", + "192.0.2.10:40123", + "[2001:db8::10]:40123", + ] { + let metadata = RequestSummaryMetadata::default().with_caller_identity( + None, + Some(caller_addr), + Some(CallerPathType::LocalHttp), + ); + + assert_eq!(metadata.caller_addr(), Some(caller_addr)); + assert_eq!(metadata.caller_path_type(), Some("local_http")); + } +} + +#[test] +fn caller_path_vocabulary_rejects_stage_transport_as_a_request_caller() { + let parsed = serde_json::from_str::("\"remote_quic_stage\""); + + assert!(parsed.is_err()); +} + +#[test] +fn authenticated_endpoint_only_caller_is_preserved_without_address_or_path() { + let endpoint_id = "ab".repeat(32); + + let metadata = + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None); + + assert_eq!(metadata.caller_endpoint_id(), Some(endpoint_id.as_str())); + assert_eq!(metadata.caller_addr(), None); + assert_eq!(metadata.caller_path_type(), None); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[test] +fn endpoint_only_caller_rejects_partial_or_unauthenticated_tuples() { + let endpoint_id = "cd".repeat(32); + for metadata in [ + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.80:11204"), + None, + ), + RequestSummaryMetadata::default().with_caller_identity( + Some("not-an-authenticated-endpoint"), + None, + None, + ), + RequestSummaryMetadata::default().with_caller_identity(None, None, None), + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ] { + assert!(!metadata.has_authenticated_remote_caller()); + assert_eq!(metadata.caller_endpoint_id(), None); + } +} + +#[test] +fn first_authenticated_endpoint_only_caller_cannot_be_overwritten() { + let first_endpoint_id = "de".repeat(32); + let later_endpoint_id = "ef".repeat(32); + let mut metadata = RequestSummaryMetadata::default().with_caller_identity( + Some(&first_endpoint_id), + None, + None, + ); + + let changed = metadata.merge_authenticated_remote_caller( + RequestSummaryMetadata::default().with_caller_identity( + Some(&later_endpoint_id), + Some("192.0.2.81:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ); + + assert!(!changed); + assert_eq!( + metadata.caller_endpoint_id(), + Some(first_endpoint_id.as_str()) + ); + assert_eq!(metadata.caller_addr(), None); + assert_eq!(metadata.caller_path_type(), None); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs index 1ef4e51f5a..2d9dcf73f1 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs @@ -14,9 +14,9 @@ use std::time::Duration; use mesh_llm_events::logging::identifiers::EventId; use mesh_llm_log_store::{ ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE, ArtifactContent, ArtifactRecord, - ArtifactRedactor, AuditEntryFilters, AuditEntryRow, Clock as StoreClock, EventRecord, - FailOpenArtifactCapture, LogStore, LogStoreError, Page, PageQuery, ProxyQuery, ProxyRecord, - QueryPage, RealClock, RequestQuery, RequestRecord, + ArtifactRedactor, AuditEntryDetail, AuditEntryFilters, AuditEntryRow, Clock as StoreClock, + EventRecord, FailOpenArtifactCapture, LogStore, LogStoreError, Page, PageQuery, ProxyQuery, + ProxyRecord, QueryPage, RealClock, RequestQuery, RequestRecord, RequestRecordWithCaller, }; #[cfg(test)] use mesh_llm_log_store::{ArtifactCaptureDisabledReason, ArtifactCaptureOutcome}; @@ -82,8 +82,8 @@ pub(crate) struct LoggingRuntimeStatus { } /// Stable, path-free explanation for metadata availability. Schema versions -/// are safe to expose to trusted-local clients and make upgrade/downgrade -/// failures actionable without leaking the database location or SQLite text. +/// are safe to expose to trusted-local clients without leaking the database +/// location or SQLite text. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum LoggingMetadataState { Ready, @@ -759,12 +759,33 @@ impl LoggingRuntimeState { RawMeshRemoteSuppressionLease::acquire(Arc::clone(&self.raw_mesh_owners), request_id) } + pub(crate) fn attribute_remote_tunneled_request( + &self, + request_id: mesh_llm_events::logging::identifiers::RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + if self.retired.load(Ordering::Acquire) { + return None; + } + let service = self.service.as_ref()?; + if !service.is_startable() { + return None; + } + super::RawMeshRemoteAttributionLease::acquire( + service, + Arc::clone(&self.raw_mesh_owners), + request_id, + metadata, + ) + } + /// Register one already-parsed management API request with bounded /// metadata. Disabled, unavailable, and retired logging stay fail-open. pub(crate) fn register_management_request( &self, request_id: mesh_llm_events::logging::identifiers::RequestId, method_route: &'static str, + source_addr: Option, ) -> Option { if self.retired.load(Ordering::Acquire) { return None; @@ -773,11 +794,15 @@ impl LoggingRuntimeState { if !service.is_startable() { return None; } - Some(ManagementRequestLifecycle::register( - service, - request_id, - method_route, - )) + Some(match source_addr { + Some(source_addr) => ManagementRequestLifecycle::register_with_source_addr( + service, + request_id, + method_route, + source_addr, + ), + None => ManagementRequestLifecycle::register(service, request_id, method_route), + }) } /// Record one static operational audit without exposing the service to diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs index 9916db3f65..fbb0e1e06f 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs @@ -45,30 +45,33 @@ impl LoggingQueryFacade { self.service.registry_ref().snapshot_active() } - pub(crate) fn request(&self, request_id: &str) -> Result, LogStoreError> { + pub(crate) fn request( + &self, + request_id: &str, + ) -> Result, LogStoreError> { #[cfg(test)] self.query_counter .point_requests .fetch_add(1, Ordering::Relaxed); - self.store.query_request(request_id) + self.store.query_request_with_caller(request_id) } pub(crate) fn requests_by_ids( &self, request_ids: &[String], - ) -> Result, LogStoreError> { + ) -> Result, LogStoreError> { #[cfg(test)] self.query_counter .batch_requests .fetch_add(1, Ordering::Relaxed); - self.store.query_requests_by_ids(request_ids) + self.store.query_requests_by_ids_with_caller(request_ids) } pub(crate) fn requests( &self, query: &RequestQuery, - ) -> Result, LogStoreError> { - self.store.query_requests(query) + ) -> Result, LogStoreError> { + self.store.query_requests_with_caller(query) } pub(crate) fn events( @@ -124,8 +127,9 @@ impl LoggingQueryFacade { limit: Option, after_cursor: Option<&str>, filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - self.store.list_audit_entries(limit, after_cursor, filters) + ) -> Result, LogStoreError> { + self.store + .list_audit_entry_details(limit, after_cursor, filters) } pub(crate) fn audit_entries_after_sequence( @@ -133,9 +137,9 @@ impl LoggingQueryFacade { after_sequence: u64, limit: usize, filters: AuditEntryFilters, - ) -> Result, LogStoreError> { + ) -> Result, LogStoreError> { self.store - .list_audit_entries_after_sequence(after_sequence, limit, filters) + .list_audit_entry_details_after_sequence(after_sequence, limit, filters) } /// Read content only through the fail-open capture owner. The route core diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs index 1e7d34e096..c1a83f8613 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs @@ -2,7 +2,11 @@ use mesh_llm_log_store::{ArtifactPrivacy, MaintenanceExecutionControl}; use std::path::Path; use super::*; -use crate::logging::TerminalOutcome; +use crate::logging::{CallerPathType, TerminalOutcome}; + +mod artifact_capture; +mod remote_caller_attribution; +mod service_lifecycle; #[derive(Default)] struct RejectPrivacy; @@ -58,589 +62,60 @@ fn artifact_config() -> mesh_llm_config::LoggingConfig { } #[test] -fn production_ingress_wiring_advertises_capture_only_when_storage_is_usable() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); - - let status = state.status(); - assert!(status.artifact_capture_available); - assert!(status.artifact_capture_ready); -} - -#[tokio::test] -async fn ingress_attachment_captures_redacted_request_and_response_bodies() { +fn remote_tunnel_suppression_merges_authenticated_caller_into_existing_parent() { let root = tempfile::tempdir().expect("temporary logging root"); let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); let state = Arc::new(LoggingRuntimeState::initialize( &foundation, - &artifact_config(), + &mesh_llm_config::LoggingConfig::default(), )); let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); let mut attachment = state.openai_ingress_attachment( request_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), - ); - - attachment.capture_request_body( - br#"{"authorization":"Bearer test-secret","model":"safe"}"#, - Some("application/json"), - ); - attachment.route_observer().capture_response_body( - br#"{"id":"chatcmpl-safe","api_key":"test-secret"}"#, - Some("application/json; charset=utf-8"), - ); - attachment.terminal(TerminalOutcome::CompletedWithStatus(201)); - - // Summary, admission, both artifact commands, and terminal state share one - // serial persistence owner. Ingress itself performs no SQLite/file work. - assert_eq!( - state - .service - .as_ref() - .expect("logging service") - .pump_sync() - .await, - 5 + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions") + .with_source(Some("direct_http")) + .with_method(Some("POST")) + .with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), ); - - let store = state.store().expect("metadata store"); - let request_key = request_id.as_uuid().to_string(); - let artifact_page = store - .query_artifacts( - &request_key, - &mesh_llm_log_store::PageQuery { - limit: 10, - cursor: None, - sort: mesh_llm_log_store::QuerySort::Ascending, - }, + let endpoint_id = "61".repeat(32); + let attribution = state + .attribute_remote_tunneled_request( + request_id, + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.61:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), ) - .expect("query artifacts"); - assert_eq!(artifact_page.items.len(), 2); - let request_artifact = artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "request") - .expect("request artifact") - .artifact_id - .clone(); - let response_artifact = artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "response") - .expect("response artifact") - .artifact_id - .clone(); - let artifacts = state.query_facade().expect("query facade"); - let request_content = artifacts - .read_artifact(&request_artifact) - .expect("request body"); - let response_content = artifacts - .read_artifact(&response_artifact) - .expect("response body"); - let request_text = String::from_utf8(request_content.bytes).expect("utf8 request artifact"); - let response_text = String::from_utf8(response_content.bytes).expect("utf8 response artifact"); - assert!(!request_text.contains("test-secret")); - assert!(!response_text.contains("test-secret")); - assert!(response_text.contains("chatcmpl-safe")); - let admitted_at = store - .get_summary(&request_key) - .unwrap() - .expect("summary") - .created_at; - assert!( - artifact_page - .items - .iter() - .all(|artifact| artifact.occurred_at >= admitted_at), - "artifact metadata must use the lifecycle clock at capture time" - ); - assert_eq!( - artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "request") - .and_then(|artifact| artifact.media_kind.as_deref()), - Some("application/json"), - "the parsed OpenAI JSON boundary supplies this closed semantic media kind" - ); - assert_eq!( - artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "response") - .and_then(|artifact| artifact.media_kind.as_deref()), - Some("application/json") - ); - let events = store.list_events_for_summary(&request_key).unwrap(); - assert_eq!(events.len(), 2, "one admitted and one terminal event"); - assert_ne!(events[0].event_id, events[1].event_id); -} + .expect("remote attribution lease"); + let lease = state + .suppress_remote_tunneled_request(request_id) + .expect("remote suppression lease"); -#[tokio::test] -async fn streaming_and_oversized_responses_persist_explicit_unavailable_metadata() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = Arc::new(LoggingRuntimeState::initialize( - &foundation, - &artifact_config(), - )); - - let streaming_id = mesh_llm_events::logging::identifiers::RequestId::new(); - let streaming = state.openai_ingress_attachment( - streaming_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), - ); - streaming.route_observer().capture_response_unavailable( - super::super::ArtifactUnavailableReason::StreamingResponseNotAssembled, - ); - - let oversized_id = mesh_llm_events::logging::identifiers::RequestId::new(); - let oversized = state.openai_ingress_attachment( - oversized_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/responses"), - ); - oversized - .route_observer() - .capture_response_body(&vec![b'x'; 4_097], Some("not a valid media type")); - - state + let active = state .service .as_ref() .expect("logging service") - .pump_sync() - .await; - let store = state.store().expect("metadata store"); - for (request_id, reason) in [ - (streaming_id, "streaming_response_not_assembled"), - (oversized_id, "capture_content_limit_exceeded"), - ] { - let page = store - .query_artifacts( - &request_id.as_uuid().to_string(), - &mesh_llm_log_store::PageQuery { - limit: 10, - cursor: None, - sort: mesh_llm_log_store::QuerySort::Ascending, - }, - ) - .expect("query unavailable artifact"); - assert_eq!(page.items.len(), 1); - let artifact = &page.items[0]; - assert_eq!(artifact.unavailable_reason.as_deref(), Some(reason)); - assert_eq!(artifact.bytes, 0); - assert!(artifact.checksum.is_none()); - assert!(!artifact.missing); - assert!(!artifact.corrupt); - } -} - -#[tokio::test] -async fn disabled_webhook_config_starts_no_delivery_scheduler() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - - let service = state - .start_persistence_worker() - .await - .expect("logging service starts"); - - assert!(!state.has_webhook_delivery_worker_for_test()); - state.shutdown_cleanup_worker().await; - assert!(service.shutdown().await); -} - -#[tokio::test] -async fn enabled_webhook_config_starts_and_retires_one_delivery_scheduler() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - webhook: mesh_llm_config::LoggingWebhookConfig { - enabled: true, - url: Some("http://127.0.0.1:9444/webhook".to_string()), - ..Default::default() - }, - ..Default::default() - }; - let state = LoggingRuntimeState::initialize(&foundation, &config); - - state - .start_persistence_worker() - .await - .expect("logging service starts"); - assert!(state.has_webhook_delivery_worker_for_test()); - - assert!(state.retire_and_shutdown().await); - assert!(!state.has_webhook_delivery_worker_for_test()); -} - -#[test] -fn metadata_only_never_opens_or_writes_artifact_capture() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let opened = Arc::new(AtomicBool::new(false)); - let opened_by_callback = Arc::clone(&opened); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &mesh_llm_config::LoggingConfig::default(), - move |_, _, _| { - opened_by_callback.store(true, Ordering::SeqCst); - Err(LogStoreError::PrivacyNotGuaranteed) - }, - ); - - assert!(!opened.load(Ordering::SeqCst)); - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "metadata-only", - request_id: "request", - kind: "body", - occurred_at: "2025-01-01T00:00:00Z", - content: b"must not be stored", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("metadata-only capture is disabled"); - assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); - assert_eq!( - std::fs::read_dir(foundation.artifact_dir()) - .expect("artifact directory") - .count(), - 0 - ); -} - -#[test] -fn metadata_only_query_facade_deletes_terminal_metadata_without_artifact_capture() { - struct NeverCancelled; - - impl MaintenanceExecutionControl for NeverCancelled { - fn is_cancelled(&self) -> bool { - false - } - } - - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - let store = state.store().expect("metadata store"); - let request_id = "00000000-0000-4000-8000-000000000143"; - store - .insert_summary( - request_id, - Some("model"), - Some("route"), - None, - None, - "2025-01-01T00:00:00Z", - None, - None, - None, - ) - .expect("summary"); - store - .conn() - .execute( - "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", - [request_id], - ) - .expect("terminal summary"); - let request = mesh_llm_log_store::DeleteOneRequest::new( - mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(143)), - request_id, - mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), - ) - .expect("delete request"); - let facade = state.query_facade().expect("query facade"); - - facade - .prepare_delete_request(&request, &NeverCancelled) - .expect("metadata-only preparation"); - let receipt = facade - .execute_prepared_delete_request(&request, &NeverCancelled) - .expect("metadata-only execution"); - + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active raw mesh parent"); assert_eq!( - receipt.state, - mesh_llm_log_store::MaintenanceReceiptState::Completed + active.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) ); - assert!(store.query_request(request_id).unwrap().is_none()); -} - -#[test] -fn metadata_only_query_facade_rejects_artifact_pointers_without_preparing_delete() { - struct NeverCancelled; - - impl MaintenanceExecutionControl for NeverCancelled { - fn is_cancelled(&self) -> bool { - false - } - } + assert_eq!(active.metadata.caller_addr(), Some("192.0.2.61:11204")); + assert_eq!(active.metadata.caller_path_type(), Some("remote_quic_http")); + assert_eq!(active.metadata.route(), Some("chat_completions")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - let store = state.store().expect("metadata store"); - let request_id = "00000000-0000-4000-8000-000000000144"; - store - .insert_summary( - request_id, - Some("model"), - Some("route"), - None, - None, - "2025-01-01T00:00:00Z", - None, - None, - None, - ) - .expect("summary"); - store - .conn() - .execute( - "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", - [request_id], - ) - .expect("terminal summary"); - store - .insert_unavailable_artifact_pointer(mesh_llm_log_store::UnavailableArtifactPointer { - artifact_id: "00000000-0000-4000-8000-000000000244", - request_id, - occurred_at: "2025-01-01T00:00:01Z", - kind: "response", - media_kind: Some("text/plain"), - version: 1, - reason: "artifact_capture_disabled", - }) - .expect("artifact pointer"); - let request = mesh_llm_log_store::DeleteOneRequest::new( - mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(144)), - request_id, - mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), - ) - .expect("delete request"); - - assert!(matches!( - state - .query_facade() - .expect("query facade") - .prepare_delete_request(&request, &NeverCancelled), - Err(LogStoreError::ArtifactDeletionUnavailable) - )); - assert!(store.query_request(request_id).unwrap().is_some()); - assert!(store.delete_one_receipt(&request).unwrap().is_none()); -} - -#[test] -fn privacy_failure_disables_only_artifacts_and_records_one_sanitized_marker() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &artifact_config(), - |artifact_root, clock, store| { - FailOpenArtifactCapture::open_with_privacy( - artifact_root, - clock, - store, - canonical_artifact_redactor(), - Arc::new(RejectPrivacy), - ) - }, - ); - - assert_eq!( - state.health(), - LoggingRuntimeHealth { - metadata_available: true, - artifact_capture_available: false, - artifact_capture_ready: false, - artifact_capture_degradation: Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE,), - } - ); - let store = state.store().expect("metadata store remains available"); - assert_eq!(marker_audit_count(&store), 1); - - store - .insert_summary( - "metadata-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - store - .insert_audit_entry( - "metadata-audit", - Some("metadata-request"), - &store.now(), - "test", - "metadata_still_available", - None, - ) - .expect("metadata audit insert"); - - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "artifact-after-disable", - request_id: "metadata-request", - kind: "request_body", - occurred_at: &store.now(), - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("disabled capture is fail-open"); - assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); - let repeated_outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "artifact-after-disable-again", - request_id: "metadata-request", - kind: "request_body", - occurred_at: &store.now(), - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("repeated disabled capture is fail-open"); - assert!(matches!( - repeated_outcome, - ArtifactCaptureOutcome::Disabled(_) - )); - assert_eq!(marker_audit_count(&store), 1); - assert!(store.get_summary("metadata-request").unwrap().is_some()); - let total_audits: i64 = store - .conn() - .query_row("SELECT COUNT(*) FROM audit_entries", [], |row| row.get(0)) - .expect("count all audits"); - assert_eq!(total_audits, 2); -} - -#[test] -fn write_time_privacy_failure_publishes_one_marker_and_keeps_metadata_available() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &artifact_config(), - |artifact_root, clock, store| { - FailOpenArtifactCapture::open_with_privacy( - artifact_root, - clock, - store, - canonical_artifact_redactor(), - Arc::new(RejectArtifactFiles), - ) - }, - ); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "write-time-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - - let occurred_at = store.now(); - let write = |artifact_id| ArtifactCaptureRequest { - artifact_id, - request_id: "write-time-request", - kind: "request_body", - occurred_at: &occurred_at, - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }; - assert!(matches!( - state.write_artifact(write("write-time-artifact")).unwrap(), - ArtifactCaptureOutcome::Disabled(_) - )); - assert!(matches!( - state - .write_artifact(write("write-time-artifact-again")) - .unwrap(), - ArtifactCaptureOutcome::Disabled(_) - )); - - assert_eq!( - state.health().artifact_capture_degradation, - Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE) - ); - assert!(state.health().metadata_available); - assert_eq!(marker_audit_count(&store), 1); -} - -#[test] -fn raw_artifact_bytes_are_redacted_before_the_capture_boundary() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "raw-artifact-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - - let secret = b"Bearer super-secret-token-0123456789"; - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "raw-artifact", - request_id: "raw-artifact-request", - kind: "request_body", - occurred_at: &store.now(), - content: secret, - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("artifact write"); - assert!(matches!( - outcome, - ArtifactCaptureOutcome::Written(receipt) if receipt.redacted - )); - - let stored = std::fs::read( - foundation - .artifact_dir() - .join("raw-artifact-request") - .join("raw-artifact"), - ) - .expect("read stored artifact"); - assert!(!stored.windows(secret.len()).any(|window| window == secret)); - assert_eq!(stored, b"[REDACTED]"); - assert!( - store - .get_artifact_pointer("raw-artifact") - .expect("artifact pointer") - .expect("pointer present") - .redacted - ); + drop(lease); + drop(attribution); + attachment.terminal(TerminalOutcome::Completed); } #[test] @@ -688,15 +163,16 @@ fn unavailable_foundation_is_sanitized_and_fail_open() { #[test] fn incompatible_schema_is_reported_without_exposing_storage_details() { + let incompatible_version = mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION + 1; let state = LoggingRuntimeState::unavailable(LoggingMetadataState::SchemaIncompatible { - found: 99, + found: incompatible_version, supported: mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION, }); let status = state.status(); assert!(!status.metadata_available); assert_eq!(status.metadata_state, "schema_incompatible"); - assert_eq!(status.schema_version, Some(99)); + assert_eq!(status.schema_version, Some(incompatible_version)); assert_eq!( status.supported_schema_version, Some(mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION) @@ -704,198 +180,6 @@ fn incompatible_schema_is_reported_without_exposing_storage_details() { assert!(state.store().is_none()); } -#[tokio::test] -async fn awaited_runtime_startup_cleanup_uses_injected_store_time_before_ready() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - retention_ttl_secs: 3_600, - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = LoggingRuntimeState::initialize_with_store_clock_for_test( - &foundation, - &config, - Arc::new(FixedStoreClock("2026-08-03T12:00:00Z")), - ); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "expired-before-startup", - None, - None, - None, - None, - "2026-08-03T10:00:00Z", - None, - None, - None, - ) - .expect("insert stale summary"); - store - .insert_summary( - "retained-after-startup", - None, - None, - None, - None, - "2026-08-03T11:30:01Z", - None, - None, - None, - ) - .expect("insert retained summary"); - for (request_id, terminal_at) in [ - ("expired-before-startup", "2026-08-03T10:00:00Z"), - ("retained-after-startup", "2026-08-03T11:30:01Z"), - ] { - store - .write_terminal_event( - request_id, - &format!("terminal-{request_id}"), - r#"{"type":"completed"}"#, - "completed", - None, - terminal_at, - ) - .expect("write deterministic terminal record"); - } - - let service = state - .start_persistence_worker() - .await - .expect("service becomes ready after cleanup outcome"); - - assert!( - store - .get_summary("expired-before-startup") - .expect("load stale summary") - .is_none(), - "startup cleanup completed before the ready service was returned" - ); - assert!( - store - .get_summary("retained-after-startup") - .expect("load retained summary") - .is_some() - ); - assert_eq!(state.status().cleanup_last_outcome, Some("completed")); - assert!(matches!( - state.status().cleanup_last_deleted_count, - Some(count) if count >= 1 - )); - - state.shutdown_cleanup_worker().await; - assert!(service.shutdown().await); -} - -#[tokio::test] -async fn concurrent_starts_publish_one_cleanup_scheduler_and_truthful_status() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); - let barrier = Arc::new(tokio::sync::Barrier::new(3)); - - let first_state = Arc::clone(&state); - let first_barrier = Arc::clone(&barrier); - let first = tokio::spawn(async move { - first_barrier.wait().await; - first_state.start_persistence_worker().await - }); - let second_state = Arc::clone(&state); - let second_barrier = Arc::clone(&barrier); - let second = tokio::spawn(async move { - second_barrier.wait().await; - second_state.start_persistence_worker().await - }); - - // Release both calls from the same scheduling boundary. The - // state-local candidate count is incremented at construction under - // the activation gate, so it proves no losing task was ever spawned. - barrier.wait().await; - let first_service = first - .await - .expect("first start task joins") - .expect("first start returns the ready service"); - let second_service = second - .await - .expect("second start task joins") - .expect("second start returns the ready service"); - - assert!(Arc::ptr_eq(&first_service, &second_service)); - assert_eq!(state.cleanup_candidate_count_for_test(), 1); - assert!(state.has_cleanup_worker_for_test()); - assert_eq!(state.status().cleanup_worker_state, "running"); - assert_eq!(state.status().persistence_worker_state, "running"); - - // Shutdown drains the one operational audit produced by the one - // startup cleanup. A losing scheduler would produce a second audit or - // leave a second task able to race a later cleanup pass. - state.shutdown_cleanup_worker().await; - assert!(first_service.shutdown().await); - assert!(!state.has_cleanup_worker_for_test()); - assert_eq!(state.status().cleanup_worker_state, "stopped"); - assert_eq!(state.status().persistence_worker_state, "stopped"); - let cleanup_audits: i64 = state - .store() - .expect("metadata store available") - .conn() - .query_row( - "SELECT COUNT(*) FROM audit_entries WHERE action = 'logging_cleanup_completed'", - [], - |row| row.get(0), - ) - .expect("count startup cleanup audits"); - assert_eq!(cleanup_audits, 1); -} - -#[tokio::test] -async fn retirement_after_cleanup_candidate_publication_leaves_no_worker_on_displaced_state() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); - let hook = state.install_cleanup_publish_hook_for_test(); - let displaced_service = Arc::clone( - state - .service - .as_ref() - .expect("healthy state owns a persistence service"), - ); - - let starting_state = Arc::clone(&state); - let start = tokio::spawn(async move { starting_state.start_persistence_worker().await }); - - // The candidate is already atomically published, but the starter has - // not yet observed readiness. Retire the displaced state in this - // exact window and prove retirement cancels and joins that task. - hook.candidate_created.wait().await; - state.retire_and_shutdown().await; - hook.resume_install.wait().await; - - assert!(start.await.expect("start task joins").is_none()); - assert!(state.is_retired()); - assert!( - !state.has_cleanup_worker_for_test(), - "a retired state must not retain a cleanup task handle" - ); - assert_eq!(state.status().cleanup_worker_state, "stopped"); - assert!(!displaced_service.is_startable()); - assert!(!displaced_service.is_spawned()); - assert_eq!( - state.status().persistence_worker_state, - "stopped", - "replacement must have joined the displaced persistence worker" - ); -} - #[test] fn store_open_failure_is_fail_open_without_exposing_the_failed_path() { let root = tempfile::tempdir().expect("temporary logging root"); @@ -934,20 +218,3 @@ fn applies_retention_and_replay_limits_together_to_the_installed_service() { state.apply_dynamic_limits(next).expect("healthy runtime"); assert_eq!(state.dynamic_limits(), Some(next)); } - -#[test] -fn openai_lifecycle_observer_snapshot_is_absent_when_disabled_or_retired() { - assert!( - LoggingRuntimeState::unavailable(LoggingMetadataState::StorageUnavailable) - .openai_lifecycle_observer() - .is_none() - ); - - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - assert!(state.openai_lifecycle_observer().is_some()); - - state.retired.store(true, Ordering::Release); - assert!(state.openai_lifecycle_observer().is_none()); -} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs new file mode 100644 index 0000000000..b5d7979355 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs @@ -0,0 +1,547 @@ +use super::*; + +#[test] +fn production_ingress_wiring_advertises_capture_only_when_storage_is_usable() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); + + let status = state.status(); + assert!(status.artifact_capture_available); + assert!(status.artifact_capture_ready); +} + +#[tokio::test] +async fn ingress_attachment_captures_redacted_request_and_response_bodies() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &artifact_config(), + )); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let mut attachment = state.openai_ingress_attachment( + request_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), + ); + + attachment.capture_request_body( + br#"{"authorization":"Bearer test-secret","model":"safe"}"#, + Some("application/json"), + ); + attachment.route_observer().capture_response_body( + br#"{"id":"chatcmpl-safe","api_key":"test-secret"}"#, + Some("application/json; charset=utf-8"), + ); + attachment.terminal(TerminalOutcome::CompletedWithStatus(201)); + + // Summary, admission, both artifact commands, and terminal state share one + // serial persistence owner. Ingress itself performs no SQLite/file work. + assert_eq!( + state + .service + .as_ref() + .expect("logging service") + .pump_sync() + .await, + 5 + ); + + let store = state.store().expect("metadata store"); + let request_key = request_id.as_uuid().to_string(); + let artifact_page = store + .query_artifacts( + &request_key, + &mesh_llm_log_store::PageQuery { + limit: 10, + cursor: None, + sort: mesh_llm_log_store::QuerySort::Ascending, + }, + ) + .expect("query artifacts"); + assert_eq!(artifact_page.items.len(), 2); + let request_artifact = artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "request") + .expect("request artifact") + .artifact_id + .clone(); + let response_artifact = artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "response") + .expect("response artifact") + .artifact_id + .clone(); + let artifacts = state.query_facade().expect("query facade"); + let request_content = artifacts + .read_artifact(&request_artifact) + .expect("request body"); + let response_content = artifacts + .read_artifact(&response_artifact) + .expect("response body"); + let request_text = String::from_utf8(request_content.bytes).expect("utf8 request artifact"); + let response_text = String::from_utf8(response_content.bytes).expect("utf8 response artifact"); + assert!(!request_text.contains("test-secret")); + assert!(!response_text.contains("test-secret")); + assert!(response_text.contains("chatcmpl-safe")); + let admitted_at = store + .get_summary(&request_key) + .unwrap() + .expect("summary") + .created_at; + assert!( + artifact_page + .items + .iter() + .all(|artifact| artifact.occurred_at >= admitted_at), + "artifact metadata must use the lifecycle clock at capture time" + ); + assert_eq!( + artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "request") + .and_then(|artifact| artifact.media_kind.as_deref()), + Some("application/json"), + "the parsed OpenAI JSON boundary supplies this closed semantic media kind" + ); + assert_eq!( + artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "response") + .and_then(|artifact| artifact.media_kind.as_deref()), + Some("application/json") + ); + let events = store.list_events_for_summary(&request_key).unwrap(); + assert_eq!(events.len(), 2, "one admitted and one terminal event"); + assert_ne!(events[0].event_id, events[1].event_id); +} + +#[tokio::test] +async fn streaming_and_oversized_responses_persist_explicit_unavailable_metadata() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &artifact_config(), + )); + + let streaming_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let streaming = state.openai_ingress_attachment( + streaming_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), + ); + streaming.route_observer().capture_response_unavailable( + crate::logging::ArtifactUnavailableReason::StreamingResponseNotAssembled, + ); + + let oversized_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let oversized = state.openai_ingress_attachment( + oversized_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/responses"), + ); + oversized + .route_observer() + .capture_response_body(&vec![b'x'; 4_097], Some("not a valid media type")); + + state + .service + .as_ref() + .expect("logging service") + .pump_sync() + .await; + let store = state.store().expect("metadata store"); + for (request_id, reason) in [ + (streaming_id, "streaming_response_not_assembled"), + (oversized_id, "capture_content_limit_exceeded"), + ] { + let page = store + .query_artifacts( + &request_id.as_uuid().to_string(), + &mesh_llm_log_store::PageQuery { + limit: 10, + cursor: None, + sort: mesh_llm_log_store::QuerySort::Ascending, + }, + ) + .expect("query unavailable artifact"); + assert_eq!(page.items.len(), 1); + let artifact = &page.items[0]; + assert_eq!(artifact.unavailable_reason.as_deref(), Some(reason)); + assert_eq!(artifact.bytes, 0); + assert!(artifact.checksum.is_none()); + assert!(!artifact.missing); + assert!(!artifact.corrupt); + } +} + +#[test] +fn metadata_only_never_opens_or_writes_artifact_capture() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let opened = Arc::new(AtomicBool::new(false)); + let opened_by_callback = Arc::clone(&opened); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &mesh_llm_config::LoggingConfig::default(), + move |_, _, _| { + opened_by_callback.store(true, Ordering::SeqCst); + Err(LogStoreError::PrivacyNotGuaranteed) + }, + ); + + assert!(!opened.load(Ordering::SeqCst)); + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "metadata-only", + request_id: "request", + kind: "body", + occurred_at: "2025-01-01T00:00:00Z", + content: b"must not be stored", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("metadata-only capture is disabled"); + assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); + assert_eq!( + std::fs::read_dir(foundation.artifact_dir()) + .expect("artifact directory") + .count(), + 0 + ); +} + +#[test] +fn metadata_only_query_facade_deletes_terminal_metadata_without_artifact_capture() { + struct NeverCancelled; + + impl MaintenanceExecutionControl for NeverCancelled { + fn is_cancelled(&self) -> bool { + false + } + } + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + let store = state.store().expect("metadata store"); + let request_id = "00000000-0000-4000-8000-000000000143"; + store + .insert_summary( + request_id, + Some("model"), + Some("route"), + None, + None, + "2025-01-01T00:00:00Z", + None, + None, + None, + ) + .expect("summary"); + store + .conn() + .execute( + "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", + [request_id], + ) + .expect("terminal summary"); + let request = mesh_llm_log_store::DeleteOneRequest::new( + mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(143)), + request_id, + mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), + ) + .expect("delete request"); + let facade = state.query_facade().expect("query facade"); + + facade + .prepare_delete_request(&request, &NeverCancelled) + .expect("metadata-only preparation"); + let receipt = facade + .execute_prepared_delete_request(&request, &NeverCancelled) + .expect("metadata-only execution"); + + assert_eq!( + receipt.state, + mesh_llm_log_store::MaintenanceReceiptState::Completed + ); + assert!(store.query_request(request_id).unwrap().is_none()); +} + +#[test] +fn metadata_only_query_facade_rejects_artifact_pointers_without_preparing_delete() { + struct NeverCancelled; + + impl MaintenanceExecutionControl for NeverCancelled { + fn is_cancelled(&self) -> bool { + false + } + } + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + let store = state.store().expect("metadata store"); + let request_id = "00000000-0000-4000-8000-000000000144"; + store + .insert_summary( + request_id, + Some("model"), + Some("route"), + None, + None, + "2025-01-01T00:00:00Z", + None, + None, + None, + ) + .expect("summary"); + store + .conn() + .execute( + "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", + [request_id], + ) + .expect("terminal summary"); + store + .insert_unavailable_artifact_pointer(mesh_llm_log_store::UnavailableArtifactPointer { + artifact_id: "00000000-0000-4000-8000-000000000244", + request_id, + occurred_at: "2025-01-01T00:00:01Z", + kind: "response", + media_kind: Some("text/plain"), + version: 1, + reason: "artifact_capture_disabled", + }) + .expect("artifact pointer"); + let request = mesh_llm_log_store::DeleteOneRequest::new( + mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(144)), + request_id, + mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), + ) + .expect("delete request"); + + assert!(matches!( + state + .query_facade() + .expect("query facade") + .prepare_delete_request(&request, &NeverCancelled), + Err(LogStoreError::ArtifactDeletionUnavailable) + )); + assert!(store.query_request(request_id).unwrap().is_some()); + assert!(store.delete_one_receipt(&request).unwrap().is_none()); +} + +#[test] +fn privacy_failure_disables_only_artifacts_and_records_one_sanitized_marker() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &artifact_config(), + |artifact_root, clock, store| { + FailOpenArtifactCapture::open_with_privacy( + artifact_root, + clock, + store, + canonical_artifact_redactor(), + Arc::new(RejectPrivacy), + ) + }, + ); + + assert_eq!( + state.health(), + LoggingRuntimeHealth { + metadata_available: true, + artifact_capture_available: false, + artifact_capture_ready: false, + artifact_capture_degradation: Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE,), + } + ); + let store = state.store().expect("metadata store remains available"); + assert_eq!(marker_audit_count(&store), 1); + + store + .insert_summary( + "metadata-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + store + .insert_audit_entry( + "metadata-audit", + Some("metadata-request"), + &store.now(), + "test", + "metadata_still_available", + None, + ) + .expect("metadata audit insert"); + + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "artifact-after-disable", + request_id: "metadata-request", + kind: "request_body", + occurred_at: &store.now(), + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("disabled capture is fail-open"); + assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); + let repeated_outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "artifact-after-disable-again", + request_id: "metadata-request", + kind: "request_body", + occurred_at: &store.now(), + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("repeated disabled capture is fail-open"); + assert!(matches!( + repeated_outcome, + ArtifactCaptureOutcome::Disabled(_) + )); + assert_eq!(marker_audit_count(&store), 1); + assert!(store.get_summary("metadata-request").unwrap().is_some()); + let total_audits: i64 = store + .conn() + .query_row("SELECT COUNT(*) FROM audit_entries", [], |row| row.get(0)) + .expect("count all audits"); + assert_eq!(total_audits, 2); +} + +#[test] +fn write_time_privacy_failure_publishes_one_marker_and_keeps_metadata_available() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &artifact_config(), + |artifact_root, clock, store| { + FailOpenArtifactCapture::open_with_privacy( + artifact_root, + clock, + store, + canonical_artifact_redactor(), + Arc::new(RejectArtifactFiles), + ) + }, + ); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "write-time-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + + let occurred_at = store.now(); + let write = |artifact_id| ArtifactCaptureRequest { + artifact_id, + request_id: "write-time-request", + kind: "request_body", + occurred_at: &occurred_at, + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }; + assert!(matches!( + state.write_artifact(write("write-time-artifact")).unwrap(), + ArtifactCaptureOutcome::Disabled(_) + )); + assert!(matches!( + state + .write_artifact(write("write-time-artifact-again")) + .unwrap(), + ArtifactCaptureOutcome::Disabled(_) + )); + + assert_eq!( + state.health().artifact_capture_degradation, + Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE) + ); + assert!(state.health().metadata_available); + assert_eq!(marker_audit_count(&store), 1); +} + +#[test] +fn raw_artifact_bytes_are_redacted_before_the_capture_boundary() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "raw-artifact-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + + let secret = b"Bearer super-secret-token-0123456789"; + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "raw-artifact", + request_id: "raw-artifact-request", + kind: "request_body", + occurred_at: &store.now(), + content: secret, + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("artifact write"); + assert!(matches!( + outcome, + ArtifactCaptureOutcome::Written(receipt) if receipt.redacted + )); + + let stored = std::fs::read( + foundation + .artifact_dir() + .join("raw-artifact-request") + .join("raw-artifact"), + ) + .expect("read stored artifact"); + assert!(!stored.windows(secret.len()).any(|window| window == secret)); + assert_eq!(stored, b"[REDACTED]"); + assert!( + store + .get_artifact_pointer("raw-artifact") + .expect("artifact pointer") + .expect("pointer present") + .redacted + ); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs new file mode 100644 index 0000000000..13a405aae7 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs @@ -0,0 +1,314 @@ +use super::*; + +fn initialized_state() -> (Arc, tempfile::TempDir) { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &mesh_llm_config::LoggingConfig::default(), + )); + (state, root) +} + +fn provisional_local_metadata() -> RequestSummaryMetadata { + RequestSummaryMetadata::from_parts( + Some("responses"), + Some("model-a"), + Some("provider-a"), + Some("engine-a"), + ) + .with_source(Some("direct_http")) + .with_method(Some("POST")) + .with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ) +} + +fn authenticated_direct_metadata(endpoint_id: &str) -> RequestSummaryMetadata { + RequestSummaryMetadata::default().with_caller_identity( + Some(endpoint_id), + Some("192.0.2.71:11204"), + Some(CallerPathType::RemoteQuicHttp), + ) +} + +fn authenticated_relay_metadata(endpoint_id: &str) -> RequestSummaryMetadata { + RequestSummaryMetadata::default().with_caller_identity( + Some(endpoint_id), + None, + Some(CallerPathType::Relay), + ) +} + +fn assert_active_caller( + state: &LoggingRuntimeState, + request_id: mesh_llm_events::logging::identifiers::RequestId, + endpoint_id: &str, + address: Option<&str>, + path_type: &str, +) { + let active = state + .service + .as_ref() + .expect("logging service") + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active request"); + assert_eq!(active.metadata.caller_endpoint_id(), Some(endpoint_id)); + assert_eq!(active.metadata.caller_addr(), address); + assert_eq!(active.metadata.caller_path_type(), Some(path_type)); + assert_eq!(active.metadata.route(), Some("responses")); + assert_eq!(active.metadata.model(), Some("model-a")); + assert_eq!(active.metadata.provider(), Some("provider-a")); + assert_eq!(active.metadata.engine(), Some("engine-a")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); +} + +#[tokio::test] +async fn production_order_authenticated_direct_survives_local_registration_and_persistence() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "71".repeat(32); + + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_direct_metadata(&endpoint_id)) + .expect("remote attribution lease"); + let suppression = state + .suppress_remote_tunneled_request(request_id) + .expect("remote suppression lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller( + &state, + request_id, + &endpoint_id, + Some("192.0.2.71:11204"), + "remote_quic_http", + ); + attachment.terminal(TerminalOutcome::Completed); + drop(suppression); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr.as_deref(), Some("192.0.2.71:11204")); + assert_eq!( + durable.caller_path_type.as_deref(), + Some("remote_quic_http") + ); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn reverse_order_authenticated_relay_atomically_clears_local_address() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "72".repeat(32); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + let first = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("first remote attribution lease"); + let second = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("idempotent remote attribution lease"); + + assert_active_caller(&state, request_id, &endpoint_id, None, "relay"); + attachment.terminal(TerminalOutcome::Completed); + drop(first); + drop(second); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type.as_deref(), Some("relay")); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn markerless_authenticated_relay_survives_production_order_without_suppression() { + let (state, _root) = initialized_state(); + let (forwarded, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), + ); + let request_id = request_id.expect("tunnel-generated request ID"); + let endpoint_id = "73".repeat(32); + + assert_eq!( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(&forwarded), + Some(request_id) + ); + + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("markerless remote attribution lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller(&state, request_id, &endpoint_id, None, "relay"); + attachment.terminal(TerminalOutcome::Completed); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type.as_deref(), Some("relay")); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn no_header_authenticated_direct_survives_active_and_durable_lifecycle() { + let (state, _root) = initialized_state(); + let (forwarded, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), + ); + let request_id = request_id.expect("tunnel-generated request ID"); + let endpoint_id = "76".repeat(32); + + assert_eq!( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(&forwarded), + Some(request_id) + ); + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_direct_metadata(&endpoint_id)) + .expect("markerless remote attribution lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller( + &state, + request_id, + &endpoint_id, + Some("192.0.2.71:11204"), + "remote_quic_http", + ); + attachment.terminal(TerminalOutcome::Completed); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr.as_deref(), Some("192.0.2.71:11204")); + assert_eq!( + durable.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + +#[tokio::test] +async fn endpoint_only_caller_survives_pending_active_terminal_and_durable_lifecycle() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "79".repeat(32); + let later_endpoint_id = "7a".repeat(32); + + let endpoint_only = state + .attribute_remote_tunneled_request( + request_id, + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None), + ) + .expect("pending endpoint-only attribution"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + let direct = state + .attribute_remote_tunneled_request( + request_id, + authenticated_direct_metadata(&later_endpoint_id), + ) + .expect("later direct attribution lease"); + let relay = state + .attribute_remote_tunneled_request( + request_id, + authenticated_relay_metadata(&later_endpoint_id), + ) + .expect("later relay attribution lease"); + + let active = state + .service + .as_ref() + .expect("logging service") + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active request"); + assert_eq!( + active.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(active.metadata.caller_addr(), None); + assert_eq!(active.metadata.caller_path_type(), None); + assert_eq!(active.metadata.route(), Some("responses")); + assert_eq!(active.metadata.model(), Some("model-a")); + assert_eq!(active.metadata.provider(), Some("provider-a")); + assert_eq!(active.metadata.engine(), Some("engine-a")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); + + attachment.terminal(TerminalOutcome::Completed); + drop(relay); + drop(direct); + drop(endpoint_only); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type, None); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs new file mode 100644 index 0000000000..c66c48944f --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs @@ -0,0 +1,250 @@ +use super::*; + +#[tokio::test] +async fn disabled_webhook_config_starts_no_delivery_scheduler() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + + let service = state + .start_persistence_worker() + .await + .expect("logging service starts"); + + assert!(!state.has_webhook_delivery_worker_for_test()); + state.shutdown_cleanup_worker().await; + assert!(service.shutdown().await); +} + +#[tokio::test] +async fn enabled_webhook_config_starts_and_retires_one_delivery_scheduler() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + webhook: mesh_llm_config::LoggingWebhookConfig { + enabled: true, + url: Some("http://127.0.0.1:9444/webhook".to_string()), + ..Default::default() + }, + ..Default::default() + }; + let state = LoggingRuntimeState::initialize(&foundation, &config); + + state + .start_persistence_worker() + .await + .expect("logging service starts"); + assert!(state.has_webhook_delivery_worker_for_test()); + + assert!(state.retire_and_shutdown().await); + assert!(!state.has_webhook_delivery_worker_for_test()); +} + +#[tokio::test] +async fn awaited_runtime_startup_cleanup_uses_injected_store_time_before_ready() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + retention_ttl_secs: 3_600, + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = LoggingRuntimeState::initialize_with_store_clock_for_test( + &foundation, + &config, + Arc::new(FixedStoreClock("2026-08-03T12:00:00Z")), + ); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "expired-before-startup", + None, + None, + None, + None, + "2026-08-03T10:00:00Z", + None, + None, + None, + ) + .expect("insert stale summary"); + store + .insert_summary( + "retained-after-startup", + None, + None, + None, + None, + "2026-08-03T11:30:01Z", + None, + None, + None, + ) + .expect("insert retained summary"); + for (request_id, terminal_at) in [ + ("expired-before-startup", "2026-08-03T10:00:00Z"), + ("retained-after-startup", "2026-08-03T11:30:01Z"), + ] { + store + .write_terminal_event( + request_id, + &format!("terminal-{request_id}"), + r#"{"type":"completed"}"#, + "completed", + None, + terminal_at, + ) + .expect("write deterministic terminal record"); + } + + let service = state + .start_persistence_worker() + .await + .expect("service becomes ready after cleanup outcome"); + + assert!( + store + .get_summary("expired-before-startup") + .expect("load stale summary") + .is_none(), + "startup cleanup completed before the ready service was returned" + ); + assert!( + store + .get_summary("retained-after-startup") + .expect("load retained summary") + .is_some() + ); + assert_eq!(state.status().cleanup_last_outcome, Some("completed")); + assert!(matches!( + state.status().cleanup_last_deleted_count, + Some(count) if count >= 1 + )); + + state.shutdown_cleanup_worker().await; + assert!(service.shutdown().await); +} + +#[tokio::test] +async fn concurrent_starts_publish_one_cleanup_scheduler_and_truthful_status() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + + let first_state = Arc::clone(&state); + let first_barrier = Arc::clone(&barrier); + let first = tokio::spawn(async move { + first_barrier.wait().await; + first_state.start_persistence_worker().await + }); + let second_state = Arc::clone(&state); + let second_barrier = Arc::clone(&barrier); + let second = tokio::spawn(async move { + second_barrier.wait().await; + second_state.start_persistence_worker().await + }); + + // Release both calls from the same scheduling boundary. The + // state-local candidate count is incremented at construction under + // the activation gate, so it proves no losing task was ever spawned. + barrier.wait().await; + let first_service = first + .await + .expect("first start task joins") + .expect("first start returns the ready service"); + let second_service = second + .await + .expect("second start task joins") + .expect("second start returns the ready service"); + + assert!(Arc::ptr_eq(&first_service, &second_service)); + assert_eq!(state.cleanup_candidate_count_for_test(), 1); + assert!(state.has_cleanup_worker_for_test()); + assert_eq!(state.status().cleanup_worker_state, "running"); + assert_eq!(state.status().persistence_worker_state, "running"); + + // Shutdown drains the one operational audit produced by the one + // startup cleanup. A losing scheduler would produce a second audit or + // leave a second task able to race a later cleanup pass. + state.shutdown_cleanup_worker().await; + assert!(first_service.shutdown().await); + assert!(!state.has_cleanup_worker_for_test()); + assert_eq!(state.status().cleanup_worker_state, "stopped"); + assert_eq!(state.status().persistence_worker_state, "stopped"); + let cleanup_audits: i64 = state + .store() + .expect("metadata store available") + .conn() + .query_row( + "SELECT COUNT(*) FROM audit_entries WHERE action = 'logging_cleanup_completed'", + [], + |row| row.get(0), + ) + .expect("count startup cleanup audits"); + assert_eq!(cleanup_audits, 1); +} + +#[tokio::test] +async fn retirement_after_cleanup_candidate_publication_leaves_no_worker_on_displaced_state() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); + let hook = state.install_cleanup_publish_hook_for_test(); + let displaced_service = Arc::clone( + state + .service + .as_ref() + .expect("healthy state owns a persistence service"), + ); + + let starting_state = Arc::clone(&state); + let start = tokio::spawn(async move { starting_state.start_persistence_worker().await }); + + // The candidate is already atomically published, but the starter has + // not yet observed readiness. Retire the displaced state in this + // exact window and prove retirement cancels and joins that task. + hook.candidate_created.wait().await; + state.retire_and_shutdown().await; + hook.resume_install.wait().await; + + assert!(start.await.expect("start task joins").is_none()); + assert!(state.is_retired()); + assert!( + !state.has_cleanup_worker_for_test(), + "a retired state must not retain a cleanup task handle" + ); + assert_eq!(state.status().cleanup_worker_state, "stopped"); + assert!(!displaced_service.is_startable()); + assert!(!displaced_service.is_spawned()); + assert_eq!( + state.status().persistence_worker_state, + "stopped", + "replacement must have joined the displaced persistence worker" + ); +} + +#[test] +fn openai_lifecycle_observer_snapshot_is_absent_when_disabled_or_retired() { + assert!( + LoggingRuntimeState::unavailable(LoggingMetadataState::StorageUnavailable) + .openai_lifecycle_observer() + .is_none() + ); + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + assert!(state.openai_lifecycle_observer().is_some()); + + state.retired.store(true, Ordering::Release); + assert!(state.openai_lifecycle_observer().is_none()); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/service.rs b/crates/mesh-llm-host-runtime/src/logging/service.rs index 06eddf6330..caf51dec23 100644 --- a/crates/mesh-llm-host-runtime/src/logging/service.rs +++ b/crates/mesh-llm-host-runtime/src/logging/service.rs @@ -29,8 +29,8 @@ mod operational_audit; mod persistence_delivery; use event_delivery::{EventDelivery, ServiceLifecycleRecorder, enqueue_event_with_delivery}; pub use operational_audit::{ - OperationalAuditContext, OperationalAuditRecord, OperationalAuditRecordBuilder, - OperationalAuditSeverity, OperationalAuditSubjectKind, + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditRecordBuilder, OperationalAuditSeverity, OperationalAuditSubjectKind, }; pub(crate) use persistence_delivery::PersistenceWorkerState; pub use persistence_delivery::WorkerHandle; @@ -740,6 +740,29 @@ impl LoggingService { } } + pub(crate) fn merge_authenticated_remote_caller( + &self, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> bool { + let Some(summary) = self + .registry + .merge_authenticated_remote_caller(&request_id.as_uuid().to_string(), metadata) + else { + return false; + }; + if self.sink.is_some() { + offer_summary_persistence( + &self.delivery, + &self.persistence_queue_drops, + &self.persistence_outstanding, + &self.metrics, + summary, + ); + } + true + } + /// Record the beginning of a transport attempt under an existing request. /// The returned branded identifier is used by its completion or failure and /// never changes the parent request lifecycle. @@ -895,29 +918,24 @@ impl LoggingService { pub(crate) fn retire(&self) { self.startable.store(false, Ordering::Release); } - pub(crate) fn is_startable(&self) -> bool { self.startable.load(Ordering::Acquire) } - #[cfg(test)] pub(crate) fn with_shutdown_drain_timeout(mut self, timeout: Duration) -> Self { self.shutdown_drain_timeout = timeout; self } - /// Check if the service is currently spawned and running. For observability / tests. #[allow(dead_code)] pub fn is_spawned(&self) -> bool { self.spawned.load(Ordering::Acquire) } - /// Clone writer for external observation of drop counters. #[allow(dead_code)] pub fn writer_ref(&self) -> Arc { Arc::clone(&self.writer) } - #[cfg(test)] pub(crate) fn worker_handle_lock_for_test( &self, diff --git a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs index 9046245de9..a02fe566fe 100644 --- a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs +++ b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs @@ -1,171 +1,8 @@ //! Bounded operational-audit records admitted by the logging service. -use std::collections::BTreeMap; +mod context; -const OPERATIONAL_AUDIT_CONTEXT_VERSION: u8 = 1; -const MAX_CONTEXT_VALUE_CHARS: usize = 256; -const MAX_NUMERIC_SUMMARIES: usize = 8; - -/// Closed subject vocabulary for diagnostic operational-audit context. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum OperationalAuditSubjectKind { - Runtime, - Model, - RuntimeInstance, - CliCommand, -} - -impl OperationalAuditSubjectKind { - pub const fn as_str(self) -> &'static str { - match self { - Self::Runtime => "runtime", - Self::Model => "model", - Self::RuntimeInstance => "runtime_instance", - Self::CliCommand => "cli_command", - } - } - - pub(crate) fn parse(value: &str) -> Option { - match value { - "runtime" => Some(Self::Runtime), - "model" => Some(Self::Model), - "runtime_instance" => Some(Self::RuntimeInstance), - "cli_command" => Some(Self::CliCommand), - _ => None, - } - } -} - -/// Versioned, bounded context shared by audit replay and durable storage. -/// -/// Values are sanitized before this type accepts them. Arbitrary detail JSON, -/// command arguments, error text, and payload bodies cannot enter this shape. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct OperationalAuditContext { - subject_kind: Option, - subject_id: Option, - operation_id: Option, - request_id: Option, - reason_code: Option<&'static str>, - outcome: Option<&'static str>, - duration_ms: Option, - numeric_summaries: BTreeMap<&'static str, u64>, -} - -impl OperationalAuditContext { - pub fn new() -> Self { - Self::default() - } - - pub fn subject(mut self, kind: OperationalAuditSubjectKind, id: &str) -> Self { - self.subject_kind = Some(kind); - self.subject_id = bounded_context_value(id); - self - } - - pub fn operation_id(mut self, operation_id: &str) -> Self { - self.operation_id = bounded_context_value(operation_id); - self - } - - pub fn request_id(mut self, request_id: &str) -> Self { - self.request_id = bounded_context_value(request_id); - self - } - - pub const fn reason_code(mut self, reason_code: &'static str) -> Self { - if Self::valid_static_code(reason_code) { - self.reason_code = Some(reason_code); - } - self - } - - pub const fn outcome(mut self, outcome: &'static str) -> Self { - if Self::valid_static_code(outcome) { - self.outcome = Some(outcome); - } - self - } - - pub const fn duration_ms(mut self, duration_ms: u64) -> Self { - self.duration_ms = Some(duration_ms); - self - } - - pub fn numeric_summary(mut self, key: &'static str, value: u64) -> Self { - if self.numeric_summaries.len() < MAX_NUMERIC_SUMMARIES && Self::valid_static_code(key) { - self.numeric_summaries.insert(key, value); - } - self - } - - pub(crate) const fn valid_static_code(value: &str) -> bool { - valid_static_code(value) - } - - pub(crate) fn fields(&self) -> serde_json::Map { - let mut fields = serde_json::Map::new(); - fields.insert( - "context_version".into(), - serde_json::json!(OPERATIONAL_AUDIT_CONTEXT_VERSION), - ); - if let Some(kind) = self.subject_kind { - fields.insert("subject_kind".into(), serde_json::json!(kind.as_str())); - } - insert_optional_string(&mut fields, "subject_id", self.subject_id.as_deref()); - insert_optional_string(&mut fields, "operation_id", self.operation_id.as_deref()); - insert_optional_string(&mut fields, "request_id", self.request_id.as_deref()); - insert_optional_string(&mut fields, "reason_code", self.reason_code); - insert_optional_string(&mut fields, "outcome", self.outcome); - if let Some(duration_ms) = self.duration_ms { - fields.insert("duration_ms".into(), serde_json::json!(duration_ms)); - } - if !self.numeric_summaries.is_empty() { - fields.insert( - "numeric_summaries".into(), - serde_json::json!(self.numeric_summaries), - ); - } - fields - } -} - -fn insert_optional_string( - fields: &mut serde_json::Map, - key: &'static str, - value: Option<&str>, -) { - if let Some(value) = value { - fields.insert(key.into(), serde_json::json!(value)); - } -} - -const fn valid_static_code(value: &str) -> bool { - if value.is_empty() || value.len() > 64 { - return false; - } - let bytes = value.as_bytes(); - let mut index = 0; - while index < bytes.len() { - let byte = bytes[index]; - if !(byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') { - return false; - } - index += 1; - } - true -} - -fn bounded_context_value(value: &str) -> Option { - let sanitized = crate::logging::policy::sanitize_paths_in_text(value); - let sanitized = crate::logging::policy::redact_urls_in_text(&sanitized); - let sanitized = crate::logging::policy::apply_redaction(&sanitized).0; - let trimmed = sanitized.trim(); - if trimmed.is_empty() { - return None; - } - Some(trimmed.chars().take(MAX_CONTEXT_VALUE_CHARS).collect()) -} +pub use context::{OperationalAuditContext, OperationalAuditPathType, OperationalAuditSubjectKind}; /// Static, bounded operational audit data admitted by the logging service. #[derive(Clone, Debug, Eq, PartialEq)] @@ -286,77 +123,3 @@ impl OperationalAuditSeverity { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn typed_context_is_sanitized_bounded_and_cardinality_limited() { - let long_subject = format!("{}?api_key=private", "model".repeat(80)); - let mut context = OperationalAuditContext::new() - .subject(OperationalAuditSubjectKind::Model, &long_subject) - .operation_id("runtime-7") - .reason_code("load_failed") - .outcome("failed") - .duration_ms(42); - for index in 0..12 { - let key = match index { - 0 => "metric_0", - 1 => "metric_1", - 2 => "metric_2", - 3 => "metric_3", - 4 => "metric_4", - 5 => "metric_5", - 6 => "metric_6", - 7 => "metric_7", - 8 => "metric_8", - 9 => "metric_9", - 10 => "metric_10", - _ => "metric_11", - }; - context = context.numeric_summary(key, index); - } - - let fields = context.fields(); - assert_eq!(fields["context_version"], 1); - assert_eq!(fields["subject_kind"], "model"); - assert_eq!(fields["operation_id"], "runtime-7"); - assert_eq!(fields["reason_code"], "load_failed"); - assert_eq!(fields["outcome"], "failed"); - assert_eq!(fields["duration_ms"], 42); - assert!(fields["subject_id"].as_str().unwrap().chars().count() <= 256); - assert!(!fields["subject_id"].as_str().unwrap().contains("private")); - assert_eq!(fields["numeric_summaries"].as_object().unwrap().len(), 8); - } - - #[test] - fn invalid_static_codes_are_not_admitted() { - let fields = OperationalAuditContext::new() - .reason_code("NOT VALID") - .outcome("also-not-valid") - .numeric_summary("bad-key", 1) - .fields(); - assert!(fields.get("reason_code").is_none()); - assert!(fields.get("outcome").is_none()); - assert!(fields.get("numeric_summaries").is_none()); - } - - #[test] - fn context_values_redact_url_credentials_and_query_secrets() { - let fields = OperationalAuditContext::new() - .subject( - OperationalAuditSubjectKind::Model, - "https://alice:top-secret@example.test/model?api_key=query-secret&safe=1", - ) - .fields(); - let subject_id = fields["subject_id"].as_str().expect("subject id"); - - assert!(!subject_id.contains("alice")); - assert!(!subject_id.contains("top-secret")); - assert!(!subject_id.contains("query-secret")); - assert!(subject_id.contains("[REDACTED]@example.test")); - assert!(subject_id.contains("api_key=[REDACTED]")); - assert!(subject_id.contains("safe=1")); - } -} diff --git a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs new file mode 100644 index 0000000000..5b66e9004e --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs @@ -0,0 +1,373 @@ +use std::{collections::BTreeMap, net::SocketAddr}; + +use mesh_llm_events::CliCommandSummary; + +const OPERATIONAL_AUDIT_CONTEXT_VERSION: u8 = 1; +const MAX_CONTEXT_VALUE_CHARS: usize = 256; +const MAX_NUMERIC_SUMMARIES: usize = 8; + +/// Closed subject vocabulary for diagnostic operational-audit context. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationalAuditSubjectKind { + Runtime, + Model, + RuntimeInstance, + CliCommand, +} + +impl OperationalAuditSubjectKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Runtime => "runtime", + Self::Model => "model", + Self::RuntimeInstance => "runtime_instance", + Self::CliCommand => "cli_command", + } + } + + pub(crate) fn parse(value: &str) -> Option { + match value { + "runtime" => Some(OperationalAuditContextSubjectKind::Public(Self::Runtime)), + "model" => Some(OperationalAuditContextSubjectKind::Public(Self::Model)), + "runtime_instance" => Some(OperationalAuditContextSubjectKind::Public( + Self::RuntimeInstance, + )), + "cli_command" => Some(OperationalAuditContextSubjectKind::Public(Self::CliCommand)), + "mesh_peer" => Some(OperationalAuditContextSubjectKind::MeshPeer), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum OperationalAuditContextSubjectKind { + Public(OperationalAuditSubjectKind), + MeshPeer, +} + +impl OperationalAuditContextSubjectKind { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Public(kind) => kind.as_str(), + Self::MeshPeer => "mesh_peer", + } + } +} + +/// Closed connection-path vocabulary for mesh peer audit context. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationalAuditPathType { + Direct, + Relay, +} + +impl OperationalAuditPathType { + pub const fn as_str(self) -> &'static str { + match self { + Self::Direct => "direct", + Self::Relay => "relay", + } + } +} + +/// Versioned, bounded context shared by audit replay and durable storage. +/// +/// Values are sanitized before this type accepts them. Arbitrary detail JSON, +/// command arguments, error text, and payload bodies cannot enter this shape. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct OperationalAuditContext { + subject_kind: Option, + subject_id: Option, + remote_addr: Option, + path_type: Option, + operation_id: Option, + request_id: Option, + reason_code: Option<&'static str>, + outcome: Option<&'static str>, + duration_ms: Option, + numeric_summaries: BTreeMap<&'static str, u64>, + command_summary: Option, +} + +impl OperationalAuditContext { + pub fn new() -> Self { + Self::default() + } + + pub fn subject(mut self, kind: OperationalAuditSubjectKind, id: &str) -> Self { + self.subject_kind = Some(OperationalAuditContextSubjectKind::Public(kind)); + self.subject_id = bounded_context_value(id); + self + } + + pub(crate) fn mesh_peer_subject(mut self, id: &str) -> Self { + self.subject_kind = Some(OperationalAuditContextSubjectKind::MeshPeer); + self.subject_id = bounded_context_value(id); + self + } + + pub fn operation_id(mut self, operation_id: &str) -> Self { + self.operation_id = bounded_context_value(operation_id); + self + } + + pub const fn network_path( + mut self, + path_type: OperationalAuditPathType, + remote_addr: Option, + ) -> Self { + self.path_type = Some(path_type); + self.remote_addr = match path_type { + OperationalAuditPathType::Direct => remote_addr, + OperationalAuditPathType::Relay => None, + }; + self + } + + pub fn request_id(mut self, request_id: &str) -> Self { + self.request_id = bounded_context_value(request_id); + self + } + + pub const fn reason_code(mut self, reason_code: &'static str) -> Self { + if Self::valid_static_code(reason_code) { + self.reason_code = Some(reason_code); + } + self + } + + pub const fn outcome(mut self, outcome: &'static str) -> Self { + if Self::valid_static_code(outcome) { + self.outcome = Some(outcome); + } + self + } + + pub const fn duration_ms(mut self, duration_ms: u64) -> Self { + self.duration_ms = Some(duration_ms); + self + } + + pub fn numeric_summary(mut self, key: &'static str, value: u64) -> Self { + if self.numeric_summaries.len() < MAX_NUMERIC_SUMMARIES && Self::valid_static_code(key) { + self.numeric_summaries.insert(key, value); + } + self + } + + pub fn command_summary(mut self, summary: &str) -> Self { + self.command_summary = + CliCommandSummary::sanitize(summary).map(|summary| summary.as_str().to_owned()); + self + } + + pub(crate) const fn valid_static_code(value: &str) -> bool { + valid_static_code(value) + } + + pub(crate) fn fields(&self) -> serde_json::Map { + let mut fields = serde_json::Map::new(); + fields.insert( + "context_version".into(), + serde_json::json!(OPERATIONAL_AUDIT_CONTEXT_VERSION), + ); + if let Some(kind) = self.subject_kind { + fields.insert("subject_kind".into(), serde_json::json!(kind.as_str())); + } + insert_optional_string(&mut fields, "subject_id", self.subject_id.as_deref()); + if let Some(remote_addr) = self.remote_addr { + fields.insert( + "remote_addr".into(), + serde_json::json!(remote_addr.to_string()), + ); + } + if let Some(path_type) = self.path_type { + fields.insert("path_type".into(), serde_json::json!(path_type.as_str())); + } + insert_optional_string(&mut fields, "operation_id", self.operation_id.as_deref()); + insert_optional_string(&mut fields, "request_id", self.request_id.as_deref()); + insert_optional_string(&mut fields, "reason_code", self.reason_code); + insert_optional_string(&mut fields, "outcome", self.outcome); + if let Some(duration_ms) = self.duration_ms { + fields.insert("duration_ms".into(), serde_json::json!(duration_ms)); + } + if !self.numeric_summaries.is_empty() { + fields.insert( + "numeric_summaries".into(), + serde_json::json!(self.numeric_summaries), + ); + } + insert_optional_string( + &mut fields, + "command_summary", + self.command_summary.as_deref(), + ); + fields + } +} + +fn insert_optional_string( + fields: &mut serde_json::Map, + key: &'static str, + value: Option<&str>, +) { + if let Some(value) = value { + fields.insert(key.into(), serde_json::json!(value)); + } +} + +const fn valid_static_code(value: &str) -> bool { + if value.is_empty() || value.len() > 64 { + return false; + } + let bytes = value.as_bytes(); + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if !(byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') { + return false; + } + index += 1; + } + true +} + +fn bounded_context_value(value: &str) -> Option { + let sanitized = crate::logging::policy::sanitize_paths_in_text(value); + let sanitized = crate::logging::policy::redact_urls_in_text(&sanitized); + let sanitized = crate::logging::policy::apply_redaction(&sanitized).0; + let trimmed = sanitized.trim(); + if trimmed.is_empty() { + return None; + } + Some(trimmed.chars().take(MAX_CONTEXT_VALUE_CHARS).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn typed_context_is_sanitized_bounded_and_cardinality_limited() { + let long_subject = format!("{}?api_key=private", "model".repeat(80)); + let mut context = OperationalAuditContext::new() + .subject(OperationalAuditSubjectKind::Model, &long_subject) + .operation_id("runtime-7") + .reason_code("load_failed") + .outcome("failed") + .duration_ms(42); + for index in 0..12 { + let key = match index { + 0 => "metric_0", + 1 => "metric_1", + 2 => "metric_2", + 3 => "metric_3", + 4 => "metric_4", + 5 => "metric_5", + 6 => "metric_6", + 7 => "metric_7", + 8 => "metric_8", + 9 => "metric_9", + 10 => "metric_10", + _ => "metric_11", + }; + context = context.numeric_summary(key, index); + } + + let fields = context.fields(); + assert_eq!(fields["context_version"], 1); + assert_eq!(fields["subject_kind"], "model"); + assert_eq!(fields["operation_id"], "runtime-7"); + assert_eq!(fields["reason_code"], "load_failed"); + assert_eq!(fields["outcome"], "failed"); + assert_eq!(fields["duration_ms"], 42); + assert!(fields["subject_id"].as_str().unwrap().chars().count() <= 256); + assert!(!fields["subject_id"].as_str().unwrap().contains("private")); + assert_eq!(fields["numeric_summaries"].as_object().unwrap().len(), 8); + } + + #[test] + fn invalid_static_codes_are_not_admitted() { + let fields = OperationalAuditContext::new() + .reason_code("NOT VALID") + .outcome("also-not-valid") + .numeric_summary("bad-key", 1) + .fields(); + assert!(fields.get("reason_code").is_none()); + assert!(fields.get("outcome").is_none()); + assert!(fields.get("numeric_summaries").is_none()); + } + + #[test] + fn context_values_redact_url_credentials_and_query_secrets() { + let fields = OperationalAuditContext::new() + .subject( + OperationalAuditSubjectKind::Model, + "https://alice:top-secret@example.test/model?api_key=query-secret&safe=1", + ) + .fields(); + let subject_id = fields["subject_id"].as_str().expect("subject id"); + + assert!(!subject_id.contains("alice")); + assert!(!subject_id.contains("top-secret")); + assert!(!subject_id.contains("query-secret")); + assert!(subject_id.contains("[REDACTED]@example.test")); + assert!(subject_id.contains("api_key=[REDACTED]")); + assert!(subject_id.contains("safe=1")); + } + + #[test] + fn mesh_peer_direct_path_preserves_identity_address_and_path_type() { + let remote_addr = "192.168.1.44:11204".parse().expect("socket address"); + + let fields = OperationalAuditContext::new() + .mesh_peer_subject("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + .network_path(OperationalAuditPathType::Direct, Some(remote_addr)) + .fields(); + + assert_eq!(fields["context_version"], 1); + assert_eq!(fields["subject_kind"], "mesh_peer"); + assert_eq!( + fields["subject_id"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert_eq!(fields["remote_addr"], "192.168.1.44:11204"); + assert_eq!(fields["path_type"], "direct"); + } + + #[test] + fn mesh_peer_relay_path_omits_observed_address() { + let relay_addr = "203.0.113.10:443".parse().expect("socket address"); + + let fields = OperationalAuditContext::new() + .mesh_peer_subject("peer-hex") + .network_path(OperationalAuditPathType::Relay, Some(relay_addr)) + .fields(); + + assert_eq!(fields["path_type"], "relay"); + assert!(fields.get("remote_addr").is_none()); + } + + #[test] + fn command_summary_is_serialized_with_context_bounds_and_redaction() { + let fields = OperationalAuditContext::new() + .command_summary("mesh-llm load name [REDACTED]") + .fields(); + let summary = fields["command_summary"].as_str().expect("command summary"); + assert!(summary.chars().count() <= 256); + assert_eq!(summary, "mesh-llm load name [REDACTED]"); + } + + #[test] + fn command_summary_context_drops_malformed_values_and_overlong_token_lists() { + let fields = OperationalAuditContext::new() + .command_summary("mesh-llm load private-model-name") + .fields(); + assert!(fields.get("command_summary").is_none()); + + let fields = OperationalAuditContext::new() + .command_summary(&format!("mesh-llm {}", "x ".repeat(32))) + .fields(); + assert!(fields.get("command_summary").is_none()); + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections.rs b/crates/mesh-llm-host-runtime/src/mesh/connections.rs index 93036d8aba..34dd91ba6c 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/connections.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/connections.rs @@ -1,6 +1,9 @@ use super::*; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +mod inbound; +mod tunnel; + pub(crate) struct NodeHardwareSnapshot { /// Accelerator-resident capacity advertised for mesh stage placement. pub(crate) vram_bytes: u64, @@ -26,6 +29,7 @@ pub(crate) struct DetectedVramLog { } pub(crate) struct AcceptedMeshStream { + pub(crate) remote: EndpointId, pub(crate) send: iroh::endpoint::SendStream, pub(crate) recv: iroh::endpoint::RecvStream, pub(crate) stream_type: u8, @@ -631,7 +635,8 @@ impl Node { "Tunnel to {} failed, broadcasting death", peer_id.fmt_short() ); - self.handle_peer_death(peer_id).await; + self.handle_peer_death_with_reason(peer_id, MeshPeerRemovalReason::TunnelOpenFailed) + .await; } result @@ -655,7 +660,6 @@ impl Node { let node = self.clone(); tokio::spawn(async move { if let Err(e) = node.handle_incoming(incoming).await { - record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed); tracing::warn!("Incoming connection error: {e}"); } }); @@ -681,7 +685,6 @@ impl Node { let node = self.clone(); tokio::spawn(Box::pin(async move { if let Err(error) = node.handle_control_incoming(incoming).await { - record_mesh_operational_event(MeshOperationalEvent::ControlHandlerFailed); tracing::debug!("Control-plane incoming connection error: {error}"); } })); @@ -744,187 +747,8 @@ impl Node { } }); } - - pub(crate) async fn handle_incoming(&self, incoming: iroh::endpoint::Incoming) -> Result<()> { - let mut accepting = incoming.accept()?; - let alpn = accepting.alpn().await?; - let conn = accepting.await?; - let remote = conn.remote_id(); - if self.handle_stage_alpn(&alpn, conn.clone(), remote).await { - return Ok(()); - } - record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted); - tracing::info!("Inbound connection from {}", remote.fmt_short()); - - // Store connection for stream dispatch (tunneling, route requests, etc.) - // Don't add to peer list yet — only gossip exchange promotes to peer. - let (was_dead, admitted) = self.remember_incoming_connection(remote, &conn).await; - self.capture_connection_event(ConnectionCaptureEvent { - event: "peer_connection_accepted", - remote, - direction: "inbound", - phase: "accept", - protocol: Some(connection_protocol(&conn)), - path_type: None, - rtt_ms: None, - admitted_peer: Some(admitted), - reason: was_dead.then_some("previously_dead"), - }); - self.capture_selected_connection_path(remote, &conn, "inbound_connection_accept_path"); - - // If this peer was previously dead, immediately gossip to restore their - // assigned/routable state in our peer list. Without this, models served by the - // reconnecting peer stay invisible until the next heartbeat (up to 60s). - if was_dead { - self.spawn_reconnect_gossip(conn.clone(), remote); - } - - self.dispatch_streams(conn, remote).await; - Ok(()) - } - - pub(crate) async fn handle_stage_alpn( - &self, - alpn: &[u8], - conn: Connection, - remote: EndpointId, - ) -> bool { - if alpn != skippy_protocol::STAGE_ALPN_V2 { - return false; - } - tracing::info!( - "Inbound skippy stage connection from {}", - remote.fmt_short() - ); - self.dispatch_stage_streams(conn, remote).await; - true - } - - pub(crate) async fn handle_control_incoming( - &self, - incoming: iroh::endpoint::Incoming, - ) -> Result<()> { - let mut accepting = incoming.accept()?; - let alpn = accepting.alpn().await?; - if alpn.as_slice() != ALPN_CONTROL_V1 { - record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); - anyhow::bail!( - "unexpected control-plane ALPN {:?}", - String::from_utf8_lossy(&alpn) - ); - } - let conn = accepting.await?; - let remote = conn.remote_id(); - record_mesh_operational_event(MeshOperationalEvent::ControlConnectionAccepted); - let permits = control_stream_semaphore(); - loop { - let permit = match permits.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => break, - }; - let (mut send, mut recv) = match conn.accept_bi().await { - Ok(streams) => streams, - Err(error) => { - tracing::debug!( - "Control-plane connection from {} closed: {error}", - remote.fmt_short() - ); - break; - } - }; - let node = self.clone(); - tokio::spawn(Box::pin(async move { - let _permit = permit; - if let Err(error) = node - .handle_control_stream(remote, &mut send, &mut recv) - .await - { - tracing::debug!( - "Control-plane stream from {} failed: {error}", - remote.fmt_short() - ); - } - })); - } - Ok(()) - } } impl Node { - pub(crate) async fn accept_mesh_stream( - &self, - conn: &Connection, - remote: EndpointId, - protocol: ControlProtocol, - ) -> Result { - let (send, mut recv) = conn.accept_bi().await.map_err(|error| { - tracing::info!("Connection to {} closed: {error}", remote.fmt_short()); - self.capture_connection_event(ConnectionCaptureEvent { - event: "peer_connection_closed", - remote, - direction: "unknown", - phase: "accept_bi", - protocol: Some(protocol), - path_type: None, - rtt_ms: None, - admitted_peer: None, - reason: Some("accept_bi_error"), - }); - })?; - let mut type_buf = [0u8; 1]; - if !matches!( - tokio::time::timeout( - MESH_STREAM_TYPE_READ_TIMEOUT, - recv.read_exact(&mut type_buf), - ) - .await, - Ok(Ok(_)) - ) { - let _ = recv.stop(0u32.into()); - return Err(()); - } - Ok(AcceptedMeshStream { - send, - recv, - stream_type: type_buf[0], - }) - } - - pub(crate) async fn admitted_mesh_stream( - &self, - remote: EndpointId, - protocol: ControlProtocol, - stream_type: u8, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> Option { - let capture_streams = self.swarm_capture_enabled(); - if stream_allowed_before_admission(stream_type, self.trust_policy) { - if capture_streams { - self.capture_stream_observation(remote, stream_type, protocol, true); - } - return Some((send, recv)); - } - let admitted = { - let state = self.state.lock().await; - state.peers.get(&remote).is_some_and(PeerInfo::is_admitted) - }; - if capture_streams { - self.capture_stream_observation(remote, stream_type, protocol, admitted); - } - if admitted { - Some((send, recv)) - } else { - self.capture_stream_rejected(remote, stream_type, protocol, "unadmitted_peer"); - tracing::warn!( - "Quarantine: stream {:#04x} from unadmitted peer {} rejected — peer must complete gossip first", - stream_type, - remote.fmt_short() - ); - drop((send, recv)); - None - } - } - pub(crate) async fn recover_closed_connection( &self, remote: EndpointId, @@ -939,7 +763,8 @@ impl Node { .await; } ClosedConnectionRecovery::RemovePeer => { - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::ClosedConnectionNoAddress) + .await; } ClosedConnectionRecovery::AlreadyReplaced => {} } @@ -957,7 +782,8 @@ impl Node { } _ => { tracing::info!("Reconnect to {} failed — removing peer", remote.fmt_short()); - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::ReconnectFailed) + .await; } } } @@ -1059,7 +885,8 @@ impl Node { "Reconnect gossip to {} failed — peer is dead, removing", remote.fmt_short() ); - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::RecoveredGossipFailed) + .await; } pub(crate) async fn recovered_connection_gossip_ok( @@ -1076,15 +903,6 @@ impl Node { .unwrap_or(false) } - /// Dispatch bi-streams on a connection by type byte - pub(crate) fn dispatch_streams( - &self, - conn: Connection, - remote: EndpointId, - ) -> std::pin::Pin + Send + '_>> { - Box::pin(self._dispatch_streams(conn, remote)) - } - pub(crate) fn spawn_gossip_stream( &self, remote: EndpointId, @@ -1281,49 +1099,6 @@ impl Node { } } impl Node { - pub(crate) async fn dispatch_mesh_stream( - &self, - remote: EndpointId, - protocol: ControlProtocol, - stream_type: u8, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if stream_type == STREAM_TUNNEL { - return self.forward_tunnel_stream(send, recv).await; - } - if stream_type == STREAM_TUNNEL_HTTP { - return self.forward_tunnel_http_stream(send, recv).await; - } - - self.spawn_non_tunnel_mesh_stream(remote, protocol, stream_type, send, recv); - true - } - - pub(crate) async fn forward_tunnel_stream( - &self, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if self.tunnel_tx.send((send, recv)).await.is_err() { - tracing::warn!("Tunnel receiver dropped"); - return false; - } - true - } - - pub(crate) async fn forward_tunnel_http_stream( - &self, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if self.tunnel_http_tx.send((send, recv)).await.is_err() { - tracing::warn!("HTTP tunnel receiver dropped"); - return false; - } - true - } - pub(crate) fn spawn_non_tunnel_mesh_stream( &self, remote: EndpointId, @@ -1380,7 +1155,8 @@ impl Node { .insert(leaving_id, std::time::Instant::now()); state.connections.remove(&leaving_id); drop(state); - self.remove_peer(leaving_id).await; + self.remove_peer(leaving_id, MeshPeerRemovalReason::CleanShutdown) + .await; } pub(crate) async fn decode_peer_leaving( @@ -1437,59 +1213,8 @@ impl Node { } } } - - pub(crate) async fn _dispatch_streams(&self, conn: Connection, remote: EndpointId) { - let protocol = connection_protocol(&conn); - let dispatcher_stable_id = conn.stable_id(); - loop { - let accepted = match self.accept_mesh_stream(&conn, remote, protocol).await { - Ok(accepted) => accepted, - Err(()) => { - self.recover_closed_connection(remote, dispatcher_stable_id) - .await; - break; - } - }; - let Some((send, recv)) = self - .admitted_mesh_stream( - remote, - protocol, - accepted.stream_type, - accepted.send, - accepted.recv, - ) - .await - else { - continue; - }; - if !self - .dispatch_mesh_stream(remote, protocol, accepted.stream_type, send, recv) - .await - { - break; - } - } - } } impl Node { - pub(crate) async fn remove_connection_if_stable_id( - &self, - peer_id: EndpointId, - conn: &Connection, - ) -> Option { - let stable_id = conn.stable_id(); - let mut state = self.state.lock().await; - if state - .connections - .get(&peer_id) - .is_some_and(|current| current.stable_id() == stable_id) - { - state.connections.remove(&peer_id) - } else { - None - } - } - pub(crate) async fn reserve_pending_connection( &self, peer_id: EndpointId, diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs new file mode 100644 index 0000000000..b735041ece --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs @@ -0,0 +1,250 @@ +use super::*; +use crate::logging::OperationalAuditContext; + +mod stage; + +impl Node { + pub(crate) async fn handle_incoming(&self, incoming: iroh::endpoint::Incoming) -> Result<()> { + let started = std::time::Instant::now(); + let mut accepting = incoming.accept().inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::AcceptSetup.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let alpn = accepting.alpn().await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::AlpnRead.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let conn = accepting.await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::Handshake.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let remote = conn.remote_id(); + if self.handle_stage_alpn(&alpn, conn.clone(), remote).await { + return Ok(()); + } + tracing::info!("Inbound connection from {}", remote.fmt_short()); + + // Store connection for stream dispatch (tunneling, route requests, etc.) + // Don't add to peer list yet — only gossip exchange promotes to peer. + let (was_dead, admitted) = self.remember_incoming_connection(remote, &conn).await; + let inbound_outcome = if was_dead { + MeshQuicInboundOutcome::Readmitted + } else { + MeshQuicInboundOutcome::Accepted + }; + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(inbound_outcome), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary("protocol_gen", u64::from(NODE_PROTOCOL_GENERATION)), + ); + self.capture_connection_event(ConnectionCaptureEvent { + event: "peer_connection_accepted", + remote, + direction: "inbound", + phase: "accept", + protocol: Some(connection_protocol(&conn)), + path_type: None, + rtt_ms: None, + admitted_peer: Some(admitted), + reason: was_dead.then_some("previously_dead"), + }); + self.capture_selected_connection_path(remote, &conn, "inbound_connection_accept_path"); + + // If this peer was previously dead, immediately gossip to restore their + // assigned/routable state in our peer list. Without this, models served by the + // reconnecting peer stay invisible until the next heartbeat (up to 60s). + if was_dead { + self.spawn_reconnect_gossip(conn.clone(), remote); + } + + self.dispatch_streams(conn, remote).await; + Ok(()) + } + + pub(crate) async fn handle_control_incoming( + &self, + incoming: iroh::endpoint::Incoming, + ) -> Result<()> { + let started = std::time::Instant::now(); + let mut accepting = incoming.accept().inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::AcceptSetup.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let alpn = accepting.alpn().await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::AlpnRead.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + if alpn.as_slice() != ALPN_CONTROL_V1 { + record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); + anyhow::bail!("unsupported control-plane ALPN"); + } + let conn = accepting.await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::Handshake.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let remote = conn.remote_id(); + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlConnectionAccepted, + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary("protocol_gen", u64::from(NODE_PROTOCOL_GENERATION)), + ); + let permits = control_stream_semaphore(); + loop { + let permit_started = std::time::Instant::now(); + let permit = match permits.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::CapacityPermit.failure_class(), + ), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .duration_ms(elapsed_ms_u64(permit_started.elapsed())), + ); + break; + } + }; + let stream_accept_started = std::time::Instant::now(); + let (mut send, mut recv) = match conn.accept_bi().await { + Ok(streams) => streams, + Err(error) => { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::StreamAccept.failure_class(), + ), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .duration_ms(elapsed_ms_u64(stream_accept_started.elapsed())), + ); + tracing::debug!( + "Control-plane connection from {} closed: {error}", + remote.fmt_short() + ); + break; + } + }; + let node = self.clone(); + let peer_context = + mesh_peer_operational_context(remote, selected_path_observation(&conn)); + tokio::spawn(Box::pin(async move { + let _permit = permit; + let dispatch_started = std::time::Instant::now(); + if let Err(error) = node + .handle_control_stream(remote, &mut send, &mut recv) + .await + { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::ControlDispatch.failure_class(), + ), + peer_context.duration_ms(elapsed_ms_u64(dispatch_started.elapsed())), + ); + tracing::debug!( + "Control-plane stream from {} failed: {error}", + remote.fmt_short() + ); + } + })); + } + Ok(()) + } + + pub(crate) async fn accept_mesh_stream( + &self, + conn: &Connection, + protocol: ControlProtocol, + ) -> Result { + let remote = conn.remote_id(); + let (send, mut recv) = conn.accept_bi().await.map_err(|error| { + tracing::info!("Connection to {} closed: {error}", remote.fmt_short()); + self.capture_connection_event(ConnectionCaptureEvent { + event: "peer_connection_closed", + remote, + direction: "unknown", + phase: "accept_bi", + protocol: Some(protocol), + path_type: None, + rtt_ms: None, + admitted_peer: None, + reason: Some("accept_bi_error"), + }); + })?; + let mut type_buf = [0u8; 1]; + if !matches!( + tokio::time::timeout( + MESH_STREAM_TYPE_READ_TIMEOUT, + recv.read_exact(&mut type_buf), + ) + .await, + Ok(Ok(_)) + ) { + let _ = recv.stop(0u32.into()); + return Err(()); + } + Ok(AcceptedMeshStream { + remote, + send, + recv, + stream_type: type_buf[0], + }) + } + + pub(crate) async fn admitted_mesh_stream( + &self, + remote: EndpointId, + protocol: ControlProtocol, + stream_type: u8, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> Option { + let capture_streams = self.swarm_capture_enabled(); + if stream_allowed_before_admission(stream_type, self.trust_policy) { + if capture_streams { + self.capture_stream_observation(remote, stream_type, protocol, true); + } + return Some((send, recv)); + } + let admitted = { + let state = self.state.lock().await; + state.peers.get(&remote).is_some_and(PeerInfo::is_admitted) + }; + if capture_streams { + self.capture_stream_observation(remote, stream_type, protocol, admitted); + } + if admitted { + Some((send, recv)) + } else { + self.capture_stream_rejected(remote, stream_type, protocol, "unadmitted_peer"); + tracing::warn!( + "Quarantine: stream {:#04x} from unadmitted peer {} rejected — peer must complete gossip first", + stream_type, + remote.fmt_short() + ); + drop((send, recv)); + None + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs new file mode 100644 index 0000000000..7357a8029a --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs @@ -0,0 +1,28 @@ +use super::super::*; + +impl Node { + pub(crate) async fn handle_stage_alpn( + &self, + alpn: &[u8], + conn: Connection, + remote: EndpointId, + ) -> bool { + if alpn != skippy_protocol::STAGE_ALPN_V2 { + return false; + } + tracing::info!( + "Inbound skippy stage connection from {}", + remote.fmt_short() + ); + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary( + "protocol_gen", + u64::from(skippy_protocol::STAGE_PROTOCOL_GENERATION), + ), + ); + self.dispatch_stage_streams(conn, remote).await; + true + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs new file mode 100644 index 0000000000..fc0386e335 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs @@ -0,0 +1,122 @@ +use super::*; + +impl Node { + /// Dispatch bi-streams on a connection by type byte + pub(crate) fn dispatch_streams( + &self, + conn: Connection, + remote: EndpointId, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(self._dispatch_streams(conn, remote)) + } + + pub(crate) async fn dispatch_mesh_stream( + &self, + remote: EndpointId, + protocol: ControlProtocol, + stream_type: u8, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if stream_type == STREAM_TUNNEL { + return self.forward_tunnel_stream(send, recv).await; + } + if stream_type == STREAM_TUNNEL_HTTP { + return self.forward_tunnel_http_stream(remote, send, recv).await; + } + + self.spawn_non_tunnel_mesh_stream(remote, protocol, stream_type, send, recv); + true + } + + pub(crate) async fn forward_tunnel_stream( + &self, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if self.tunnel_tx.send((send, recv)).await.is_err() { + tracing::warn!("Tunnel receiver dropped"); + return false; + } + true + } + + pub(crate) async fn forward_tunnel_http_stream( + &self, + remote: EndpointId, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if self + .tunnel_http_tx + .send((remote, send, recv)) + .await + .is_err() + { + tracing::warn!("HTTP tunnel receiver dropped"); + return false; + } + true + } + + pub(crate) async fn _dispatch_streams(&self, conn: Connection, remote: EndpointId) { + let protocol = connection_protocol(&conn); + let dispatcher_stable_id = conn.stable_id(); + loop { + let accepted = match self.accept_mesh_stream(&conn, protocol).await { + Ok(accepted) => accepted, + Err(()) => { + self.recover_closed_connection(remote, dispatcher_stable_id) + .await; + break; + } + }; + let Some((send, recv)) = self + .admitted_mesh_stream( + accepted.remote, + protocol, + accepted.stream_type, + accepted.send, + accepted.recv, + ) + .await + else { + continue; + }; + if !self + .dispatch_mesh_stream(accepted.remote, protocol, accepted.stream_type, send, recv) + .await + { + break; + } + } + } + + pub(crate) async fn authenticated_peer_path( + &self, + remote: EndpointId, + ) -> Option { + let conn = self.state.lock().await.connections.get(&remote).cloned()?; + (conn.remote_id() == remote) + .then(|| selected_path_observation(&conn)) + .flatten() + } + + pub(crate) async fn remove_connection_if_stable_id( + &self, + peer_id: EndpointId, + conn: &Connection, + ) -> Option { + let stable_id = conn.stable_id(); + let mut state = self.state.lock().await; + if state + .connections + .get(&peer_id) + .is_some_and(|current| current.stable_id() == stable_id) + { + state.connections.remove(&peer_id) + } else { + None + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs index d40f33ac5c..8322e83323 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs @@ -2,11 +2,12 @@ //! and peer list management (add/remove/update). use super::{ - DEAD_PEER_TTL, DisplayLatencySource, InviteTokenMaterial, MeshOperationalEvent, ModelDemand, - ModelRuntimeDescriptor, Node, NodeRole, PEER_CONNECT_AND_GOSSIP_TIMEOUT, PEER_STALE_SECS, - PeerAnnouncement, PeerInfo, ServedModelDescriptor, SignedNodeOwnership, connect_mesh, - elapsed_ms_u64, emit_mesh_info, infer_remote_served_descriptors, parse_invite_token, - record_mesh_operational_event, + DEAD_PEER_TTL, DisplayLatencySource, InviteTokenMaterial, MeshOperationalEvent, + MeshPeerRemovalReason, MeshPolicyRejectionReason, ModelDemand, ModelRuntimeDescriptor, Node, + NodeRole, PEER_CONNECT_AND_GOSSIP_TIMEOUT, PEER_STALE_SECS, PeerAnnouncement, PeerInfo, + ServedModelDescriptor, SignedNodeOwnership, connect_mesh, elapsed_ms_u64, emit_mesh_info, + infer_remote_served_descriptors, mesh_peer_operational_context, parse_invite_token, + record_mesh_operational_event, record_mesh_operational_event_with_context, }; use crate::crypto::{OwnershipSummary, verify_node_ownership}; use crate::mesh::peer_state::{PropagatedLatencyObservation, policy_accepts_peer}; @@ -519,7 +520,12 @@ impl Node { ); } if self - .add_peer_after_direct_requirements_validated(remote, addr.clone(), ann) + .add_peer_after_direct_requirements_validated( + remote, + addr.clone(), + ann, + context.negotiated_protocol_generation, + ) .await { if let Some(ref their_id) = ann.mesh_id { @@ -884,8 +890,14 @@ impl Node { let _ = self.peer_change_tx.send(admitted_count); } drop(state); - if newly_rejected { - record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected); + if newly_rejected + && let Some(reason) = + MeshPolicyRejectionReason::from_ownership_status(&owner_summary.status) + { + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipPolicyRejected(reason), + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await), + ); } true } @@ -971,7 +983,11 @@ impl Node { .count(); drop(state); self.capture_peer_observation("peer_direct_add", &peer, "direct", None); - record_mesh_operational_event(MeshOperationalEvent::GossipDirectPeerPromoted); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipDirectPeerPromoted, + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await) + .numeric_summary("direct_peers", count as u64), + ); let _ = self.peer_change_tx.send(count); self.emit_plugin_mesh_event( crate::plugin::proto::mesh_event::Kind::PeerUp, @@ -1571,7 +1587,7 @@ impl Node { Ok(()) } - pub(super) async fn remove_peer(&self, id: EndpointId) { + pub(super) async fn remove_peer(&self, id: EndpointId, reason: MeshPeerRemovalReason) { let mut state = self.state.lock().await; // Always clear any rejection-tracking entry so the map stays bounded. state.policy_rejected_peers.remove(&id); @@ -1598,14 +1614,18 @@ impl Node { self.capture_peer_lifecycle_event(PeerLifecycleCaptureEvent { event: "peer_removed", peer: id, - reason: "remove_peer", + reason: reason.reason_code(), reporter: None, last_seen_age_ms: Some(last_seen_age_ms), last_mentioned_age_ms: Some(last_mentioned_age_ms), had_connection: Some(had_connection), bridge_id, }); - record_mesh_operational_event(MeshOperationalEvent::GossipPeerRemoved); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipPeerRemoved(reason), + mesh_peer_operational_context(id, peer.selected_path) + .numeric_summary("direct_peers", count as u64), + ); let _ = self.peer_change_tx.send(count); self.emit_plugin_mesh_event( crate::plugin::proto::mesh_event::Kind::PeerDown, @@ -1651,8 +1671,13 @@ impl Node { } return; } - self.add_peer_after_direct_requirements_validated(id, addr, ann) - .await; + self.add_peer_after_direct_requirements_validated( + id, + addr, + ann, + negotiated_protocol_generation, + ) + .await; } pub(crate) async fn add_peer_after_direct_requirements_validated( @@ -1660,6 +1685,7 @@ impl Node { id: EndpointId, addr: EndpointAddr, ann: &PeerAnnouncement, + negotiated_protocol_generation: Option, ) -> bool { // Reject ingest from peers below the supported version floor. They // are not added to local state, do not appear in /api/status, and @@ -1671,7 +1697,15 @@ impl Node { id.fmt_short(), ann.version ); - record_mesh_operational_event(MeshOperationalEvent::GossipIncompatibleVersionRejected); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await) + .numeric_summary("local_gen", u64::from(NODE_PROTOCOL_GENERATION)) + .numeric_summary( + "peer_gen", + u64::from(negotiated_protocol_generation.unwrap_or_default()), + ), + ); self.remove_disallowed_peer(id).await; return false; } diff --git a/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs b/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs index 58450143f8..e8a94504e6 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs @@ -5,9 +5,10 @@ //! PeerDown messages are broadcast to the mesh when a peer is confirmed dead. use super::{ - ConnectionCaptureEvent, ControlProtocol, DEAD_PEER_TTL, ModelRuntimeDescriptor, Node, - PEER_DOWN_REPORTER_COOLDOWN_SECS, PEER_STALE_SECS, PeerInfo, PeerLifecycleCaptureEvent, - ServedModelDescriptor, connect_mesh, connection_protocol, endpoint_id_hex, + ConnectionCaptureEvent, ControlProtocol, DEAD_PEER_TTL, MeshPeerRemovalReason, + ModelRuntimeDescriptor, Node, PEER_DOWN_REPORTER_COOLDOWN_SECS, PEER_STALE_SECS, PeerInfo, + PeerLifecycleCaptureEvent, ServedModelDescriptor, connect_mesh, connection_protocol, + endpoint_id_hex, }; use crate::protocol::{ NODE_PROTOCOL_GENERATION, STREAM_PEER_DOWN, STREAM_PEER_LEAVING, write_len_prefixed, @@ -960,7 +961,8 @@ impl Node { None, ) .await; - self.remove_peer(stale_id).await; + self.remove_peer(stale_id, MeshPeerRemovalReason::StaleDirectAndTransitive) + .await; self.state.lock().await.connections.remove(&stale_id); } } @@ -1014,6 +1016,15 @@ impl Node { /// Handle a peer death: remove from state, broadcast to all other peers. pub async fn handle_peer_death(&self, dead_id: EndpointId) { + self.handle_peer_death_with_reason(dead_id, MeshPeerRemovalReason::HeartbeatUnreachable) + .await; + } + + pub(crate) async fn handle_peer_death_with_reason( + &self, + dead_id: EndpointId, + reason: MeshPeerRemovalReason, + ) { super::emit_mesh_warning(format!( "⚠️ Peer {} died — removing and broadcasting", dead_id.fmt_short() @@ -1029,11 +1040,11 @@ impl Node { self.capture_peer_lifecycle_snapshot( "peer_dead_marked", dead_id, - "handle_peer_death", + reason.reason_code(), None, ) .await; - self.remove_peer(dead_id).await; + self.remove_peer(dead_id, reason).await; self.broadcast_peer_down(dead_id).await; } diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index f42aa1618e..da9a96d511 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -110,7 +110,13 @@ use connections::*; pub(crate) use host_role_claims::{HostRoleClaim, HostRoleClaims}; use model_identity::*; use node_identity::*; -use operational_logging::{MeshOperationalEvent, record_mesh_operational_event}; +#[cfg(test)] +use operational_logging::capture_mesh_operational_audits; +use operational_logging::{ + MeshHandlerFailureBoundary, MeshOperationalEvent, MeshPeerRemovalReason, + MeshPolicyRejectionReason, MeshQuicInboundOutcome, mesh_peer_operational_context, + record_mesh_operational_event, record_mesh_operational_event_with_context, +}; use owner_control::*; use owner_lifecycle_cache::*; use peer_state::*; diff --git a/crates/mesh-llm-host-runtime/src/mesh/node.rs b/crates/mesh-llm-host-runtime/src/mesh/node.rs index 9b6cf897df..7d680ba913 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/node.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/node.rs @@ -115,8 +115,11 @@ pub struct Node { pub(crate) runtime_data_producer: crate::runtime_data::RuntimeDataProducer, pub(crate) tunnel_tx: tokio::sync::mpsc::Sender<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, - pub(crate) tunnel_http_tx: - tokio::sync::mpsc::Sender<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, + pub(crate) tunnel_http_tx: tokio::sync::mpsc::Sender<( + EndpointId, + iroh::endpoint::SendStream, + iroh::endpoint::RecvStream, + )>, pub(crate) stage_transport_tx: tokio::sync::mpsc::Sender<( EndpointId, iroh::endpoint::SendStream, diff --git a/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs b/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs index d2a33e8911..c42a2a3412 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs @@ -1,203 +1,305 @@ //! Bounded, metadata-only operational audit vocabulary for mesh boundaries. //! -//! These events deliberately accept no peer, endpoint, address, token, ALPN, -//! or error values. The service receives only the static level and code below. +//! Dynamic context is restricted to authenticated peer identity and selected +//! connection-path observations. Tokens, ALPN bytes, hostnames, and raw errors +//! never enter this adapter. +//! Pre-authentication failures remain identity-free; they carry only static +//! outcome/reason codes and an elapsed duration when that boundary owns a timer. #[cfg(test)] use crate::logging::LoggingService; -use crate::logging::{OperationalAuditRecord, OperationalAuditSeverity}; +use crate::logging::{ + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, +}; +use crate::mesh::SelectedPathObservation; +use iroh::EndpointId; -const OPERATIONAL_AUDIT_INFO: &str = "info"; -const OPERATIONAL_AUDIT_WARNING: &str = "warning"; +mod vocabulary; + +#[cfg(test)] +use vocabulary::MeshHandlerFailureClass; +pub(crate) use vocabulary::{ + MeshHandlerFailureBoundary, MeshOperationalEvent, MeshPeerRemovalReason, + MeshPolicyRejectionReason, MeshQuicInboundOutcome, +}; +use vocabulary::{OPERATIONAL_AUDIT_INFO, OPERATIONAL_AUDIT_WARNING}; const OPERATIONAL_AUDIT_SOURCE: &str = "mesh"; -fn operational_audit_record(code: &'static str, level: &'static str) -> OperationalAuditRecord { +fn operational_audit_record( + event: MeshOperationalEvent, + context: Option, +) -> OperationalAuditRecord { + let level = event.level(); let severity = match level { OPERATIONAL_AUDIT_INFO => OperationalAuditSeverity::Info, OPERATIONAL_AUDIT_WARNING => OperationalAuditSeverity::Warning, _ => OperationalAuditSeverity::Error, }; - OperationalAuditRecord::builder(OPERATIONAL_AUDIT_SOURCE, code) + let record = OperationalAuditRecord::builder(OPERATIONAL_AUDIT_SOURCE, event.code()) .severity(severity) - .build() -} - -/// Static outcomes that are safe to publish through the local operational log. -/// -/// Variants and codes follow the reviewed mesh audit vocabulary. `QuicInboundAccepted` -/// describes a non-Skippy inbound QUIC connection accepted after the Skippy ALPN -/// exclusion; there is no `quic_alpn_accepted` code because no explicit mesh-ALPN -/// validation branch exists in the inbound path. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum MeshOperationalEvent { - QuicHandlerFailed, - QuicInboundAccepted, - ControlHandlerFailed, - ControlAlpnRejected, - ControlConnectionAccepted, - GossipPolicyRejected, - GossipDirectPeerPromoted, - GossipIncompatibleVersionRejected, - GossipPeerRemoved, - AutoJoinSucceeded, - AutoJoinFailed, -} - -impl MeshOperationalEvent { - const fn level(self) -> &'static str { - match self { - Self::QuicInboundAccepted - | Self::ControlConnectionAccepted - | Self::GossipDirectPeerPromoted - | Self::GossipPeerRemoved - | Self::AutoJoinSucceeded => OPERATIONAL_AUDIT_INFO, - Self::QuicHandlerFailed - | Self::ControlHandlerFailed - | Self::ControlAlpnRejected - | Self::GossipPolicyRejected - | Self::GossipIncompatibleVersionRejected - | Self::AutoJoinFailed => OPERATIONAL_AUDIT_WARNING, - } + .build(); + if matches!( + event, + MeshOperationalEvent::AutoJoinSucceeded | MeshOperationalEvent::AutoJoinFailed + ) { + return record; } + let Some(context) = context else { + return record; + }; + record.with_context(event.decorate_context(context)) +} - const fn code(self) -> &'static str { - match self { - Self::QuicHandlerFailed => "mesh_quic_handler_failed", - Self::QuicInboundAccepted => "mesh_quic_inbound_accepted", - Self::ControlHandlerFailed => "mesh_control_handler_failed", - Self::ControlAlpnRejected => "mesh_control_alpn_rejected", - Self::ControlConnectionAccepted => "mesh_control_connection_accepted", - Self::GossipPolicyRejected => "gossip_policy_rejected", - Self::GossipDirectPeerPromoted => "gossip_direct_peer_promoted", - Self::GossipIncompatibleVersionRejected => "gossip_incompatible_version_rejected", - Self::GossipPeerRemoved => "gossip_peer_removed", - Self::AutoJoinSucceeded => "mesh_auto_join_succeeded", - Self::AutoJoinFailed => "mesh_auto_join_failed", - } +pub(crate) fn mesh_peer_operational_context( + peer: EndpointId, + path: Option, +) -> OperationalAuditContext { + let context = OperationalAuditContext::new().mesh_peer_subject(&hex::encode(peer.as_bytes())); + let Some(path) = path else { + return context; + }; + match path.path_type { + "direct" => context.network_path( + OperationalAuditPathType::Direct, + path.observed_direct_remote_addr, + ), + "relay" => context.network_path(OperationalAuditPathType::Relay, None), + _ => context, } } /// Record one mesh boundary result through the process-local logging service. /// Logging is optional and this intentionally never affects mesh serving. pub(crate) fn record_mesh_operational_event(event: MeshOperationalEvent) { + let record = operational_audit_record(event, Some(OperationalAuditContext::new())); + #[cfg(test)] + capture_mesh_operational_audit_for_test(&record); let Some(state) = crate::logging_runtime_state() else { return; }; - let _ = state.write_operational_audit(operational_audit_record(event.code(), event.level())); + let _ = state.write_operational_audit(record); +} + +pub(crate) fn record_mesh_operational_event_with_context( + event: MeshOperationalEvent, + context: OperationalAuditContext, +) { + let record = operational_audit_record(event, Some(context)); + #[cfg(test)] + capture_mesh_operational_audit_for_test(&record); + let Some(state) = crate::logging_runtime_state() else { + return; + }; + let _ = state.write_operational_audit(record); +} + +#[cfg(test)] +static MESH_OPERATIONAL_AUDIT_CAPTURE: std::sync::Mutex< + Option>, +> = std::sync::Mutex::new(None); + +#[cfg(test)] +pub(crate) struct MeshOperationalAuditCaptureGuard { + sender: tokio::sync::mpsc::UnboundedSender, +} + +#[cfg(test)] +impl Drop for MeshOperationalAuditCaptureGuard { + fn drop(&mut self) { + let mut capture = MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock"); + if capture + .as_ref() + .is_some_and(|sender| sender.same_channel(&self.sender)) + { + *capture = None; + } + } +} + +#[cfg(test)] +pub(crate) fn capture_mesh_operational_audits() -> ( + tokio::sync::mpsc::UnboundedReceiver, + MeshOperationalAuditCaptureGuard, +) { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); + *MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock") = Some(sender.clone()); + (receiver, MeshOperationalAuditCaptureGuard { sender }) +} + +#[cfg(test)] +fn capture_mesh_operational_audit_for_test(record: &OperationalAuditRecord) { + if let Some(sender) = MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock") + .as_ref() + { + let _ = sender.send(record.clone()); + } } #[cfg(test)] -fn record_mesh_operational_event_with_service( +fn record_mesh_operational_event_with_context_and_service( service: &LoggingService, event: MeshOperationalEvent, + context: OperationalAuditContext, ) { - let _ = service.write_operational_audit(operational_audit_record(event.code(), event.level())); + let _ = service.write_operational_audit(operational_audit_record(event, Some(context))); } #[cfg(test)] mod tests { use super::{ - MeshOperationalEvent, record_mesh_operational_event, - record_mesh_operational_event_with_service, + MeshHandlerFailureBoundary, MeshHandlerFailureClass, MeshOperationalEvent, + MeshPeerRemovalReason, MeshPolicyRejectionReason, MeshQuicInboundOutcome, + capture_mesh_operational_audits, mesh_peer_operational_context, + record_mesh_operational_event, record_mesh_operational_event_with_context_and_service, }; - use crate::logging::{LoggingService, ServiceConfig}; - - fn recorded_audits(service: &LoggingService) -> Vec { - service - .bus_ref() - .drain() - .into_iter() - .map(|entry| { - let audit: serde_json::Value = - serde_json::from_str(&entry.payload).expect("audit payload"); - serde_json::json!({ - "kind": "audit", - "level": audit["severity"], - "message": audit["code"], - }) - }) - .collect() + use crate::crypto::OwnershipStatus; + use crate::logging::{LoggingService, OperationalAuditContext, ServiceConfig}; + use crate::mesh::SelectedPathObservation; + use iroh::{EndpointId, SecretKey}; + use serial_test::serial; + use tokio::sync::mpsc::error::TryRecvError; + + #[test] + #[serial] + fn stale_capture_guard_does_not_clear_replacement() { + let (_receiver_a, guard_a) = capture_mesh_operational_audits(); + let (mut receiver_b, guard_b) = capture_mesh_operational_audits(); + + drop(guard_a); + record_mesh_operational_event(MeshOperationalEvent::ControlConnectionAccepted); + + let captured = loop { + match receiver_b.try_recv() { + Ok(record) if record.code() == "mesh_control_connection_accepted" => break record, + Ok(_) => {} + Err(error) => panic!("replacement capture must receive the audit: {error}"), + } + }; + assert_eq!(captured.code(), "mesh_control_connection_accepted"); + + drop(guard_b); + record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); + + loop { + match receiver_b.try_recv() { + Ok(record) => assert_ne!(record.code(), "mesh_control_alpn_rejected"), + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => panic!("active capture registration remains"), + } + } } #[test] - fn mesh_boundary_outcomes_emit_exact_static_audits_without_raw_metadata() { + fn mesh_boundary_outcomes_emit_authenticated_peer_context_without_raw_secrets() { let service = LoggingService::new_disabled(ServiceConfig::default()); - let events = [ - MeshOperationalEvent::QuicInboundAccepted, - MeshOperationalEvent::ControlAlpnRejected, - MeshOperationalEvent::ControlHandlerFailed, + let peer = EndpointId::from(SecretKey::from_bytes(&[0x42; 32]).public()); + let peer_hex = hex::encode(peer.as_bytes()); + let direct_addr = "192.0.2.42:11204".parse().expect("direct address"); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(17), + observed_direct_remote_addr: Some(direct_addr), + }), + ) + .numeric_summary("direct_peers", 3); + record_mesh_operational_event_with_context_and_service( + &service, MeshOperationalEvent::GossipDirectPeerPromoted, - MeshOperationalEvent::GossipPolicyRejected, - MeshOperationalEvent::AutoJoinFailed, - ]; - - for event in events { - record_mesh_operational_event_with_service(&service, event); - } + context, + ); - let audits = recorded_audits(&service); + let audit: serde_json::Value = serde_json::from_str( + &service + .bus_ref() + .drain() + .into_iter() + .next() + .expect("peer audit") + .payload, + ) + .expect("audit payload"); + assert_eq!(audit["kind"], "audit"); + assert_eq!(audit["severity"], "info"); + assert_eq!(audit["code"], "gossip_direct_peer_promoted"); + assert_eq!(audit["subject_kind"], "mesh_peer"); + assert_eq!(audit["subject_id"], peer_hex); + assert_eq!(audit["path_type"], "direct"); + assert_eq!(audit["remote_addr"], direct_addr.to_string()); + assert_eq!(audit["outcome"], "promoted"); + assert!(audit.get("duration_ms").is_none()); assert_eq!( - audits, - vec![ - serde_json::json!({ - "kind": "audit", - "level": "info", - "message": "mesh_quic_inbound_accepted", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_control_alpn_rejected", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_control_handler_failed", - }), - serde_json::json!({ - "kind": "audit", - "level": "info", - "message": "gossip_direct_peer_promoted", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "gossip_policy_rejected", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_auto_join_failed", - }), - ] + audit["numeric_summaries"], + serde_json::json!({"direct_peers": 3}) ); - let serialized = serde_json::to_string(&audits).expect("serialized audit payloads"); + let serialized = serde_json::to_string(&audit).expect("serialized audit payload"); for raw_value in [ "node=untrusted-lab-host", - "peer=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "token=mesh-secret-bootstrap-token", + "mesh-llm/1-private-alpn", + "connection refused at secret.example.test", + "untrusted-lab-host.example.test", ] { assert!( !serialized.contains(raw_value), - "raw mesh metadata must not enter the audit payload" + "raw secret, ALPN, error, and hostname data must not enter the audit payload" ); } } #[test] - fn mesh_operational_vocabulary_is_bounded_and_identifier_free() { + fn relay_peer_context_never_records_an_address() { + let peer = EndpointId::from(SecretKey::from_bytes(&[0x24; 32]).public()); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "relay", + rtt_ms: Some(31), + observed_direct_remote_addr: Some( + "203.0.113.10:443".parse().expect("relay-shaped address"), + ), + }), + ); + let service = LoggingService::new_disabled(ServiceConfig::default()); + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + context, + ); + + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("audit payload"); + assert_eq!(audit["subject_id"], hex::encode(peer.as_bytes())); + assert_eq!(audit["path_type"], "relay"); + assert!(audit.get("remote_addr").is_none()); + assert!(audit.get("numeric_summaries").is_none()); + assert_eq!(audit["outcome"], "accepted"); + } + + #[test] + fn mesh_operational_vocabulary_is_bounded_and_static() { let events = [ - MeshOperationalEvent::QuicHandlerFailed, - MeshOperationalEvent::QuicInboundAccepted, - MeshOperationalEvent::ControlHandlerFailed, + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Capacity), MeshOperationalEvent::ControlAlpnRejected, MeshOperationalEvent::ControlConnectionAccepted, - MeshOperationalEvent::GossipPolicyRejected, + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), MeshOperationalEvent::GossipDirectPeerPromoted, MeshOperationalEvent::GossipIncompatibleVersionRejected, - MeshOperationalEvent::GossipPeerRemoved, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), MeshOperationalEvent::AutoJoinSucceeded, MeshOperationalEvent::AutoJoinFailed, ]; @@ -214,19 +316,373 @@ mod tests { } } + #[test] + fn peer_removal_reason_vocabulary_is_exactly_the_bounded_control_flow_causes() { + let cases = [ + ( + MeshPeerRemovalReason::StaleDirectAndTransitive, + "stale_direct_and_transitive", + ), + ( + MeshPeerRemovalReason::HeartbeatUnreachable, + "heartbeat_unreachable", + ), + ( + MeshPeerRemovalReason::PeerDownProbeFailed, + "peer_down_probe_failed", + ), + ( + MeshPeerRemovalReason::ClosedConnectionNoAddress, + "closed_connection_no_address", + ), + (MeshPeerRemovalReason::ReconnectFailed, "reconnect_failed"), + ( + MeshPeerRemovalReason::RecoveredGossipFailed, + "recovered_gossip_failed", + ), + (MeshPeerRemovalReason::CleanShutdown, "clean_shutdown"), + ( + MeshPeerRemovalReason::TunnelOpenFailed, + "tunnel_open_failed", + ), + ]; + + for (reason, expected_code) in cases { + let code = reason.reason_code(); + assert_eq!(code, expected_code); + assert!(code.len() <= 48, "reason code must stay bounded: {code}"); + assert!( + code.bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_'), + "reason code must be a static identifier: {code}" + ); + } + } + + #[test] + fn gossip_peer_removed_audit_carries_only_cause_count_and_peer_path_identity() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + let peer = EndpointId::from(SecretKey::from_bytes(&[0x51; 32]).public()); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(11), + observed_direct_remote_addr: Some( + "192.0.2.51:1151".parse().expect("direct peer address"), + ), + }), + ) + .numeric_summary("direct_peers", 2); + + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::HeartbeatUnreachable), + context, + ); + + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("removal audit JSON"); + assert_eq!(audit["reason_code"], "heartbeat_unreachable"); + assert_eq!( + audit["numeric_summaries"], + serde_json::json!({"direct_peers": 2}) + ); + assert_eq!(audit["subject_id"], hex::encode(peer.as_bytes())); + assert_eq!(audit["path_type"], "direct"); + assert_eq!(audit["remote_addr"], "192.0.2.51:1151"); + for diagnostic_only_field in [ + "last_seen_age_ms", + "last_mentioned_age_ms", + "had_connection", + "bridge", + "reporter", + ] { + assert!(audit.get(diagnostic_only_field).is_none()); + } + } + + #[test] + fn reviewed_audit_outcomes_and_rejection_reasons_are_exact() { + let cases = [ + ( + MeshOperationalEvent::GossipDirectPeerPromoted, + "promoted", + None, + ), + ( + MeshOperationalEvent::ControlAlpnRejected, + "rejected", + Some("alpn_unsupported"), + ), + ( + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), + "rejected", + Some("owner_attestation_required"), + ), + ( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + "rejected", + Some("protocol_version_unsupported"), + ), + ( + MeshOperationalEvent::GossipPeerRemoved( + MeshPeerRemovalReason::HeartbeatUnreachable, + ), + "removed", + Some("heartbeat_unreachable"), + ), + ( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Readmitted), + "readmitted", + None, + ), + ]; + + for (event, expected_outcome, expected_reason) in cases { + assert_eq!(event.outcome(), expected_outcome); + assert_eq!(event.reason_code(), expected_reason); + } + } + + #[test] + fn pre_auth_alpn_rejection_stays_sparse() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::ControlAlpnRejected, + OperationalAuditContext::new(), + ); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("pre-auth audit JSON"); + + assert_eq!(audit["outcome"], "rejected"); + assert_eq!(audit["reason_code"], "alpn_unsupported"); + for unavailable_field in [ + "subject_kind", + "subject_id", + "remote_addr", + "path_type", + "duration_ms", + "numeric_summaries", + ] { + assert!(audit.get(unavailable_field).is_none()); + } + } + + #[test] + fn reviewed_audit_numeric_summaries_and_handler_duration_are_exact() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + let cases = [ + ( + MeshOperationalEvent::GossipDirectPeerPromoted, + OperationalAuditContext::new().numeric_summary("direct_peers", 4), + serde_json::json!({"direct_peers": 4}), + None, + ), + ( + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), + OperationalAuditContext::new().numeric_summary("direct_peers", 3), + serde_json::json!({"direct_peers": 3}), + None, + ), + ( + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), + OperationalAuditContext::new(), + serde_json::json!({}), + None, + ), + ( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + OperationalAuditContext::new() + .numeric_summary("peer_gen", 0) + .numeric_summary("local_gen", 1), + serde_json::json!({"local_gen": 1, "peer_gen": 0}), + None, + ), + ( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + OperationalAuditContext::new().numeric_summary("protocol_gen", 1), + serde_json::json!({"protocol_gen": 1}), + None, + ), + ( + MeshOperationalEvent::ControlConnectionAccepted, + OperationalAuditContext::new().numeric_summary("protocol_gen", 1), + serde_json::json!({"protocol_gen": 1}), + None, + ), + ( + MeshOperationalEvent::ControlAlpnRejected, + OperationalAuditContext::new(), + serde_json::json!({}), + None, + ), + ( + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), + OperationalAuditContext::new().duration_ms(29), + serde_json::json!({}), + Some(29), + ), + ( + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Internal), + OperationalAuditContext::new().duration_ms(37), + serde_json::json!({}), + Some(37), + ), + ]; + + for (event, context, expected_summaries, expected_duration) in cases { + record_mesh_operational_event_with_context_and_service(&service, event, context); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = + serde_json::from_str(&payload).expect("reviewed audit JSON"); + assert_eq!( + audit + .get("numeric_summaries") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + expected_summaries, + ); + assert_eq!( + audit.get("duration_ms").and_then(serde_json::Value::as_u64), + expected_duration + ); + } + } + + #[test] + fn auto_join_audits_remain_generic_and_context_free() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + + for event in [ + MeshOperationalEvent::AutoJoinSucceeded, + MeshOperationalEvent::AutoJoinFailed, + ] { + record_mesh_operational_event_with_context_and_service( + &service, + event, + OperationalAuditContext::new() + .outcome("must_not_escape") + .reason_code("must_not_escape") + .numeric_summary("must_not_escape", 1), + ); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = + serde_json::from_str(&payload).expect("generic auto-join audit JSON"); + + for context_field in [ + "context_version", + "outcome", + "reason_code", + "numeric_summaries", + ] { + assert!(audit.get(context_field).is_none()); + } + } + } + + #[test] + fn handler_failure_vocabulary_is_exactly_the_production_backed_classes() { + let classes = [ + MeshHandlerFailureClass::Capacity, + MeshHandlerFailureClass::Internal, + ]; + assert_eq!( + classes.map(MeshHandlerFailureClass::reason_code), + ["capacity", "internal"] + ); + } + + #[test] + fn ownership_status_maps_exhaustively_to_gate_specific_policy_reasons() { + let cases = [ + (OwnershipStatus::Verified, None), + ( + OwnershipStatus::Unsigned, + Some("owner_attestation_required"), + ), + (OwnershipStatus::Expired, Some("owner_attestation_expired")), + ( + OwnershipStatus::InvalidSignature, + Some("owner_attestation_invalid"), + ), + ( + OwnershipStatus::MismatchedNodeId, + Some("owner_attestation_node_mismatch"), + ), + (OwnershipStatus::RevokedOwner, Some("owner_revoked")), + ( + OwnershipStatus::RevokedCert, + Some("owner_attestation_revoked"), + ), + (OwnershipStatus::RevokedNodeId, Some("owner_node_revoked")), + ( + OwnershipStatus::UnsupportedProtocol, + Some("owner_attestation_protocol_unsupported"), + ), + (OwnershipStatus::UntrustedOwner, Some("owner_untrusted")), + ]; + + for (status, expected_reason) in cases { + assert_eq!( + MeshPolicyRejectionReason::from_ownership_status(&status) + .map(MeshPolicyRejectionReason::reason_code), + expected_reason, + ); + } + } + + #[test] + fn handler_failure_boundaries_map_only_when_truthfully_classifiable() { + let cases = [ + ( + MeshHandlerFailureBoundary::AcceptSetup, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::AlpnRead, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::Handshake, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::CapacityPermit, + MeshHandlerFailureClass::Capacity, + ), + ( + MeshHandlerFailureBoundary::StreamAccept, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::ControlDispatch, + MeshHandlerFailureClass::Internal, + ), + ]; + for (boundary, expected) in cases { + assert_eq!(boundary.failure_class(), expected); + } + } + #[test] fn mesh_operational_vocabulary_maps_each_variant_to_its_reviewed_code() { let cases = [ ( - MeshOperationalEvent::QuicHandlerFailed, + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), "mesh_quic_handler_failed", ), ( - MeshOperationalEvent::QuicInboundAccepted, + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), "mesh_quic_inbound_accepted", ), ( - MeshOperationalEvent::ControlHandlerFailed, + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Internal), "mesh_control_handler_failed", ), ( @@ -238,7 +694,9 @@ mod tests { "mesh_control_connection_accepted", ), ( - MeshOperationalEvent::GossipPolicyRejected, + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), "gossip_policy_rejected", ), ( @@ -250,7 +708,7 @@ mod tests { "gossip_incompatible_version_rejected", ), ( - MeshOperationalEvent::GossipPeerRemoved, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), "gossip_peer_removed", ), ( @@ -276,9 +734,15 @@ mod tests { // When the process-local logging runtime state is absent the adapter // must be a no-op; when a concurrent logging test installed state the // bounded write path must equally never panic (fail-open by contract). - record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed); + record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureClass::Internal, + )); record_mesh_operational_event(MeshOperationalEvent::AutoJoinFailed); - record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected); - record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted); + record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + )); + record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted( + MeshQuicInboundOutcome::Accepted, + )); } } diff --git a/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs b/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs new file mode 100644 index 0000000000..a90cd5f06c --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs @@ -0,0 +1,228 @@ +use crate::crypto::OwnershipStatus; +use crate::logging::OperationalAuditContext; + +pub(super) const OPERATIONAL_AUDIT_INFO: &str = "info"; +pub(super) const OPERATIONAL_AUDIT_WARNING: &str = "warning"; + +/// Static outcomes that are safe to publish through the local operational log. +/// +/// Variants and codes follow the reviewed mesh audit vocabulary. `QuicInboundAccepted` +/// covers authenticated mesh and Skippy-stage QUIC connections; its `protocol_gen` +/// summary is the generation of the negotiated protocol family. There is no +/// `quic_alpn_accepted` code because no explicit mesh-ALPN validation branch exists. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshOperationalEvent { + QuicHandlerFailed(MeshHandlerFailureClass), + QuicInboundAccepted(MeshQuicInboundOutcome), + ControlHandlerFailed(MeshHandlerFailureClass), + ControlAlpnRejected, + ControlConnectionAccepted, + GossipPolicyRejected(MeshPolicyRejectionReason), + GossipDirectPeerPromoted, + GossipIncompatibleVersionRejected, + GossipPeerRemoved(MeshPeerRemovalReason), + AutoJoinSucceeded, + AutoJoinFailed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshPeerRemovalReason { + StaleDirectAndTransitive, + HeartbeatUnreachable, + PeerDownProbeFailed, + ClosedConnectionNoAddress, + ReconnectFailed, + RecoveredGossipFailed, + CleanShutdown, + TunnelOpenFailed, +} + +impl MeshPeerRemovalReason { + pub(crate) const fn reason_code(self) -> &'static str { + match self { + Self::StaleDirectAndTransitive => "stale_direct_and_transitive", + Self::HeartbeatUnreachable => "heartbeat_unreachable", + Self::PeerDownProbeFailed => "peer_down_probe_failed", + Self::ClosedConnectionNoAddress => "closed_connection_no_address", + Self::ReconnectFailed => "reconnect_failed", + Self::RecoveredGossipFailed => "recovered_gossip_failed", + Self::CleanShutdown => "clean_shutdown", + Self::TunnelOpenFailed => "tunnel_open_failed", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshQuicInboundOutcome { + Accepted, + Readmitted, +} + +impl MeshQuicInboundOutcome { + const fn outcome(self) -> &'static str { + match self { + Self::Accepted => "accepted", + Self::Readmitted => "readmitted", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshHandlerFailureClass { + Capacity, + Internal, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshPolicyRejectionReason { + AttestationRequired, + AttestationExpired, + AttestationInvalid, + AttestationNodeMismatch, + Revoked, + AttestationRevoked, + NodeRevoked, + AttestationProtocolUnsupported, + Untrusted, +} + +impl MeshPolicyRejectionReason { + pub(crate) const fn from_ownership_status(status: &OwnershipStatus) -> Option { + match status { + OwnershipStatus::Verified => None, + OwnershipStatus::Unsigned => Some(Self::AttestationRequired), + OwnershipStatus::Expired => Some(Self::AttestationExpired), + OwnershipStatus::InvalidSignature => Some(Self::AttestationInvalid), + OwnershipStatus::MismatchedNodeId => Some(Self::AttestationNodeMismatch), + OwnershipStatus::RevokedOwner => Some(Self::Revoked), + OwnershipStatus::RevokedCert => Some(Self::AttestationRevoked), + OwnershipStatus::RevokedNodeId => Some(Self::NodeRevoked), + OwnershipStatus::UnsupportedProtocol => Some(Self::AttestationProtocolUnsupported), + OwnershipStatus::UntrustedOwner => Some(Self::Untrusted), + } + } + + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::AttestationRequired => "owner_attestation_required", + Self::AttestationExpired => "owner_attestation_expired", + Self::AttestationInvalid => "owner_attestation_invalid", + Self::AttestationNodeMismatch => "owner_attestation_node_mismatch", + Self::Revoked => "owner_revoked", + Self::AttestationRevoked => "owner_attestation_revoked", + Self::NodeRevoked => "owner_node_revoked", + Self::AttestationProtocolUnsupported => "owner_attestation_protocol_unsupported", + Self::Untrusted => "owner_untrusted", + } + } +} + +impl MeshHandlerFailureClass { + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::Capacity => "capacity", + Self::Internal => "internal", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshHandlerFailureBoundary { + AcceptSetup, + AlpnRead, + Handshake, + CapacityPermit, + StreamAccept, + ControlDispatch, +} + +impl MeshHandlerFailureBoundary { + pub(crate) const fn failure_class(self) -> MeshHandlerFailureClass { + match self { + Self::AcceptSetup + | Self::AlpnRead + | Self::Handshake + | Self::StreamAccept + | Self::ControlDispatch => MeshHandlerFailureClass::Internal, + Self::CapacityPermit => MeshHandlerFailureClass::Capacity, + } + } +} + +impl MeshOperationalEvent { + pub(super) const fn level(self) -> &'static str { + match self { + Self::QuicInboundAccepted(_) + | Self::ControlConnectionAccepted + | Self::GossipDirectPeerPromoted + | Self::GossipPeerRemoved(_) + | Self::AutoJoinSucceeded => OPERATIONAL_AUDIT_INFO, + Self::QuicHandlerFailed(_) + | Self::ControlHandlerFailed(_) + | Self::ControlAlpnRejected + | Self::GossipPolicyRejected(_) + | Self::GossipIncompatibleVersionRejected + | Self::AutoJoinFailed => OPERATIONAL_AUDIT_WARNING, + } + } + + pub(super) const fn code(self) -> &'static str { + match self { + Self::QuicHandlerFailed(_) => "mesh_quic_handler_failed", + Self::QuicInboundAccepted(_) => "mesh_quic_inbound_accepted", + Self::ControlHandlerFailed(_) => "mesh_control_handler_failed", + Self::ControlAlpnRejected => "mesh_control_alpn_rejected", + Self::ControlConnectionAccepted => "mesh_control_connection_accepted", + Self::GossipPolicyRejected(_) => "gossip_policy_rejected", + Self::GossipDirectPeerPromoted => "gossip_direct_peer_promoted", + Self::GossipIncompatibleVersionRejected => "gossip_incompatible_version_rejected", + Self::GossipPeerRemoved(_) => "gossip_peer_removed", + Self::AutoJoinSucceeded => "mesh_auto_join_succeeded", + Self::AutoJoinFailed => "mesh_auto_join_failed", + } + } + + pub(super) const fn outcome(self) -> &'static str { + match self { + Self::QuicInboundAccepted(outcome) => outcome.outcome(), + Self::ControlConnectionAccepted => "accepted", + Self::GossipDirectPeerPromoted => "promoted", + Self::GossipPeerRemoved(_) => "removed", + Self::AutoJoinSucceeded => "succeeded", + Self::QuicHandlerFailed(_) | Self::ControlHandlerFailed(_) | Self::AutoJoinFailed => { + "failed" + } + Self::ControlAlpnRejected + | Self::GossipPolicyRejected(_) + | Self::GossipIncompatibleVersionRejected => "rejected", + } + } + + pub(super) const fn reason_code(self) -> Option<&'static str> { + match self { + Self::QuicHandlerFailed(class) | Self::ControlHandlerFailed(class) => { + Some(class.reason_code()) + } + Self::ControlAlpnRejected => Some("alpn_unsupported"), + Self::GossipPolicyRejected(reason) => Some(reason.reason_code()), + Self::GossipIncompatibleVersionRejected => Some("protocol_version_unsupported"), + Self::GossipPeerRemoved(reason) => Some(reason.reason_code()), + Self::AutoJoinFailed => Some("candidate_failed"), + Self::QuicInboundAccepted(_) + | Self::ControlConnectionAccepted + | Self::GossipDirectPeerPromoted + | Self::AutoJoinSucceeded => None, + } + } + + pub(super) const fn decorate_context( + self, + mut context: OperationalAuditContext, + ) -> OperationalAuditContext { + context = context.outcome(self.outcome()); + match self.reason_code() { + Some(reason) => context.reason_code(reason), + None => context, + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs index 8b03009574..01180ee750 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs @@ -1277,7 +1277,8 @@ impl Node { state.dead_peers.insert(id, std::time::Instant::now()); state.connections.remove(&id); drop(state); - self.remove_peer(id).await; + self.remove_peer(id, MeshPeerRemovalReason::PeerDownProbeFailed) + .await; } pub(crate) async fn record_peer_down_rejection(&self, remote: EndpointId, dead_id: EndpointId) { diff --git a/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs b/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs index dcb727b9fe..b59cfe52d6 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs @@ -367,7 +367,11 @@ pub(crate) fn artifact_transfer_allowed_by_topology( /// Channels returned by Node::start for inbound tunnel streams. pub struct TunnelChannels { pub rpc: tokio::sync::mpsc::Receiver<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, - pub http: tokio::sync::mpsc::Receiver<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, + pub http: tokio::sync::mpsc::Receiver<( + EndpointId, + iroh::endpoint::SendStream, + iroh::endpoint::RecvStream, + )>, pub stage: tokio::sync::mpsc::Receiver<( EndpointId, iroh::endpoint::SendStream, diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs index 7ed8509cb0..08a1bad4a1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs @@ -16,6 +16,110 @@ use tokio::sync::{mpsc, watch}; mod direct_path; +#[tokio::test] +async fn inbound_http_tunnel_channel_retains_authenticated_remote_endpoint() { + async fn start_node() -> (Node, TunnelChannels) { + let relay_urls = Vec::new(); + let relay_auths = HashMap::new(); + Node::start( + super::NodeRole::Client, + RelayConfig { + urls: &relay_urls, + auths: &relay_auths, + policy: RelayPolicy::Disabled, + }, + QuicBindSelection { + ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + port: Some(0), + }, + Some(0.0), + false, + None, + None, + crate::MeshRequirements::unrestricted(), + ) + .await + .expect("start tunnel identity test node") + } + + let (sender, _sender_channels) = start_node().await; + let (receiver, mut receiver_channels) = start_node().await; + sender.start_accepting(); + receiver.start_accepting(); + sender + .connect_to_peer(receiver.endpoint_addr_for_advertisement()) + .await + .expect("connect tunnel identity test nodes"); + + let _outbound = sender + .open_http_tunnel(receiver.id()) + .await + .expect("open authenticated HTTP tunnel"); + let (remote, _send, _recv) = tokio::time::timeout( + std::time::Duration::from_secs(5), + receiver_channels.http.recv(), + ) + .await + .expect("HTTP tunnel should reach receiver") + .expect("HTTP tunnel channel should remain open"); + + assert_eq!(remote, sender.id()); +} + +#[tokio::test] +#[serial] +async fn stage_alpn_emits_authenticated_direct_peer_audit_before_stream_dispatch() { + let (mut audits, _capture) = super::capture_mesh_operational_audits(); + let sender = make_test_node(super::NodeRole::Worker) + .await + .expect("sender node"); + let expected_sender_id = hex::encode(sender.id().as_bytes()); + let receiver = make_test_node(super::NodeRole::Worker) + .await + .expect("receiver node"); + receiver.start_accepting(); + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + mesh_peer_operational_context(receiver.id(), None), + ); + + let _connection = sender + .endpoint + .connect( + receiver.endpoint_addr_for_advertisement(), + skippy_protocol::STAGE_ALPN_V2, + ) + .await + .expect("authenticated stage connection"); + + let audit = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let record = audits.recv().await.expect("audit capture remains open"); + if record.code() == "mesh_quic_inbound_accepted" + && record.context().is_some_and(|context| { + let fields = context.fields(); + fields["subject_id"].as_str() == Some(expected_sender_id.as_str()) + }) + { + return record; + } + } + }) + .await + .expect("stage audit should be emitted before waiting for a stream"); + + let fields = audit.context().expect("stage audit context").fields(); + assert_eq!(fields["subject_kind"], "mesh_peer"); + assert_eq!(fields["subject_id"], expected_sender_id); + assert_eq!(fields["path_type"], "direct"); + assert!(fields["remote_addr"].as_str().is_some()); + assert_eq!(fields["outcome"], "accepted"); + assert_eq!( + fields["numeric_summaries"]["protocol_gen"], + skippy_protocol::STAGE_PROTOCOL_GENERATION + ); +} + #[tokio::test] async fn owner_control_stream_work_is_bounded_per_connection() { let permits = control_stream_semaphore(); diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs index c572fbad64..40808b8a96 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs @@ -4,6 +4,7 @@ use crate::mesh::connections::encode_endpoint_addr_token; use crate::mesh::now_secs; use iroh::SecretKey; use mesh_llm_types::mesh::DEMAND_TTL_SECS; +use serial_test::serial; use std::collections::HashMap; pub(crate) fn test_endpoint_id(seed: u8) -> EndpointId { @@ -72,853 +73,20 @@ pub(crate) fn test_peer(ts: Option) -> PeerInfo { ) } -#[test] -pub(crate) fn test_merge_none_to_some() { - let mut existing = test_peer(None); - let ann = test_announcement(Some(100)); +mod merge_and_refresh { + use super::*; - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_some_to_none_keeps_existing() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(None); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_earlier_incoming_wins() { - let mut existing = test_peer(Some(200)); - let ann = test_announcement(Some(100)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_later_incoming_loses() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(Some(200)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_equal_values_unchanged() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(Some(100)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_first_joined_mesh_ts() { - let old_peer = test_peer(Some(100)); - let new_peer = test_peer(Some(200)); - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_explicit_model_interests() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_stage_status_list_support() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.stage_status_list_supported = !old_peer.stage_status_list_supported; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_stage_protocol_generation_support() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.stage_protocol_generation_supported = !old_peer.stage_protocol_generation_supported; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_explicit_model_interests() { - let mut existing = test_peer(Some(100)); - let mut ann = test_announcement(Some(100)); - ann.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!( - existing.explicit_model_interests, - vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] - ); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_stage_status_list_support() { - let mut existing = test_peer(Some(100)); - existing.stage_status_list_supported = false; - let mut ann = test_announcement(Some(100)); - ann.stage_status_list_supported = true; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert!(existing.stage_status_list_supported); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_stage_protocol_generation_support() { - let mut existing = test_peer(Some(100)); - existing.stage_protocol_generation_supported = false; - let mut ann = test_announcement(Some(100)); - ann.stage_protocol_generation_supported = true; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert!(existing.stage_protocol_generation_supported); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_advertised_model_throughput() { - let mut existing = test_peer(Some(100)); - let mut ann = test_announcement(Some(100)); - ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { - model_name: "qwen".to_string(), - avg_tokens_per_second_milli: 35_000, - throughput_samples: 4, - }]; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!( - existing.advertised_model_throughput, - ann.advertised_model_throughput - ); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_stage_status_list_support() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x44); - let addr = test_addr(0x44); - let mut ann = test_announcement(Some(100)); - ann.stage_status_list_supported = false; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.stage_status_list_supported = true; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert!(peer.stage_status_list_supported); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_stage_protocol_generation_support() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x45); - let addr = test_addr(0x45); - let mut ann = test_announcement(Some(100)); - ann.stage_protocol_generation_supported = false; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.stage_protocol_generation_supported = true; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert!(peer.stage_protocol_generation_supported); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_advertised_model_throughput() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x46); - let addr = test_addr(0x46); - let mut ann = test_announcement(Some(100)); - ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { - model_name: "qwen".to_string(), - avg_tokens_per_second_milli: 20_000, - throughput_samples: 2, - }]; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.advertised_model_throughput[0].avg_tokens_per_second_milli = 48_000; - ann.advertised_model_throughput[0].throughput_samples = 9; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert_eq!( - peer.advertised_model_throughput, - ann.advertised_model_throughput - ); -} - -#[tokio::test] -pub(crate) async fn test_collect_announcements_includes_self_explicit_model_interests() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - node.set_explicit_model_interests(vec![ - "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), - "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), - ]) - .await; - - let announcements = node.collect_announcements().await; - let self_announcement = announcements - .iter() - .find(|announcement| announcement.addr.id == node.id()) - .expect("self announcement must be present"); - - assert_eq!( - self_announcement.explicit_model_interests, - vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] - ); -} - -#[tokio::test] -pub(crate) async fn gossip_admission_advertisement_matrix() { - use crate::runtime::activity_policy::{ - activity_advertisement_decision, ActivityPolicyState, - }; - use mesh_llm_config::ActivityAdvertisement; - - let accepting = crate::proto::node::InferenceAdmissionState::Accepting; - let paused = crate::proto::node::InferenceAdmissionState::RemotePaused; - let cases = [ - (ActivityAdvertisement::None, false, None, false), - (ActivityAdvertisement::None, true, None, false), - (ActivityAdvertisement::AvailabilityOnly, false, None, false), - (ActivityAdvertisement::AvailabilityOnly, true, None, true), - ( - ActivityAdvertisement::CoarseState, - false, - Some(accepting), - false, - ), - ( - ActivityAdvertisement::CoarseState, - true, - Some(paused), - true, - ), - ]; - for (mode, blocking, expected_state, expected_withdrawal) in cases { - let state = if blocking { - ActivityPolicyState::RemotePaused - } else { - ActivityPolicyState::Accepting - }; - let decision = activity_advertisement_decision(true, mode, state, false); - assert_eq!(decision.admission_state, expected_state, "mode={mode:?}"); - assert_eq!( - decision.withdraw_model_availability, expected_withdrawal, - "mode={mode:?}" - ); - } - - let private = activity_advertisement_decision( - true, - ActivityAdvertisement::PrivateCoarseState, - ActivityPolicyState::AllPaused, - false, - ); - assert_eq!( - private.admission_state, - Some(crate::proto::node::InferenceAdmissionState::AllPaused) - ); - assert!(private.withdraw_model_availability); - - let public = activity_advertisement_decision( - true, - ActivityAdvertisement::PrivateCoarseState, - ActivityPolicyState::AllPaused, - true, - ); - assert_eq!(public.admission_state, None); - assert!(public.withdraw_model_availability); - - let disabled = activity_advertisement_decision( - false, - ActivityAdvertisement::CoarseState, - ActivityPolicyState::AllPaused, - false, - ); - assert_eq!(disabled.admission_state, None); - assert!(!disabled.withdraw_model_availability); -} - -#[tokio::test] -pub(crate) async fn legacy_peer_admission_and_known_empty_withdrawal() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x71); - let addr = test_addr(0x71); - - let mut legacy_add = test_announcement(Some(100)); - legacy_add.addr = addr.clone(); - legacy_add.role = NodeRole::Host { http_port: 9337 }; - legacy_add.version = None; - legacy_add.serving_models = vec!["legacy-model".to_string()]; - legacy_add.hosted_models = None; - legacy_add.inference_admission_state = None; - node.add_peer(peer_id, addr.clone(), &legacy_add, None).await; - - let state = node.state.lock().await; - let peer = state - .peers - .get(&peer_id) - .expect("legacy-style peer should be admitted"); - assert_eq!(peer.version, None); - assert!(!peer.hosted_models_known); - assert!(peer.routes_http_model("legacy-model")); - drop(state); - - let mut legacy_withdrawal = test_announcement(Some(100)); - legacy_withdrawal.addr = addr.clone(); - legacy_withdrawal.role = NodeRole::Host { http_port: 9337 }; - legacy_withdrawal.version = None; - legacy_withdrawal.serving_models = Vec::new(); - legacy_withdrawal.hosted_models = Some(Vec::new()); - legacy_withdrawal.inference_admission_state = None; - node.add_peer(peer_id, addr, &legacy_withdrawal, None).await; - - let state = node.state.lock().await; - let peer = state - .peers - .get(&peer_id) - .expect("legacy-withdrawal update should be applied"); - assert_eq!(peer.hosted_models, Vec::::new()); - assert!(peer.hosted_models_known); - assert!(!peer.routes_http_model("legacy-model")); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_handles_floor() { - // At or above the floor — allowed. - assert!(version_allowed_for_rebroadcast(Some("0.60.0"))); - assert!(version_allowed_for_rebroadcast(Some("0.60.2"))); - assert!(version_allowed_for_rebroadcast(Some("0.64.0"))); - assert!(version_allowed_for_rebroadcast(Some("0.65.1"))); - assert!(version_allowed_for_rebroadcast(Some("1.0.0"))); - // Below the floor — refused. - assert!(!version_allowed_for_rebroadcast(Some("0.57.0"))); - assert!(!version_allowed_for_rebroadcast(Some("0.55.1"))); - assert!(!version_allowed_for_rebroadcast(Some("0.58.0"))); - assert!(!version_allowed_for_rebroadcast(Some("0.59.99"))); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_handles_metadata_and_prerelease() { - // Build metadata is stripped. - assert!(version_allowed_for_rebroadcast(Some( - "0.65.1+skippy.20260504.kv.2" - ))); - assert!(!version_allowed_for_rebroadcast(Some("0.57.0+anything"))); - // Pre-release tags are stripped — 0.63.0-rc5 still passes. - assert!(version_allowed_for_rebroadcast(Some("0.63.0-rc5"))); - assert!(!version_allowed_for_rebroadcast(Some("0.58.0-beta"))); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_is_conservative_on_unknown() { - // Unparseable / missing / empty — preserved (don't drop legacy nodes - // that never advertised a version). - assert!(version_allowed_for_rebroadcast(None)); - assert!(version_allowed_for_rebroadcast(Some(""))); - assert!(version_allowed_for_rebroadcast(Some(" "))); - assert!(version_allowed_for_rebroadcast(Some("garbage"))); - assert!(version_allowed_for_rebroadcast(Some("0"))); - assert!(version_allowed_for_rebroadcast(Some("0.x"))); -} - -#[tokio::test] -pub(crate) async fn transitive_ingest_rejects_below_version_floor() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let old_addr = test_addr(0x57); - let new_addr = test_addr(0x65); - let old_id = old_addr.id; - let new_id = new_addr.id; - - let mut old_ann = test_announcement(None); - old_ann.addr = old_addr.clone(); - old_ann.role = NodeRole::Client; - old_ann.version = Some("0.57.0".to_string()); - let mut new_ann = test_announcement(None); - new_ann.addr = new_addr.clone(); - new_ann.role = NodeRole::Client; - new_ann.version = Some("0.65.0".to_string()); - // Give the v0.65.0 client a demand signal so the idle-transitive- - // client filter (a separate gate) doesn't drop it — this test - // exercises the version floor specifically. - new_ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - let bridge = test_endpoint_id(0xBB); - node.update_transitive_peer(old_id, &old_addr, &old_ann, bridge) - .await; - node.update_transitive_peer(new_id, &new_addr, &new_ann, bridge) - .await; - - // Old peer must NOT be in local state — it was rejected at ingest. - // New peer must be present. - { - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&old_id), - "v0.57.0 peer must be rejected at ingest, not appear in local state" - ); - assert!( - state.peers.contains_key(&new_id), - "v0.65.0 peer should be added to local state" - ); - } - - // Outbound gossip must also exclude the old peer. - let announcements = node.collect_announcements().await; - assert!( - !announcements.iter().any(|a| a.addr.id == old_id), - "v0.57.0 peer must not appear in outbound gossip" - ); - assert!( - announcements.iter().any(|a| a.addr.id == new_id), - "v0.65.0 peer should appear in outbound gossip" - ); -} - -#[test] -pub(crate) fn peer_is_idle_transitive_client_basic_shapes() { - // Empty idle client: no hostname, no direct measurement, no - // interests → caught. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - assert!(peer_is_idle_transitive_client(&ann)); - - // Real idle user with a hostname → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.hostname = Some("Sams-MacBook-Pro.local".into()); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Hostname-less client that someone directly measured → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.latency_source = Some(crate::proto::node::LatencySource::Direct); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Estimated latency (propagated guess, not direct) — still caught; - // only Direct counts as proof of contact. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.latency_source = Some(crate::proto::node::LatencySource::Estimated); - assert!(peer_is_idle_transitive_client(&ann)); - - // Client asking for a model → kept (demand signal). - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Client somehow advertising serving → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Client advertising hosted → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.hosted_models = Some(vec!["Qwen3-8B-Q4_K_M".to_string()]); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Host → never caught regardless of other fields. - let mut ann = test_announcement(None); - ann.role = NodeRole::Host { http_port: 9337 }; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Worker → never caught. - let mut ann = test_announcement(None); - ann.role = NodeRole::Worker; - assert!(!peer_is_idle_transitive_client(&ann)); -} - -#[tokio::test] -pub(crate) async fn transitive_ingest_drops_idle_clients_but_keeps_clients_with_demand() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let idle_addr = test_addr(0xC1); - let demand_addr = test_addr(0xC2); - let host_addr = test_addr(0xC3); - let idle_id = idle_addr.id; - let demand_id = demand_addr.id; - let host_id = host_addr.id; - - // Idle client — should be dropped at transitive ingest. - let mut idle = test_announcement(None); - idle.addr = idle_addr.clone(); - idle.role = NodeRole::Client; - idle.version = Some("0.65.1".to_string()); - - // Client asking for a model — must be kept (demand signal). - let mut with_demand = test_announcement(None); - with_demand.addr = demand_addr.clone(); - with_demand.role = NodeRole::Client; - with_demand.version = Some("0.65.1".to_string()); - with_demand.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - // Host — must be kept (real compute). - let mut host = test_announcement(None); - host.addr = host_addr.clone(); - host.role = NodeRole::Host { http_port: 9337 }; - host.version = Some("0.65.1".to_string()); - host.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - let bridge = test_endpoint_id(0xBB); - node.update_transitive_peer(idle_id, &idle_addr, &idle, bridge) - .await; - node.update_transitive_peer(demand_id, &demand_addr, &with_demand, bridge) - .await; - node.update_transitive_peer(host_id, &host_addr, &host, bridge) - .await; - - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&idle_id), - "idle transitive client must be rejected" - ); - assert!( - state.peers.contains_key(&demand_id), - "client with requested_models must be kept (demand signal)" - ); - assert!( - state.peers.contains_key(&host_id), - "host must be kept (real compute)" - ); -} - -#[tokio::test] -pub(crate) async fn direct_add_peer_admits_idle_clients() { - // Idle clients we actually directly contact are still admitted. - // The predicate is for transitive ingest only — a direct connection - // is proof of life and the peer is observable. - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let addr = test_addr(0xC4); - let id = addr.id; - - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.role = NodeRole::Client; - ann.version = Some("0.65.1".to_string()); - // No requested, no serving, no hosted — pure idle client. - - node.add_peer(id, addr, &ann, None).await; - - let state = node.state.lock().await; - assert!( - state.peers.contains_key(&id), - "direct idle client must be admitted (direct contact is proof of life)" - ); -} - -#[tokio::test] -pub(crate) async fn direct_add_peer_rejects_below_version_floor() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let addr = test_addr(0x57); - let id = addr.id; - - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.role = NodeRole::Client; - ann.version = Some("0.57.0".to_string()); - - node.add_peer(id, addr, &ann, None).await; - - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&id), - "direct add of v0.57.0 peer must be rejected (no local state entry)" - ); -} - -#[tokio::test] -pub(crate) async fn rejected_direct_peer_does_not_apply_mesh_state_or_demand() { - // Given: a direct announcement that passes requirement validation but is - // rejected by the supported-version admission policy. - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let addr = test_addr(0x59); - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.version = Some("0.57.0".to_string()); - ann.mesh_id = Some("rejected-mesh".to_string()); - ann.model_demand.insert( - "rejected-demand".to_string(), - ModelDemand { - last_active: now_secs(), - request_count: 1, - }, - ); - - // When: the direct payload is applied through the production gossip path. - node.apply_announced_peers(addr.id, &[(addr.clone(), ann)], None, None, true) - .await - .expect("policy rejection should not fail the gossip frame"); - - // Then: admission rejection must happen before mesh-wide side effects. - assert_eq!(node.mesh_id().await, None); - assert!(node.get_demand().is_empty()); - assert!(!node.state.lock().await.peers.contains_key(&addr.id)); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub(crate) async fn inbound_gossip_rejection_preserves_dead_peer_state() -> Result<()> { - // Given: a host that still considers the direct sender dead, and a direct - // gossip payload whose sender announcement fails admission. - let host = Node::new_for_tests(NodeRole::Worker).await?; - let policy = crate::MeshGenesisPolicy::new( - "test-owner", - current_time_unix_ms(), - crate::MeshRequirements::default(), - ) - .expect("test policy should be valid"); - let mesh_id = policy.policy_derived_mesh_id().expect("mesh id"); - let policy_hash = policy.canonical_hash_hex().expect("policy hash"); - host.set_active_mesh_policy_for_tests(policy).await; - - let sender_id = test_endpoint_id(0x58); - host.state - .lock() - .await - .dead_peers - .insert(sender_id, std::time::Instant::now()); - - let mut sender_announcement = test_announcement(None); - sender_announcement.addr = EndpointAddr { - id: sender_id, - addrs: Default::default(), - }; - sender_announcement.role = NodeRole::Client; - sender_announcement.version = Some(crate::VERSION.to_string()); - sender_announcement.mesh_id = Some(mesh_id); - sender_announcement.mesh_policy_hash = Some(policy_hash); - let announcements = [(sender_announcement.addr.clone(), sender_announcement)]; - - // When: the same production phase used by `handle_gossip_stream` processes - // the rejected direct announcement. - host.validate_and_capture_inbound_gossip( - ControlProtocol::ProtoV1, - &announcements, - AnnouncedPeerContext::direct(sender_id, Some(NODE_PROTOCOL_GENERATION)), - ) - .await - .expect_err("rejected gossip must fail admission"); - - // Then: admission rejection is recorded, but liveness recovery has not - // cleared the sender from dead_peers. - let state = host.state.lock().await; - assert!( - state.dead_peers.contains_key(&sender_id), - "rejected inbound gossip must not clear dead-peer state" - ); - assert!( - state.requirement_rejected_peers.contains(&sender_id), - "rejected inbound gossip should still be tracked as an admission rejection" - ); - assert!( - !state.peers.contains_key(&sender_id), - "rejected inbound gossip must not admit the sender" - ); - - Ok(()) -} - -#[tokio::test] -pub(crate) async fn future_demand_timestamps_are_active_without_underflow() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let now = now_secs(); - node.merge_remote_demand(&HashMap::from([( - "future-demand".to_string(), - ModelDemand { - last_active: now + DEMAND_TTL_SECS + 60, - request_count: 3, - }, - )])); - - let active = node.active_demand().await; - assert!(active.contains_key("future-demand")); - - node.gc_demand().await; - assert!(node.get_demand().contains_key("future-demand")); + include!("gossip/merge_and_refresh.rs"); } -/// Regression test for the `--auto` startup wedge: when a transitive -/// gossip payload includes peers that would be rejected at ingest -/// (version-floor or idle-transitive-client), `maybe_connect_discovered_peer` -/// must skip the dial. Otherwise each unreachable ghost address triggers -/// a 30 s `connect_to_peer` timeout sequentially in the dial loop, -/// wedging the surrounding gossip exchange (and the `attempt_run_auto_join` -/// that initiated it) for tens of minutes. -/// -/// The function returns without panicking and without dialing within a -/// generous time bound — a real dial to a fake address would block on -/// the 30 s `PEER_CONNECT_AND_GOSSIP_TIMEOUT`. We assert the result is -/// reached well under that bound and that no connection entry was created. -#[tokio::test] -pub(crate) async fn maybe_connect_discovered_peer_skips_filtered_announcements() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let my_role = NodeRole::Worker; +mod admission { + use super::*; - // Below-floor version — must be skipped without dialing. - let old_addr = test_addr(0x57); - let old_id = old_addr.id; - let mut old_ann = test_announcement(None); - old_ann.addr = old_addr.clone(); - old_ann.role = NodeRole::Client; - old_ann.version = Some("0.57.0".to_string()); - - // Idle transitive client (matching version, but no hostname / no - // direct measurement / no model interests) — must also be skipped. - let idle_addr = test_addr(0xC1); - let idle_id = idle_addr.id; - let mut idle_ann = test_announcement(None); - idle_ann.addr = idle_addr.clone(); - idle_ann.role = NodeRole::Client; - idle_ann.version = Some("0.65.1".to_string()); - - // Both calls together must return well under the 30 s connect - // timeout. If the dial-loop skip is missing, each call will block - // on PEER_CONNECT_AND_GOSSIP_TIMEOUT (30 s) attempting to dial the - // fake test address. - tokio::time::timeout(std::time::Duration::from_secs(5), async { - node.maybe_connect_discovered_peer(&my_role, old_addr, &old_ann, true, false) - .await; - node.maybe_connect_discovered_peer(&my_role, idle_addr, &idle_ann, true, false) - .await; - }) - .await - .expect("filtered peers must be skipped quickly, not dialed"); - - // No connection was attempted (no entry in state.connections), and - // no peer was added (the filtered announcements never reach add_peer - // or update_transitive_peer through this path). - let state = node.state.lock().await; - assert!( - !state.connections.contains_key(&old_id), - "below-floor peer must not be dialed" - ); - assert!( - !state.connections.contains_key(&idle_id), - "idle transitive client must not be dialed" - ); - assert!( - !state.peers.contains_key(&old_id), - "below-floor peer must not be added (this path is dial-only)" - ); - assert!( - !state.peers.contains_key(&idle_id), - "idle transitive client must not be added (this path is dial-only)" - ); + include!("gossip/admission.rs"); } -#[tokio::test] -pub(crate) async fn client_auto_join_probe_returns_none_for_single_candidate() { - let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); - let token = encode_endpoint_addr_token(&test_addr(0x42)); - - let selected = node - .join_first_responsive_candidate(&[(token, Some("single".to_string()))]) - .await - .unwrap(); - - assert!(selected.is_none()); -} - -#[tokio::test] -pub(crate) async fn client_auto_join_probe_candidate_collection_filters_unusable_tokens() { - let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); - let valid_addr = test_addr(0x42); - let dead_addr = test_addr(0x43); - let self_token = encode_endpoint_addr_token(&node.endpoint_addr_for_advertisement()); - let dead_token = encode_endpoint_addr_token(&dead_addr); - let valid_token = encode_endpoint_addr_token(&valid_addr); - - node.state - .lock() - .await - .dead_peers - .insert(dead_addr.id, std::time::Instant::now()); - - let candidates = node - .collect_join_probe_candidates(&[ - ("not-an-invite-token".to_string(), None), - (self_token, None), - (dead_token, None), - (valid_token, Some("usable".to_string())), - ]) - .await; +mod discovery { + use super::*; - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].addr.id, valid_addr.id); - assert_eq!(candidates[0].mesh_name.as_deref(), Some("usable")); + include!("gossip/discovery.rs"); } diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs new file mode 100644 index 0000000000..345479ae45 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs @@ -0,0 +1,531 @@ +#[tokio::test] +pub(crate) async fn gossip_admission_advertisement_matrix() { + use crate::runtime::activity_policy::{ + activity_advertisement_decision, ActivityPolicyState, + }; + use mesh_llm_config::ActivityAdvertisement; + + let accepting = crate::proto::node::InferenceAdmissionState::Accepting; + let paused = crate::proto::node::InferenceAdmissionState::RemotePaused; + let cases = [ + (ActivityAdvertisement::None, false, None, false), + (ActivityAdvertisement::None, true, None, false), + (ActivityAdvertisement::AvailabilityOnly, false, None, false), + (ActivityAdvertisement::AvailabilityOnly, true, None, true), + ( + ActivityAdvertisement::CoarseState, + false, + Some(accepting), + false, + ), + ( + ActivityAdvertisement::CoarseState, + true, + Some(paused), + true, + ), + ]; + for (mode, blocking, expected_state, expected_withdrawal) in cases { + let state = if blocking { + ActivityPolicyState::RemotePaused + } else { + ActivityPolicyState::Accepting + }; + let decision = activity_advertisement_decision(true, mode, state, false); + assert_eq!(decision.admission_state, expected_state, "mode={mode:?}"); + assert_eq!( + decision.withdraw_model_availability, expected_withdrawal, + "mode={mode:?}" + ); + } + + let private = activity_advertisement_decision( + true, + ActivityAdvertisement::PrivateCoarseState, + ActivityPolicyState::AllPaused, + false, + ); + assert_eq!( + private.admission_state, + Some(crate::proto::node::InferenceAdmissionState::AllPaused) + ); + assert!(private.withdraw_model_availability); + + let public = activity_advertisement_decision( + true, + ActivityAdvertisement::PrivateCoarseState, + ActivityPolicyState::AllPaused, + true, + ); + assert_eq!(public.admission_state, None); + assert!(public.withdraw_model_availability); + + let disabled = activity_advertisement_decision( + false, + ActivityAdvertisement::CoarseState, + ActivityPolicyState::AllPaused, + false, + ); + assert_eq!(disabled.admission_state, None); + assert!(!disabled.withdraw_model_availability); +} + +#[tokio::test] +pub(crate) async fn legacy_peer_admission_and_known_empty_withdrawal() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x71); + let addr = test_addr(0x71); + + let mut legacy_add = test_announcement(Some(100)); + legacy_add.addr = addr.clone(); + legacy_add.role = NodeRole::Host { http_port: 9337 }; + legacy_add.version = None; + legacy_add.serving_models = vec!["legacy-model".to_string()]; + legacy_add.hosted_models = None; + legacy_add.inference_admission_state = None; + node.add_peer(peer_id, addr.clone(), &legacy_add, None).await; + + let state = node.state.lock().await; + let peer = state + .peers + .get(&peer_id) + .expect("legacy-style peer should be admitted"); + assert_eq!(peer.version, None); + assert!(!peer.hosted_models_known); + assert!(peer.routes_http_model("legacy-model")); + drop(state); + + let mut legacy_withdrawal = test_announcement(Some(100)); + legacy_withdrawal.addr = addr.clone(); + legacy_withdrawal.role = NodeRole::Host { http_port: 9337 }; + legacy_withdrawal.version = None; + legacy_withdrawal.serving_models = Vec::new(); + legacy_withdrawal.hosted_models = Some(Vec::new()); + legacy_withdrawal.inference_admission_state = None; + node.add_peer(peer_id, addr, &legacy_withdrawal, None).await; + + let state = node.state.lock().await; + let peer = state + .peers + .get(&peer_id) + .expect("legacy-withdrawal update should be applied"); + assert_eq!(peer.hosted_models, Vec::::new()); + assert!(peer.hosted_models_known); + assert!(!peer.routes_http_model("legacy-model")); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_handles_floor() { + // At or above the floor — allowed. + assert!(version_allowed_for_rebroadcast(Some("0.60.0"))); + assert!(version_allowed_for_rebroadcast(Some("0.60.2"))); + assert!(version_allowed_for_rebroadcast(Some("0.64.0"))); + assert!(version_allowed_for_rebroadcast(Some("0.65.1"))); + assert!(version_allowed_for_rebroadcast(Some("1.0.0"))); + // Below the floor — refused. + assert!(!version_allowed_for_rebroadcast(Some("0.57.0"))); + assert!(!version_allowed_for_rebroadcast(Some("0.55.1"))); + assert!(!version_allowed_for_rebroadcast(Some("0.58.0"))); + assert!(!version_allowed_for_rebroadcast(Some("0.59.99"))); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_handles_metadata_and_prerelease() { + // Build metadata is stripped. + assert!(version_allowed_for_rebroadcast(Some( + "0.65.1+skippy.20260504.kv.2" + ))); + assert!(!version_allowed_for_rebroadcast(Some("0.57.0+anything"))); + // Pre-release tags are stripped — 0.63.0-rc5 still passes. + assert!(version_allowed_for_rebroadcast(Some("0.63.0-rc5"))); + assert!(!version_allowed_for_rebroadcast(Some("0.58.0-beta"))); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_is_conservative_on_unknown() { + // Unparseable / missing / empty — preserved (don't drop legacy nodes + // that never advertised a version). + assert!(version_allowed_for_rebroadcast(None)); + assert!(version_allowed_for_rebroadcast(Some(""))); + assert!(version_allowed_for_rebroadcast(Some(" "))); + assert!(version_allowed_for_rebroadcast(Some("garbage"))); + assert!(version_allowed_for_rebroadcast(Some("0"))); + assert!(version_allowed_for_rebroadcast(Some("0.x"))); +} + +#[tokio::test] +pub(crate) async fn transitive_ingest_rejects_below_version_floor() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let old_addr = test_addr(0x57); + let new_addr = test_addr(0x65); + let old_id = old_addr.id; + let new_id = new_addr.id; + + let mut old_ann = test_announcement(None); + old_ann.addr = old_addr.clone(); + old_ann.role = NodeRole::Client; + old_ann.version = Some("0.57.0".to_string()); + let mut new_ann = test_announcement(None); + new_ann.addr = new_addr.clone(); + new_ann.role = NodeRole::Client; + new_ann.version = Some("0.65.0".to_string()); + // Give the v0.65.0 client a demand signal so the idle-transitive- + // client filter (a separate gate) doesn't drop it — this test + // exercises the version floor specifically. + new_ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + let bridge = test_endpoint_id(0xBB); + node.update_transitive_peer(old_id, &old_addr, &old_ann, bridge) + .await; + node.update_transitive_peer(new_id, &new_addr, &new_ann, bridge) + .await; + + // Old peer must NOT be in local state — it was rejected at ingest. + // New peer must be present. + { + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&old_id), + "v0.57.0 peer must be rejected at ingest, not appear in local state" + ); + assert!( + state.peers.contains_key(&new_id), + "v0.65.0 peer should be added to local state" + ); + } + + // Outbound gossip must also exclude the old peer. + let announcements = node.collect_announcements().await; + assert!( + !announcements.iter().any(|a| a.addr.id == old_id), + "v0.57.0 peer must not appear in outbound gossip" + ); + assert!( + announcements.iter().any(|a| a.addr.id == new_id), + "v0.65.0 peer should appear in outbound gossip" + ); +} + +#[test] +pub(crate) fn peer_is_idle_transitive_client_basic_shapes() { + // Empty idle client: no hostname, no direct measurement, no + // interests → caught. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + assert!(peer_is_idle_transitive_client(&ann)); + + // Real idle user with a hostname → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.hostname = Some("Sams-MacBook-Pro.local".into()); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Hostname-less client that someone directly measured → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.latency_source = Some(crate::proto::node::LatencySource::Direct); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Estimated latency (propagated guess, not direct) — still caught; + // only Direct counts as proof of contact. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.latency_source = Some(crate::proto::node::LatencySource::Estimated); + assert!(peer_is_idle_transitive_client(&ann)); + + // Client asking for a model → kept (demand signal). + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Client somehow advertising serving → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Client advertising hosted → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.hosted_models = Some(vec!["Qwen3-8B-Q4_K_M".to_string()]); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Host → never caught regardless of other fields. + let mut ann = test_announcement(None); + ann.role = NodeRole::Host { http_port: 9337 }; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Worker → never caught. + let mut ann = test_announcement(None); + ann.role = NodeRole::Worker; + assert!(!peer_is_idle_transitive_client(&ann)); +} + +#[tokio::test] +pub(crate) async fn transitive_ingest_drops_idle_clients_but_keeps_clients_with_demand() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let idle_addr = test_addr(0xC1); + let demand_addr = test_addr(0xC2); + let host_addr = test_addr(0xC3); + let idle_id = idle_addr.id; + let demand_id = demand_addr.id; + let host_id = host_addr.id; + + // Idle client — should be dropped at transitive ingest. + let mut idle = test_announcement(None); + idle.addr = idle_addr.clone(); + idle.role = NodeRole::Client; + idle.version = Some("0.65.1".to_string()); + + // Client asking for a model — must be kept (demand signal). + let mut with_demand = test_announcement(None); + with_demand.addr = demand_addr.clone(); + with_demand.role = NodeRole::Client; + with_demand.version = Some("0.65.1".to_string()); + with_demand.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + // Host — must be kept (real compute). + let mut host = test_announcement(None); + host.addr = host_addr.clone(); + host.role = NodeRole::Host { http_port: 9337 }; + host.version = Some("0.65.1".to_string()); + host.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + let bridge = test_endpoint_id(0xBB); + node.update_transitive_peer(idle_id, &idle_addr, &idle, bridge) + .await; + node.update_transitive_peer(demand_id, &demand_addr, &with_demand, bridge) + .await; + node.update_transitive_peer(host_id, &host_addr, &host, bridge) + .await; + + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&idle_id), + "idle transitive client must be rejected" + ); + assert!( + state.peers.contains_key(&demand_id), + "client with requested_models must be kept (demand signal)" + ); + assert!( + state.peers.contains_key(&host_id), + "host must be kept (real compute)" + ); +} + +#[tokio::test] +pub(crate) async fn direct_add_peer_admits_idle_clients() { + // Idle clients we actually directly contact are still admitted. + // The predicate is for transitive ingest only — a direct connection + // is proof of life and the peer is observable. + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let addr = test_addr(0xC4); + let id = addr.id; + + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.role = NodeRole::Client; + ann.version = Some("0.65.1".to_string()); + // No requested, no serving, no hosted — pure idle client. + + node.add_peer(id, addr, &ann, None).await; + + let state = node.state.lock().await; + assert!( + state.peers.contains_key(&id), + "direct idle client must be admitted (direct contact is proof of life)" + ); +} + +#[tokio::test] +#[serial] +pub(crate) async fn direct_add_peer_rejects_below_version_floor() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let addr = test_addr(0x57); + let id = addr.id; + + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.role = NodeRole::Client; + ann.version = Some("0.57.0".to_string()); + + node.add_peer(id, addr, &ann, None).await; + + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&id), + "direct add of v0.57.0 peer must be rejected (no local state entry)" + ); +} + +#[tokio::test] +#[serial] +pub(crate) async fn below_version_floor_audit_is_direct_only() { + let direct_addr = test_addr(0x57); + let mut direct_ann = test_announcement(None); + direct_ann.addr = direct_addr.clone(); + direct_ann.role = NodeRole::Client; + direct_ann.version = Some("0.57.0".to_string()); + + let (mut direct_audits, direct_capture) = + crate::mesh::capture_mesh_operational_audits(); + let direct_node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + direct_node + .add_peer(direct_addr.id, direct_addr, &direct_ann, None) + .await; + + let direct_subject_id = hex::encode(direct_ann.addr.id.as_bytes()); + let mut direct_records = Vec::new(); + while let Ok(record) = direct_audits.try_recv() { + if record.code() == "gossip_incompatible_version_rejected" + && record + .context() + .is_some_and(|context| context.fields()["subject_id"] == direct_subject_id) + { + direct_records.push(record); + } + } + assert_eq!(direct_records.len(), 1); + let direct_record = &direct_records[0]; + assert_eq!(direct_record.code(), "gossip_incompatible_version_rejected"); + let direct_context = direct_record.context().expect("direct peer context").fields(); + assert_eq!( + direct_context["reason_code"], + "protocol_version_unsupported" + ); + assert_eq!(direct_context["outcome"], "rejected"); + assert_eq!(direct_context["subject_kind"], "mesh_peer"); + assert_eq!( + direct_context["subject_id"], + direct_subject_id + ); + drop(direct_capture); + + let transitive_addr = test_addr(0x58); + let mut transitive_ann = test_announcement(None); + transitive_ann.addr = transitive_addr.clone(); + transitive_ann.role = NodeRole::Host { http_port: 9337 }; + transitive_ann.version = Some("0.57.0".to_string()); + let bridge = test_endpoint_id(0xBB); + + let (mut transitive_audits, _transitive_capture) = + crate::mesh::capture_mesh_operational_audits(); + let transitive_node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + transitive_node + .update_transitive_peer( + transitive_addr.id, + &transitive_addr, + &transitive_ann, + bridge, + ) + .await; + + let transitive_subject_id = hex::encode(transitive_ann.addr.id.as_bytes()); + let mut transitive_records = 0; + while let Ok(record) = transitive_audits.try_recv() { + if record.code() == "gossip_incompatible_version_rejected" + && record + .context() + .is_some_and(|context| context.fields()["subject_id"] == transitive_subject_id) + { + transitive_records += 1; + } + } + assert_eq!(transitive_records, 0); +} + +#[tokio::test] +pub(crate) async fn rejected_direct_peer_does_not_apply_mesh_state_or_demand() { + // Given: a direct announcement that passes requirement validation but is + // rejected by the supported-version admission policy. + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let addr = test_addr(0x59); + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.version = Some("0.57.0".to_string()); + ann.mesh_id = Some("rejected-mesh".to_string()); + ann.model_demand.insert( + "rejected-demand".to_string(), + ModelDemand { + last_active: now_secs(), + request_count: 1, + }, + ); + + // When: the direct payload is applied through the production gossip path. + node.apply_announced_peers(addr.id, &[(addr.clone(), ann)], None, None, true) + .await + .expect("policy rejection should not fail the gossip frame"); + + // Then: admission rejection must happen before mesh-wide side effects. + assert_eq!(node.mesh_id().await, None); + assert!(node.get_demand().is_empty()); + assert!(!node.state.lock().await.peers.contains_key(&addr.id)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub(crate) async fn inbound_gossip_rejection_preserves_dead_peer_state() -> Result<()> { + // Given: a host that still considers the direct sender dead, and a direct + // gossip payload whose sender announcement fails admission. + let host = Node::new_for_tests(NodeRole::Worker).await?; + let policy = crate::MeshGenesisPolicy::new( + "test-owner", + current_time_unix_ms(), + crate::MeshRequirements::default(), + ) + .expect("test policy should be valid"); + let mesh_id = policy.policy_derived_mesh_id().expect("mesh id"); + let policy_hash = policy.canonical_hash_hex().expect("policy hash"); + host.set_active_mesh_policy_for_tests(policy).await; + + let sender_id = test_endpoint_id(0x58); + host.state + .lock() + .await + .dead_peers + .insert(sender_id, std::time::Instant::now()); + + let mut sender_announcement = test_announcement(None); + sender_announcement.addr = EndpointAddr { + id: sender_id, + addrs: Default::default(), + }; + sender_announcement.role = NodeRole::Client; + sender_announcement.version = Some(crate::VERSION.to_string()); + sender_announcement.mesh_id = Some(mesh_id); + sender_announcement.mesh_policy_hash = Some(policy_hash); + let announcements = [(sender_announcement.addr.clone(), sender_announcement)]; + + // When: the same production phase used by `handle_gossip_stream` processes + // the rejected direct announcement. + host.validate_and_capture_inbound_gossip( + ControlProtocol::ProtoV1, + &announcements, + AnnouncedPeerContext::direct(sender_id, Some(NODE_PROTOCOL_GENERATION)), + ) + .await + .expect_err("rejected gossip must fail admission"); + + // Then: admission rejection is recorded, but liveness recovery has not + // cleared the sender from dead_peers. + let state = host.state.lock().await; + assert!( + state.dead_peers.contains_key(&sender_id), + "rejected inbound gossip must not clear dead-peer state" + ); + assert!( + state.requirement_rejected_peers.contains(&sender_id), + "rejected inbound gossip should still be tracked as an admission rejection" + ); + assert!( + !state.peers.contains_key(&sender_id), + "rejected inbound gossip must not admit the sender" + ); + + Ok(()) +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs new file mode 100644 index 0000000000..024b4755c8 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs @@ -0,0 +1,129 @@ +#[tokio::test] +pub(crate) async fn future_demand_timestamps_are_active_without_underflow() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let now = now_secs(); + node.merge_remote_demand(&HashMap::from([( + "future-demand".to_string(), + ModelDemand { + last_active: now + DEMAND_TTL_SECS + 60, + request_count: 3, + }, + )])); + + let active = node.active_demand().await; + assert!(active.contains_key("future-demand")); + + node.gc_demand().await; + assert!(node.get_demand().contains_key("future-demand")); +} + +/// Regression test for the `--auto` startup wedge: when a transitive +/// gossip payload includes peers that would be rejected at ingest +/// (version-floor or idle-transitive-client), `maybe_connect_discovered_peer` +/// must skip the dial. Otherwise each unreachable ghost address triggers +/// a 30 s `connect_to_peer` timeout sequentially in the dial loop, +/// wedging the surrounding gossip exchange (and the `attempt_run_auto_join` +/// that initiated it) for tens of minutes. +/// +/// The function returns without panicking and without dialing within a +/// generous time bound — a real dial to a fake address would block on +/// the 30 s `PEER_CONNECT_AND_GOSSIP_TIMEOUT`. We assert the result is +/// reached well under that bound and that no connection entry was created. +#[tokio::test] +pub(crate) async fn maybe_connect_discovered_peer_skips_filtered_announcements() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let my_role = NodeRole::Worker; + + // Below-floor version — must be skipped without dialing. + let old_addr = test_addr(0x57); + let old_id = old_addr.id; + let mut old_ann = test_announcement(None); + old_ann.addr = old_addr.clone(); + old_ann.role = NodeRole::Client; + old_ann.version = Some("0.57.0".to_string()); + + // Idle transitive client (matching version, but no hostname / no + // direct measurement / no model interests) — must also be skipped. + let idle_addr = test_addr(0xC1); + let idle_id = idle_addr.id; + let mut idle_ann = test_announcement(None); + idle_ann.addr = idle_addr.clone(); + idle_ann.role = NodeRole::Client; + idle_ann.version = Some("0.65.1".to_string()); + + // Both calls together must return well under the 30 s connect + // timeout. If the dial-loop skip is missing, each call will block + // on PEER_CONNECT_AND_GOSSIP_TIMEOUT (30 s) attempting to dial the + // fake test address. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + node.maybe_connect_discovered_peer(&my_role, old_addr, &old_ann, true, false) + .await; + node.maybe_connect_discovered_peer(&my_role, idle_addr, &idle_ann, true, false) + .await; + }) + .await + .expect("filtered peers must be skipped quickly, not dialed"); + + // No connection was attempted (no entry in state.connections), and + // no peer was added (the filtered announcements never reach add_peer + // or update_transitive_peer through this path). + let state = node.state.lock().await; + assert!( + !state.connections.contains_key(&old_id), + "below-floor peer must not be dialed" + ); + assert!( + !state.connections.contains_key(&idle_id), + "idle transitive client must not be dialed" + ); + assert!( + !state.peers.contains_key(&old_id), + "below-floor peer must not be added (this path is dial-only)" + ); + assert!( + !state.peers.contains_key(&idle_id), + "idle transitive client must not be added (this path is dial-only)" + ); +} + +#[tokio::test] +pub(crate) async fn client_auto_join_probe_returns_none_for_single_candidate() { + let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); + let token = encode_endpoint_addr_token(&test_addr(0x42)); + + let selected = node + .join_first_responsive_candidate(&[(token, Some("single".to_string()))]) + .await + .unwrap(); + + assert!(selected.is_none()); +} + +#[tokio::test] +pub(crate) async fn client_auto_join_probe_candidate_collection_filters_unusable_tokens() { + let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); + let valid_addr = test_addr(0x42); + let dead_addr = test_addr(0x43); + let self_token = encode_endpoint_addr_token(&node.endpoint_addr_for_advertisement()); + let dead_token = encode_endpoint_addr_token(&dead_addr); + let valid_token = encode_endpoint_addr_token(&valid_addr); + + node.state + .lock() + .await + .dead_peers + .insert(dead_addr.id, std::time::Instant::now()); + + let candidates = node + .collect_join_probe_candidates(&[ + ("not-an-invite-token".to_string(), None), + (self_token, None), + (dead_token, None), + (valid_token, Some("usable".to_string())), + ]) + .await; + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].addr.id, valid_addr.id); + assert_eq!(candidates[0].mesh_name.as_deref(), Some("usable")); +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs new file mode 100644 index 0000000000..27ca0dcaef --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs @@ -0,0 +1,265 @@ +#[test] +pub(crate) fn test_merge_none_to_some() { + let mut existing = test_peer(None); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_some_to_none_keeps_existing() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(None); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_earlier_incoming_wins() { + let mut existing = test_peer(Some(200)); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_later_incoming_loses() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(Some(200)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_equal_values_unchanged() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_first_joined_mesh_ts() { + let old_peer = test_peer(Some(100)); + let new_peer = test_peer(Some(200)); + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_explicit_model_interests() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_stage_status_list_support() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.stage_status_list_supported = !old_peer.stage_status_list_supported; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_stage_protocol_generation_support() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.stage_protocol_generation_supported = !old_peer.stage_protocol_generation_supported; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_explicit_model_interests() { + let mut existing = test_peer(Some(100)); + let mut ann = test_announcement(Some(100)); + ann.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!( + existing.explicit_model_interests, + vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] + ); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_stage_status_list_support() { + let mut existing = test_peer(Some(100)); + existing.stage_status_list_supported = false; + let mut ann = test_announcement(Some(100)); + ann.stage_status_list_supported = true; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert!(existing.stage_status_list_supported); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_stage_protocol_generation_support() { + let mut existing = test_peer(Some(100)); + existing.stage_protocol_generation_supported = false; + let mut ann = test_announcement(Some(100)); + ann.stage_protocol_generation_supported = true; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert!(existing.stage_protocol_generation_supported); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_advertised_model_throughput() { + let mut existing = test_peer(Some(100)); + let mut ann = test_announcement(Some(100)); + ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { + model_name: "qwen".to_string(), + avg_tokens_per_second_milli: 35_000, + throughput_samples: 4, + }]; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!( + existing.advertised_model_throughput, + ann.advertised_model_throughput + ); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_stage_status_list_support() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x44); + let addr = test_addr(0x44); + let mut ann = test_announcement(Some(100)); + ann.stage_status_list_supported = false; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.stage_status_list_supported = true; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert!(peer.stage_status_list_supported); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_stage_protocol_generation_support() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x45); + let addr = test_addr(0x45); + let mut ann = test_announcement(Some(100)); + ann.stage_protocol_generation_supported = false; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.stage_protocol_generation_supported = true; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert!(peer.stage_protocol_generation_supported); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_advertised_model_throughput() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x46); + let addr = test_addr(0x46); + let mut ann = test_announcement(Some(100)); + ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { + model_name: "qwen".to_string(), + avg_tokens_per_second_milli: 20_000, + throughput_samples: 2, + }]; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.advertised_model_throughput[0].avg_tokens_per_second_milli = 48_000; + ann.advertised_model_throughput[0].throughput_samples = 9; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert_eq!( + peer.advertised_model_throughput, + ann.advertised_model_throughput + ); +} + +#[tokio::test] +pub(crate) async fn test_collect_announcements_includes_self_explicit_model_interests() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + node.set_explicit_model_interests(vec![ + "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), + "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), + ]) + .await; + + let announcements = node.collect_announcements().await; + let self_announcement = announcements + .iter() + .find(|announcement| announcement.addr.id == node.id()) + .expect("self announcement must be present"); + + assert_eq!( + self_announcement.explicit_model_interests, + vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] + ); +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs index 73a971c85e..ea344b5ee1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs @@ -914,6 +914,11 @@ async fn reconnect_gossip_failure_removes_zombie_peer() { let node = make_test_node(super::NodeRole::Worker) .await .expect("test node must start"); + let capture_dir = tempfile::tempdir().expect("swarm capture tempdir"); + let recorder = crate::capture::SwarmCaptureRecorder::new(capture_dir.path().join("capture")) + .expect("swarm capture recorder"); + let capture_path = recorder.path().to_path_buf(); + node.set_swarm_capture_recorder(Some(recorder)); node.insert_test_peer(make_test_peer_info(peer_id)).await; @@ -930,6 +935,25 @@ async fn reconnect_gossip_failure_removes_zombie_peer() { "zombie peer must be removed when reconnect gossip fails (relay-connected but process dead)" ); + let capture_line = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if let Ok(contents) = std::fs::read_to_string(&capture_path) + && let Some(line) = contents.lines().find(|line| line.contains("peer_removed")) + { + break line.to_string(); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("peer removal should reach swarm capture"); + let capture: serde_json::Value = + serde_json::from_str(&capture_line).expect("peer removal capture JSON"); + assert_eq!(capture["fields"]["reason"], "recovered_gossip_failed"); + assert!(capture["fields"]["last_seen_age_ms"].is_u64()); + assert!(capture["fields"]["last_mentioned_age_ms"].is_u64()); + assert_eq!(capture["fields"]["had_connection"], false); + let peer_key2 = SecretKey::from_bytes(&[0xF1; 32]); let peer_id2 = EndpointId::from(peer_key2.public()); node.insert_test_peer(make_test_peer_info(peer_id2)).await; diff --git a/crates/mesh-llm-host-runtime/src/models/catalog.rs b/crates/mesh-llm-host-runtime/src/models/catalog.rs index 5e8fc9746e..8a1e0834d7 100644 --- a/crates/mesh-llm-host-runtime/src/models/catalog.rs +++ b/crates/mesh-llm-host-runtime/src/models/catalog.rs @@ -942,20 +942,64 @@ pub async fn download_hf_repo_file_with_progress_label( revision: revision.clone(), file: file.to_string(), }; - let HfAssetsDownload { - mut paths, - transfer_stats, - } = download_hf_assets(label, vec![asset.clone()], progress).await?; - paths.sort(); - let path = paths - .iter() - .find(|path| path_suffix_matches_ignore_case(path, &asset.file)) - .cloned() - .ok_or_else(|| { - anyhow::anyhow!( - "Downloaded Hugging Face asset not found in cache: {repo}/{file}@{revision}" + let mut retried_dangling_symlink = false; + let (path, paths, transfer_stats) = loop { + let HfAssetsDownload { + mut paths, + transfer_stats, + } = download_hf_assets(label, vec![asset.clone()], progress).await?; + paths.sort(); + let path = paths + .iter() + .find(|path| path_suffix_matches_ignore_case(path, &asset.file)) + .cloned() + .ok_or_else(|| { + anyhow::anyhow!( + "Downloaded Hugging Face asset not found in cache: {repo}/{file}@{revision}" + ) + })?; + let resolution_error = match std::fs::canonicalize(&path) { + Ok(_) => break (path, paths, transfer_stats), + Err(err) => err, + }; + if resolution_error.kind() != std::io::ErrorKind::NotFound { + return Err(resolution_error).with_context(|| { + format!( + "Resolve downloaded Hugging Face asset {repo}/{file}@{revision} at {}", + path.display() + ) + }); + } + let is_symlink = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata.file_type().is_symlink(), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, + Err(err) => { + return Err(err).with_context(|| { + format!("Inspect downloaded Hugging Face asset {}", path.display()) + }); + } + }; + if !is_symlink { + return Err(resolution_error).with_context(|| { + format!( + "Resolve downloaded Hugging Face asset {repo}/{file}@{revision} at {}", + path.display() + ) + }); + } + std::fs::remove_file(&path).with_context(|| { + format!( + "Remove dangling Hugging Face snapshot pointer {}", + path.display() ) })?; + if retried_dangling_symlink { + anyhow::bail!( + "Downloaded Hugging Face asset remained a dangling snapshot symlink after one retry: {repo}/{file}@{revision}" + ); + } + retried_dangling_symlink = true; + }; let display_name = Path::new(&asset.file) .file_name() .and_then(|value| value.to_str()) @@ -1523,6 +1567,167 @@ mod tests { assert_eq!(download.paths, vec![shard_one, shard_two, shard_three]); } + #[cfg(unix)] + #[tokio::test] + async fn download_hf_repo_file_retries_dangling_snapshot_symlink_once() { + use std::os::unix::fs::symlink; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let snapshot_dir = cache.path().join("snapshots/commit"); + let blob_dir = cache.path().join("blobs"); + let snapshot = snapshot_dir.join("model.gguf"); + let blob = blob_dir.join("etag"); + std::fs::create_dir_all(&snapshot_dir).unwrap(); + std::fs::create_dir_all(&blob_dir).unwrap(); + symlink("../../blobs/etag", &snapshot).unwrap(); + assert!( + std::fs::symlink_metadata(&snapshot) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("dangling-snapshot-retry-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let snapshot = snapshot.clone(); + let blob = blob.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![snapshot.clone()]), + 1 => { + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + std::fs::write(&blob, b"gguf").unwrap(); + symlink("../../blobs/etag", &snapshot).unwrap(); + Ok(vec![snapshot.clone()]) + } + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let download = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await + .unwrap(); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a dangling Ok(path) must not be trusted" + ); + assert_eq!(download.path, snapshot); + assert!(download.path.exists()); + assert_eq!(std::fs::read(&download.path).unwrap(), b"gguf"); + } + + #[cfg(unix)] + #[tokio::test] + async fn download_hf_repo_file_removes_second_dangling_snapshot_symlink() { + use std::os::unix::fs::symlink; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let snapshot_dir = cache.path().join("snapshots/commit"); + let snapshot = snapshot_dir.join("model.gguf"); + std::fs::create_dir_all(&snapshot_dir).unwrap(); + symlink("../../blobs/first-etag", &snapshot).unwrap(); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("double-dangling-snapshot-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let snapshot = snapshot.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![snapshot.clone()]), + 1 => { + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + symlink("../../blobs/second-etag", &snapshot).unwrap(); + Ok(vec![snapshot.clone()]) + } + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let result = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await; + + assert!(result.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + } + + #[tokio::test] + async fn download_hf_repo_file_does_not_retry_missing_non_symlink_path() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let fixture = cache.path().join("fixture"); + let missing = fixture.join("model.gguf"); + let sentinel = fixture.join("sentinel"); + std::fs::create_dir_all(&fixture).unwrap(); + std::fs::write(&sentinel, b"keep").unwrap(); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("missing-non-symlink-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let missing = missing.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![missing.clone()]), + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let result = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await; + + assert!(result.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(fixture.is_dir()); + assert_eq!(std::fs::read(sentinel).unwrap(), b"keep"); + assert!(!missing.exists()); + } + #[test] fn is_mlx_primary_asset_includes_first_shard() { assert!(is_mlx_primary_asset("model.safetensors")); diff --git a/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs b/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs index 59e5824a99..fc0678fd95 100644 --- a/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs +++ b/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs @@ -167,6 +167,9 @@ fn synthetic_local_gguf_ref_for_test(path: &Path) -> String { async fn bare_name_resolves_from_remote_catalog() { let query = "RemoteOnlyResolverFallbackModel-Q4_K_M"; let source_file = "RemoteOnlyResolverFallbackModel-Q4_K_M.gguf"; + let cache = tempfile::tempdir().unwrap(); + let downloaded = cache.path().join(source_file); + std::fs::write(&downloaded, b"gguf").unwrap(); let _catalog_guard = crate::models::remote_catalog::set_catalog_entries_for_test(vec![remote_catalog_entry( @@ -177,14 +180,17 @@ async fn bare_name_resolves_from_remote_catalog() { )]); let _download_guard = catalog::set_download_hf_assets_label_override( query.to_string(), - Arc::new(move |_| Ok(vec![PathBuf::from(format!("/tmp/{source_file}"))])), + Arc::new({ + let downloaded = downloaded.clone(); + move |_| Ok(vec![downloaded.clone()]) + }), ); let resolved = resolve_model_spec_with_progress(Path::new(query), false) .await .unwrap(); - assert_eq!(resolved, PathBuf::from(format!("/tmp/{source_file}"))); + assert_eq!(resolved, downloaded); } #[tokio::test] @@ -192,6 +198,9 @@ async fn bare_name_resolves_from_remote_catalog() { async fn bare_name_resolution_prefers_remote_catalog_over_baked_catalog() { let query = "Qwen3-8B-Q4_K_M"; let source_file = "RemotePreferred-Q4_K_M.gguf"; + let cache = tempfile::tempdir().unwrap(); + let downloaded = cache.path().join(source_file); + std::fs::write(&downloaded, b"gguf").unwrap(); let _catalog_guard = crate::models::remote_catalog::set_catalog_entries_for_test(vec![remote_catalog_entry( query, @@ -201,14 +210,17 @@ async fn bare_name_resolution_prefers_remote_catalog_over_baked_catalog() { )]); let _download_guard = catalog::set_download_hf_assets_label_override( "Remote Preferred Catalog Model".to_string(), - Arc::new(move |_| Ok(vec![PathBuf::from(format!("/tmp/{source_file}"))])), + Arc::new({ + let downloaded = downloaded.clone(); + move |_| Ok(vec![downloaded.clone()]) + }), ); let resolved = resolve_model_spec_with_progress(Path::new(query), false) .await .unwrap(); - assert_eq!(resolved, PathBuf::from(format!("/tmp/{source_file}"))); + assert_eq!(resolved, downloaded); } #[test] @@ -805,6 +817,13 @@ fn format_huggingface_display_ref_uses_selector_for_split_gguf() { #[serial] async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { let fixture = load_gemma_live_fixture(); + let cache = tempfile::tempdir().unwrap(); + let bf16_dir = cache.path().join("BF16"); + let shard_one = bf16_dir.join("gemma-4-31B-it-BF16-00001-of-00002.gguf"); + let shard_two = bf16_dir.join("gemma-4-31B-it-BF16-00002-of-00002.gguf"); + std::fs::create_dir_all(&bf16_dir).unwrap(); + std::fs::write(&shard_one, b"gguf-one").unwrap(); + std::fs::write(&shard_two, b"gguf-two").unwrap(); let _siblings_guard = RepoSiblingEntriesOverrideGuard::set(Arc::new({ let repo = fixture.repo.clone(); let siblings = fixture @@ -832,11 +851,10 @@ async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { })); let _download_guard = catalog::set_download_hf_assets_label_override( "unsloth/gemma-4-31B-it-GGUF:BF16".to_string(), - Arc::new(|_| { - Ok(vec![ - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00001-of-00002.gguf"), - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00002-of-00002.gguf"), - ]) + Arc::new({ + let shard_one = shard_one.clone(); + let shard_two = shard_two.clone(); + move |_| Ok(vec![shard_one.clone(), shard_two.clone()]) }), ); @@ -844,10 +862,7 @@ async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { .await .unwrap(); - assert_eq!( - resolved, - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00001-of-00002.gguf") - ); + assert_eq!(resolved, shard_one); assert_eq!( *planned.lock().unwrap(), vec![ diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index 72ac27463a..08c84ec00b 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -1,5 +1,5 @@ use crate::inference::{election, pipeline}; -use crate::logging::{OpenAiLifecycleAttachment, OpenAiRouteObserver}; +use crate::logging::{CallerPathType, OpenAiLifecycleAttachment, OpenAiRouteObserver}; use crate::mesh; use crate::network::affinity; use crate::network::openai::auto_route; @@ -927,13 +927,20 @@ async fn handle_buffered_api_request( tcp_stream: tokio::net::TcpStream, mut request: proxy::BufferedHttpRequest, ctx: ProxyConnectionContext<'_>, + source_addr: Option, ) { // Claim the parent at host OpenAI ingress. All downstream dispatch sees // only a metadata observer; this scope remains the sole terminal owner. + let caller_addr = source_addr.map(|addr| addr.to_string()); let request_metadata = crate::logging::RequestSummaryMetadata::from_openai_ingress_path(&request.client_path) .with_source(Some("direct_http")) - .with_method(Some(&request.method)); + .with_method(Some(&request.method)) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let mut lifecycle = crate::logging_runtime_state() .map(|state| state.openai_ingress_attachment(request.request_id, request_metadata)) .unwrap_or_else(OpenAiLifecycleAttachment::unowned); @@ -1058,6 +1065,7 @@ async fn handle_api_proxy_connection( targets: election::ModelTargets, affinity: affinity::AffinityRouter, ) { + let source_addr = tcp_stream.peer_addr().ok(); let plugin_manager = node.plugin_manager().await; match proxy::read_http_request_with_plugin_manager_with_context( &mut tcp_stream, @@ -1072,8 +1080,13 @@ async fn handle_api_proxy_connection( affinity: &affinity, plugin_manager: plugin_manager.as_ref(), }; - handle_buffered_api_request(tcp_stream, request, ProxyConnectionContext { route }) - .await; + handle_buffered_api_request( + tcp_stream, + request, + ProxyConnectionContext { route }, + source_addr, + ) + .await; } Err(error) => { let _ = super::parse_failure::send_read_failure(tcp_stream, &error).await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs index cc12d300b6..66d7f23499 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs @@ -44,6 +44,10 @@ async fn parsed_missing_model_error_persists_the_client_visible_response_artifac let mut client = tokio::net::TcpStream::connect(address) .await .expect("connect ingress client"); + let caller_addr = client + .local_addr() + .expect("ingress caller address") + .to_string(); client .write_all(request.as_bytes()) .await @@ -60,8 +64,25 @@ async fn parsed_missing_model_error_persists_the_client_visible_response_artifac ); let state = crate::logging_runtime_state().expect("installed logging runtime"); + let active = state + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); state.pump_persistence_for_test().await; let request_key = request_id.as_uuid().to_string(); + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_key) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); let artifacts = state .store() .expect("metadata store") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index bfc2009dfa..a67173ed69 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -24,6 +24,10 @@ const MAX_OBJECT_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024; const MAX_CHUNKED_WIRE_BYTES: usize = MAX_BODY_BYTES * 6 + 64 * 1024; const MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_OBJECT_UPLOAD_BODY_BYTES * 6 + 64 * 1024; pub(super) const MAX_HEADERS: usize = 64; +const CRLF: &[u8] = b"\r\n"; +const LF: &[u8] = b"\n"; +const CRLF_HEADER_TERMINATOR: &[u8] = b"\r\n\r\n"; +const LF_HEADER_TERMINATOR: &[u8] = b"\n\n"; #[derive(Debug, Clone, Copy)] pub(super) struct HttpReadLimits { @@ -650,6 +654,65 @@ pub(crate) fn canonical_request_id_from_header_prefix(prefix: &[u8]) -> Option Option<(usize, &'static [u8])> { + let crlf = prefix + .windows(CRLF_HEADER_TERMINATOR.len()) + .position(|window| window == CRLF_HEADER_TERMINATOR) + .map(|offset| (offset + CRLF_HEADER_TERMINATOR.len(), CRLF)); + let lf = prefix + .windows(LF_HEADER_TERMINATOR.len()) + .position(|window| window == LF_HEADER_TERMINATOR) + .map(|offset| (offset + LF_HEADER_TERMINATOR.len(), LF)); + + match (crlf, lf) { + (Some(crlf), Some(lf)) => Some(if crlf.0 <= lf.0 { crlf } else { lf }), + (Some(terminator), None) | (None, Some(terminator)) => Some(terminator), + (None, None) => None, + } +} + +pub(crate) fn ensure_canonical_request_id_in_header_prefix( + mut prefix: Vec, +) -> (Vec, Option) { + let mut headers_buf = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut request = httparse::Request::new(&mut headers_buf); + let Ok(httparse::Status::Complete(header_end)) = request.parse(&prefix) else { + return (prefix, None); + }; + if header_end > MAX_HEADER_BYTES { + return (prefix, None); + } + + let request_id_header_count = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")) + .count(); + if let Some(request_id) = canonical_request_id_from_headers(request.headers) { + return (prefix, Some(request_id)); + } + if request_id_header_count != 0 || request.headers.len() >= MAX_HEADERS { + return (prefix, None); + } + + let Some((terminator_end, line_ending)) = http_header_terminator(&prefix[..header_end]) else { + return (prefix, None); + }; + if terminator_end != header_end { + return (prefix, None); + } + + let request_id = RequestId::new(); + let mut header = format!("x-request-id: {}", request_id.as_uuid()).into_bytes(); + header.extend_from_slice(line_ending); + if header_end.saturating_add(header.len()) > MAX_HEADER_BYTES { + return (prefix, None); + } + let insertion_offset = header_end - line_ending.len(); + prefix.splice(insertion_offset..insertion_offset, header); + (prefix, Some(request_id)) +} + /// Parse the private raw-lifecycle assertion from a complete, bounded HTTP /// header prefix. The marker is accepted only once and only when its UUID /// exactly matches the one canonical `x-request-id` header. diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index 2075101129..6f006c29ef 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -5,7 +5,8 @@ use crate::inference::election; use crate::logging::{ - OpenAiLifecycleAttachment, OpenAiRouteAttempt, OpenAiRouteObserver, ProxyAttemptFinish, + CallerPathType, OpenAiLifecycleAttachment, OpenAiRouteAttempt, OpenAiRouteObserver, + ProxyAttemptFinish, }; use crate::mesh; use crate::network::affinity::{ @@ -234,11 +235,20 @@ fn capture_path_for_request(request: &BufferedHttpRequest) -> &str { &request.client_path } -fn attach_request_logging(request: &mut BufferedHttpRequest) -> OpenAiLifecycleAttachment { +fn attach_request_logging( + request: &mut BufferedHttpRequest, + source_addr: Option, +) -> OpenAiLifecycleAttachment { + let caller_addr = source_addr.map(|addr| addr.to_string()); let metadata = crate::logging::RequestSummaryMetadata::from_openai_ingress_path(&request.client_path) .with_source(Some("mesh_forwarded")) - .with_method(Some(&request.method)); + .with_method(Some(&request.method)) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let lifecycle = crate::logging_runtime_state() .map(|state| state.openai_ingress_attachment(request.request_id, metadata)) .unwrap_or_else(OpenAiLifecycleAttachment::unowned); @@ -310,7 +320,7 @@ pub async fn handle_mesh_request( }; // The parsed host ingress owns the parent. Downstream route code receives // only the attachment's metadata observer and cannot terminalize it. - let mut lifecycle = attach_request_logging(&mut request); + let mut lifecycle = attach_request_logging(&mut request, source_addr); if node.swarm_capture_enabled() { node.capture_http_request(crate::mesh::HttpCaptureEvent { event: "openai_ingress_http_request", diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs index 584ba0c84f..9af95db2a7 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs @@ -197,6 +197,10 @@ async fn passive_missing_model_error_persists_the_client_visible_response_artifa let mut client = tokio::net::TcpStream::connect(address) .await .expect("connect passive client"); + let caller_addr = client + .local_addr() + .expect("passive caller address") + .to_string(); client .write_all(request.as_bytes()) .await @@ -213,7 +217,24 @@ async fn passive_missing_model_error_persists_the_client_visible_response_artifa ); let state = crate::logging_runtime_state().expect("installed logging runtime"); + let active = state + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); state.pump_persistence_for_test().await; + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); let artifacts = state .store() .expect("metadata store") diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel.rs b/crates/mesh-llm-host-runtime/src/network/tunnel.rs index 5a8e370eb0..e1e4d137c0 100644 --- a/crates/mesh-llm-host-runtime/src/network/tunnel.rs +++ b/crates/mesh-llm-host-runtime/src/network/tunnel.rs @@ -12,7 +12,7 @@ use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; -const TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES: usize = 8 * 1024; +mod inbound_http; /// Global byte counter for tunnel traffic static BYTES_TRANSFERRED: AtomicU64 = AtomicU64::new(0); @@ -39,6 +39,7 @@ impl Manager { iroh::endpoint::RecvStream, )>, mut tunnel_http_rx: tokio::sync::mpsc::Receiver<( + EndpointId, iroh::endpoint::SendStream, iroh::endpoint::RecvStream, )>, @@ -58,7 +59,7 @@ impl Manager { let http_port_ref = mgr.http_port.clone(); let http_node = mgr.node.clone(); tokio::spawn(async move { - while let Some((send, recv)) = tunnel_http_rx.recv().await { + while let Some((remote, send, recv)) = tunnel_http_rx.recv().await { let port = http_port_ref.load(Ordering::Relaxed); if port == 0 { tracing::warn!("Inbound HTTP tunnel but no OpenAI surface running, dropping"); @@ -66,7 +67,10 @@ impl Manager { } let node = http_node.clone(); tokio::spawn(async move { - if let Err(e) = handle_inbound_http_stream(node, send, recv, port).await { + if let Err(e) = + inbound_http::handle_inbound_http_stream(node, remote, send, recv, port) + .await + { tracing::warn!("Inbound HTTP tunnel stream error: {e}"); } }); @@ -99,73 +103,6 @@ impl Manager { } } -/// Handle an inbound HTTP tunnel bi-stream: connect to the local API proxy and relay. -async fn handle_inbound_http_stream( - node: Node, - quic_send: iroh::endpoint::SendStream, - mut quic_recv: iroh::endpoint::RecvStream, - http_port: u16, -) -> Result<()> { - // Admission check for remote QUIC HTTP ingress. - match node - .activity_policy_guard - .check_admission(crate::runtime::IngressType::RemoteQuicHttp) - { - crate::runtime::AdmissionResult::Allowed => {} - crate::runtime::AdmissionResult::Paused { reason, .. } => { - tracing::debug!(reason, "Inbound HTTP tunnel rejected by activity policy"); - anyhow::bail!("remote inference paused: {reason}"); - } - } - - tracing::info!("Inbound HTTP tunnel stream -> API proxy :{http_port}"); - let mut tcp_stream = TcpStream::connect(format!("127.0.0.1:{http_port}")).await?; - tcp_stream.set_nodelay(true)?; - let _inflight = node.begin_inflight_request(); - - // Only raw mesh ingress that successfully claimed a parent emits the - // private assertion. Direct API requests have it stripped before they can - // reach this tunnel, so they retain normal target frontend ownership. - let prefix = read_tunneled_http_header_prefix(&mut quic_recv).await?; - let _remote_suppression = - crate::network::openai::request_parse::raw_lifecycle_owner_from_header_prefix(&prefix) - .and_then(|request_id| { - crate::logging_runtime_state() - .and_then(|state| state.suppress_remote_tunneled_request(request_id)) - }); - tcp_stream.write_all(&prefix).await?; - - let (tcp_read, tcp_write) = tokio::io::split(tcp_stream); - relay_bidirectional(tcp_read, tcp_write, quic_send, quic_recv).await -} - -/// Read at most one bounded HTTP header prefix without changing the bytes that -/// the existing relay will subsequently forward. A read may include a small -/// amount of body data from the same transport chunk; it is forwarded directly -/// to the local API proxy and is never retained by logging state. -async fn read_tunneled_http_header_prefix(reader: &mut R) -> Result> -where - R: AsyncRead + Unpin, -{ - let mut prefix = Vec::with_capacity(TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES); - let mut chunk = [0u8; TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES]; - let max_header_bytes = crate::network::openai::request_parse::MAX_HEADER_BYTES; - - while prefix.len() < max_header_bytes { - if prefix.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - let read_cap = (max_header_bytes - prefix.len()).min(chunk.len()); - let bytes_read = reader.read(&mut chunk[..read_cap]).await?; - if bytes_read == 0 { - break; - } - prefix.extend_from_slice(&chunk[..bytes_read]); - } - - Ok(prefix) -} - async fn handle_inbound_stage_transport( node: Node, remote: EndpointId, @@ -428,18 +365,6 @@ fn relay_remaining_chunks_error(total: u64, err: std::io::Error) -> Result<()> { mod tests { use super::*; - #[tokio::test] - async fn tunnel_prefetch_forwards_a_complete_bounded_header_prefix() { - let (mut writer, mut reader) = tokio::io::duplex(4096); - let request = b"POST /v1/chat/completions HTTP/1.1\r\nx-request-id: 550e8400-e29b-41d4-a716-446655440000\r\n\r\n"; - writer.write_all(request).await.unwrap(); - - let prefix = read_tunneled_http_header_prefix(&mut reader).await.unwrap(); - - assert_eq!(prefix, request); - assert!(prefix.len() <= crate::network::openai::request_parse::MAX_HEADER_BYTES); - } - /// Simulate relay_bidirectional behavior when one direction finishes /// before the other — the scenario that caused the remote proxy bug. /// diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs new file mode 100644 index 0000000000..8a40ac0e2b --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs @@ -0,0 +1,130 @@ +use crate::mesh::Node; +use anyhow::{Context, Result}; +use iroh::EndpointId; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +const TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES: usize = 8 * 1024; +const TUNNELED_HTTP_HEADER_READ_TIMEOUT: Duration = Duration::from_secs(5); + +#[cfg(test)] +mod tests; + +/// Handle an inbound HTTP tunnel bi-stream: connect to the local API proxy and relay. +pub(super) async fn handle_inbound_http_stream( + node: Node, + remote: EndpointId, + quic_send: iroh::endpoint::SendStream, + mut quic_recv: iroh::endpoint::RecvStream, + http_port: u16, +) -> Result<()> { + // Admission check for remote QUIC HTTP ingress. + match node + .activity_policy_guard + .check_admission(crate::runtime::IngressType::RemoteQuicHttp) + { + crate::runtime::AdmissionResult::Allowed => {} + crate::runtime::AdmissionResult::Paused { reason, .. } => { + tracing::debug!(reason, "Inbound HTTP tunnel rejected by activity policy"); + anyhow::bail!("remote inference paused: {reason}"); + } + } + + tracing::info!("Inbound HTTP tunnel stream -> API proxy :{http_port}"); + let mut tcp_stream = TcpStream::connect(format!("127.0.0.1:{http_port}")).await?; + tcp_stream.set_nodelay(true)?; + let _inflight = node.begin_inflight_request(); + + // Only raw mesh ingress that successfully claimed a parent emits the + // private assertion. Direct API requests have it stripped before they can + // reach this tunnel, so they retain normal target frontend ownership. + let prefix = read_tunneled_http_header_prefix(&mut quic_recv).await?; + let (prefix, _) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix(prefix); + let caller_metadata = + remote_tunnel_request_metadata(remote, node.authenticated_peer_path(remote).await); + let (attribution_request_id, suppression_request_id) = remote_tunnel_request_ids(&prefix); + let logging_state = crate::logging_runtime_state(); + let _remote_attribution = attribution_request_id.and_then(|request_id| { + logging_state + .as_ref() + .and_then(|state| state.attribute_remote_tunneled_request(request_id, caller_metadata)) + }); + let _remote_suppression = suppression_request_id.and_then(|request_id| { + logging_state + .as_ref() + .and_then(|state| state.suppress_remote_tunneled_request(request_id)) + }); + tcp_stream.write_all(&prefix).await?; + + let (tcp_read, tcp_write) = tokio::io::split(tcp_stream); + super::relay_bidirectional(tcp_read, tcp_write, quic_send, quic_recv).await +} + +fn remote_tunnel_request_ids( + prefix: &[u8], +) -> ( + Option, + Option, +) { + ( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(prefix), + crate::network::openai::request_parse::raw_lifecycle_owner_from_header_prefix(prefix), + ) +} + +fn remote_tunnel_request_metadata( + remote: EndpointId, + path: Option, +) -> crate::logging::RequestSummaryMetadata { + let endpoint_id = hex::encode(remote.as_bytes()); + let (addr, path_type) = match path { + Some(observation) if observation.path_type == "direct" => ( + observation + .observed_direct_remote_addr + .map(|addr| addr.to_string()), + Some(crate::logging::CallerPathType::RemoteQuicHttp), + ), + Some(observation) if observation.path_type == "relay" => { + (None, Some(crate::logging::CallerPathType::Relay)) + } + Some(_) | None => (None, None), + }; + crate::logging::RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + addr.as_deref(), + path_type, + ) +} + +/// Read at most one bounded HTTP header prefix without changing the bytes that +/// the existing relay will subsequently forward. A read may include a small +/// amount of body data from the same transport chunk; it is forwarded directly +/// to the local API proxy and is never retained by logging state. +async fn read_tunneled_http_header_prefix(reader: &mut R) -> Result> +where + R: AsyncRead + Unpin, +{ + tokio::time::timeout(TUNNELED_HTTP_HEADER_READ_TIMEOUT, async { + let mut prefix = Vec::with_capacity(TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES); + let mut chunk = [0u8; TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES]; + let max_header_bytes = crate::network::openai::request_parse::MAX_HEADER_BYTES; + + while prefix.len() < max_header_bytes { + if crate::network::openai::request_parse::http_header_terminator(&prefix).is_some() { + break; + } + let read_cap = (max_header_bytes - prefix.len()).min(chunk.len()); + let bytes_read = reader.read(&mut chunk[..read_cap]).await?; + if bytes_read == 0 { + break; + } + prefix.extend_from_slice(&chunk[..bytes_read]); + } + + Ok(prefix) + }) + .await + .context("timed out reading tunneled HTTP header prefix")? +} diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs new file mode 100644 index 0000000000..0ef57c37d4 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs @@ -0,0 +1,387 @@ +use super::*; +use std::time::Duration; +use tokio::io::AsyncWriteExt; + +#[test] +fn tunneled_request_metadata_uses_authenticated_peer_and_selected_direct_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x61; 32]).public()); + let direct_addr = "192.0.2.61:11204".parse().expect("direct address"); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(9), + observed_direct_remote_addr: Some(direct_addr), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert_eq!( + metadata.caller_addr(), + Some(direct_addr.to_string().as_str()) + ); + assert_eq!(metadata.caller_path_type(), Some("remote_quic_http")); +} + +#[test] +fn tunneled_request_metadata_omits_address_for_relay_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x62; 32]).public()); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "relay", + rtt_ms: Some(27), + observed_direct_remote_addr: Some( + "203.0.113.62:443".parse().expect("relay-shaped address"), + ), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert_eq!(metadata.caller_path_type(), Some("relay")); +} + +#[test] +fn tunneled_request_metadata_preserves_endpoint_when_path_is_missing() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x63; 32]).public()); + let metadata = remote_tunnel_request_metadata(remote, None); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert!(metadata.caller_path_type().is_none()); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[test] +fn tunneled_request_metadata_preserves_endpoint_for_unrecognized_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x64; 32]).public()); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "unknown", + rtt_ms: None, + observed_direct_remote_addr: Some( + "198.51.100.64:443" + .parse() + .expect("unrecognized path address"), + ), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert!(metadata.caller_path_type().is_none()); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[tokio::test] +async fn tunnel_prefetch_forwards_a_complete_bounded_header_prefix() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let request = b"POST /v1/chat/completions HTTP/1.1\r\nx-request-id: 550e8400-e29b-41d4-a716-446655440000\r\n\r\n"; + writer.write_all(request).await.unwrap(); + + let prefix = read_tunneled_http_header_prefix(&mut reader).await.unwrap(); + + assert_eq!(prefix, request); + assert!(prefix.len() <= crate::network::openai::request_parse::MAX_HEADER_BYTES); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_completes_on_lf_only_header_terminator() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let request = b"GET /v1/models HTTP/1.1\nHost: localhost\n\n"; + writer.write_all(request).await.unwrap(); + + let prefix = tokio::time::timeout( + Duration::from_secs(1), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("LF-only header pre-read must complete while the stream remains open") + .unwrap(); + + assert_eq!(prefix, request); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_times_out_when_peer_is_silent() { + let (_writer, mut reader) = tokio::io::duplex(4096); + + let result = tokio::time::timeout( + Duration::from_secs(6), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("header pre-read must finish within the regression guard"); + + let error = result.expect_err("a silent peer must exceed the header deadline"); + assert!( + error + .to_string() + .contains("timed out reading tunneled HTTP header prefix") + ); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_slow_partial_progress_does_not_reset_deadline() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let writer_task = tokio::spawn(async move { + for fragment in [ + b"POST /v1/responses HTTP/1.1\r\n".as_slice(), + b"Host: localhost\r\n".as_slice(), + b"Content-Length: 2\r\n".as_slice(), + b"\r\n{}".as_slice(), + ] { + writer.write_all(fragment).await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + } + }); + + let result = tokio::time::timeout( + Duration::from_secs(7), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("header pre-read must finish within the regression guard"); + + let error = result.expect_err("partial progress must not reset the header deadline"); + assert!( + error + .to_string() + .contains("timed out reading tunneled HTTP header prefix") + ); + writer_task.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_preserves_fragmented_header_and_body_overread() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let fragments: [&[u8]; 3] = [ + b"POST /v1/responses HTTP/1.1\r\n", + b"Host: localhost\r\nContent-Length: 2\r\n", + b"\r\n{}", + ]; + let writer_task = tokio::spawn(async move { + for fragment in fragments { + writer.write_all(fragment).await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); + + let prefix = read_tunneled_http_header_prefix(&mut reader) + .await + .expect("fragmented header must complete before the deadline"); + + assert_eq!( + prefix, + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}" + ); + writer_task.await.unwrap(); +} + +#[test] +fn tunnel_prefix_without_request_id_gets_one_canonical_id() { + let original = b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}"; + + let (rewritten, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.to_vec(), + ); + + let request_id = request_id.expect("generated canonical request ID"); + assert_eq!( + remote_tunnel_request_ids(&rewritten), + (Some(request_id), None) + ); + assert_eq!( + rewritten + .windows(b"x-request-id:".len()) + .filter(|window| window.eq_ignore_ascii_case(b"x-request-id:")) + .count(), + 1 + ); + assert!(rewritten.ends_with(b"\r\n\r\n{}")); +} + +#[test] +fn lf_only_tunnel_prefix_inserts_request_id_on_its_own_header_line() { + let original = b"GET /v1/models HTTP/1.1\nHost: localhost\n\n"; + + let (rewritten, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.to_vec(), + ); + + let request_id = request_id.expect("generated canonical request ID"); + let mut headers = [httparse::EMPTY_HEADER; 8]; + let mut request = httparse::Request::new(&mut headers); + assert_eq!( + request + .parse(&rewritten) + .expect("rewritten request must parse"), + httparse::Status::Complete(rewritten.len()) + ); + assert_eq!( + request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")) + .count(), + 1 + ); + let request_id_line = format!("\nx-request-id: {}\n\n", request_id.as_uuid()); + assert!( + rewritten + .windows(request_id_line.len()) + .any(|window| window == request_id_line.as_bytes()) + ); + assert!(!rewritten.contains(&b'\r')); +} + +#[test] +fn tunnel_prefix_preserves_existing_canonical_request_id_byte_for_byte() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let original = format!( + "POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ) + .into_bytes(); + + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + + assert_eq!(selected, Some(request_id)); + assert_eq!(rewritten, original); +} + +#[test] +fn duplicate_or_malformed_tunnel_request_id_fails_open_without_rewrite() { + let first = mesh_llm_events::logging::identifiers::RequestId::new(); + let second = mesh_llm_events::logging::identifiers::RequestId::new(); + let duplicate = format!( + "GET /v1/models HTTP/1.1\r\nx-request-id: {}\r\nx-request-id: {}\r\n\r\n", + first.as_uuid(), + second.as_uuid() + ) + .into_bytes(); + let malformed = b"GET /v1/models HTTP/1.1\r\nx-request-id: client-controlled\r\n\r\n".to_vec(); + + for original in [duplicate, malformed] { + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + assert_eq!(selected, None); + assert_eq!(rewritten, original); + assert_eq!(remote_tunnel_request_ids(&rewritten), (None, None)); + } +} + +#[test] +fn full_bounded_tunnel_header_without_request_id_fails_open() { + let start = b"GET /v1/models HTTP/1.1\r\nx-padding: "; + let ending = b"\r\n\r\n"; + let padding_length = + crate::network::openai::request_parse::MAX_HEADER_BYTES - start.len() - ending.len(); + let mut original = start.to_vec(); + original.extend(std::iter::repeat_n(b'x', padding_length)); + original.extend_from_slice(ending); + + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + + assert_eq!( + original.len(), + crate::network::openai::request_parse::MAX_HEADER_BYTES + ); + assert_eq!(selected, None); + assert_eq!(rewritten, original); + assert_eq!(remote_tunnel_request_ids(&rewritten), (None, None)); +} + +#[test] +fn markerless_canonical_request_id_allows_attribution_without_suppression() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let prefix = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ); + + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), None) + ); +} + +#[test] +fn near_limit_markerless_headers_keep_canonical_attribution() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let start = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\nx-padding: ", + request_id.as_uuid() + ); + let ending = "\r\n\r\n"; + let padding_length = + crate::network::openai::request_parse::MAX_HEADER_BYTES - start.len() - ending.len(); + let prefix = format!("{start}{}{ending}", "x".repeat(padding_length)); + + assert_eq!( + prefix.len(), + crate::network::openai::request_parse::MAX_HEADER_BYTES + ); + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), None) + ); +} + +#[test] +fn matching_private_marker_enables_single_parent_suppression() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let prefix = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {0}\r\n{1}: {0}\r\n\r\n", + request_id.as_uuid(), + crate::network::openai::request_parse::RAW_LIFECYCLE_OWNER_HEADER, + ); + + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), Some(request_id)) + ); +} + +#[test] +fn malformed_incomplete_and_maximal_header_prefixes_fail_open() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let malformed = b"POST /v1/responses HTTP/1.1\r\nx-request-id: not-a-uuid\r\n\r\n"; + let incomplete = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\n", + request_id.as_uuid() + ); + let maximal = vec![b'x'; crate::network::openai::request_parse::MAX_HEADER_BYTES]; + + for prefix in [ + malformed.as_slice(), + incomplete.as_bytes(), + maximal.as_slice(), + ] { + assert_eq!(remote_tunnel_request_ids(prefix), (None, None)); + } +} diff --git a/crates/mesh-llm-host-runtime/tests/audit_test.rs b/crates/mesh-llm-host-runtime/tests/audit_test.rs index 48fd5b30c2..bc83cd796c 100644 --- a/crates/mesh-llm-host-runtime/tests/audit_test.rs +++ b/crates/mesh-llm-host-runtime/tests/audit_test.rs @@ -1,8 +1,33 @@ use mesh_llm_events::audit::AuditLevel; use mesh_llm_events::audit::AuditLogFormat; +use mesh_llm_host_runtime::OperationalAuditSubjectKind; #[test] fn test_audit_types_exported() { let _ = AuditLogFormat::JsonLines; let _ = AuditLevel::Info; } + +#[test] +fn operational_audit_subject_kind_is_source_compatible() { + let subjects = [ + (OperationalAuditSubjectKind::Runtime, "runtime"), + (OperationalAuditSubjectKind::Model, "model"), + ( + OperationalAuditSubjectKind::RuntimeInstance, + "runtime_instance", + ), + (OperationalAuditSubjectKind::CliCommand, "cli_command"), + ]; + + for (subject, expected) in subjects { + let released_value = match subject { + OperationalAuditSubjectKind::Runtime => "runtime", + OperationalAuditSubjectKind::Model => "model", + OperationalAuditSubjectKind::RuntimeInstance => "runtime_instance", + OperationalAuditSubjectKind::CliCommand => "cli_command", + }; + assert_eq!(released_value, expected); + assert_eq!(subject.as_str(), expected); + } +} diff --git a/crates/mesh-llm-log-store/Cargo.toml b/crates/mesh-llm-log-store/Cargo.toml index f64074187a..7171f3fbbb 100644 --- a/crates/mesh-llm-log-store/Cargo.toml +++ b/crates/mesh-llm-log-store/Cargo.toml @@ -18,7 +18,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } data-encoding = "2.6" hex = "0.4" mesh-llm-events = { version = "0.76.0-rc7", path = "../mesh-llm-events" } -rusqlite = { version = "0.37", features = ["bundled"] } +rusqlite = { version = "0.37", features = ["bundled", "functions"] } serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/mesh-llm-log-store/README.md b/crates/mesh-llm-log-store/README.md index 81cbd6d8c3..f89ac5de75 100644 --- a/crates/mesh-llm-log-store/README.md +++ b/crates/mesh-llm-log-store/README.md @@ -8,3 +8,24 @@ artifact storage, and the audited maintenance operations used by the host runtime's trusted local logging APIs. The crate keeps persistence policy and storage details below the runtime and transport layers so callers can query, retain, and clean up logs without coupling those APIs to SQLite internals. + +## Schema lifecycle + +New empty databases are initialized atomically to the complete current schema. +Reopening a current database performs no schema work and preserves its data. A +database with unrecognized objects at version 0 is rejected without being reset +or migrated. + +The only public-schema compatibility exception is the exact released physical +schema with source `user_version` 3 or 11. Its tables, columns, indexes, foreign +keys, checks, partial-index predicates, and `AUTOINCREMENT` semantics must match +the released structure exactly, its `application_id` must be zero, and it must +not contain private lineage objects. The store then adds the three nullable +caller identity columns in place and atomically adopts the database into the +private lineage at epoch 1 without rebuilding tables or copying rows. + +Lookalikes, partial schemas, and the same physical schema under any source marker +other than 3 or 11 fail closed without mutation. Future private schema changes +must be registered as contiguous, forward-only steps; each step commits its +schema, data, and `user_version` together so a later failure can resume from the +last committed version. diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs index 462e2061d1..0768f4e6c3 100644 --- a/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs @@ -1,11 +1,7 @@ use super::*; -// ════════════════════════════════════ -// MIGRATION TESTS -// ════════════════════════════════════ - #[test] -fn fresh_db_migrates_to_latest() { +fn fresh_db_initializes_to_current_version() { let (store, _, _tmp) = open_store(); assert_eq!(store.schema_version(), CURRENT_VERSION); } @@ -137,12 +133,12 @@ fn sqlite_root_database_and_sidecars_have_only_current_user_acl() { } #[test] -fn reopen_preserves_data_and_skips_migrations() { +fn current_schema_reopen_preserves_data_without_schema_work() { let tmp = tempfile::tempdir().expect("create temp dir"); let clock: Arc = Arc::new(TestClock::default()); // Insert data. - let store1 = LogStore::open(tmp.path(), clock.clone()).expect("open v1"); + let store1 = LogStore::open(tmp.path(), clock.clone()).expect("open current schema"); store1 .insert_summary( "s-001", @@ -173,7 +169,7 @@ fn reopen_preserves_data_and_skips_migrations() { } #[test] -fn migrations_are_idempotent_on_reopen() { +fn current_schema_is_a_noop_on_reopen() { let tmp = tempfile::tempdir().expect("create temp dir"); let clock: Arc = Arc::new(TestClock::default()); @@ -184,6 +180,123 @@ fn migrations_are_idempotent_on_reopen() { } } +#[test] +fn unknown_nonempty_version_zero_is_rejected_before_database_mutation() { + assert_unknown_schema_rejected_before_mutation(0); +} + +#[test] +fn unknown_future_version_is_rejected_before_database_mutation() { + assert_unknown_schema_rejected_before_mutation(2); +} + +#[test] +fn markerless_identified_version_one_is_rejected_before_wal_or_schema_mutation() { + let root = tempfile::tempdir().expect("database root"); + let database = root.path().join("log_store.db"); + let connection = rusqlite::Connection::open(&database).expect("open fixture database"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;", + ) + .expect("seed forged schema"); + drop(connection); + + let clock: Arc = Arc::new(TestClock::default()); + let error = match LogStore::open(root.path(), clock) { + Ok(_) => panic!("markerless schema must fail closed"), + Err(error) => error, + }; + + assert!(matches!( + error, + LogStoreError::SchemaIncompatible { + found: 1, + supported: 1 + } + )); + assert!(!root.path().join("log_store.db-wal").exists()); + assert!(!root.path().join("log_store.db-shm").exists()); + let connection = rusqlite::Connection::open(database).expect("reopen fixture database"); + assert_eq!( + connection + .pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) + .expect("journal mode"), + "delete" + ); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"), + 1 + ); +} + +fn assert_unknown_schema_rejected_before_mutation(version: u32) { + let root = tempfile::tempdir().expect("database root"); + let database = root.path().join("log_store.db"); + let connection = rusqlite::Connection::open(&database).expect("open fixture database"); + connection + .execute_batch(&format!( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); PRAGMA user_version = {version};" + )) + .expect("seed unknown schema"); + drop(connection); + + let clock: Arc = Arc::new(TestClock::default()); + let error = match LogStore::open(root.path(), clock) { + Ok(_) => panic!("unknown schema must fail closed"), + Err(error) => error, + }; + + assert!(matches!( + error, + LogStoreError::SchemaIncompatible { found, supported: 1 } if found == version + )); + let connection = rusqlite::Connection::open(database).expect("reopen fixture database"); + assert_eq!( + connection + .pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .expect("schema version"), + version + ); + assert_eq!( + connection + .pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) + .expect("journal mode"), + "delete" + ); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('summaries', 'audit_entries')", + [], + |row| row.get::<_, i64>(0), + ) + .expect("log table count"), + 0 + ); +} + // ════════════════════════════════════ // TERMINAL EVENT TRANSACTION TESTS // ════════════════════════════════════ diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs index 775d745a4e..258907b9dc 100644 --- a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs @@ -1,573 +1,5 @@ use super::*; -// ════════════════════════════════════ -// EMPTY / SINGLE ITEM PAGINATION TESTS -// ════════════════════════════════════ - -#[test] -fn empty_table_pagination() { - let (store, _, _tmp) = open_store(); - - let page = store.list_summaries(10, None).unwrap(); - assert!(page.items.is_empty()); - assert!(page.next_cursor.is_none()); - - // Also test lifecycle events and artifacts. - let ev_page = store.list_lifecycle_events(10, None).unwrap(); - assert!(ev_page.items.is_empty()); - - let art_page = store.list_artifact_pointers(10, None).unwrap(); - assert!(art_page.items.is_empty()); -} - -#[test] -fn single_item_pagination() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "only-one", - Some("llama3"), - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - - let page = store.list_summaries(10, None).unwrap(); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].request_id, "only-one"); - assert!(page.next_cursor.is_none()); -} - -// ════════════════════════════════════ -// SUMMARY STATUS COUNTS TEST -// ════════════════════════════════════ - -#[test] -fn summary_status_counts() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "s-active-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - - // Insert + terminal update. - store - .insert_summary( - "s-completed-1", - None, - Some("route-a"), - Some("provider-x"), - Some("engine-y"), - &clock.now(), - None, - None, - None, - ) - .unwrap(); - let payload = r#"{"type":"completed","status_code":200}"#; - store - .write_terminal_event( - "s-completed-1", - "evt-c1", - payload, - "completed", - Some(200), - &clock.now(), - ) - .unwrap(); - - // Failed terminal. - store - .insert_summary( - "s-failed-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - let failed_payload = r#"{"type":"failed","error":"timeout"}"#; - store - .write_terminal_event( - "s-failed-1", - "evt-f1", - failed_payload, - "failed", - None, - &clock.now(), - ) - .unwrap(); - - let counts = store.count_summaries_by_status().unwrap(); - assert_eq!(counts.len(), 3); // active, completed, failed states - - // Verify specific counts. - for (state, count) in &counts { - match state.as_str() { - "active" => assert_eq!(*count, 1), - "completed" => assert_eq!(*count, 1), - "failed" => assert_eq!(*count, 1), - _ => panic!("unexpected state: {}", state), - } - } -} - -// ════════════════════════════════════ -// HAPPY PATH INSERT + COUNT TESTS -// ════════════════════════════════════ - -#[test] -fn artifact_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_artifact_pointer( - "art-1", - "req-1", - &clock.now(), - "log", - Some(r#"{"size": 42}"#), - ) - .expect("insert artifact"); - - assert_eq!(store.count_table("artifact_pointers").unwrap(), 1); - - // Duplicate PK should fail with AlreadyExists. - let err = store - .insert_artifact_pointer("art-1", "req-1", &clock.now(), "log", None) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); -} - -#[test] -fn proxy_record_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_proxy_record( - "att-1", - "req-1", - &clock.now(), - "http://target.api", - Some("provider-x"), - Some("engine-y"), - Some(&clock.now()), - Some(&clock.now()), - Some(200), - None, - ) - .expect("insert proxy record"); - - assert_eq!(store.count_table("proxy_records").unwrap(), 1); - - // Duplicate PK fails. - let err = store - .insert_proxy_record( - "att-1", - "req-1", - &clock.now(), - "http://other.api", - None, - None, - None, - None, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); -} - -#[test] -fn audit_entry_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - // With request_id. - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_summary( - "req-2", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_audit_entry( - "aud-1", - Some("req-1"), - &clock.now(), - "user-alice", - "model_added", - Some(r#"{"model":"llama3"}"#), - ) - .expect("insert audit with request_id"); - - // Without request_id (standalone). - store - .insert_audit_entry("aud-2", None, &clock.now(), "system", "startup", None) - .expect("insert audit without request_id"); - - assert_eq!(store.count_table("audit_entries").unwrap(), 2); - - // Duplicate PK fails. - let err = store - .insert_audit_entry( - "aud-1", - Some("req-1"), - &clock.now(), - "user-bob", - "other_action", - None, - ) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); - - // UNIQUE(request_id, entry_id) — same request_id + different entry_id should work. - store - .insert_audit_entry( - "aud-3", - Some("req-1"), - &clock.now(), - "user-carol", - "action_3", - None, - ) - .expect("different entry_id with same request_id is fine"); - - // UNIQUE(request_id, entry_id) — different request_id + different entry_id works. - store - .insert_audit_entry( - "aud-5", - Some("req-2"), - &clock.now(), - "user-carol", - "action_3", - None, - ) - .expect("different request and entry are fine"); - - // Different entry_id always works (entry_id is PK). - store - .insert_audit_entry( - "aud-4", - Some("req-1"), - &clock.now(), - "user-dave", - "action_4", - None, - ) - .expect("another unique entry_id with same request_id is fine"); - - assert_eq!(store.count_table("audit_entries").unwrap(), 5); -} - -#[test] -fn audit_entries_page_by_occurred_at_and_entry_id_without_detail_leakage() { - let (store, _, _tmp) = open_store(); - let timestamp = "2025-06-15T12:00:00Z"; - for entry_id in ["audit-0001", "audit-0003", "audit-0002"] { - store - .insert_audit_entry( - entry_id, - None, - timestamp, - "runtime", - "runtime_ready", - Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), - ) - .unwrap(); - } - - let first_page = store - .list_audit_entries(Some(2), None, AuditEntryFilters::default()) - .unwrap(); - assert_eq!( - first_page - .items - .iter() - .map(|entry| entry.entry_id.as_str()) - .collect::>(), - vec!["audit-0003", "audit-0002"] - ); - assert_eq!(first_page.items[0].severity, Some(AuditEntrySeverity::Info)); - assert!( - !format!("{:?}", first_page.items).contains("SENTINEL-AUDIT-SECRET"), - "detail_json must never cross the AuditEntryRow boundary" - ); - - let second_page = store - .list_audit_entries( - Some(2), - first_page.next_cursor.as_deref(), - AuditEntryFilters::default(), - ) - .unwrap(); - assert_eq!(second_page.items.len(), 1); - assert_eq!(second_page.items[0].entry_id, "audit-0001"); - assert!(second_page.next_cursor.is_none()); -} - -#[test] -fn audit_entries_reconcile_forward_by_durable_sequence() { - let (store, clock, tmp) = open_store(); - let external_writer = LogStore::open(tmp.path(), clock).expect("open independent writer"); - for entry_id in ["audit-sequence-1", "audit-sequence-2", "audit-sequence-3"] { - external_writer - .insert_audit_entry( - entry_id, - None, - "2026-08-12T12:00:00Z", - "runtime", - "runtime_ready", - Some(r#"{"severity":"info"}"#), - ) - .unwrap(); - } - - let first = store - .list_audit_entries_after_sequence(0, 2, AuditEntryFilters::default()) - .unwrap(); - assert_eq!( - first - .iter() - .map(|entry| entry.entry_id.as_str()) - .collect::>(), - ["audit-sequence-1", "audit-sequence-2"] - ); - let second = store - .list_audit_entries_after_sequence( - u64::try_from(first[1].sequence).unwrap(), - 2, - AuditEntryFilters::default(), - ) - .unwrap(); - assert_eq!(second[0].entry_id, "audit-sequence-3"); -} - -#[test] -fn audit_entries_project_only_versioned_bounded_context() { - let (store, _, _tmp) = open_store(); - store - .insert_audit_entry( - "audit-context-0001", - None, - "2026-08-12T12:00:00Z", - "runtime", - "runtime_model_ready", - Some( - r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"numeric_summaries":{"bytes":4096},"secret":"SENTINEL-AUDIT-SECRET"}"#, - ), - ) - .unwrap(); - - let page = store - .list_audit_entries(Some(1), None, AuditEntryFilters::default()) - .unwrap(); - let row = &page.items[0]; - assert_eq!(row.context_version, Some(1)); - assert_eq!(row.subject_kind.as_deref(), Some("model")); - assert_eq!(row.subject_id.as_deref(), Some("local-gguf/sha256-safe")); - assert_eq!(row.operation_id.as_deref(), Some("runtime-7")); - assert_eq!(row.outcome.as_deref(), Some("ready")); - assert_eq!(row.duration_ms, Some(42)); - assert_eq!(row.numeric_summaries.get("bytes"), Some(&4096)); - assert!(!format!("{row:?}").contains("SENTINEL-AUDIT-SECRET")); -} - -#[test] -fn audit_entry_query_rejects_malformed_cursor_and_out_of_range_limit() { - let (store, _, _tmp) = open_store(); - - let cursor_error = store - .list_audit_entries(Some(1), Some("not-a-cursor"), AuditEntryFilters::default()) - .unwrap_err(); - assert!(matches!(cursor_error, LogStoreError::CursorMalformed(_))); - - for limit in [Some(0), Some(101)] { - let limit_error = store - .list_audit_entries(limit, None, AuditEntryFilters::default()) - .unwrap_err(); - assert!(matches!(limit_error, LogStoreError::InvalidQuery(_))); - } -} - -#[test] -fn audit_entry_query_handles_nullable_request_empty_page_and_allowlisted_filters() { - let (store, _, _tmp) = open_store(); - let empty_page = store - .list_audit_entries(None, None, AuditEntryFilters::default()) - .unwrap(); - assert!(empty_page.items.is_empty()); - assert!(empty_page.next_cursor.is_none()); - - store - .insert_audit_entry( - "audit-null-request", - None, - "2025-06-15T12:00:00Z", - "runtime", - "runtime_ready", - Some(r#"{"severity":"info"}"#), - ) - .unwrap(); - store - .insert_audit_entry( - "audit-mesh-warning", - None, - "2025-06-15T12:00:01Z", - "mesh", - "mesh_quic_handler_failed", - Some(r#"{"severity":"warning"}"#), - ) - .unwrap(); - - let page = store - .list_audit_entries( - None, - None, - AuditEntryFilters { - source: Some(AuditEntrySource::Mesh), - severity: Some(AuditEntrySeverity::Warning), - ..AuditEntryFilters::default() - }, - ) - .unwrap(); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].entry_id, "audit-mesh-warning"); - assert!(page.items[0].request_id.is_none()); -} - -#[test] -fn nullable_column_conversion_errors_are_not_silenced() { - let (store, clock, _tmp) = open_store(); - store - .insert_summary( - "req-invalid-column", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .expect("insert summary"); - store - .conn() - .execute( - "UPDATE summaries SET status_code = 'not-an-integer' WHERE request_id = 'req-invalid-column'", - [], - ) - .expect("store invalid status code"); - - assert!(matches!( - store.get_summary("req-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_summaries(10, None), - Err(LogStoreError::QueryFailed(_)) - )); - - store - .conn() - .execute( - "UPDATE summaries SET status_code = NULL WHERE request_id = 'req-invalid-column'", - [], - ) - .expect("restore nullable summary column"); - store - .insert_artifact_pointer( - "art-invalid-column", - "req-invalid-column", - &clock.now(), - "log", - None, - ) - .expect("insert artifact pointer"); - store - .conn() - .execute( - "UPDATE artifact_pointers SET checksum = x'00' WHERE artifact_id = 'art-invalid-column'", - [], - ) - .expect("store invalid checksum type"); - - assert!(matches!( - store.get_artifact_pointer("art-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_artifact_pointers_for_request("req-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_artifact_pointers(10, None), - Err(LogStoreError::QueryFailed(_)) - )); -} +mod basic; +mod query; +mod sanitization; diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs new file mode 100644 index 0000000000..e404aeb122 --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs @@ -0,0 +1,326 @@ +use super::*; + +// ════════════════════════════════════ +// EMPTY / SINGLE ITEM PAGINATION TESTS +// ════════════════════════════════════ + +#[test] +fn empty_table_pagination() { + let (store, _, _tmp) = open_store(); + + let page = store.list_summaries(10, None).unwrap(); + assert!(page.items.is_empty()); + assert!(page.next_cursor.is_none()); + + // Also test lifecycle events and artifacts. + let ev_page = store.list_lifecycle_events(10, None).unwrap(); + assert!(ev_page.items.is_empty()); + + let art_page = store.list_artifact_pointers(10, None).unwrap(); + assert!(art_page.items.is_empty()); +} + +#[test] +fn single_item_pagination() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "only-one", + Some("llama3"), + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + + let page = store.list_summaries(10, None).unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].request_id, "only-one"); + assert!(page.next_cursor.is_none()); +} + +// ════════════════════════════════════ +// SUMMARY STATUS COUNTS TEST +// ════════════════════════════════════ + +#[test] +fn summary_status_counts() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "s-active-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + + // Insert + terminal update. + store + .insert_summary( + "s-completed-1", + None, + Some("route-a"), + Some("provider-x"), + Some("engine-y"), + &clock.now(), + None, + None, + None, + ) + .unwrap(); + let payload = r#"{"type":"completed","status_code":200}"#; + store + .write_terminal_event( + "s-completed-1", + "evt-c1", + payload, + "completed", + Some(200), + &clock.now(), + ) + .unwrap(); + + // Failed terminal. + store + .insert_summary( + "s-failed-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + let failed_payload = r#"{"type":"failed","error":"timeout"}"#; + store + .write_terminal_event( + "s-failed-1", + "evt-f1", + failed_payload, + "failed", + None, + &clock.now(), + ) + .unwrap(); + + let counts = store.count_summaries_by_status().unwrap(); + assert_eq!(counts.len(), 3); // active, completed, failed states + + // Verify specific counts. + for (state, count) in &counts { + match state.as_str() { + "active" => assert_eq!(*count, 1), + "completed" => assert_eq!(*count, 1), + "failed" => assert_eq!(*count, 1), + _ => panic!("unexpected state: {}", state), + } + } +} + +// ════════════════════════════════════ +// HAPPY PATH INSERT + COUNT TESTS +// ════════════════════════════════════ + +#[test] +fn artifact_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_artifact_pointer( + "art-1", + "req-1", + &clock.now(), + "log", + Some(r#"{"size": 42}"#), + ) + .expect("insert artifact"); + + assert_eq!(store.count_table("artifact_pointers").unwrap(), 1); + + // Duplicate PK should fail with AlreadyExists. + let err = store + .insert_artifact_pointer("art-1", "req-1", &clock.now(), "log", None) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); +} + +#[test] +fn proxy_record_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_proxy_record( + "att-1", + "req-1", + &clock.now(), + "http://target.api", + Some("provider-x"), + Some("engine-y"), + Some(&clock.now()), + Some(&clock.now()), + Some(200), + None, + ) + .expect("insert proxy record"); + + assert_eq!(store.count_table("proxy_records").unwrap(), 1); + + // Duplicate PK fails. + let err = store + .insert_proxy_record( + "att-1", + "req-1", + &clock.now(), + "http://other.api", + None, + None, + None, + None, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); +} + +#[test] +fn audit_entry_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + // With request_id. + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_summary( + "req-2", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_audit_entry( + "aud-1", + Some("req-1"), + &clock.now(), + "user-alice", + "model_added", + Some(r#"{"model":"llama3"}"#), + ) + .expect("insert audit with request_id"); + + // Without request_id (standalone). + store + .insert_audit_entry("aud-2", None, &clock.now(), "system", "startup", None) + .expect("insert audit without request_id"); + + assert_eq!(store.count_table("audit_entries").unwrap(), 2); + + // Duplicate PK fails. + let err = store + .insert_audit_entry( + "aud-1", + Some("req-1"), + &clock.now(), + "user-bob", + "other_action", + None, + ) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); + + // UNIQUE(request_id, entry_id) — same request_id + different entry_id should work. + store + .insert_audit_entry( + "aud-3", + Some("req-1"), + &clock.now(), + "user-carol", + "action_3", + None, + ) + .expect("different entry_id with same request_id is fine"); + + // UNIQUE(request_id, entry_id) — different request_id + different entry_id works. + store + .insert_audit_entry( + "aud-5", + Some("req-2"), + &clock.now(), + "user-carol", + "action_3", + None, + ) + .expect("different request and entry are fine"); + + // Different entry_id always works (entry_id is PK). + store + .insert_audit_entry( + "aud-4", + Some("req-1"), + &clock.now(), + "user-dave", + "action_4", + None, + ) + .expect("another unique entry_id with same request_id is fine"); + + assert_eq!(store.count_table("audit_entries").unwrap(), 5); +} diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs new file mode 100644 index 0000000000..1695ad8f2b --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs @@ -0,0 +1,478 @@ +use super::*; + +#[test] +fn audit_entries_page_by_occurred_at_and_entry_id_without_detail_leakage() { + let (store, _, _tmp) = open_store(); + let timestamp = "2025-06-15T12:00:00Z"; + for entry_id in ["audit-0001", "audit-0003", "audit-0002"] { + store + .insert_audit_entry( + entry_id, + None, + timestamp, + "runtime", + "runtime_ready", + Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), + ) + .unwrap(); + } + + let first_page = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + first_page + .items + .iter() + .map(|entry| entry.entry_id.as_str()) + .collect::>(), + vec!["audit-0003", "audit-0002"] + ); + assert_eq!(first_page.items[0].severity, Some(AuditEntrySeverity::Info)); + assert!( + !format!("{:?}", first_page.items).contains("SENTINEL-AUDIT-SECRET"), + "detail_json must never cross the AuditEntryRow boundary" + ); + + let second_page = store + .list_audit_entries( + Some(2), + first_page.next_cursor.as_deref(), + AuditEntryFilters::default(), + ) + .unwrap(); + assert_eq!(second_page.items.len(), 1); + assert_eq!(second_page.items[0].entry_id, "audit-0001"); + assert!(second_page.next_cursor.is_none()); +} + +#[test] +fn audit_entries_reconcile_forward_by_durable_sequence() { + let (store, clock, tmp) = open_store(); + let external_writer = LogStore::open(tmp.path(), clock).expect("open independent writer"); + for entry_id in ["audit-sequence-1", "audit-sequence-2", "audit-sequence-3"] { + external_writer + .insert_audit_entry( + entry_id, + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"severity":"info"}"#), + ) + .unwrap(); + } + + let first = store + .list_audit_entries_after_sequence(0, 2, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + first + .iter() + .map(|entry| entry.entry_id.as_str()) + .collect::>(), + ["audit-sequence-1", "audit-sequence-2"] + ); + let second = store + .list_audit_entries_after_sequence( + u64::try_from(first[1].sequence).unwrap(), + 2, + AuditEntryFilters::default(), + ) + .unwrap(); + assert_eq!(second[0].entry_id, "audit-sequence-3"); +} + +#[test] +fn audit_entries_project_only_versioned_bounded_context() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-context-0001", + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_model_ready", + Some( + r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"numeric_summaries":{"bytes":4096},"secret":"SENTINEL-AUDIT-SECRET"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entries(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + let row = &page.items[0]; + assert_eq!(row.context_version, Some(1)); + assert_eq!(row.subject_kind.as_deref(), Some("model")); + assert_eq!(row.subject_id.as_deref(), Some("local-gguf/sha256-safe")); + assert_eq!(row.operation_id.as_deref(), Some("runtime-7")); + assert_eq!(row.outcome.as_deref(), Some("ready")); + assert_eq!(row.duration_ms, Some(42)); + assert_eq!(row.numeric_summaries.get("bytes"), Some(&4096)); + assert!(!format!("{row:?}").contains("SENTINEL-AUDIT-SECRET")); +} + +#[test] +fn audit_entries_drop_malformed_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-malformed", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm gpus --draft run-benchmark --backend cuda"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_duplicate_command_summary_flags_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-duplicate", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm models list --json --json"}"#), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_deep_malformed_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-deep", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm load unload status discover rotate-key setup --port 1234"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_noncanonical_and_raw_relay_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + for (request_id, summary) in [ + ( + "audit-command-summary-whitespace", + "mesh-llm models list --json", + ), + ( + "audit-command-summary-raw-relay", + "mesh-llm load name [REDACTED] --relay private-relay", + ), + ] { + store + .insert_audit_entry( + request_id, + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(&format!( + r#"{{"context_version":1,"command_summary":"{summary}"}}"# + )), + ) + .unwrap(); + } + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items.iter().all(|item| item.command_summary.is_none())); +} + +#[test] +fn audit_entries_preserve_valid_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-valid", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + page.items[0].command_summary.as_deref(), + Some("mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]") + ); +} + +#[test] +fn audit_entries_preserve_direct_mesh_peer_identity_and_path() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-direct", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","remote_addr":"192.168.1.44:11204","path_type":"direct"}"#, + ), + ) + .expect("insert direct peer audit"); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .expect("list direct peer audit"); + let row = &page.items[0]; + + assert_eq!(row.entry.subject_kind.as_deref(), Some("mesh_peer")); + assert_eq!( + row.entry.subject_id.as_deref(), + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ); + assert_eq!(row.remote_addr.as_deref(), Some("192.168.1.44:11204")); + assert_eq!(row.path_type.as_deref(), Some("direct")); +} + +#[test] +fn audit_entries_omit_relay_address_and_parse_legacy_path_as_absent() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-relay", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-relay","remote_addr":"203.0.113.10:443","path_type":"relay"}"#, + ), + ) + .expect("insert relay peer audit"); + store + .insert_audit_entry( + "audit-legacy", + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"context_version":1,"subject_kind":"runtime","subject_id":"host"}"#), + ) + .expect("insert legacy audit"); + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list relay and legacy audits"); + let relay = &page.items[0]; + let legacy = &page.items[1]; + + assert_eq!(relay.path_type.as_deref(), Some("relay")); + assert!(relay.remote_addr.is_none()); + assert!(legacy.path_type.is_none()); + assert!(legacy.remote_addr.is_none()); +} + +#[test] +fn audit_entries_drop_unknown_paths_and_invalid_direct_addresses() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-unknown-path", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-unknown","remote_addr":"192.168.1.44:11204","path_type":"proxy"}"#, + ), + ) + .expect("insert unknown peer path audit"); + store + .insert_audit_entry( + "audit-peer-invalid-address", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-direct","remote_addr":"not-a-socket-address","path_type":"direct"}"#, + ), + ) + .expect("insert invalid direct address audit"); + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list bounded peer path audits"); + let unknown_path = &page.items[0]; + let invalid_address = &page.items[1]; + + assert!(unknown_path.path_type.is_none()); + assert!(unknown_path.remote_addr.is_none()); + assert_eq!(invalid_address.path_type.as_deref(), Some("direct")); + assert!(invalid_address.remote_addr.is_none()); +} + +#[test] +fn audit_entry_query_rejects_malformed_cursor_and_out_of_range_limit() { + let (store, _, _tmp) = open_store(); + + let cursor_error = store + .list_audit_entries(Some(1), Some("not-a-cursor"), AuditEntryFilters::default()) + .unwrap_err(); + assert!(matches!(cursor_error, LogStoreError::CursorMalformed(_))); + + for limit in [Some(0), Some(101)] { + let limit_error = store + .list_audit_entries(limit, None, AuditEntryFilters::default()) + .unwrap_err(); + assert!(matches!(limit_error, LogStoreError::InvalidQuery(_))); + } +} + +#[test] +fn audit_entry_query_handles_nullable_request_empty_page_and_allowlisted_filters() { + let (store, _, _tmp) = open_store(); + let empty_page = store + .list_audit_entries(None, None, AuditEntryFilters::default()) + .unwrap(); + assert!(empty_page.items.is_empty()); + assert!(empty_page.next_cursor.is_none()); + + store + .insert_audit_entry( + "audit-null-request", + None, + "2025-06-15T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"severity":"info"}"#), + ) + .unwrap(); + store + .insert_audit_entry( + "audit-mesh-warning", + None, + "2025-06-15T12:00:01Z", + "mesh", + "mesh_quic_handler_failed", + Some(r#"{"severity":"warning"}"#), + ) + .unwrap(); + + let page = store + .list_audit_entries( + None, + None, + AuditEntryFilters { + source: Some(AuditEntrySource::Mesh), + severity: Some(AuditEntrySeverity::Warning), + ..AuditEntryFilters::default() + }, + ) + .unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].entry_id, "audit-mesh-warning"); + assert!(page.items[0].request_id.is_none()); +} + +#[test] +fn nullable_column_conversion_errors_are_not_silenced() { + let (store, clock, _tmp) = open_store(); + store + .insert_summary( + "req-invalid-column", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .expect("insert summary"); + store + .conn() + .execute( + "UPDATE summaries SET status_code = 'not-an-integer' WHERE request_id = 'req-invalid-column'", + [], + ) + .expect("store invalid status code"); + + assert!(matches!( + store.get_summary("req-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_summaries(10, None), + Err(LogStoreError::QueryFailed(_)) + )); + + store + .conn() + .execute( + "UPDATE summaries SET status_code = NULL WHERE request_id = 'req-invalid-column'", + [], + ) + .expect("restore nullable summary column"); + store + .insert_artifact_pointer( + "art-invalid-column", + "req-invalid-column", + &clock.now(), + "log", + None, + ) + .expect("insert artifact pointer"); + store + .conn() + .execute( + "UPDATE artifact_pointers SET checksum = x'00' WHERE artifact_id = 'art-invalid-column'", + [], + ) + .expect("store invalid checksum type"); + + assert!(matches!( + store.get_artifact_pointer("art-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_artifact_pointers_for_request("req-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_artifact_pointers(10, None), + Err(LogStoreError::QueryFailed(_)) + )); +} diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs new file mode 100644 index 0000000000..abccd98378 --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs @@ -0,0 +1,201 @@ +use super::*; + +#[test] +fn insert_audit_entry_sanitizes_detail_before_sqlite_persistence() { + // Given: valid structured detail containing every sensitive value class. + let (store, clock, _tmp) = open_store(); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let detail = serde_json::json!({ + "safe": {"outcome": "completed", "count": 3}, + "nested": {"password": "store-secret-value"}, + "note": "request failed with Bearer store-credential-value", + "path": format!("{home}/private/audit.json"), + }) + .to_string(); + + // When: the public repository method persists the detail. + store + .insert_audit_entry( + "audit-sanitized-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&detail), + ) + .expect("insert audit detail"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-sanitized-detail'", + [], + |row| row.get(0), + ) + .expect("read raw audit detail"); + + // Then: the raw column retains safe structure without sensitive originals. + assert!(!raw_detail.contains("store-secret-value")); + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("store-credential-value")); + assert!(!raw_detail.contains(&home)); + let stored: serde_json::Value = + serde_json::from_str(&raw_detail).expect("stored detail remains valid JSON"); + assert_eq!(stored["safe"]["outcome"], "completed"); + assert_eq!(stored["safe"]["count"], 3); +} + +#[test] +fn insert_audit_entry_sanitizes_actor_and_action_before_sqlite_persistence() { + let (store, clock, _tmp) = open_store(); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let source = format!("{home}/runtime\nsource"); + let code = format!("Bearer {}", "x".repeat(1_100)); + + store + .insert_audit_entry( + "audit-sanitized-scalars", + None, + &clock.now(), + &source, + &code, + None, + ) + .expect("insert audit scalars"); + store + .insert_audit_entry( + "audit-bounded-scalars", + None, + &clock.now(), + "runtime", + &"x".repeat(1_100), + None, + ) + .expect("insert oversized audit scalar"); + let stored: (String, String) = store + .conn() + .query_row( + "SELECT actor, action FROM audit_entries WHERE entry_id = 'audit-sanitized-scalars'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("read raw audit scalars"); + + assert_eq!(stored.0, "~/runtimesource"); + assert_eq!(stored.1, "[REDACTED]"); + assert!(!stored.0.contains(&home)); + assert!(stored.0.chars().count() <= 1_024); + assert!(stored.1.chars().count() <= 1_024); + let bounded: String = store + .conn() + .query_row( + "SELECT action FROM audit_entries WHERE entry_id = 'audit-bounded-scalars'", + [], + |row| row.get(0), + ) + .expect("read bounded audit scalar"); + assert!(bounded.ends_with("... [TRUNCATED]")); + assert!(bounded.chars().count() <= 1_024); +} + +#[test] +fn insert_audit_entry_replaces_malformed_sensitive_detail_before_sqlite_persistence() { + // Given: malformed JSON containing a credential marker and value. + let (store, clock, _tmp) = open_store(); + let malformed = r#"{"note":"Bearer malformed-store-credential""#; + + // When: the public repository method persists the detail. + store + .insert_audit_entry( + "audit-malformed-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(malformed), + ) + .expect("insert malformed audit detail"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-malformed-detail'", + [], + |row| row.get(0), + ) + .expect("read raw malformed audit detail"); + + // Then: SQLite contains only the fixed valid-JSON replacement. + assert_eq!(raw_detail, r#""[REDACTED]""#); + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("malformed-store-credential")); +} + +#[test] +fn insert_audit_entry_bounds_raw_detail_and_sanitizes_colliding_keys_before_persistence() { + // Given: oversized raw detail plus valid detail with unsafe and colliding oversized keys. + let (store, clock, _tmp) = open_store(); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let oversized_raw = serde_json::json!({ + "sentinel": "RAW-SQLITE-OVERSIZED-SENTINEL", + "padding": "x".repeat(256 * 1024), + }) + .to_string(); + let collision_prefix = "k".repeat(1_024); + let unsafe_keys = serde_json::json!({ + format!("{home}/private/audit\nkey"): "path-value", + "Bearer raw-store-key-private": "sensitive-field-value", + format!("{collision_prefix}a"): "collision-a", + format!("{collision_prefix}b"): "collision-b", + }) + .to_string(); + + // When: the public repository method persists both raw detail documents. + store + .insert_audit_entry( + "audit-oversized-raw-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&oversized_raw), + ) + .expect("insert oversized raw detail"); + store + .insert_audit_entry( + "audit-unsafe-key-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&unsafe_keys), + ) + .expect("insert unsafe key detail"); + let oversized_stored: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-oversized-raw-detail'", + [], + |row| row.get(0), + ) + .expect("read oversized raw detail"); + let keys_stored: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-unsafe-key-detail'", + [], + |row| row.get(0), + ) + .expect("read unsafe key detail"); + + // Then: SQLite retains no oversized sentinel or unsafe key text and keeps both safe values. + assert_eq!(oversized_stored, r#""[TRUNCATED]""#); + assert!(!oversized_stored.contains("RAW-SQLITE-OVERSIZED-SENTINEL")); + assert!(!keys_stored.contains(&home)); + assert!(!keys_stored.contains("Bearer")); + assert!(!keys_stored.contains("raw-store-key-private")); + let stored: serde_json::Value = + serde_json::from_str(&keys_stored).expect("stored detail remains valid JSON"); + let object = stored.as_object().expect("stored detail remains an object"); + assert!(object.keys().all(|key| key.chars().count() <= 1_024)); + assert!(object.values().any(|value| value == "collision-a")); + assert!(object.values().any(|value| value == "collision-b")); +} diff --git a/crates/mesh-llm-log-store/src/audit_detail.rs b/crates/mesh-llm-log-store/src/audit_detail.rs new file mode 100644 index 0000000000..fa11137950 --- /dev/null +++ b/crates/mesh-llm-log-store/src/audit_detail.rs @@ -0,0 +1,16 @@ +use crate::repositories::AuditEntryRow; + +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AuditEntryDetail { + pub entry: AuditEntryRow, + pub remote_addr: Option, + pub path_type: Option, + pub command_summary: Option, +} + +impl From for AuditEntryRow { + fn from(detail: AuditEntryDetail) -> Self { + detail.entry + } +} diff --git a/crates/mesh-llm-log-store/src/lib.rs b/crates/mesh-llm-log-store/src/lib.rs index 9879897af4..e01edbc985 100644 --- a/crates/mesh-llm-log-store/src/lib.rs +++ b/crates/mesh-llm-log-store/src/lib.rs @@ -18,6 +18,7 @@ mod space_maintenance_tests; mod artifact_privacy; mod artifact_repository; mod artifacts; +mod audit_detail; mod capture; mod cursor; mod error; @@ -25,6 +26,7 @@ mod maintenance; mod migrations; mod query; mod repositories; +mod schema; mod store; mod timestamps; @@ -35,6 +37,7 @@ pub use artifact_repository::UnavailableArtifactPointer; pub use artifacts::{ ArtifactContent, ArtifactFileStore, ArtifactRedactor, ArtifactStatus, ArtifactWriteReceipt, }; +pub use audit_detail::AuditEntryDetail; pub use capture::{ ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE, ArtifactCaptureDisabledReason, ArtifactCaptureHealthMarker, ArtifactCaptureOutcome, FailOpenArtifactCapture, @@ -50,7 +53,7 @@ pub use maintenance::{ pub use migrations::CURRENT_VERSION as LOG_STORE_SCHEMA_VERSION; pub use query::{ ArtifactRecord, EventRecord, MAX_QUERY_LIMIT, PageQuery, ProxyQuery, ProxyRecord, QueryPage, - QuerySort, RequestOutcome, RequestQuery, RequestRecord, + QuerySort, RequestOutcome, RequestQuery, RequestRecord, RequestRecordWithCaller, }; pub use repositories::{ AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, CascadeArtifactPointer, diff --git a/crates/mesh-llm-log-store/src/maintenance/execution.rs b/crates/mesh-llm-log-store/src/maintenance/execution.rs index d02f424bc8..dd784910dc 100644 --- a/crates/mesh-llm-log-store/src/maintenance/execution.rs +++ b/crates/mesh-llm-log-store/src/maintenance/execution.rs @@ -922,18 +922,20 @@ fn write_audit( reason: &str, ) -> Result<(), LogStoreError> { let occurred_at = canonical_persisted_timestamp(occurred_at)?; - let detail = serde_json::json!({ - "actor": "trusted_local_operator", - "source": "logs_api", - "result": result, - "reason": reason, - "operationId": operation_id.to_string(), - }) - .to_string(); + let detail = mesh_llm_events::audit::SanitizedAuditDetailJson::sanitize( + &serde_json::json!({ + "actor": "trusted_local_operator", + "source": "logs_api", + "result": result, + "reason": reason, + "operationId": operation_id.to_string(), + }) + .to_string(), + ); transaction .execute( "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?1, NULL, ?2, ?3, ?4, ?5)", - rusqlite::params![entry_id, occurred_at, "logs_api", action, detail], + rusqlite::params![entry_id, occurred_at, "logs_api", action, detail.as_str()], ) .map_err(LogStoreError::Sqlite)?; Ok(()) diff --git a/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs b/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs index c05c589d82..22fcd33b88 100644 --- a/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs +++ b/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs @@ -32,6 +32,48 @@ fn preview_snapshots_bounded_targets_and_rejects_invalid_typed_input() { ); } +#[test] +fn preview_cleanup_sanitizes_audit_detail_inside_transaction() { + // Given: a valid maintenance request whose reason contains a credential and home path. + let (_root, artifacts) = fixture(); + let store = artifacts.store_ref(); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let request = CleanupPreviewRequest { + reason: MaintenanceReason::try_from( + format!("Bearer maintenance-credential at {home}/private/audit.json").as_str(), + ) + .expect("valid maintenance reason"), + ..request(32, "2025-02-01T00:00:00Z") + }; + + // When: preview writes its audit row in the receipt transaction. + let receipt = store + .preview_cleanup(&request, &NeverCancelled) + .expect("preview cleanup"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = ?1", + [receipt + .preview_audit_id + .as_deref() + .expect("preview audit ID")], + |row| row.get(0), + ) + .expect("raw maintenance audit detail"); + + // Then: sensitive originals are absent and safe transaction fields remain structured. + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("maintenance-credential")); + assert!(!raw_detail.contains(&home)); + let stored: serde_json::Value = + serde_json::from_str(&raw_detail).expect("maintenance detail remains valid JSON"); + assert_eq!(stored["actor"], "trusted_local_operator"); + assert_eq!(stored["source"], "logs_api"); + assert_eq!(stored["result"], "previewed"); + assert_eq!(stored["operationId"], request.operation_id.to_string()); +} + #[test] fn cleanup_targets_visible_requests_instead_of_hidden_management_traffic() { let (_root, artifacts) = fixture(); diff --git a/crates/mesh-llm-log-store/src/migrations.rs b/crates/mesh-llm-log-store/src/migrations.rs index 68586634f0..de207bcc45 100644 --- a/crates/mesh-llm-log-store/src/migrations.rs +++ b/crates/mesh-llm-log-store/src/migrations.rs @@ -1,903 +1,180 @@ -//! Forward-only schema migrations for log_store. +//! Initial schema installation and future forward migrations for log_store. -use rusqlite::Connection; +use rusqlite::{Connection, Transaction, TransactionBehavior}; -mod legacy_v10; +mod lineage; +mod released_schema; -/// Current forward-only schema version for the integrated local logging feature. -/// -/// Versions 4 through 10 were used by pre-integration development builds. The -/// version remains monotonic so a v10 development store can be identified and -/// upgraded without mistaking it for an unknown future schema. -pub const CURRENT_VERSION: u32 = 11; +type MigrationFn = fn(&Connection) -> Result<(), rusqlite::Error>; -const MIGRATIONS_V1: &str = r#" -CREATE TABLE IF NOT EXISTS summaries ( - request_id TEXT PRIMARY KEY, - state TEXT NOT NULL DEFAULT 'active' - CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), - created_at TEXT NOT NULL, - terminal_at TEXT, - route TEXT, - model TEXT, - provider TEXT, - engine TEXT, - status_code INTEGER, - error_msg TEXT, - tenant_id TEXT, - account_id TEXT, - user_id TEXT -); - -CREATE INDEX IF NOT EXISTS idx_summaries_created ON summaries (created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_state ON summaries (state); - -CREATE TABLE IF NOT EXISTS lifecycle_events ( - event_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - payload_json TEXT NOT NULL DEFAULT '{}', - event_type TEXT NOT NULL DEFAULT 'unknown', - is_terminal INTEGER NOT NULL DEFAULT 0 - CHECK (is_terminal IN (0, 1)), - - UNIQUE(request_id, event_id) -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_terminal_event_one_per_request -ON lifecycle_events (request_id) -WHERE is_terminal = 1; - -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request ON lifecycle_events (request_id); -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); - -CREATE TABLE IF NOT EXISTS artifact_pointers ( - artifact_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - kind TEXT NOT NULL, - metadata_json TEXT, - media_kind TEXT, - checksum TEXT, - bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), - version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), - redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), - truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), - stored_at TEXT, - missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), - corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), - - UNIQUE(request_id, artifact_id) -); - -CREATE INDEX IF NOT EXISTS idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); - -CREATE TABLE IF NOT EXISTS proxy_records ( - attempt_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - target TEXT NOT NULL, - provider TEXT, - engine TEXT, - started_at TEXT, - completed_at TEXT, - status_code INTEGER, - error_msg TEXT, - - UNIQUE(request_id, attempt_id) -); - -CREATE INDEX IF NOT EXISTS idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); - -CREATE TABLE IF NOT EXISTS audit_entries ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), - entry_id TEXT NOT NULL UNIQUE, - request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, - actor TEXT NOT NULL, - action TEXT NOT NULL, - detail_json TEXT, - - UNIQUE(request_id, entry_id) -); - -CREATE INDEX IF NOT EXISTS idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); - -CREATE TABLE IF NOT EXISTS webhook_deliveries ( - delivery_id TEXT PRIMARY KEY, - request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - terminal_outcome TEXT NOT NULL - CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), - terminal_status_code INTEGER - CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), - occurred_at TEXT NOT NULL, - target_url TEXT NOT NULL, - attempt_number INTEGER NOT NULL DEFAULT 0 - CHECK (attempt_number BETWEEN 0 AND 20), - status_code INTEGER - CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), - response_body TEXT, - error_msg TEXT, - state TEXT NOT NULL DEFAULT 'succeeded' - CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), - created_at TEXT NOT NULL DEFAULT '', - updated_at TEXT NOT NULL DEFAULT '', - next_attempt_at TEXT, - lease_expires_at TEXT, - claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), - max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), - last_error_code TEXT - CHECK (last_error_code IS NULL OR last_error_code IN - ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), - - UNIQUE(request_id, delivery_id) -); - -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_eligible -ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); - -CREATE TABLE IF NOT EXISTS cleanup_runs ( - run_id TEXT PRIMARY KEY, - occurred_at TEXT NOT NULL, - policy_name TEXT NOT NULL, - cutoff_before TEXT NOT NULL, - deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), - duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) -); - -CREATE INDEX IF NOT EXISTS idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); - -CREATE TABLE IF NOT EXISTS maintenance_operations ( - operation_id TEXT PRIMARY KEY, - action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), - cutoff_before TEXT NOT NULL, - request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), - reason TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), - planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), - planned_events INTEGER NOT NULL CHECK (planned_events >= 0), - planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), - planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), - planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), - executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), - executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), - executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), - executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), - executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), - has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), - created_at TEXT NOT NULL, - completed_at TEXT, - selection_fingerprint TEXT NOT NULL DEFAULT '', - artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), - artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), - artifact_file_failure_class TEXT - CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), - preview_audit_id TEXT, - execution_audit_id TEXT, - cleanup_filters_json TEXT NOT NULL DEFAULT '{}' -); - -CREATE TABLE IF NOT EXISTS maintenance_operation_targets ( - operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - request_id TEXT NOT NULL, - PRIMARY KEY (operation_id, request_id), - UNIQUE (operation_id, ordinal) -); - -CREATE INDEX IF NOT EXISTS idx_maintenance_operation_targets_operation -ON maintenance_operation_targets (operation_id, ordinal); - -CREATE TABLE IF NOT EXISTS pending_artifact_deletions ( - artifact_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_pending_artifact_deletions_request -ON pending_artifact_deletions (request_id, artifact_id); -"#; - -// Query and retention indexes are intentionally added in a second migration: -// V1 is already durable in the field, and these indexes must be available to -// upgraded stores without rebuilding their logging history. -const MIGRATIONS_V2: &str = r#" -CREATE INDEX IF NOT EXISTS idx_summaries_terminal_order -ON summaries (state, COALESCE(terminal_at, created_at), request_id); -CREATE INDEX IF NOT EXISTS idx_summaries_route_created -ON summaries (route, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_model_created -ON summaries (model, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_provider_created -ON summaries (provider, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_engine_created -ON summaries (engine, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_status_created -ON summaries (status_code, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_state_created -ON summaries (state, created_at DESC, request_id DESC); - -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_occurred -ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); -CREATE INDEX IF NOT EXISTS idx_artifact_pointers_request_occurred -ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); - -CREATE INDEX IF NOT EXISTS idx_proxy_records_request_occurred -ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_provider_occurred -ON proxy_records (provider, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_engine_occurred -ON proxy_records (engine, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_status_occurred -ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); - -CREATE INDEX IF NOT EXISTS idx_audit_entries_actor_occurred -ON audit_entries (actor, occurred_at DESC, entry_id DESC); -CREATE INDEX IF NOT EXISTS idx_audit_entries_severity_occurred -ON audit_entries ( - CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, - occurred_at DESC, - entry_id DESC -); +#[derive(Clone, Copy)] +struct Migration { + version: u32, + apply: MigrationFn, +} -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_ready -ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) -WHERE state IN ('pending', 'retry', 'manual_retry'); -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_expired_lease -ON webhook_deliveries (lease_expires_at, created_at, delivery_id) -WHERE state = 'in_flight'; -"#; +struct MigrationPlan<'a> { + target: u32, + initialize: MigrationFn, + migrations: &'a [Migration], +} -// Artifact bodies can be intentionally unavailable even though their -// metadata record is healthy. Keep that state separate from the existing -// missing/corrupt file-health flags so API consumers never have to infer an -// intentional omission from a null checksum. -const MIGRATIONS_V3: &str = r#" -ALTER TABLE artifact_pointers ADD COLUMN unavailable_reason TEXT - CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( - 'streaming_response_not_assembled', - 'response_body_not_bounded', - 'capture_content_limit_exceeded', - 'capture_memory_budget_exceeded', - 'artifact_capture_disabled', - 'artifact_capture_failed' - )); -"#; +const MIGRATIONS: &[Migration] = &[]; +const APPLICATION_ID: u32 = 0x4D4C4F47; -/// Install the complete V1 schema and its version marker atomically. -pub fn apply_migrations(conn: &Connection) -> Result<(), rusqlite::Error> { - let current_ver: i32 = conn.query_row("PRAGMA user_version", [], |r| r.get(0))?; +/// Current schema version for the integrated local logging feature. +pub const CURRENT_VERSION: u32 = 1; - if !accepted_schema_version(conn, current_ver as u32)? { - return Err(rusqlite::Error::InvalidQuery); - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SchemaClassification { + Fresh, + Private { version: u32 }, + ReleasedSchema { source_version: u32 }, + Incompatible { found: u32 }, +} - match current_ver { - 0 => { - apply_migration_transactionally(conn, 1, MIGRATIONS_V1)?; - apply_migration_transactionally(conn, 2, MIGRATIONS_V2)?; - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 1 => { - apply_migration_transactionally(conn, 2, MIGRATIONS_V2)?; - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 2 => { - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 3 => mark_current(conn), - legacy_v10::LEGACY_VERSION => legacy_v10::migrate(conn), - _ => Ok(()), - } +pub fn apply_migrations(connection: &Connection) -> Result<(), rusqlite::Error> { + run_migrations( + connection, + MigrationPlan { + target: CURRENT_VERSION, + initialize: crate::schema::initialize, + migrations: MIGRATIONS, + }, + ) } -/// Return the version pair when the database cannot be migrated safely. -/// -/// The v10 development lineage is accepted only when its structural -/// fingerprint matches. This check is read-only so callers can surface an -/// actionable compatibility state before any migration is attempted. pub(crate) fn incompatible_schema( - conn: &Connection, + connection: &Connection, ) -> Result, rusqlite::Error> { - let found: u32 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?; - let supported = CURRENT_VERSION; - let compatible = accepted_schema_version(conn, found)?; - Ok((!compatible).then_some((found, supported))) -} - -/// Whether `version` is a schema the store can open: a blank-slate version -/// the migration steps can upgrade, the current version, or the -/// fingerprint-guarded v10 development lineage. -fn accepted_schema_version(conn: &Connection, version: u32) -> Result { - Ok(matches!(version, 0..=3) - || version == CURRENT_VERSION - || (version == legacy_v10::LEGACY_VERSION as u32 && legacy_v10::matches(conn)?)) -} - -fn mark_current(conn: &Connection) -> Result<(), rusqlite::Error> { - apply_migration_transactionally(conn, CURRENT_VERSION as i32, "") + incompatible_schema_for_target(connection, CURRENT_VERSION) } -fn apply_migration_transactionally( - conn: &Connection, - version: i32, - migration: &str, -) -> Result<(), rusqlite::Error> { - let transaction = conn.unchecked_transaction()?; - transaction.execute_batch(migration)?; - transaction.execute_batch(&format!("PRAGMA user_version = {version}"))?; - transaction.commit() +fn incompatible_schema_for_target( + connection: &Connection, + target: u32, +) -> Result, rusqlite::Error> { + let classification = classify_schema(connection)?; + let compatible = match classification { + SchemaClassification::Fresh => true, + SchemaClassification::Private { version } => version <= target, + SchemaClassification::ReleasedSchema { .. } => target == CURRENT_VERSION, + SchemaClassification::Incompatible { .. } => false, + }; + Ok((!compatible).then_some((schema_version(connection)?, target))) } -#[cfg(test)] -mod tests { - use super::*; - - fn table_columns(connection: &Connection, table: &str) -> Vec { - let mut statement = connection - .prepare(&format!("PRAGMA table_info('{table}')")) - .expect("prepare table info"); - statement - .query_map([], |row| row.get(1)) - .expect("query table info") - .collect::, _>>() - .expect("collect table columns") +fn classify_schema(connection: &Connection) -> Result { + if let Some(source_version) = released_schema::source_version(connection)? { + return Ok(SchemaClassification::ReleasedSchema { source_version }); } - - fn schema_object_names(connection: &Connection, object_type: &str) -> Vec { - let mut statement = connection - .prepare( - "SELECT name FROM sqlite_master \ - WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", - ) - .expect("prepare schema object query"); - statement - .query_map([object_type], |row| row.get(0)) - .expect("query schema objects") - .collect::, _>>() - .expect("collect schema objects") + let found = schema_version(connection)?; + let identity = application_id(connection)?; + if found == 0 && identity == 0 && !has_user_objects(connection)? { + return Ok(SchemaClassification::Fresh); } - - fn schema_sql(connection: &Connection, object_type: &str, name: &str) -> String { - connection - .query_row( - "SELECT sql FROM sqlite_master WHERE type = ?1 AND name = ?2", - [object_type, name], - |row| row.get(0), - ) - .expect("schema SQL") - } - - const EXPECTED_TABLES: &[&str] = &[ - "artifact_pointers", - "audit_entries", - "cleanup_runs", - "lifecycle_events", - "maintenance_operation_targets", - "maintenance_operations", - "pending_artifact_deletions", - "proxy_records", - "summaries", - "webhook_deliveries", - ]; - - const EXPECTED_TABLE_COLUMNS: &[(&str, &[&str])] = &[ - ( - "summaries", - &[ - "request_id", - "state", - "created_at", - "terminal_at", - "route", - "model", - "provider", - "engine", - "status_code", - "error_msg", - "tenant_id", - "account_id", - "user_id", - ], - ), - ( - "lifecycle_events", - &[ - "event_id", - "request_id", - "occurred_at", - "payload_json", - "event_type", - "is_terminal", - ], - ), - ( - "artifact_pointers", - &[ - "artifact_id", - "request_id", - "occurred_at", - "kind", - "metadata_json", - "media_kind", - "checksum", - "bytes", - "version", - "redacted", - "truncated", - "stored_at", - "missing", - "corrupt", - "unavailable_reason", - ], - ), - ( - "proxy_records", - &[ - "attempt_id", - "request_id", - "occurred_at", - "target", - "provider", - "engine", - "started_at", - "completed_at", - "status_code", - "error_msg", - ], - ), - ( - "audit_entries", - &[ - "sequence", - "entry_id", - "request_id", - "occurred_at", - "actor", - "action", - "detail_json", - ], - ), - ( - "webhook_deliveries", - &[ - "delivery_id", - "request_id", - "terminal_outcome", - "terminal_status_code", - "occurred_at", - "target_url", - "attempt_number", - "status_code", - "response_body", - "error_msg", - "state", - "created_at", - "updated_at", - "next_attempt_at", - "lease_expires_at", - "claim_generation", - "max_attempts", - "last_error_code", - ], - ), - ( - "cleanup_runs", - &[ - "run_id", - "occurred_at", - "policy_name", - "cutoff_before", - "deleted_count", - "duration_ms", - ], - ), - ( - "maintenance_operations", - &[ - "operation_id", - "action", - "cutoff_before", - "request_limit", - "reason", - "state", - "planned_requests", - "planned_events", - "planned_artifacts", - "planned_proxy_records", - "planned_database_rows", - "executed_requests", - "executed_events", - "executed_artifacts", - "executed_proxy_records", - "executed_database_rows", - "has_more", - "created_at", - "completed_at", - "selection_fingerprint", - "artifact_files_removed", - "artifact_files_failed", - "artifact_file_failure_class", - "preview_audit_id", - "execution_audit_id", - "cleanup_filters_json", - ], - ), - ( - "maintenance_operation_targets", - &["operation_id", "ordinal", "request_id"], - ), - ("pending_artifact_deletions", &["artifact_id", "request_id"]), - ]; - - const EXPECTED_INDEXES: &[&str] = &[ - "idx_artifact_pointers_occurred", - "idx_artifact_pointers_request_occurred", - "idx_audit_entries_actor_occurred", - "idx_audit_entries_occurred", - "idx_audit_entries_severity_occurred", - "idx_cleanup_runs_occurred", - "idx_lifecycle_events_occurred", - "idx_lifecycle_events_request", - "idx_lifecycle_events_request_occurred", - "idx_lifecycle_events_request_terminal", - "idx_maintenance_operation_targets_operation", - "idx_pending_artifact_deletions_request", - "idx_proxy_records_engine_occurred", - "idx_proxy_records_occurred", - "idx_proxy_records_provider_occurred", - "idx_proxy_records_request_occurred", - "idx_proxy_records_status_occurred", - "idx_summaries_created", - "idx_summaries_engine_created", - "idx_summaries_model_created", - "idx_summaries_provider_created", - "idx_summaries_route_created", - "idx_summaries_state", - "idx_summaries_state_created", - "idx_summaries_status_created", - "idx_summaries_terminal_order", - "idx_terminal_event_one_per_request", - "idx_webhook_deliveries_eligible", - "idx_webhook_deliveries_expired_lease", - "idx_webhook_deliveries_occurred", - "idx_webhook_deliveries_ready", - ]; - - fn assert_schema_identity(connection: &Connection) { - assert_eq!( - CURRENT_VERSION, 11, - "the canonical schema follows the pre-integration v10 lineage" - ); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, u32>(0)) - .expect("schema version"), - 11 - ); - assert_eq!(schema_object_names(connection, "table"), EXPECTED_TABLES); - assert_eq!(schema_object_names(connection, "index"), EXPECTED_INDEXES); - } - - fn assert_schema_columns(connection: &Connection) { - for (table, expected) in EXPECTED_TABLE_COLUMNS { - assert_eq!(table_columns(connection, table), *expected, "{table}"); - } + if found > 0 && identity == APPLICATION_ID && lineage::is_valid(connection)? { + return Ok(SchemaClassification::Private { version: found }); } + Ok(SchemaClassification::Incompatible { found }) +} - fn assert_schema_constraints(connection: &Connection) { - let lifecycle = schema_sql(connection, "table", "lifecycle_events"); - assert!(lifecycle.contains("CHECK (is_terminal IN (0, 1))")); - assert!( - schema_sql(connection, "index", "idx_terminal_event_one_per_request") - .contains("WHERE is_terminal = 1") - ); - let webhook = schema_sql(connection, "table", "webhook_deliveries"); - for constraint in [ - "terminal_outcome IN", - "terminal_status_code BETWEEN 100 AND 599", - "status_code BETWEEN 100 AND 599", - "state IN", - "attempt_number BETWEEN 0 AND 20", - "max_attempts BETWEEN 1 AND 20", - "claim_generation >= 0", - "last_error_code IN", - ] { - assert!( - webhook.contains(constraint), - "missing webhook constraint {constraint}" - ); +fn run_migrations(connection: &Connection, plan: MigrationPlan<'_>) -> Result<(), rusqlite::Error> { + validate_registry(plan.target, plan.migrations)?; + let classification = classify_schema(connection)?; + let mut current = match classification { + SchemaClassification::Fresh => 0, + SchemaClassification::Private { version } if version <= plan.target => version, + SchemaClassification::ReleasedSchema { source_version } + if plan.target == CURRENT_VERSION => + { + released_schema::import(connection, source_version)?; + CURRENT_VERSION } - assert!( - schema_sql(connection, "table", "summaries") - .contains("state IN ('active', 'completed', 'failed'") - ); - let artifacts = schema_sql(connection, "table", "artifact_pointers"); - for constraint in [ - "bytes >= 0", - "version >= 1", - "redacted IN (0, 1)", - "truncated IN (0, 1)", - "missing IN (0, 1)", - "corrupt IN (0, 1)", - ] { - assert!( - artifacts.contains(constraint), - "missing artifact constraint {constraint}" - ); + SchemaClassification::Private { .. } + | SchemaClassification::ReleasedSchema { .. } + | SchemaClassification::Incompatible { .. } => return Err(rusqlite::Error::InvalidQuery), + }; + if matches!(classification, SchemaClassification::Fresh) { + let transaction = immediate_transaction(connection)?; + if schema_version(&transaction)? != 0 + || application_id(&transaction)? != 0 + || has_user_objects(&transaction)? + { + return Err(rusqlite::Error::InvalidQuery); } - let maintenance = schema_sql(connection, "table", "maintenance_operations"); - for constraint in [ - "action IN ('cleanup', 'delete_one')", - "request_limit BETWEEN 1 AND 100", - "state IN ('previewed', 'completed', 'partial')", - "has_more IN (0, 1)", - "artifact_file_failure_class IN ('io', 'unsafe_path')", - ] { - assert!( - maintenance.contains(constraint), - "missing maintenance constraint {constraint}" - ); + set_application_id(&transaction, APPLICATION_ID)?; + (plan.initialize)(&transaction)?; + lineage::install(&transaction)?; + set_schema_version(&transaction, 1)?; + transaction.commit()?; + current = 1; + } + for migration in plan + .migrations + .iter() + .filter(|migration| migration.version > current) + { + let transaction = immediate_transaction(connection)?; + if schema_version(&transaction)? != migration.version - 1 + || application_id(&transaction)? != APPLICATION_ID + || !lineage::is_valid(&transaction)? + { + return Err(rusqlite::Error::InvalidQuery); } - assert!( - !schema_sql(connection, "table", "pending_artifact_deletions").contains("REFERENCES"), - "pending work must outlive deleted pointer and summary owners" - ); - } - - fn assert_fresh_only_migration_sql() { - assert!(!MIGRATIONS_V1.contains("ALTER TABLE")); - assert!(!MIGRATIONS_V1.contains("UPDATE ")); - assert!(!MIGRATIONS_V1.contains("DROP INDEX")); - } - - #[test] - fn fresh_database_is_exactly_complete_current_schema() { - let connection = Connection::open_in_memory().expect("open database"); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .expect("enable foreign keys"); - apply_migrations(&connection).expect("apply fresh schema"); - - assert_schema_identity(&connection); - - assert_schema_columns(&connection); - assert_schema_constraints(&connection); - assert_fresh_only_migration_sql(); - } - - #[test] - fn rejects_future_schema_versions_without_modifying_them() { - let connection = Connection::open_in_memory().unwrap(); - connection.pragma_update(None, "user_version", 99).unwrap(); - assert!(matches!( - apply_migrations(&connection), - Err(rusqlite::Error::InvalidQuery) - )); - let version: i32 = connection - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .unwrap(); - assert_eq!(version, 99); - } - - #[test] - fn rejects_unrecognized_v10_schema_without_modifying_it() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch( - "CREATE TABLE sentinel (value TEXT); \ - INSERT INTO sentinel VALUES ('unchanged'); \ - PRAGMA user_version = 10;", - ) - .unwrap(); - - assert!(matches!( - apply_migrations(&connection), - Err(rusqlite::Error::InvalidQuery) - )); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, i32>(0)) - .unwrap(), - 10 - ); - assert_eq!( - connection - .query_row("SELECT value FROM sentinel", [], |row| row - .get::<_, String>(0)) - .unwrap(), - "unchanged" - ); + (migration.apply)(&transaction)?; + set_schema_version(&transaction, migration.version)?; + transaction.commit()?; } + Ok(()) +} - #[test] - fn failed_migration_does_not_advance_schema_version() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute("CREATE TABLE summaries (request_id TEXT PRIMARY KEY)", []) - .unwrap(); - assert!(apply_migrations(&connection).is_err()); - let version: i32 = connection - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .unwrap(); - assert_eq!(version, 0); +fn validate_registry(target: u32, migrations: &[Migration]) -> Result<(), rusqlite::Error> { + let terminal = target.checked_add(1).ok_or(rusqlite::Error::InvalidQuery)?; + let mut expected = 2_u32; + for migration in migrations { + if migration.version != expected { + return Err(rusqlite::Error::InvalidQuery); + } + expected = expected + .checked_add(1) + .ok_or(rusqlite::Error::InvalidQuery)?; } - - #[test] - fn forced_v1_failure_rolls_back_partial_schema_and_user_version() { - let connection = Connection::open_in_memory().expect("open database"); - let forced_failure = r#" - CREATE TABLE must_rollback (id INTEGER PRIMARY KEY); - CREATE TABLE invalid_sql ( - "#; - - assert!(apply_migration_transactionally(&connection, 1, forced_failure).is_err()); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, u32>(0)) - .expect("schema version"), - 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'must_rollback'", - [], - |row| row.get::<_, u32>(0), - ) - .expect("partial table count"), - 0 - ); + if expected != terminal { + return Err(rusqlite::Error::InvalidQuery); } + Ok(()) +} - #[test] - fn schema_v1_upgrades_to_current_and_current_schema_is_a_noop() { - let connection = Connection::open_in_memory().expect("open database"); - connection - .execute_batch(MIGRATIONS_V1) - .expect("install V1 schema"); - connection.pragma_update(None, "user_version", 1).unwrap(); - apply_migrations(&connection).expect("upgrade V1 to current"); - assert_schema_identity(&connection); +fn immediate_transaction(connection: &Connection) -> Result, rusqlite::Error> { + Transaction::new_unchecked(connection, TransactionBehavior::Immediate) +} - connection - .execute_batch( - "CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('unchanged');", - ) - .expect("seed sentinel"); - apply_migrations(&connection).expect("current schema is a no-op"); - assert_eq!( - connection - .query_row("SELECT value FROM sentinel", [], |row| row - .get::<_, String>(0)) - .expect("sentinel"), - "unchanged" - ); - } +fn schema_version(connection: &Connection) -> Result { + let value: i32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + Ok(u32::from_ne_bytes(value.to_ne_bytes())) +} - #[test] - fn seeded_v2_artifact_row_survives_v3_with_nullable_unavailable_reason() { - let root = tempfile::tempdir().expect("database root"); - let connection = Connection::open(root.path().join("log_store.db")).expect("open database"); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .expect("enable foreign keys"); - connection - .execute_batch(MIGRATIONS_V1) - .expect("install V1 schema"); - connection - .execute_batch(MIGRATIONS_V2) - .expect("install V2 schema"); - connection.pragma_update(None, "user_version", 2).unwrap(); - connection - .execute( - "INSERT INTO summaries (request_id, created_at) \ - VALUES (?1, ?2)", - [ - "00000000-0000-4000-8000-000000000401", - "2025-01-01T00:00:00.000000000Z", - ], - ) - .expect("seed V2 summary"); - connection - .execute( - "INSERT INTO artifact_pointers \ - (artifact_id, request_id, occurred_at, kind, media_kind, checksum, bytes, version, redacted) \ - VALUES (?1, ?2, ?3, 'response', 'application/json', ?4, 2, 1, 1)", - rusqlite::params![ - "00000000-0000-4000-8000-000000000402", - "00000000-0000-4000-8000-000000000401", - "2025-01-01T00:00:00.100000000Z", - "ab" - ], - ) - .expect("seed V2 artifact"); +fn set_schema_version(connection: &Connection, version: u32) -> Result<(), rusqlite::Error> { + connection.pragma_update(None, "user_version", version) +} - apply_migrations(&connection).expect("upgrade V2 to V3"); - assert_schema_identity(&connection); - let row = connection - .query_row( - "SELECT request_id, kind, media_kind, checksum, bytes, redacted, unavailable_reason \ - FROM artifact_pointers WHERE artifact_id = ?1", - ["00000000-0000-4000-8000-000000000402"], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, i64>(4)?, - row.get::<_, i32>(5)?, - row.get::<_, Option>(6)?, - )) - }, - ) - .expect("read migrated artifact row"); - assert_eq!( - row, - ( - "00000000-0000-4000-8000-000000000401".to_string(), - "response".to_string(), - Some("application/json".to_string()), - Some("ab".to_string()), - 2, - 1, - None, - ) - ); - drop(connection); - let store = crate::LogStore::reopen_at(root.path(), std::sync::Arc::new(crate::RealClock)) - .expect("reopen migrated store through the production path"); - let page = store - .query_artifacts( - "00000000-0000-4000-8000-000000000401", - &crate::PageQuery { - limit: 10, - cursor: None, - sort: crate::QuerySort::Ascending, - }, - ) - .expect("read migrated artifact through repository query"); - assert_eq!(page.items.len(), 1); - assert_eq!( - page.items[0].artifact_id, - "00000000-0000-4000-8000-000000000402" - ); - assert_eq!(page.items[0].unavailable_reason, None); - assert!(!page.items[0].missing); - assert!(!page.items[0].corrupt); - } +fn application_id(connection: &Connection) -> Result { + let value: i32 = connection.pragma_query_value(None, "application_id", |row| row.get(0))?; + Ok(u32::from_ne_bytes(value.to_ne_bytes())) +} - #[test] - fn initial_webhook_schema_bounds_terminal_status_codes() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .unwrap(); - apply_migrations(&connection).unwrap(); - connection - .execute( - "INSERT INTO summaries (request_id, created_at) VALUES ('request-1', '2026-08-04T12:00:00Z')", - [], - ) - .unwrap(); +fn set_application_id(connection: &Connection, application_id: u32) -> Result<(), rusqlite::Error> { + connection.pragma_update(None, "application_id", application_id) +} - let error = connection - .execute( - "INSERT INTO webhook_deliveries \ - (delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, target_url, attempt_number) \ - VALUES ('delivery-1', 'request-1', 'completed', 99, '2026-08-04T12:00:00Z', 'configured_webhook', 0)", - [], - ) - .expect_err("invalid terminal status must violate the V1 schema check"); - assert!(matches!( - error, - rusqlite::Error::SqliteFailure(_, Some(message)) - if message.contains("terminal_status_code") - )); - } +fn has_user_objects(connection: &Connection) -> Result { + connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type IN ('table', 'index', 'view', 'trigger') AND name NOT LIKE 'sqlite_%')", + [], + |row| row.get(0), + ) } + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs b/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs deleted file mode 100644 index 57b4cd7566..0000000000 --- a/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Compatibility migration for local stores created by pre-integration builds. - -use rusqlite::Connection; - -pub(super) const LEGACY_VERSION: i32 = 10; - -/// Match the abandoned v10 development schema by structure, not only by its -/// version marker. This keeps an unrelated future schema at version 10 from -/// being rewritten. -pub(super) fn matches(conn: &Connection) -> Result { - conn.query_row( - r#" - SELECT - EXISTS(SELECT 1 FROM pragma_table_info('lifecycle_events') WHERE name = 'payload_json') - AND NOT EXISTS(SELECT 1 FROM pragma_table_info('lifecycle_events') WHERE name = 'event_type') - AND EXISTS(SELECT 1 FROM pragma_table_info('audit_entries') WHERE name = 'source') - AND NOT EXISTS(SELECT 1 FROM pragma_table_info('audit_entries') WHERE name = 'sequence') - AND EXISTS(SELECT 1 FROM pragma_table_info('maintenance_operations') WHERE name = 'intent_hash') - AND EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'maintenance_previews') - "#, - [], - |row| row.get(0), - ) -} - -/// Replace the pre-integration schema transactionally while retaining request, -/// lifecycle, artifact, proxy, audit, webhook, and cleanup records. Obsolete -/// maintenance receipts are intentionally retired; their audit entries remain. -pub(super) fn migrate(conn: &Connection) -> Result<(), rusqlite::Error> { - let transaction = conn.unchecked_transaction()?; - transaction.execute_batch(LEGACY_PREPARE_SQL)?; - transaction.execute_batch(super::MIGRATIONS_V1)?; - transaction.execute_batch(super::MIGRATIONS_V2)?; - transaction.execute_batch(super::MIGRATIONS_V3)?; - transaction.execute_batch(LEGACY_COPY_SQL)?; - transaction.execute_batch(&format!("PRAGMA user_version = {}", super::CURRENT_VERSION))?; - transaction.commit() -} - -const LEGACY_PREPARE_SQL: &str = r#" -PRAGMA defer_foreign_keys = ON; - -DROP TRIGGER IF EXISTS maintenance_operation_targets_resolution_insert; -DROP TRIGGER IF EXISTS maintenance_operation_targets_resolution_update; - -DROP TABLE IF EXISTS maintenance_operation_remaining_requests; -DROP TABLE IF EXISTS maintenance_preview_targets; -DROP TABLE IF EXISTS maintenance_operation_targets; -DROP TABLE IF EXISTS maintenance_operations; -DROP TABLE IF EXISTS maintenance_previews; -DROP TABLE IF EXISTS pending_artifact_deletions; - -ALTER TABLE summaries RENAME TO legacy_v10_summaries; -ALTER TABLE lifecycle_events RENAME TO legacy_v10_lifecycle_events; -ALTER TABLE artifact_pointers RENAME TO legacy_v10_artifact_pointers; -ALTER TABLE proxy_records RENAME TO legacy_v10_proxy_records; -ALTER TABLE audit_entries RENAME TO legacy_v10_audit_entries; -ALTER TABLE webhook_deliveries RENAME TO legacy_v10_webhook_deliveries; -ALTER TABLE cleanup_runs RENAME TO legacy_v10_cleanup_runs; - -DROP INDEX IF EXISTS idx_artifact_pointers_occurred; -DROP INDEX IF EXISTS idx_audit_entries_occurred; -DROP INDEX IF EXISTS idx_audit_entries_operation_id; -DROP INDEX IF EXISTS idx_cleanup_runs_occurred; -DROP INDEX IF EXISTS idx_lifecycle_events_occurred; -DROP INDEX IF EXISTS idx_lifecycle_events_request; -DROP INDEX IF EXISTS idx_maintenance_operation_remaining_requests; -DROP INDEX IF EXISTS idx_maintenance_operation_targets_operation; -DROP INDEX IF EXISTS idx_maintenance_preview_targets_operation; -DROP INDEX IF EXISTS idx_proxy_records_occurred; -DROP INDEX IF EXISTS idx_proxy_records_request_attempt; -DROP INDEX IF EXISTS idx_summaries_created; -DROP INDEX IF EXISTS idx_summaries_state; -DROP INDEX IF EXISTS idx_terminal_event_one_per_request; -DROP INDEX IF EXISTS idx_webhook_deliveries_occurred; -"#; - -const LEGACY_COPY_SQL: &str = r#" -INSERT INTO summaries ( - request_id, state, created_at, terminal_at, route, model, provider, engine, - status_code, error_msg, tenant_id, account_id, user_id -) -SELECT - request_id, - CASE - WHEN state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped') THEN state - ELSE 'failed' - END, - created_at, terminal_at, route, model, provider, engine, status_code, - error_msg, tenant_id, account_id, user_id -FROM legacy_v10_summaries; - -INSERT INTO lifecycle_events ( - event_id, request_id, occurred_at, payload_json, event_type, is_terminal -) -SELECT - event_id, - request_id, - occurred_at, - payload_json, - CASE - WHEN json_valid(payload_json) THEN COALESCE(json_extract(payload_json, '$.type'), 'unknown') - ELSE 'unknown' - END, - CASE - WHEN json_valid(payload_json) - AND json_extract(payload_json, '$.type') IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped') - THEN 1 - ELSE 0 - END -FROM legacy_v10_lifecycle_events; - -INSERT INTO artifact_pointers ( - artifact_id, request_id, occurred_at, kind, metadata_json, media_kind, - checksum, bytes, version, redacted, truncated, stored_at, missing, corrupt, - unavailable_reason -) -SELECT - artifact_id, request_id, occurred_at, kind, metadata_json, media_kind, - checksum, MAX(bytes, 0), MAX(version, 1), - CASE WHEN redacted = 1 THEN 1 ELSE 0 END, - CASE WHEN truncated = 1 THEN 1 ELSE 0 END, - stored_at, - CASE WHEN missing = 1 THEN 1 ELSE 0 END, - CASE WHEN corrupt = 1 THEN 1 ELSE 0 END, - NULL -FROM legacy_v10_artifact_pointers; - -INSERT INTO proxy_records ( - attempt_id, request_id, occurred_at, target, provider, engine, started_at, - completed_at, status_code, error_msg -) -SELECT - attempt_id, request_id, occurred_at, target, provider, engine, started_at, - completed_at, status_code, error_msg -FROM legacy_v10_proxy_records; - -INSERT INTO audit_entries ( - sequence, entry_id, request_id, occurred_at, actor, action, detail_json -) -SELECT - ROW_NUMBER() OVER (ORDER BY occurred_at, entry_id), - entry_id, - request_id, - occurred_at, - CASE - WHEN actor IN ('logging_service', 'runtime', 'mesh', 'cli', 'logs_api') THEN actor - ELSE 'logging_service' - END, - action, - json_patch( - json_patch( - json_patch( - json_patch( - CASE WHEN json_valid(detail_json) THEN detail_json ELSE '{}' END, - CASE WHEN result IS NOT NULL THEN json_object('outcome', result) ELSE '{}' END - ), - CASE WHEN source IS NOT NULL THEN json_object('source', source) ELSE '{}' END - ), - CASE WHEN reason IS NOT NULL THEN json_object('reason_code', reason) ELSE '{}' END - ), - CASE WHEN operation_id IS NOT NULL THEN json_object('operation_id', operation_id) ELSE '{}' END - ) -FROM legacy_v10_audit_entries; - -INSERT INTO webhook_deliveries ( - delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, - target_url, attempt_number, status_code, state, created_at, updated_at, - max_attempts -) -SELECT - delivery.delivery_id, - delivery.request_id, - CASE - WHEN summary.state IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped') THEN summary.state - ELSE 'failed' - END, - CASE WHEN summary.status_code BETWEEN 100 AND 599 THEN summary.status_code ELSE NULL END, - delivery.occurred_at, - 'configured_webhook', - MIN(MAX(delivery.attempt_number, 0), 20), - CASE WHEN delivery.status_code BETWEEN 100 AND 599 THEN delivery.status_code ELSE NULL END, - CASE WHEN delivery.status_code BETWEEN 200 AND 299 THEN 'succeeded' ELSE 'dead_letter' END, - delivery.occurred_at, - delivery.occurred_at, - MIN(MAX(delivery.attempt_number, 1), 20) -FROM legacy_v10_webhook_deliveries AS delivery -LEFT JOIN legacy_v10_summaries AS summary ON summary.request_id = delivery.request_id; - -INSERT INTO cleanup_runs ( - run_id, occurred_at, policy_name, cutoff_before, deleted_count, duration_ms -) -SELECT - run_id, occurred_at, policy_name, cutoff_before, MAX(deleted_count, 0), - CASE WHEN duration_ms >= 0 THEN duration_ms ELSE NULL END -FROM legacy_v10_cleanup_runs; - -DROP TABLE legacy_v10_lifecycle_events; -DROP TABLE legacy_v10_artifact_pointers; -DROP TABLE legacy_v10_proxy_records; -DROP TABLE legacy_v10_webhook_deliveries; -DROP TABLE legacy_v10_audit_entries; -DROP TABLE legacy_v10_cleanup_runs; -DROP TABLE legacy_v10_summaries; -"#; - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use rusqlite::Connection; - - use crate::{LogStore, RealClock}; - - #[test] - fn fingerprinted_v10_store_upgrades_and_preserves_ledger_rows() { - let root = tempfile::tempdir().expect("legacy database root"); - let database = root.path().join("log_store.db"); - let connection = Connection::open(&database).expect("open legacy database"); - connection - .execute_batch(LEGACY_FIXTURE_SQL) - .expect("seed legacy database"); - drop(connection); - - let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("upgrade legacy store"); - - assert_eq!(store.schema_version(), super::super::CURRENT_VERSION); - assert!(store.get_summary("legacy-request").unwrap().is_some()); - let connection = store.conn(); - let event: (String, i64) = connection - .query_row( - "SELECT event_type, is_terminal FROM lifecycle_events WHERE event_id = 'legacy-event'", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read migrated event"); - assert_eq!(event, ("completed".to_string(), 1)); - let audit: (i64, String) = connection - .query_row( - "SELECT sequence, actor FROM audit_entries WHERE entry_id = 'legacy-audit'", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read migrated audit"); - assert_eq!(audit, (1, "logging_service".to_string())); - let default_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit'", - [], - |row| row.get(0), - ) - .expect("read migrated audit detail"); - let default_detail: serde_json::Value = - serde_json::from_str(&default_detail).expect("migrated detail is valid JSON"); - assert_eq!(default_detail["outcome"], "succeeded"); - assert_eq!(default_detail["source"], "system"); - let explicit_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit-explicit'", - [], - |row| row.get(0), - ) - .expect("read migrated explicit audit detail"); - let explicit_detail: serde_json::Value = - serde_json::from_str(&explicit_detail).expect("migrated explicit detail is valid JSON"); - assert_eq!(explicit_detail["custom"], "kept"); - assert_eq!(explicit_detail["outcome"], "failed"); - assert_eq!(explicit_detail["source"], "mesh"); - assert_eq!(explicit_detail["reason_code"], "model_load_failed"); - assert_eq!(explicit_detail["operation_id"], "op-42"); - let partial_null_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit-partial-null'", - [], - |row| row.get(0), - ) - .expect("read migrated partial-null audit detail"); - let partial_null_detail: serde_json::Value = serde_json::from_str(&partial_null_detail) - .expect("migrated partial-null detail is valid JSON"); - // `reason` and `operation_id` are NULL on the legacy row; the merge - // must not clobber the pre-existing detail_json keys with nulls. - assert_eq!(partial_null_detail["outcome"], "succeeded"); - assert_eq!(partial_null_detail["source"], "mesh"); - assert_eq!(partial_null_detail["reason_code"], "pre-existing-reason"); - assert_eq!(partial_null_detail["operation_id"], "pre-existing-op"); - let obsolete_tables: i64 = connection - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name LIKE 'legacy_v10_%'", - [], - |row| row.get(0), - ) - .expect("count legacy tables"); - assert_eq!(obsolete_tables, 0); - } - - const LEGACY_FIXTURE_SQL: &str = r#" - PRAGMA foreign_keys = ON; - CREATE TABLE summaries ( - request_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active', created_at TEXT NOT NULL, - terminal_at TEXT, route TEXT, model TEXT, provider TEXT, engine TEXT, status_code INTEGER, - error_msg TEXT, tenant_id TEXT, account_id TEXT, user_id TEXT - ); - CREATE TABLE lifecycle_events ( - event_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}', UNIQUE(request_id, event_id) - ); - CREATE TABLE artifact_pointers ( - artifact_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, kind TEXT NOT NULL, metadata_json TEXT, media_kind TEXT, checksum TEXT, - bytes INTEGER NOT NULL DEFAULT 0, version INTEGER NOT NULL DEFAULT 1, redacted INTEGER NOT NULL DEFAULT 0, - truncated INTEGER NOT NULL DEFAULT 0, stored_at TEXT, missing INTEGER NOT NULL DEFAULT 0, - corrupt INTEGER NOT NULL DEFAULT 0, UNIQUE(request_id, artifact_id) - ); - CREATE TABLE proxy_records ( - attempt_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, target TEXT NOT NULL, provider TEXT, engine TEXT, started_at TEXT, - completed_at TEXT, status_code INTEGER, error_msg TEXT, attempt_number INTEGER NOT NULL DEFAULT 0, - UNIQUE(request_id, attempt_id) - ); - CREATE TABLE audit_entries ( - entry_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, actor TEXT NOT NULL, action TEXT NOT NULL, detail_json TEXT, - source TEXT NOT NULL DEFAULT 'system', reason TEXT, result TEXT NOT NULL DEFAULT 'succeeded', - operation_id TEXT, UNIQUE(request_id, entry_id) - ); - CREATE TABLE webhook_deliveries ( - delivery_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, target_url TEXT NOT NULL, attempt_number INTEGER NOT NULL, - status_code INTEGER, response_body TEXT, error_msg TEXT, UNIQUE(request_id, delivery_id) - ); - CREATE TABLE cleanup_runs ( - run_id TEXT PRIMARY KEY, occurred_at TEXT NOT NULL, policy_name TEXT NOT NULL, - cutoff_before TEXT NOT NULL, deleted_count INTEGER NOT NULL DEFAULT 0, duration_ms INTEGER - ); - CREATE TABLE maintenance_previews (operation_id TEXT PRIMARY KEY); - CREATE TABLE maintenance_operations (operation_id TEXT PRIMARY KEY, intent_hash TEXT NOT NULL); - INSERT INTO summaries ( - request_id, state, created_at, terminal_at, route, model, provider, engine, status_code - ) VALUES ( - 'legacy-request', 'completed', '2026-08-03T03:50:26.386Z', '2026-08-03T03:50:26.387Z', - 'chat_completions', 'legacy-model', 'local', 'skippy', 200 - ); - INSERT INTO lifecycle_events (event_id, request_id, occurred_at, payload_json) - VALUES ( - 'legacy-event', 'legacy-request', '2026-08-03T03:50:26.387Z', - '{"type":"completed","status_code":200}' - ); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action) - VALUES ('legacy-audit', '2026-08-03T03:50:27Z', 'system', 'cleanup_preview'); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action, source, reason, result, operation_id, detail_json) - VALUES ( - 'legacy-audit-explicit', '2026-08-03T03:50:28Z', 'runtime', 'model_start', - 'mesh', 'model_load_failed', 'failed', 'op-42', - '{"custom":"kept"}' - ); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action, source, reason, result, operation_id, detail_json) - VALUES ( - 'legacy-audit-partial-null', '2026-08-03T03:50:29Z', 'runtime', 'model_start', - 'mesh', NULL, 'succeeded', NULL, - '{"reason_code":"pre-existing-reason","operation_id":"pre-existing-op"}' - ); - PRAGMA user_version = 10; - "#; -} diff --git a/crates/mesh-llm-log-store/src/migrations/lineage.rs b/crates/mesh-llm-log-store/src/migrations/lineage.rs new file mode 100644 index 0000000000..87d89f8e3d --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/lineage.rs @@ -0,0 +1,81 @@ +use rusqlite::Connection; +use rusqlite::types::ValueRef; + +const TABLE_NAME: &str = "_mesh_llm_log_store_lineage"; + +pub(super) fn install(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE _mesh_llm_log_store_lineage ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + epoch INTEGER NOT NULL CHECK (epoch = 1) + ) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage (id, epoch) VALUES (1, 1);", + ) +} + +pub(super) fn is_valid(connection: &Connection) -> Result { + if !has_exact_private_object(connection)? + || !has_expected_table_shape(connection)? + || !has_expected_columns(connection)? + { + return Ok(false); + } + has_expected_row(connection) +} + +fn has_exact_private_object(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 1 + AND COUNT(*) FILTER ( + WHERE type = 'table' AND name = ?1 AND tbl_name = ?1 AND sql IS NOT NULL + ) = 1 + FROM sqlite_schema + WHERE name GLOB '_mesh_llm_log_store_*'", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_table_shape(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 1 + FROM pragma_table_list + WHERE schema = 'main' AND name = ?1 AND type = 'table' + AND ncol = 2 AND wr = 1 AND strict = 0", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_columns(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 2 + AND COUNT(*) FILTER ( + WHERE cid = 0 AND name = 'id' AND type = 'INTEGER' + AND \"notnull\" = 1 AND dflt_value IS NULL AND pk = 1 AND hidden = 0 + ) = 1 + AND COUNT(*) FILTER ( + WHERE cid = 1 AND name = 'epoch' AND type = 'INTEGER' + AND \"notnull\" = 1 AND dflt_value IS NULL AND pk = 0 AND hidden = 0 + ) = 1 + FROM pragma_table_xinfo(?1)", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_row(connection: &Connection) -> Result { + let mut statement = connection.prepare("SELECT * FROM _mesh_llm_log_store_lineage")?; + if statement.column_count() != 2 { + return Ok(false); + } + let mut rows = statement.query([])?; + let Some(row) = rows.next()? else { + return Ok(false); + }; + let row_is_valid = matches!( + (row.get_ref(0)?, row.get_ref(1)?), + (ValueRef::Integer(1), ValueRef::Integer(1)) + ); + Ok(row_is_valid && rows.next()?.is_none()) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema.rs b/crates/mesh-llm-log-store/src/migrations/released_schema.rs new file mode 100644 index 0000000000..63414bfa2e --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema.rs @@ -0,0 +1,53 @@ +use rusqlite::Connection; + +mod fingerprint; + +const SOURCE_VERSIONS: [u32; 2] = [3, 11]; + +pub(super) fn source_version(connection: &Connection) -> Result, rusqlite::Error> { + let source_version = super::schema_version(connection)?; + Ok(matches_source(connection, source_version)?.then_some(source_version)) +} + +fn matches_source(connection: &Connection, source_version: u32) -> Result { + Ok(SOURCE_VERSIONS.contains(&source_version) + && super::schema_version(connection)? == source_version + && super::application_id(connection)? == 0 + && !fingerprint::has_private_objects(connection)? + && fingerprint::matches(connection)?) +} + +pub(super) fn import(connection: &Connection, source_version: u32) -> Result<(), rusqlite::Error> { + import_with_hook(connection, source_version, |_| Ok(())) +} + +fn import_with_hook( + connection: &Connection, + source_version: u32, + before_commit: impl FnOnce(&Connection) -> Result<(), rusqlite::Error>, +) -> Result<(), rusqlite::Error> { + let transaction = super::immediate_transaction(connection)?; + if !matches_source(&transaction, source_version)? { + return Err(rusqlite::Error::InvalidQuery); + } + transaction.execute_batch( + "ALTER TABLE summaries ADD COLUMN caller_endpoint_id TEXT; + ALTER TABLE summaries ADD COLUMN caller_addr TEXT; + ALTER TABLE summaries ADD COLUMN caller_path_type TEXT;", + )?; + super::lineage::install(&transaction)?; + super::set_application_id(&transaction, super::APPLICATION_ID)?; + super::set_schema_version(&transaction, super::CURRENT_VERSION)?; + before_commit(&transaction)?; + transaction.commit() +} + +#[cfg(test)] +pub(super) fn import_with_before_commit_failure( + connection: &Connection, + source_version: u32, +) -> Result<(), rusqlite::Error> { + import_with_hook(connection, source_version, |_| { + Err(rusqlite::Error::InvalidQuery) + }) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs new file mode 100644 index 0000000000..073ad6f506 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs @@ -0,0 +1,206 @@ +use rusqlite::Connection; + +mod contract; +mod create_table; +mod predicate; +mod semantic_contract; + +pub(super) fn has_private_objects(connection: &Connection) -> Result { + connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name GLOB '_mesh_llm_log_store_*')", + [], + |row| row.get(0), + ) +} + +pub(super) fn matches(connection: &Connection) -> Result { + Ok(has_exact_objects(connection)? + && has_exact_tables(connection)? + && has_exact_indexes(connection)? + && has_exact_foreign_keys(connection)? + && has_exact_create_table_semantics(connection)? + && predicate::matches(connection)?) +} + +fn has_exact_objects(connection: &Connection) -> Result { + let unexpected: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema + WHERE type IN ('view', 'trigger') AND name NOT LIKE 'sqlite_%')", + [], + |row| row.get(0), + )?; + if unexpected { + return Ok(false); + } + let tables = object_names(connection, "table")?; + let indexes = object_names(connection, "index")?; + Ok( + names_match(&tables, contract::TABLES.iter().map(|table| table.name)) + && names_match(&indexes, contract::INDEXES.iter().map(|index| index.name)), + ) +} + +fn object_names( + connection: &Connection, + object_type: &str, +) -> Result, rusqlite::Error> { + let mut statement = connection.prepare( + "SELECT name FROM sqlite_schema + WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + statement + .query_map([object_type], |row| row.get(0))? + .collect() +} + +fn names_match<'a>(actual: &[String], expected: impl Iterator) -> bool { + actual.iter().map(String::as_str).eq(expected) +} + +fn has_exact_tables(connection: &Connection) -> Result { + for table in contract::TABLES { + let shape = connection.query_row( + "SELECT ncol, wr, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name = ?1", + [table.name], + |row| { + Ok(( + row.get::<_, usize>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ); + let Ok(shape) = shape else { + return Ok(false); + }; + let columns = column_signature(connection, table.name)?; + if shape.1 + || shape.2 + || columns.split('|').count() != shape.0 + || columns != table.columns + || implicit_index_signature(connection, table.name)? != table.implicit_indexes + { + return Ok(false); + } + } + Ok(true) +} + +fn column_signature(connection: &Connection, table: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%s:%s:%d:%s:%d:%d', name, type, \"notnull\", + COALESCE(dflt_value, '-'), pk, hidden) AS signature + FROM pragma_table_xinfo(?1) ORDER BY cid + )", + [table], + |row| row.get(0), + ) +} + +fn implicit_index_signature( + connection: &Connection, + table: &str, +) -> Result { + let mut statement = connection.prepare( + "SELECT name, \"unique\", origin, partial FROM pragma_index_list(?1) + WHERE origin != 'c' ORDER BY name", + )?; + let indexes = statement + .query_map([table], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + )) + })? + .collect::, _>>()?; + indexes + .into_iter() + .map(|(name, unique, origin, partial)| { + Ok(format!( + "{table}:{name}:{}:{origin}:{}:{}", + u8::from(unique), + u8::from(partial), + index_column_signature(connection, &name)? + )) + }) + .collect::, rusqlite::Error>>() + .map(|signatures| signatures.join("\n")) +} + +fn has_exact_indexes(connection: &Connection) -> Result { + for index in contract::INDEXES { + let metadata = connection.query_row( + "SELECT \"unique\", origin, partial FROM pragma_index_list(?1) WHERE name = ?2", + [index.table, index.name], + |row| { + Ok(( + row.get::<_, bool>(0)?, + row.get::<_, String>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ); + let Ok(metadata) = metadata else { + return Ok(false); + }; + if metadata != (index.unique, "c".to_owned(), index.partial) + || index_column_signature(connection, index.name)? != index.columns + { + return Ok(false); + } + } + Ok(true) +} + +fn index_column_signature(connection: &Connection, index: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%d:%s:%d:%s', cid, COALESCE(name, '-'), \"desc\", coll) AS signature + FROM pragma_index_xinfo(?1) WHERE key = 1 ORDER BY seqno + )", + [index], + |row| row.get(0), + ) +} + +fn has_exact_foreign_keys(connection: &Connection) -> Result { + for table in contract::TABLES { + if foreign_key_signature(connection, table.name)? != table.foreign_keys { + return Ok(false); + } + } + Ok(true) +} + +fn foreign_key_signature(connection: &Connection, table: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%s:%s:%s:%s:%s:%s', \"table\", \"from\", \"to\", + on_update, on_delete, match) AS signature + FROM pragma_foreign_key_list(?1) ORDER BY id, seq + )", + [table], + |row| row.get(0), + ) +} + +fn has_exact_create_table_semantics(connection: &Connection) -> Result { + for expected in semantic_contract::TABLES { + let sql = connection.query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [expected.name], + |row| row.get::<_, String>(0), + )?; + let Some(actual) = create_table::parse(&sql) else { + return Ok(false); + }; + if !expected.matches(&actual) { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs new file mode 100644 index 0000000000..6c65d957af --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs @@ -0,0 +1,259 @@ +pub(super) struct Table { + pub(super) name: &'static str, + pub(super) columns: &'static str, + pub(super) foreign_keys: &'static str, + pub(super) implicit_indexes: &'static str, +} + +pub(super) const TABLES: &[Table] = &[ + Table { + name: "artifact_pointers", + columns: "artifact_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|kind:TEXT:1:-:0:0|metadata_json:TEXT:0:-:0:0|media_kind:TEXT:0:-:0:0|checksum:TEXT:0:-:0:0|bytes:INTEGER:1:0:0:0|version:INTEGER:1:1:0:0|redacted:INTEGER:1:0:0:0|truncated:INTEGER:1:0:0:0|stored_at:TEXT:0:-:0:0|missing:INTEGER:1:0:0:0|corrupt:INTEGER:1:0:0:0|unavailable_reason:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "artifact_pointers:sqlite_autoindex_artifact_pointers_1:1:pk:0:0:artifact_id:0:BINARY\nartifact_pointers:sqlite_autoindex_artifact_pointers_2:1:u:0:1:request_id:0:BINARY|0:artifact_id:0:BINARY", + }, + Table { + name: "audit_entries", + columns: "sequence:INTEGER:0:-:1:0|entry_id:TEXT:1:-:0:0|request_id:TEXT:0:-:0:0|occurred_at:TEXT:1:-:0:0|actor:TEXT:1:-:0:0|action:TEXT:1:-:0:0|detail_json:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:SET NULL:NONE", + implicit_indexes: "audit_entries:sqlite_autoindex_audit_entries_1:1:u:0:1:entry_id:0:BINARY\naudit_entries:sqlite_autoindex_audit_entries_2:1:u:0:2:request_id:0:BINARY|1:entry_id:0:BINARY", + }, + Table { + name: "cleanup_runs", + columns: "run_id:TEXT:0:-:1:0|occurred_at:TEXT:1:-:0:0|policy_name:TEXT:1:-:0:0|cutoff_before:TEXT:1:-:0:0|deleted_count:INTEGER:1:0:0:0|duration_ms:INTEGER:0:-:0:0", + foreign_keys: "", + implicit_indexes: "cleanup_runs:sqlite_autoindex_cleanup_runs_1:1:pk:0:0:run_id:0:BINARY", + }, + Table { + name: "lifecycle_events", + columns: "event_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|payload_json:TEXT:1:'{}':0:0|event_type:TEXT:1:'unknown':0:0|is_terminal:INTEGER:1:0:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "lifecycle_events:sqlite_autoindex_lifecycle_events_1:1:pk:0:0:event_id:0:BINARY\nlifecycle_events:sqlite_autoindex_lifecycle_events_2:1:u:0:1:request_id:0:BINARY|0:event_id:0:BINARY", + }, + Table { + name: "maintenance_operation_targets", + columns: "operation_id:TEXT:1:-:1:0|ordinal:INTEGER:1:-:0:0|request_id:TEXT:1:-:2:0", + foreign_keys: "maintenance_operations:operation_id:operation_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "maintenance_operation_targets:sqlite_autoindex_maintenance_operation_targets_1:1:pk:0:0:operation_id:0:BINARY|2:request_id:0:BINARY\nmaintenance_operation_targets:sqlite_autoindex_maintenance_operation_targets_2:1:u:0:0:operation_id:0:BINARY|1:ordinal:0:BINARY", + }, + Table { + name: "maintenance_operations", + columns: "operation_id:TEXT:0:-:1:0|action:TEXT:1:-:0:0|cutoff_before:TEXT:1:-:0:0|request_limit:INTEGER:1:-:0:0|reason:TEXT:1:-:0:0|state:TEXT:1:-:0:0|planned_requests:INTEGER:1:-:0:0|planned_events:INTEGER:1:-:0:0|planned_artifacts:INTEGER:1:-:0:0|planned_proxy_records:INTEGER:1:-:0:0|planned_database_rows:INTEGER:1:-:0:0|executed_requests:INTEGER:1:0:0:0|executed_events:INTEGER:1:0:0:0|executed_artifacts:INTEGER:1:0:0:0|executed_proxy_records:INTEGER:1:0:0:0|executed_database_rows:INTEGER:1:0:0:0|has_more:INTEGER:1:-:0:0|created_at:TEXT:1:-:0:0|completed_at:TEXT:0:-:0:0|selection_fingerprint:TEXT:1:'':0:0|artifact_files_removed:INTEGER:1:0:0:0|artifact_files_failed:INTEGER:1:0:0:0|artifact_file_failure_class:TEXT:0:-:0:0|preview_audit_id:TEXT:0:-:0:0|execution_audit_id:TEXT:0:-:0:0|cleanup_filters_json:TEXT:1:'{}':0:0", + foreign_keys: "", + implicit_indexes: "maintenance_operations:sqlite_autoindex_maintenance_operations_1:1:pk:0:0:operation_id:0:BINARY", + }, + Table { + name: "pending_artifact_deletions", + columns: "artifact_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0", + foreign_keys: "", + implicit_indexes: "pending_artifact_deletions:sqlite_autoindex_pending_artifact_deletions_1:1:pk:0:0:artifact_id:0:BINARY", + }, + Table { + name: "proxy_records", + columns: "attempt_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|target:TEXT:1:-:0:0|provider:TEXT:0:-:0:0|engine:TEXT:0:-:0:0|started_at:TEXT:0:-:0:0|completed_at:TEXT:0:-:0:0|status_code:INTEGER:0:-:0:0|error_msg:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "proxy_records:sqlite_autoindex_proxy_records_1:1:pk:0:0:attempt_id:0:BINARY\nproxy_records:sqlite_autoindex_proxy_records_2:1:u:0:1:request_id:0:BINARY|0:attempt_id:0:BINARY", + }, + Table { + name: "summaries", + columns: "request_id:TEXT:0:-:1:0|state:TEXT:1:'active':0:0|created_at:TEXT:1:-:0:0|terminal_at:TEXT:0:-:0:0|route:TEXT:0:-:0:0|model:TEXT:0:-:0:0|provider:TEXT:0:-:0:0|engine:TEXT:0:-:0:0|status_code:INTEGER:0:-:0:0|error_msg:TEXT:0:-:0:0|tenant_id:TEXT:0:-:0:0|account_id:TEXT:0:-:0:0|user_id:TEXT:0:-:0:0", + foreign_keys: "", + implicit_indexes: "summaries:sqlite_autoindex_summaries_1:1:pk:0:0:request_id:0:BINARY", + }, + Table { + name: "webhook_deliveries", + columns: "delivery_id:TEXT:0:-:1:0|request_id:TEXT:0:-:0:0|terminal_outcome:TEXT:1:-:0:0|terminal_status_code:INTEGER:0:-:0:0|occurred_at:TEXT:1:-:0:0|target_url:TEXT:1:-:0:0|attempt_number:INTEGER:1:0:0:0|status_code:INTEGER:0:-:0:0|response_body:TEXT:0:-:0:0|error_msg:TEXT:0:-:0:0|state:TEXT:1:'succeeded':0:0|created_at:TEXT:1:'':0:0|updated_at:TEXT:1:'':0:0|next_attempt_at:TEXT:0:-:0:0|lease_expires_at:TEXT:0:-:0:0|claim_generation:INTEGER:1:0:0:0|max_attempts:INTEGER:1:1:0:0|last_error_code:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:SET NULL:NONE", + implicit_indexes: "webhook_deliveries:sqlite_autoindex_webhook_deliveries_1:1:pk:0:0:delivery_id:0:BINARY\nwebhook_deliveries:sqlite_autoindex_webhook_deliveries_2:1:u:0:1:request_id:0:BINARY|0:delivery_id:0:BINARY", + }, +]; + +pub(super) struct Index { + pub(super) name: &'static str, + pub(super) table: &'static str, + pub(super) unique: bool, + pub(super) partial: bool, + pub(super) columns: &'static str, +} + +macro_rules! index { + ($name:literal, $table:literal, $columns:literal) => { + Index { + name: $name, + table: $table, + unique: false, + partial: false, + columns: $columns, + } + }; +} + +pub(super) const INDEXES: &[Index] = &[ + index!( + "idx_artifact_pointers_occurred", + "artifact_pointers", + "2:occurred_at:1:BINARY|0:artifact_id:1:BINARY" + ), + index!( + "idx_artifact_pointers_request_occurred", + "artifact_pointers", + "1:request_id:0:BINARY|2:occurred_at:0:BINARY|0:artifact_id:0:BINARY" + ), + index!( + "idx_audit_entries_actor_occurred", + "audit_entries", + "4:actor:0:BINARY|3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_audit_entries_occurred", + "audit_entries", + "3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_audit_entries_severity_occurred", + "audit_entries", + "-2:-:0:BINARY|3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_cleanup_runs_occurred", + "cleanup_runs", + "1:occurred_at:1:BINARY|0:run_id:1:BINARY" + ), + index!( + "idx_lifecycle_events_occurred", + "lifecycle_events", + "2:occurred_at:1:BINARY|0:event_id:1:BINARY" + ), + index!( + "idx_lifecycle_events_request", + "lifecycle_events", + "1:request_id:0:BINARY" + ), + index!( + "idx_lifecycle_events_request_occurred", + "lifecycle_events", + "1:request_id:0:BINARY|2:occurred_at:0:BINARY|0:event_id:0:BINARY" + ), + index!( + "idx_lifecycle_events_request_terminal", + "lifecycle_events", + "1:request_id:0:BINARY|5:is_terminal:0:BINARY" + ), + index!( + "idx_maintenance_operation_targets_operation", + "maintenance_operation_targets", + "0:operation_id:0:BINARY|1:ordinal:0:BINARY" + ), + index!( + "idx_pending_artifact_deletions_request", + "pending_artifact_deletions", + "1:request_id:0:BINARY|0:artifact_id:0:BINARY" + ), + index!( + "idx_proxy_records_engine_occurred", + "proxy_records", + "5:engine:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_occurred", + "proxy_records", + "2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_provider_occurred", + "proxy_records", + "4:provider:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_request_occurred", + "proxy_records", + "1:request_id:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_status_occurred", + "proxy_records", + "8:status_code:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_summaries_created", + "summaries", + "2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_engine_created", + "summaries", + "7:engine:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_model_created", + "summaries", + "5:model:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_provider_created", + "summaries", + "6:provider:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_route_created", + "summaries", + "4:route:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!("idx_summaries_state", "summaries", "1:state:0:BINARY"), + index!( + "idx_summaries_state_created", + "summaries", + "1:state:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_status_created", + "summaries", + "8:status_code:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_terminal_order", + "summaries", + "1:state:0:BINARY|-2:-:0:BINARY|0:request_id:0:BINARY" + ), + Index { + name: "idx_terminal_event_one_per_request", + table: "lifecycle_events", + unique: true, + partial: true, + columns: "1:request_id:0:BINARY", + }, + index!( + "idx_webhook_deliveries_eligible", + "webhook_deliveries", + "10:state:0:BINARY|13:next_attempt_at:0:BINARY|14:lease_expires_at:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY" + ), + Index { + name: "idx_webhook_deliveries_expired_lease", + table: "webhook_deliveries", + unique: false, + partial: true, + columns: "14:lease_expires_at:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY", + }, + index!( + "idx_webhook_deliveries_occurred", + "webhook_deliveries", + "4:occurred_at:1:BINARY|0:delivery_id:1:BINARY" + ), + Index { + name: "idx_webhook_deliveries_ready", + table: "webhook_deliveries", + unique: false, + partial: true, + columns: "-2:-:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY", + }, +]; + +#[cfg(test)] +mod tests { + use super::INDEXES; + + #[test] + fn indexes_are_lexicographically_ordered() { + assert!(INDEXES.windows(2).all(|pair| pair[0].name < pair[1].name)); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs new file mode 100644 index 0000000000..90836e7ab3 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs @@ -0,0 +1,212 @@ +use super::predicate::token; + +mod autoincrement; + +#[derive(Debug, Eq, PartialEq)] +pub(super) enum Collation { + Binary, + Named(String), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum Deferrability { + NotDeferrable, + Deferrable, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum InitialTiming { + Immediate, + Deferred, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct Column { + pub(super) collation: Collation, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct ForeignKey { + pub(super) columns: Vec, + pub(super) deferrability: Deferrability, + pub(super) initial_timing: InitialTiming, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct Table { + pub(super) columns: Vec, + pub(super) foreign_keys: Vec, + pub(super) autoincrement_column: Option, +} + +pub(super) fn parse(sql: &str) -> Option { + let clauses = top_level_clauses(sql)?; + let mut columns = Vec::new(); + let mut foreign_keys = Vec::new(); + let mut autoincrement_column = None; + for clause in clauses { + let tokens = token::tokenize(clause); + let first = tokens.first()?.as_str(); + if is_table_constraint(first) { + if autoincrement::contains_keyword(&tokens) { + return None; + } + if let Some(foreign_key) = table_foreign_key(&tokens)? { + foreign_keys.push(foreign_key); + } + continue; + } + + let column_name = tokens.first()?.clone(); + if let autoincrement::Clause::Column(column) = autoincrement::parse(&tokens)? + && autoincrement_column.replace(column).is_some() + { + return None; + } + columns.push(Column { + collation: column_collation(&tokens)?, + }); + if top_level_position(&tokens, "REFERENCES").is_some() { + foreign_keys.push(foreign_key(&tokens, vec![column_name])?); + } + } + Some(Table { + columns, + foreign_keys, + autoincrement_column, + }) +} + +fn is_table_constraint(first: &str) -> bool { + matches!( + first, + "CONSTRAINT" | "PRIMARY" | "UNIQUE" | "CHECK" | "FOREIGN" + ) +} + +fn column_collation(tokens: &[String]) -> Option { + let Some(position) = top_level_position(tokens, "COLLATE") else { + return Some(Collation::Binary); + }; + match tokens.get(position + 1)?.as_str() { + "BINARY" => Some(Collation::Binary), + name => Some(Collation::Named(name.to_owned())), + } +} + +fn table_foreign_key(tokens: &[String]) -> Option> { + let Some(foreign) = top_level_position(tokens, "FOREIGN") else { + return Some(None); + }; + if tokens.get(foreign + 1).map(String::as_str) != Some("KEY") + || tokens.get(foreign + 2).map(String::as_str) != Some("(") + { + return None; + } + let close = tokens[foreign + 3..] + .iter() + .position(|value| value == ")")? + + foreign + + 3; + let columns = tokens[foreign + 3..close].to_vec(); + if columns.is_empty() { + return None; + } + foreign_key(tokens, columns).map(Some) +} + +fn foreign_key(tokens: &[String], columns: Vec) -> Option { + let references = top_level_position(tokens, "REFERENCES")?; + let (deferrability, initial_timing) = foreign_key_timing(&tokens[references + 1..])?; + Some(ForeignKey { + columns, + deferrability, + initial_timing, + }) +} + +fn foreign_key_timing(tokens: &[String]) -> Option<(Deferrability, InitialTiming)> { + let mut deferrability = Deferrability::NotDeferrable; + let mut initial_timing = InitialTiming::Immediate; + let mut depth = 0_u32; + for (position, value) in tokens.iter().enumerate() { + match value.as_str() { + "(" => depth += 1, + ")" => depth = depth.checked_sub(1)?, + "DEFERRABLE" if depth == 0 => { + deferrability = if position > 0 && tokens[position - 1] == "NOT" { + Deferrability::NotDeferrable + } else { + Deferrability::Deferrable + }; + } + "INITIALLY" if depth == 0 => { + initial_timing = match tokens.get(position + 1)?.as_str() { + "IMMEDIATE" => InitialTiming::Immediate, + "DEFERRED" => InitialTiming::Deferred, + _ => return None, + }; + } + _ => {} + } + } + Some((deferrability, initial_timing)) +} + +fn top_level_position(tokens: &[String], expected: &str) -> Option { + let mut depth = 0_u32; + for (position, value) in tokens.iter().enumerate() { + match value.as_str() { + "(" => depth += 1, + ")" => depth = depth.checked_sub(1)?, + _ if depth == 0 && value == expected => return Some(position), + _ => {} + } + } + None +} + +fn top_level_clauses(sql: &str) -> Option> { + let bytes = sql.as_bytes(); + let body_start = bytes.iter().position(|value| *value == b'(')? + 1; + let mut clauses = Vec::new(); + let mut clause_start = body_start; + let mut depth = 1_u32; + let mut quote = None; + let mut position = body_start; + while position < bytes.len() { + if let Some(closing) = quote { + if bytes[position] == closing { + if closing != b']' && bytes.get(position + 1) == Some(&closing) { + position += 2; + continue; + } + quote = None; + } + position += 1; + continue; + } + match bytes[position] { + b'\'' | b'"' | b'`' => quote = Some(bytes[position]), + b'[' => quote = Some(b']'), + b'(' => depth += 1, + b')' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + clauses.push(sql[clause_start..position].trim()); + return Some(clauses); + } + } + b',' if depth == 1 => { + clauses.push(sql[clause_start..position].trim()); + clause_start = position + 1; + } + _ => {} + } + position += 1; + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs new file mode 100644 index 0000000000..86eb9537ea --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs @@ -0,0 +1,27 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) enum Clause { + Absent, + Column(String), +} + +pub(super) fn parse(tokens: &[String]) -> Option { + let positions = tokens + .iter() + .enumerate() + .filter_map(|(position, token)| (token == "AUTOINCREMENT").then_some(position)) + .collect::>(); + match positions.as_slice() { + [] => Some(Clause::Absent), + [4] if tokens.get(1).map(String::as_str) == Some("INTEGER") + && tokens.get(2).map(String::as_str) == Some("PRIMARY") + && tokens.get(3).map(String::as_str) == Some("KEY") => + { + Some(Clause::Column(tokens.first()?.clone())) + } + _ => None, + } +} + +pub(super) fn contains_keyword(tokens: &[String]) -> bool { + tokens.iter().any(|token| token == "AUTOINCREMENT") +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs new file mode 100644 index 0000000000..d67ae3820a --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs @@ -0,0 +1,99 @@ +use super::*; + +#[test] +fn parser_treats_omitted_and_explicit_binary_column_collation_equally() { + // Given + let omitted = "CREATE TABLE sample (value TEXT)"; + let explicit = "CREATE TABLE sample (value TEXT COLLATE binary)"; + + // When + let omitted = parse(omitted).expect("parse omitted collation"); + let explicit = parse(explicit).expect("parse explicit collation"); + + // Then + assert_eq!(omitted.columns, explicit.columns); + assert_eq!(omitted.columns[0].collation, Collation::Binary); +} + +#[test] +fn parser_reads_inline_and_table_foreign_key_timing() { + // Given + let sql = "CREATE TABLE sample ( + parent_id TEXT REFERENCES parents(id) NOT DEFERRABLE INITIALLY IMMEDIATE, + backup_id TEXT, + FOREIGN KEY (backup_id) REFERENCES parents(id) DEFERRABLE INITIALLY DEFERRED + )"; + + // When + let table = parse(sql).expect("parse foreign keys"); + + // Then + assert_eq!( + table.foreign_keys, + [ + ForeignKey { + columns: vec!["PARENT_ID".to_owned()], + deferrability: Deferrability::NotDeferrable, + initial_timing: InitialTiming::Immediate, + }, + ForeignKey { + columns: vec!["BACKUP_ID".to_owned()], + deferrability: Deferrability::Deferrable, + initial_timing: InitialTiming::Deferred, + }, + ] + ); +} + +#[test] +fn parser_ignores_commas_and_parentheses_inside_nested_or_quoted_regions() { + // Given + let sql = "CREATE TABLE sample ( + value TEXT DEFAULT 'kept, with (punctuation)', + state TEXT CHECK (state IN ('one', 'two')) + )"; + + // When + let table = parse(sql).expect("parse nested and quoted regions"); + + // Then + assert_eq!(table.columns.len(), 2); +} + +#[test] +fn parser_records_integer_primary_key_autoincrement_semantics() { + // Given + let autoincrement = + "CREATE TABLE sample (sequence INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)"; + let rowid_reuse = "CREATE TABLE sample (sequence INTEGER PRIMARY KEY, value TEXT)"; + + // When + let autoincrement = parse(autoincrement).expect("parse autoincrement primary key"); + let rowid_reuse = parse(rowid_reuse).expect("parse reusable rowid primary key"); + + // Then + assert_eq!( + autoincrement.autoincrement_column.as_deref(), + Some("SEQUENCE") + ); + assert_eq!(rowid_reuse.autoincrement_column, None); +} + +#[test] +fn parser_rejects_impossible_or_misplaced_autoincrement_semantics() { + const INVALID_TABLES: &[&str] = &[ + "CREATE TABLE sample (sequence TEXT PRIMARY KEY AUTOINCREMENT)", + "CREATE TABLE sample (sequence INTEGER AUTOINCREMENT PRIMARY KEY)", + "CREATE TABLE sample (sequence INTEGER, PRIMARY KEY (sequence) AUTOINCREMENT)", + "CREATE TABLE sample (sequence INTEGER PRIMARY KEY AUTOINCREMENT AUTOINCREMENT)", + "CREATE TABLE sample (first INTEGER PRIMARY KEY AUTOINCREMENT, second INTEGER PRIMARY KEY AUTOINCREMENT)", + ]; + + for sql in INVALID_TABLES { + // When + let parsed = parse(sql); + + // Then + assert_eq!(parsed, None, "accepted invalid AUTOINCREMENT clause: {sql}"); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs new file mode 100644 index 0000000000..a58452c24f --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs @@ -0,0 +1,182 @@ +use rusqlite::Connection; + +pub(super) mod token; + +struct Predicate { + object_type: &'static str, + name: &'static str, + check_count: usize, + required: &'static [&'static str], +} + +pub(super) fn matches(connection: &Connection) -> Result { + for object in PREDICATES { + let sql = connection.query_row( + "SELECT sql FROM sqlite_schema WHERE type = ?1 AND name = ?2", + [object.object_type, object.name], + |row| row.get::<_, String>(0), + )?; + let tokens = token::tokenize(&sql); + let expected_checks = object.required[..object.check_count] + .iter() + .map(|predicate| token::tokenize(predicate)) + .collect::>(); + let exact_partial = object + .required + .iter() + .find(|predicate| predicate.trim_start().starts_with("WHERE ")) + .is_none_or(|predicate| token::tail(&tokens, "WHERE") == token::tokenize(predicate)); + if token::checks(&tokens) != Some(expected_checks) + || !exact_partial + || object + .required + .iter() + .any(|predicate| !token::contains(&tokens, &token::tokenize(predicate))) + { + return Ok(false); + } + } + Ok(true) +} + +const PREDICATES: &[Predicate] = &[ + Predicate { + object_type: "table", + name: "summaries", + check_count: 1, + required: &[ + "state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')", + ], + }, + Predicate { + object_type: "table", + name: "lifecycle_events", + check_count: 1, + required: &["is_terminal IN (0, 1)", "UNIQUE(request_id, event_id)"], + }, + Predicate { + object_type: "table", + name: "artifact_pointers", + check_count: 7, + required: &[ + "bytes >= 0", + "version >= 1", + "redacted IN (0, 1)", + "truncated IN (0, 1)", + "missing IN (0, 1)", + "corrupt IN (0, 1)", + "unavailable_reason IS NULL OR unavailable_reason IN ('streaming_response_not_assembled', 'response_body_not_bounded', 'capture_content_limit_exceeded', 'capture_memory_budget_exceeded', 'artifact_capture_disabled', 'artifact_capture_failed')", + "UNIQUE(request_id, artifact_id)", + ], + }, + Predicate { + object_type: "table", + name: "proxy_records", + check_count: 0, + required: &["UNIQUE(request_id, attempt_id)"], + }, + Predicate { + object_type: "table", + name: "audit_entries", + check_count: 1, + required: &[ + "sequence > 0", + "entry_id TEXT NOT NULL UNIQUE", + "UNIQUE(request_id, entry_id)", + ], + }, + Predicate { + object_type: "table", + name: "webhook_deliveries", + check_count: 8, + required: &[ + "terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')", + "terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599", + "attempt_number BETWEEN 0 AND 20", + "status_code IS NULL OR status_code BETWEEN 100 AND 599", + "state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')", + "claim_generation >= 0", + "max_attempts BETWEEN 1 AND 20", + "last_error_code IS NULL OR last_error_code IN ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')", + "UNIQUE(request_id, delivery_id)", + ], + }, + Predicate { + object_type: "table", + name: "cleanup_runs", + check_count: 2, + required: &[ + "deleted_count >= 0", + "duration_ms IS NULL OR duration_ms >= 0", + ], + }, + Predicate { + object_type: "table", + name: "maintenance_operations", + check_count: 17, + required: &[ + "action IN ('cleanup', 'delete_one')", + "request_limit BETWEEN 1 AND 100", + "state IN ('previewed', 'completed', 'partial')", + "planned_requests >= 0", + "planned_events >= 0", + "planned_artifacts >= 0", + "planned_proxy_records >= 0", + "planned_database_rows >= 0", + "executed_requests >= 0", + "executed_events >= 0", + "executed_artifacts >= 0", + "executed_proxy_records >= 0", + "executed_database_rows >= 0", + "has_more IN (0, 1)", + "artifact_files_removed >= 0", + "artifact_files_failed >= 0", + "artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')", + ], + }, + Predicate { + object_type: "table", + name: "maintenance_operation_targets", + check_count: 1, + required: &[ + "ordinal >= 0", + "PRIMARY KEY (operation_id, request_id)", + "UNIQUE (operation_id, ordinal)", + ], + }, + Predicate { + object_type: "index", + name: "idx_summaries_terminal_order", + check_count: 0, + required: &["COALESCE(terminal_at, created_at)"], + }, + Predicate { + object_type: "index", + name: "idx_audit_entries_severity_occurred", + check_count: 0, + required: &[ + "CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END", + ], + }, + Predicate { + object_type: "index", + name: "idx_terminal_event_one_per_request", + check_count: 0, + required: &["WHERE is_terminal = 1"], + }, + Predicate { + object_type: "index", + name: "idx_webhook_deliveries_ready", + check_count: 0, + required: &[ + "COALESCE(next_attempt_at, created_at)", + "WHERE state IN ('pending', 'retry', 'manual_retry')", + ], + }, + Predicate { + object_type: "index", + name: "idx_webhook_deliveries_expired_lease", + check_count: 0, + required: &["WHERE state = 'in_flight'"], + }, +]; diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs new file mode 100644 index 0000000000..bbb641c1f6 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs @@ -0,0 +1,120 @@ +pub(in super::super) fn tokenize(sql: &str) -> Vec { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut start = 0; + while start < bytes.len() { + if bytes[start].is_ascii_whitespace() || bytes[start] == b',' { + start += 1; + continue; + } + if matches!(bytes[start], b'(' | b')') { + tokens.push(char::from(bytes[start]).to_string()); + start += 1; + continue; + } + if matches!(bytes[start], b'\'' | b'"' | b'`' | b'[') { + let end = quoted_end(bytes, start); + tokens.push(sql[start..end].to_owned()); + start = end; + continue; + } + let mut end = start + 1; + if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') { + if end < bytes.len() && matches!(bytes[end], b'=' | b'>') { + end += 1; + } + } else { + while end < bytes.len() + && !bytes[end].is_ascii_whitespace() + && !matches!(bytes[end], b',' | b'(' | b')' | b'<' | b'>' | b'=' | b'!') + { + end += 1; + } + } + tokens.push(sql[start..end].to_ascii_uppercase()); + start = end; + } + tokens +} + +fn quoted_end(bytes: &[u8], start: usize) -> usize { + let opening = bytes[start]; + let closing = if opening == b'[' { b']' } else { opening }; + let mut end = start + 1; + while end < bytes.len() { + if bytes[end] != closing { + end += 1; + continue; + } + if closing != b']' && bytes.get(end + 1) == Some(&closing) { + end += 2; + continue; + } + return end + 1; + } + bytes.len() +} + +pub(super) fn contains(actual: &[String], expected: &[String]) -> bool { + actual + .windows(expected.len()) + .any(|window| window == expected) +} + +pub(super) fn tail(tokens: &[String], start: &str) -> Vec { + tokens + .iter() + .position(|token| token == start) + .map_or_else(Vec::new, |position| tokens[position..].to_vec()) +} + +pub(super) fn checks(tokens: &[String]) -> Option>> { + let mut expressions = Vec::new(); + let mut cursor = 0; + while let Some(offset) = tokens[cursor..].iter().position(|token| token == "CHECK") { + let check = cursor + offset; + if tokens.get(check + 1).map(String::as_str) != Some("(") { + return None; + } + let mut depth = 1_u32; + let mut end = check + 2; + while depth > 0 { + match tokens.get(end).map(String::as_str) { + Some("(") => depth += 1, + Some(")") => depth -= 1, + Some(_) => {} + None => return None, + } + end += 1; + } + expressions.push(tokens[check + 2..end - 1].to_vec()); + cursor = end; + } + Some(expressions) +} + +#[cfg(test)] +mod tests { + use super::tokenize; + + #[test] + fn quoted_sql_string_is_one_case_preserving_token_when_it_contains_sql_punctuation() { + // Given + let sql = "state IN ('Active value, (pending)! It''s exact.')"; + + // When + let tokens = tokenize(sql); + + // Then + assert_eq!( + tokens, + [ + "STATE", + "IN", + "(", + "'Active value, (pending)! It''s exact.'", + ")" + ] + ); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs new file mode 100644 index 0000000000..caf095c261 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs @@ -0,0 +1,84 @@ +use super::create_table::{Collation, Deferrability, ForeignKey, InitialTiming, Table}; + +pub(super) struct TableContract { + pub(super) name: &'static str, + column_collation: Collation, + foreign_keys: &'static [ForeignKeyContract], + autoincrement_column: Option<&'static str>, +} + +struct ForeignKeyContract { + columns: &'static [&'static str], + deferrability: Deferrability, + initial_timing: InitialTiming, +} + +impl TableContract { + pub(super) fn matches(&self, table: &Table) -> bool { + table + .columns + .iter() + .all(|column| column.collation == self.column_collation) + && table.foreign_keys.len() == self.foreign_keys.len() + && table + .foreign_keys + .iter() + .zip(self.foreign_keys) + .all(|(actual, expected)| expected.matches(actual)) + && table.autoincrement_column.as_deref() == self.autoincrement_column + } +} + +impl ForeignKeyContract { + fn matches(&self, foreign_key: &ForeignKey) -> bool { + foreign_key + .columns + .iter() + .map(String::as_str) + .eq(self.columns.iter().copied()) + && foreign_key.deferrability == self.deferrability + && foreign_key.initial_timing == self.initial_timing + } +} + +macro_rules! foreign_key { + ($column:literal) => { + ForeignKeyContract { + columns: &[$column], + deferrability: Deferrability::NotDeferrable, + initial_timing: InitialTiming::Immediate, + } + }; +} + +macro_rules! table { + ($name:literal, $autoincrement_column:expr, $foreign_keys:expr) => { + TableContract { + name: $name, + column_collation: Collation::Binary, + foreign_keys: $foreign_keys, + autoincrement_column: $autoincrement_column, + } + }; +} + +pub(super) const TABLES: &[TableContract] = &[ + table!("artifact_pointers", None, &[foreign_key!("REQUEST_ID")]), + table!( + "audit_entries", + Some("SEQUENCE"), + &[foreign_key!("REQUEST_ID")] + ), + table!("cleanup_runs", None, &[]), + table!("lifecycle_events", None, &[foreign_key!("REQUEST_ID")]), + table!( + "maintenance_operation_targets", + None, + &[foreign_key!("OPERATION_ID")] + ), + table!("maintenance_operations", None, &[]), + table!("pending_artifact_deletions", None, &[]), + table!("proxy_records", None, &[foreign_key!("REQUEST_ID")]), + table!("summaries", None, &[]), + table!("webhook_deliveries", None, &[foreign_key!("REQUEST_ID")]), +]; diff --git a/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs b/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs new file mode 100644 index 0000000000..068032cdbb --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::{LogStore, RealClock}; +use std::sync::Arc; + +fn initialized_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept');", + ) +} + +fn no_op_migration(_: &Connection) -> Result<(), rusqlite::Error> { + Ok(()) +} + +fn initialize_to_version(connection: &Connection, version: u32) { + let migrations = [ + Migration { + version: 2, + apply: no_op_migration, + }, + Migration { + version: 3, + apply: no_op_migration, + }, + ]; + let migration_count = usize::try_from(version.saturating_sub(1)).expect("migration count"); + run_migrations( + connection, + MigrationPlan { + target: version, + initialize: initialized_schema, + migrations: &migrations[..migration_count], + }, + ) + .expect("initialize synthetic schema"); +} + +fn assert_inspection_preserves_database( + connection: &Connection, + target: u32, + expected: Option<(u32, u32)>, +) { + let identity = application_id(connection).expect("application identity"); + let found = schema_version(connection).expect("schema version"); + let object_count = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"); + + assert_eq!( + incompatible_schema_for_target(connection, target).expect("inspect schema"), + expected + ); + assert_eq!( + application_id(connection).expect("application identity"), + identity + ); + assert_eq!(schema_version(connection).expect("schema version"), found); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"), + object_count + ); +} + +#[test] +fn initialized_version_one_is_compatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + assert_inspection_preserves_database(&connection, 2, None); +} + +#[test] +fn initialized_version_two_is_compatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 2); + assert_inspection_preserves_database(&connection, 2, None); +} + +#[test] +fn initialized_version_three_is_incompatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 3); + assert_inspection_preserves_database(&connection, 2, Some((3, 2))); +} + +#[test] +fn nonempty_version_zero_is_incompatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialized_schema(&connection).expect("seed unknown schema"); + assert_inspection_preserves_database(&connection, 2, Some((0, 2))); +} + +#[test] +fn missing_identity_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + connection + .pragma_update(None, "application_id", 0_u32) + .expect("clear application identity"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn wrong_identity_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + connection + .pragma_update(None, "application_id", 0x4E4F5045_u32) + .expect("replace application identity"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn markerless_identified_version_one_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialized_schema(&connection).expect("seed unknown schema"); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("seed application identity"); + connection + .pragma_update(None, "user_version", 1_u32) + .expect("seed schema version"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn high_bit_header_values_are_classified_as_incompatible() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .pragma_update(None, "user_version", -2_147_483_647_i32) + .expect("seed high-bit schema version"); + connection + .pragma_update(None, "application_id", -2_147_483_646_i32) + .expect("seed high-bit application identity"); + + assert_eq!( + incompatible_schema_for_target(&connection, 2).expect("inspect high-bit schema"), + Some((0x8000_0001, 2)) + ); +} + +#[test] +fn released_schema_marker_eleven_store_upgrades_without_losing_rows() { + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + { + let connection = Connection::open(&database_path).expect("open released database"); + super::released_schema_fixture::install(&connection); + } + + let store = LogStore::open(root.path(), Arc::new(RealClock)) + .expect("upgrade released version eleven store"); + + assert_eq!(store.schema_version(), CURRENT_VERSION); + super::released_schema_fixture::assert_seeded_history(&store.conn()); + drop(store); + + let reopened = LogStore::open(root.path(), Arc::new(RealClock)) + .expect("reopen upgraded version eleven store"); + assert_eq!(reopened.schema_version(), CURRENT_VERSION); + assert_eq!( + table_columns(&reopened.conn(), "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + "caller_endpoint_id", + "caller_addr", + "caller_path_type", + ] + ); + super::released_schema_fixture::assert_seeded_history(&reopened.conn()); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/mod.rs b/crates/mesh-llm-log-store/src/migrations/tests/mod.rs new file mode 100644 index 0000000000..d188f76141 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/mod.rs @@ -0,0 +1,78 @@ +use super::*; +use rusqlite::Connection; + +mod compatibility; +mod released_schema_fixture; +mod released_schema_import; +mod runner; +mod schema_contract; +mod sqlite_autoincrement; + +pub(super) const EXPECTED_INDEXES: &[&str] = &[ + "idx_artifact_pointers_occurred", + "idx_artifact_pointers_request_occurred", + "idx_audit_entries_actor_occurred", + "idx_audit_entries_occurred", + "idx_audit_entries_severity_occurred", + "idx_cleanup_runs_occurred", + "idx_lifecycle_events_occurred", + "idx_lifecycle_events_request", + "idx_lifecycle_events_request_occurred", + "idx_lifecycle_events_request_terminal", + "idx_maintenance_operation_targets_operation", + "idx_pending_artifact_deletions_request", + "idx_proxy_records_engine_occurred", + "idx_proxy_records_occurred", + "idx_proxy_records_provider_occurred", + "idx_proxy_records_request_occurred", + "idx_proxy_records_status_occurred", + "idx_summaries_created", + "idx_summaries_engine_created", + "idx_summaries_model_created", + "idx_summaries_provider_created", + "idx_summaries_route_created", + "idx_summaries_state", + "idx_summaries_state_created", + "idx_summaries_status_created", + "idx_summaries_terminal_order", + "idx_terminal_event_one_per_request", + "idx_webhook_deliveries_eligible", + "idx_webhook_deliveries_expired_lease", + "idx_webhook_deliveries_occurred", + "idx_webhook_deliveries_ready", +]; + +fn table_columns(connection: &Connection, table: &str) -> Vec { + let mut statement = connection + .prepare(&format!("PRAGMA table_info('{table}')")) + .expect("prepare table info"); + statement + .query_map([], |row| row.get(1)) + .expect("query table info") + .collect::, _>>() + .expect("collect table columns") +} + +pub(super) fn schema_object_names(connection: &Connection, object_type: &str) -> Vec { + let mut statement = connection + .prepare( + "SELECT name FROM sqlite_master \ + WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .expect("prepare schema objects"); + statement + .query_map([object_type], |row| row.get(0)) + .expect("query schema objects") + .collect::, _>>() + .expect("collect schema objects") +} + +fn schema_sql(connection: &Connection, object_type: &str, name: &str) -> String { + connection + .query_row( + "SELECT sql FROM sqlite_master WHERE type = ?1 AND name = ?2", + [object_type, name], + |row| row.get(0), + ) + .expect("schema SQL") +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs new file mode 100644 index 0000000000..82dc1b0b08 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs @@ -0,0 +1,352 @@ +use rusqlite::Connection; + +pub(super) const SOURCE_VERSION_THREE: u32 = 3; +pub(super) const SOURCE_VERSION_ELEVEN: u32 = 11; +pub(super) const SOURCE_VERSIONS: [u32; 2] = [SOURCE_VERSION_THREE, SOURCE_VERSION_ELEVEN]; + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct SeededHistory { + summary: String, + lifecycle_event: String, + audit_entry: String, +} + +pub(super) fn connection() -> Connection { + let connection = Connection::open_in_memory().expect("open released database"); + install(&connection); + connection +} + +pub(super) fn install(connection: &Connection) { + install_at_marker(connection, SOURCE_VERSION_ELEVEN); +} + +pub(super) fn install_at_marker(connection: &Connection, marker: u32) { + assert!(SOURCE_VERSIONS.contains(&marker)); + connection + .execute_batch(RELEASED_SCHEMA) + .expect("install frozen released schema"); + connection + .pragma_update(None, "user_version", marker) + .expect("set frozen released schema marker"); +} + +pub(super) fn install_without_audit_autoincrement(connection: &Connection) { + let schema = + RELEASED_SCHEMA.replace("INTEGER PRIMARY KEY AUTOINCREMENT", "INTEGER PRIMARY KEY"); + connection + .execute_batch(&schema) + .expect("install frozen released schema without audit autoincrement"); + connection + .pragma_update(None, "user_version", SOURCE_VERSION_ELEVEN) + .expect("set frozen released schema marker"); +} + +pub(super) fn seeded_history(connection: &Connection) -> SeededHistory { + SeededHistory { + summary: connection + .query_row( + "SELECT json_array(request_id, state, created_at, terminal_at, route, model, + provider, engine, status_code, error_msg, tenant_id, account_id, + user_id) + FROM summaries WHERE request_id = 'released-request'", + [], + |row| row.get(0), + ) + .expect("snapshot released summary"), + lifecycle_event: connection + .query_row( + "SELECT json_array(event_id, request_id, occurred_at, payload_json, event_type, + is_terminal) + FROM lifecycle_events WHERE event_id = 'released-event'", + [], + |row| row.get(0), + ) + .expect("snapshot released lifecycle event"), + audit_entry: connection + .query_row( + "SELECT json_array(sequence, entry_id, request_id, occurred_at, actor, action, + detail_json) + FROM audit_entries WHERE entry_id = 'released-audit'", + [], + |row| row.get(0), + ) + .expect("snapshot released audit entry"), + } +} + +pub(super) fn assert_seeded_history(connection: &Connection) { + assert_eq!( + connection + .query_row( + "SELECT state FROM summaries WHERE request_id = 'released-request'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released summary"), + "completed" + ); + assert_eq!( + connection + .query_row( + "SELECT event_type FROM lifecycle_events WHERE event_id = 'released-event'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released lifecycle event"), + "completed" + ); + assert_eq!( + connection + .query_row( + "SELECT action FROM audit_entries WHERE entry_id = 'released-audit'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released audit entry"), + "request.completed" + ); +} + +const RELEASED_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS summaries ( + request_id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'active' + CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), + created_at TEXT NOT NULL, + terminal_at TEXT, + route TEXT, + model TEXT, + provider TEXT, + engine TEXT, + status_code INTEGER, + error_msg TEXT, + tenant_id TEXT, + account_id TEXT, + user_id TEXT +); + +CREATE INDEX IF NOT EXISTS idx_summaries_created ON summaries (created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_state ON summaries (state); + +CREATE TABLE IF NOT EXISTS lifecycle_events ( + event_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + event_type TEXT NOT NULL DEFAULT 'unknown', + is_terminal INTEGER NOT NULL DEFAULT 0 + CHECK (is_terminal IN (0, 1)), + UNIQUE(request_id, event_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_terminal_event_one_per_request +ON lifecycle_events (request_id) +WHERE is_terminal = 1; +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request ON lifecycle_events (request_id); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); + +CREATE TABLE IF NOT EXISTS artifact_pointers ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + kind TEXT NOT NULL, + metadata_json TEXT, + media_kind TEXT, + checksum TEXT, + bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), + version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), + redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), + truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), + stored_at TEXT, + missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), + corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), + UNIQUE(request_id, artifact_id) +); + +CREATE INDEX IF NOT EXISTS idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); + +CREATE TABLE IF NOT EXISTS proxy_records ( + attempt_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + target TEXT NOT NULL, + provider TEXT, + engine TEXT, + started_at TEXT, + completed_at TEXT, + status_code INTEGER, + error_msg TEXT, + UNIQUE(request_id, attempt_id) +); + +CREATE INDEX IF NOT EXISTS idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); + +CREATE TABLE IF NOT EXISTS audit_entries ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), + entry_id TEXT NOT NULL UNIQUE, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + detail_json TEXT, + UNIQUE(request_id, entry_id) +); + +CREATE INDEX IF NOT EXISTS idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); + +CREATE TABLE IF NOT EXISTS webhook_deliveries ( + delivery_id TEXT PRIMARY KEY, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + terminal_outcome TEXT NOT NULL + CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), + terminal_status_code INTEGER + CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), + occurred_at TEXT NOT NULL, + target_url TEXT NOT NULL, + attempt_number INTEGER NOT NULL DEFAULT 0 + CHECK (attempt_number BETWEEN 0 AND 20), + status_code INTEGER + CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), + response_body TEXT, + error_msg TEXT, + state TEXT NOT NULL DEFAULT 'succeeded' + CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), + created_at TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT '', + next_attempt_at TEXT, + lease_expires_at TEXT, + claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), + max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), + last_error_code TEXT + CHECK (last_error_code IS NULL OR last_error_code IN + ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), + UNIQUE(request_id, delivery_id) +); + +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_eligible +ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); + +CREATE TABLE IF NOT EXISTS cleanup_runs ( + run_id TEXT PRIMARY KEY, + occurred_at TEXT NOT NULL, + policy_name TEXT NOT NULL, + cutoff_before TEXT NOT NULL, + deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), + duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) +); + +CREATE INDEX IF NOT EXISTS idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); + +CREATE TABLE IF NOT EXISTS maintenance_operations ( + operation_id TEXT PRIMARY KEY, + action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), + cutoff_before TEXT NOT NULL, + request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), + reason TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), + planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), + planned_events INTEGER NOT NULL CHECK (planned_events >= 0), + planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), + planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), + planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), + executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), + executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), + executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), + executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), + executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), + has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), + created_at TEXT NOT NULL, + completed_at TEXT, + selection_fingerprint TEXT NOT NULL DEFAULT '', + artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), + artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), + artifact_file_failure_class TEXT + CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), + preview_audit_id TEXT, + execution_audit_id TEXT, + cleanup_filters_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS maintenance_operation_targets ( + operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + request_id TEXT NOT NULL, + PRIMARY KEY (operation_id, request_id), + UNIQUE (operation_id, ordinal) +); + +CREATE INDEX IF NOT EXISTS idx_maintenance_operation_targets_operation +ON maintenance_operation_targets (operation_id, ordinal); + +CREATE TABLE IF NOT EXISTS pending_artifact_deletions ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_pending_artifact_deletions_request +ON pending_artifact_deletions (request_id, artifact_id); + +CREATE INDEX IF NOT EXISTS idx_summaries_terminal_order +ON summaries (state, COALESCE(terminal_at, created_at), request_id); +CREATE INDEX IF NOT EXISTS idx_summaries_route_created +ON summaries (route, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_model_created +ON summaries (model, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_provider_created +ON summaries (provider, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_engine_created +ON summaries (engine, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_status_created +ON summaries (status_code, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_state_created +ON summaries (state, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_occurred +ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); +CREATE INDEX IF NOT EXISTS idx_artifact_pointers_request_occurred +ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_request_occurred +ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_provider_occurred +ON proxy_records (provider, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_engine_occurred +ON proxy_records (engine, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_status_occurred +ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_entries_actor_occurred +ON audit_entries (actor, occurred_at DESC, entry_id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_entries_severity_occurred +ON audit_entries ( + CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, + occurred_at DESC, + entry_id DESC +); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_ready +ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) +WHERE state IN ('pending', 'retry', 'manual_retry'); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_expired_lease +ON webhook_deliveries (lease_expires_at, created_at, delivery_id) +WHERE state = 'in_flight'; + +ALTER TABLE artifact_pointers ADD COLUMN unavailable_reason TEXT + CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( + 'streaming_response_not_assembled', + 'response_body_not_bounded', + 'capture_content_limit_exceeded', + 'capture_memory_budget_exceeded', + 'artifact_capture_disabled', + 'artifact_capture_failed' + )); + +INSERT INTO summaries (request_id, state, created_at, terminal_at) +VALUES ('released-request', 'completed', '2026-08-01T00:00:00Z', '2026-08-01T00:00:01Z'); +INSERT INTO lifecycle_events + (event_id, request_id, occurred_at, payload_json, event_type, is_terminal) +VALUES + ('released-event', 'released-request', '2026-08-01T00:00:01Z', '{}', 'completed', 1); +INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) +VALUES + ('released-audit', 'released-request', '2026-08-01T00:00:02Z', 'system', 'request.completed', '{}'); +"#; diff --git a/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs new file mode 100644 index 0000000000..58971de28e --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs @@ -0,0 +1,574 @@ +use super::released_schema_fixture::{ + self, SOURCE_VERSION_ELEVEN, SOURCE_VERSION_THREE, SOURCE_VERSIONS, +}; +use super::*; + +fn released_connection() -> Connection { + released_schema_fixture::connection() +} + +fn assert_released_state_is_unchanged(connection: &Connection, source_version: u32) { + assert_eq!( + schema_version(connection).expect("schema version"), + source_version + ); + assert_eq!(application_id(connection).expect("application identity"), 0); + released_schema_fixture::assert_seeded_history(connection); + assert!(!lineage::is_valid(connection).expect("inspect lineage")); +} + +fn assert_rejected_without_mutation(connection: &Connection) { + let objects_before = schema_object_names(connection, "table"); + let source_version = schema_version(connection).expect("source version"); + + // When + let compatibility = incompatible_schema(connection).expect("inspect compatibility"); + + // Then + assert_eq!(compatibility, Some((source_version, CURRENT_VERSION))); + assert_eq!(schema_object_names(connection, "table"), objects_before); + assert_released_state_is_unchanged(connection, source_version); +} + +fn assert_imported_identity_and_caller_columns(connection: &Connection) { + assert_eq!( + schema_version(connection).expect("schema version"), + CURRENT_VERSION + ); + assert_eq!( + application_id(connection).expect("application identity"), + APPLICATION_ID + ); + assert!(lineage::is_valid(connection).expect("inspect lineage")); + let caller_values = connection + .query_row( + "SELECT caller_endpoint_id, caller_addr, caller_path_type + FROM summaries WHERE request_id = 'released-request'", + [], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .expect("read imported caller columns"); + assert_eq!(caller_values, (None, None, None)); +} + +#[test] +fn released_schema_source_versions_are_classified_for_import_without_mutation() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { source_version } + ); + assert_eq!(compatibility, None); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn released_schema_is_accepted_only_for_the_current_production_target() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + + // When + let compatibility = incompatible_schema_for_target(&connection, CURRENT_VERSION + 1) + .expect("inspect non-production target compatibility"); + + // Then + assert_eq!(compatibility, Some((source_version, CURRENT_VERSION + 1))); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn released_schema_without_if_not_exists_is_classified_for_import_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, ' IF NOT EXISTS', '') + WHERE sql IS NOT NULL; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("remove optional creation clause"); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_predicates_match_independently_of_sql_whitespace() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >=\n0') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + ) + .expect("reformat released predicate"); + + // When + let classification = classify_schema(&connection).expect("classify reformatted schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_with_extra_missing_or_altered_objects_is_rejected() { + const MUTATIONS: &[&str] = &[ + "CREATE TABLE unrelated (value TEXT);", + "DROP TABLE cleanup_runs;", + "DROP INDEX idx_summaries_state; + CREATE INDEX idx_summaries_state ON summaries (created_at);", + "CREATE VIEW released_view AS SELECT request_id FROM summaries;", + "CREATE TRIGGER released_trigger AFTER INSERT ON summaries BEGIN SELECT 1; END;", + "ALTER TABLE cleanup_runs ADD COLUMN lookalike TEXT;", + "CREATE TABLE _mesh_llm_log_store_lookalike (value TEXT);", + ]; + + for mutation in MUTATIONS { + // Given + let connection = released_connection(); + connection + .execute_batch(mutation) + .expect("mutate released schema"); + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn released_schema_without_audit_autoincrement_is_rejected_without_mutation() { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_without_audit_autoincrement(&connection); + let tables_before = schema_object_names(&connection, "table"); + let summary_columns_before = table_columns(&connection, "summaries"); + let history_before = released_schema_fixture::seeded_history(&connection); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + compatibility, + Some((SOURCE_VERSION_ELEVEN, CURRENT_VERSION)) + ); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_eq!( + table_columns(&connection, "summaries"), + summary_columns_before + ); + assert_eq!( + schema_version(&connection).expect("schema version"), + SOURCE_VERSION_ELEVEN + ); + assert_eq!( + application_id(&connection).expect("application identity"), + 0 + ); + assert!(!lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + history_before + ); +} + +#[test] +fn released_schema_with_altered_foreign_key_metadata_is_rejected() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema + SET sql = replace(sql, 'ON DELETE CASCADE', 'ON DELETE SET NULL') + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("alter released foreign key"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_changed_quoted_check_literal_case_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'CHECK (state IN (''active'',', + 'CHECK (state IN (''ACTIVE'',' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("change released check literal case"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_deferrable_foreign_key_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'ON DELETE CASCADE', + 'ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED' + ) + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("defer released foreign key"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_collated_nonindexed_column_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'error_msg TEXT,', + 'error_msg TEXT COLLATE NOCASE,' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("collate released nonindexed column"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_explicit_binary_nonindexed_column_collation_is_accepted() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'error_msg TEXT,', + 'error_msg TEXT COLLATE BINARY,' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("make released binary collation explicit"); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_with_near_match_implicit_unique_index_is_rejected() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'UNIQUE(request_id, event_id)', + 'UNIQUE(request_id, event_id, occurred_at) /* UNIQUE(request_id, event_id) */' + ) + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("alter released implicit unique index"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_altered_check_or_partial_index_predicate_is_rejected() { + const MUTATIONS: &[&str] = &[ + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >= -1') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >= 0 OR bytes = -1') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'WHERE is_terminal = 1', 'WHERE is_terminal >= 0') + WHERE type = 'index' AND name = 'idx_terminal_event_one_per_request'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'WHERE is_terminal = 1', + 'WHERE is_terminal = 1 OR request_id IS NULL' + ) + WHERE type = 'index' AND name = 'idx_terminal_event_one_per_request'; + PRAGMA writable_schema = OFF;", + ]; + + for mutation in MUTATIONS { + // Given + let connection = released_connection(); + connection + .execute_batch(mutation) + .expect("alter released predicate"); + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn released_schema_with_nonzero_application_identity_is_rejected() { + // Given + let connection = released_connection(); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("set private identity"); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + compatibility, + Some((SOURCE_VERSION_ELEVEN, CURRENT_VERSION)) + ); + assert_eq!( + application_id(&connection).expect("application identity"), + APPLICATION_ID + ); + assert_eq!( + schema_version(&connection).expect("schema version"), + SOURCE_VERSION_ELEVEN + ); +} + +#[test] +fn released_schema_with_any_partial_caller_column_state_is_rejected() { + const CALLER_COLUMNS: &[&str] = &[ + "caller_endpoint_id TEXT", + "caller_addr TEXT", + "caller_path_type TEXT", + ]; + + for count in 1..=CALLER_COLUMNS.len() { + // Given + let connection = released_connection(); + for column in &CALLER_COLUMNS[..count] { + connection + .execute_batch(&format!("ALTER TABLE summaries ADD COLUMN {column};")) + .expect("add caller column"); + } + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn all_other_source_versions_are_rejected_without_mutation() { + for version in [0_u32, 1, 2, 4, 10, 12, 99] { + // Given + let connection = released_connection(); + connection + .pragma_update(None, "user_version", version) + .expect("set unsupported version"); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!(compatibility, Some((version, CURRENT_VERSION))); + assert_eq!( + schema_version(&connection).expect("schema version"), + version + ); + assert_eq!( + application_id(&connection).expect("application identity"), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM summaries", [], |row| row + .get::<_, u32>(0)) + .expect("summary count"), + 1 + ); + } +} + +#[test] +fn forced_released_schema_import_failure_rolls_back_both_source_versions() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + let tables_before = schema_object_names(&connection, "table"); + + // When + let error = released_schema::import_with_before_commit_failure(&connection, source_version) + .expect_err("force failure before import commit"); + + // Then + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_eq!( + table_columns(&connection, "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + ] + ); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn importer_revalidates_the_classified_source_version_before_writing() { + // Given + let connection = released_connection(); + let tables_before = schema_object_names(&connection, "table"); + + // When + let error = released_schema::import(&connection, SOURCE_VERSION_THREE) + .expect_err("reject mismatched classified source version"); + + // Then + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn production_import_installs_private_identity_lineage_and_nullable_columns() { + // Given + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + let connection = Connection::open(&database_path).expect("open released database"); + released_schema_fixture::install(&connection); + drop(connection); + + // When + let store = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("import released store"); + + // Then + let connection = store.conn(); + assert_imported_identity_and_caller_columns(&connection); + released_schema_fixture::assert_seeded_history(&connection); +} + +#[test] +fn production_import_accepts_published_marker_three_and_preserves_rows_after_reopen() { + // Given + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + let released_history = { + let connection = Connection::open(&database_path).expect("open released database"); + released_schema_fixture::install_at_marker(&connection, SOURCE_VERSION_THREE); + released_schema_fixture::seeded_history(&connection) + }; + + // When + let store = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("import published marker-three store"); + + // Then + { + let connection = store.conn(); + assert_imported_identity_and_caller_columns(&connection); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + released_history + ); + } + drop(store); + + let reopened = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("reopen imported marker-three store"); + let connection = reopened.conn(); + assert_imported_identity_and_caller_columns(&connection); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + released_history + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/runner.rs b/crates/mesh-llm-log-store/src/migrations/tests/runner.rs new file mode 100644 index 0000000000..13c5f73e72 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/runner.rs @@ -0,0 +1,323 @@ +use super::*; + +mod identity; + +fn synthetic_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch("CREATE TABLE baseline (value TEXT NOT NULL);") +} + +fn failing_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE initialization_partial (value TEXT); INSERT INTO missing_table VALUES (1);", + ) +} + +fn conflicting_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE baseline (value TEXT NOT NULL); + CREATE TABLE _mesh_llm_log_store_lineage ( + id INTEGER NOT NULL PRIMARY KEY, + epoch INTEGER NOT NULL + ) WITHOUT ROWID;", + ) +} + +fn migration_v2(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v2 (value TEXT NOT NULL); INSERT INTO future_v2 VALUES ('v2');", + ) +} + +fn failing_migration_v2(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v2_partial (value TEXT); INSERT INTO missing_table VALUES (2);", + ) +} + +fn failing_migration_v3(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v3_partial (value TEXT); INSERT INTO missing_table VALUES (3);", + ) +} + +fn migration_v3(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v3 (value TEXT NOT NULL); INSERT INTO future_v3 VALUES ('v3');", + ) +} + +fn version(connection: &Connection) -> u32 { + connection + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("schema version") +} + +fn application_id(connection: &Connection) -> u32 { + connection + .pragma_query_value(None, "application_id", |row| row.get(0)) + .expect("application identity") +} + +fn user_objects(connection: &Connection) -> i64 { + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'index', 'view', 'trigger') AND name NOT LIKE 'sqlite_%'", + [], + |row| row.get(0), + ) + .expect("user object count") +} + +fn plan<'a>( + target: u32, + initialize: MigrationFn, + migrations: &'a [Migration], +) -> MigrationPlan<'a> { + MigrationPlan { + target, + initialize, + migrations, + } +} + +#[test] +fn empty_version_zero_bootstraps_atomically() { + let connection = Connection::open_in_memory().expect("open database"); + + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 2); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn nonempty_version_zero_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch("CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('kept');") + .expect("seed unknown schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(user_objects(&connection), 1); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn preexisting_lineage_marker_makes_version_zero_nonfresh() { + let connection = Connection::open_in_memory().expect("open database"); + lineage::install(&connection).expect("seed lineage marker"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 1); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn production_rejects_version_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); PRAGMA application_id = 0x4D4C4F47; \ + PRAGMA user_version = 2;", + ) + .expect("seed future schema"); + + assert!(apply_migrations(&connection).is_err()); + + assert_eq!(version(&connection), 2); + assert_eq!(user_objects(&connection), 1); +} + +#[test] +fn invalid_registries_are_rejected_before_initialization() { + let invalid_registries: &[(u32, &[Migration])] = &[ + (2, &[]), + ( + 3, + &[Migration { + version: 2, + apply: migration_v2, + }], + ), + ( + 3, + &[ + Migration { + version: 3, + apply: migration_v3, + }, + Migration { + version: 2, + apply: migration_v2, + }, + ], + ), + ( + 3, + &[ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 2, + apply: migration_v2, + }, + ], + ), + ( + 2, + &[Migration { + version: 1, + apply: migration_v2, + }], + ), + ( + 3, + &[ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: migration_v3, + }, + Migration { + version: 4, + apply: migration_v3, + }, + ], + ), + ]; + + for (target, registry) in invalid_registries { + let connection = Connection::open_in_memory().expect("open database"); + assert!( + run_migrations( + &connection, + plan(*target, synthetic_initial_schema, registry), + ) + .is_err() + ); + assert_eq!(version(&connection), 0); + assert_eq!(user_objects(&connection), 0); + } +} + +#[test] +fn synthetic_version_two_step_commits_schema_data_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + + run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)) + .expect("migrate to v2"); + + assert_eq!(version(&connection), 2); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 3); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v2", [], |row| row + .get::<_, String>(0)) + .expect("v2 row"), + "v2" + ); +} + +#[test] +fn failed_version_two_step_rolls_back_its_schema_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: failing_migration_v2, + }]; + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 2); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn committed_version_two_resumes_with_corrected_version_three() { + let connection = Connection::open_in_memory().expect("open database"); + let failing = [ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: failing_migration_v3, + }, + ]; + assert!(run_migrations(&connection, plan(3, synthetic_initial_schema, &failing)).is_err()); + assert_eq!(version(&connection), 2); + assert_eq!(user_objects(&connection), 3); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + + let corrected = [ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: migration_v3, + }, + ]; + run_migrations(&connection, plan(3, synthetic_initial_schema, &corrected)).expect("resume v3"); + + assert_eq!(version(&connection), 3); + assert_eq!(user_objects(&connection), 4); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v3", [], |row| row + .get::<_, String>(0)) + .expect("v3 row"), + "v3" + ); +} + +#[test] +fn failing_initialization_rolls_back_schema_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + + assert!(run_migrations(&connection, plan(1, failing_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 0); +} + +#[test] +fn initialization_lineage_conflict_rolls_back_every_object_and_pragma() { + let connection = Connection::open_in_memory().expect("open database"); + + assert!(run_migrations(&connection, plan(1, conflicting_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 0); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs b/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs new file mode 100644 index 0000000000..241d164725 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs @@ -0,0 +1,192 @@ +use super::*; + +#[test] +fn initialized_version_one_is_a_data_preserving_noop() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + connection + .execute("INSERT INTO baseline VALUES ('kept')", []) + .expect("seed current schema data"); + + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("reopen schema"); + + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM baseline", [], |row| row + .get::<_, String>(0)) + .expect("preserved row"), + "kept" + ); +} + +#[test] +fn markerless_identified_version_one_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;", + ) + .expect("seed forged schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 1); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn nonzero_version_without_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('kept'); \ + PRAGMA user_version = 1;", + ) + .expect("seed foreign schema"); + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), 0); + assert!(!schema_object_names(&connection, "table").contains(&"future_v2".to_owned())); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn nonzero_version_with_wrong_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); PRAGMA application_id = 0x4E4F5045; \ + PRAGMA user_version = 1;", + ) + .expect("seed foreign schema"); + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), 0x4E4F5045); + assert_eq!(user_objects(&connection), 1); + assert!(!schema_object_names(&connection, "table").contains(&"future_v2".to_owned())); +} + +#[test] +fn empty_version_zero_with_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("seed identity without schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 0); +} + +#[test] +fn identified_version_one_advances_to_future_target() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + + run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)) + .expect("migrate identified schema"); + + assert_eq!(version(&connection), 2); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v2", [], |row| row + .get::<_, String>(0)) + .expect("v2 row"), + "v2" + ); +} + +#[test] +fn malformed_lineage_states_are_rejected_without_migration() { + const CASES: &[&str] = &[ + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 2);", + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL, extra INTEGER) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 1, 0);", + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 1); + INSERT INTO _mesh_llm_log_store_lineage VALUES (2, 1);", + "CREATE VIEW _mesh_llm_log_store_lineage AS SELECT 1 AS id, 1 AS epoch;", + ]; + for marker_sql in CASES { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch(&format!( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + {marker_sql} + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;" + )) + .expect("seed malformed lineage"); + let objects_before = user_objects(&connection); + + let error = run_migrations(&connection, plan(1, synthetic_initial_schema, &[])) + .expect_err("reject malformed lineage"); + + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), objects_before); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + } +} + +#[test] +fn extra_private_object_invalidates_an_initialized_lineage() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + connection + .execute_batch("CREATE VIEW _mesh_llm_log_store_extra AS SELECT 1 AS value;") + .expect("seed extra private object"); + + let error = run_migrations(&connection, plan(1, synthetic_initial_schema, &[])) + .expect_err("reject extra private object"); + + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert!( + schema_object_names(&connection, "view").contains(&"_mesh_llm_log_store_extra".to_owned()) + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs b/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs new file mode 100644 index 0000000000..cf0403a206 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs @@ -0,0 +1,129 @@ +use super::*; + +const EXPECTED_TABLES: &[&str] = &[ + "artifact_pointers", + "audit_entries", + "cleanup_runs", + "lifecycle_events", + "maintenance_operation_targets", + "maintenance_operations", + "pending_artifact_deletions", + "proxy_records", + "summaries", + "webhook_deliveries", +]; + +#[test] +fn fresh_database_is_exactly_complete_current_schema() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch("PRAGMA foreign_keys = ON;") + .expect("enable foreign keys"); + + apply_migrations(&connection).expect("apply fresh schema"); + + assert_eq!(CURRENT_VERSION, 1); + assert!(MIGRATIONS.is_empty()); + assert_eq!( + connection + .pragma_query_value(None, "application_id", |row| row.get::<_, u32>(0)) + .expect("application identity"), + APPLICATION_ID + ); + assert_eq!( + connection + .pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .expect("schema version"), + 1 + ); + let application_tables = schema_object_names(&connection, "table") + .into_iter() + .filter(|name| name != "_mesh_llm_log_store_lineage") + .collect::>(); + assert_eq!(application_tables, EXPECTED_TABLES); + assert_eq!( + private_schema_object_names(&connection), + ["_mesh_llm_log_store_lineage"] + ); + assert!(lineage::is_valid(&connection).expect("inspect lineage marker")); + assert!( + schema_sql(&connection, "table", "_mesh_llm_log_store_lineage").contains("WITHOUT ROWID") + ); + assert_eq!(schema_object_names(&connection, "index"), EXPECTED_INDEXES); + assert_eq!( + table_columns(&connection, "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + "caller_endpoint_id", + "caller_addr", + "caller_path_type", + ] + ); + assert_eq!( + table_columns(&connection, "artifact_pointers") + .last() + .map(String::as_str), + Some("unavailable_reason") + ); + assert!(schema_sql(&connection, "table", "lifecycle_events").contains("is_terminal IN (0, 1)")); + assert!( + schema_sql(&connection, "index", "idx_terminal_event_one_per_request") + .contains("WHERE is_terminal = 1") + ); + assert!( + schema_sql(&connection, "table", "webhook_deliveries") + .contains("terminal_status_code BETWEEN 100 AND 599") + ); + assert!(!schema_sql(&connection, "table", "pending_artifact_deletions").contains("REFERENCES")); +} + +fn private_schema_object_names(connection: &Connection) -> Vec { + let mut statement = connection + .prepare( + "SELECT name FROM sqlite_schema + WHERE name GLOB '_mesh_llm_log_store_*' ORDER BY name", + ) + .expect("prepare private schema objects"); + statement + .query_map([], |row| row.get(0)) + .expect("query private schema objects") + .collect::, _>>() + .expect("collect private schema objects") +} + +#[test] +fn initial_webhook_schema_bounds_terminal_status_codes() { + let connection = Connection::open_in_memory().expect("open database"); + apply_migrations(&connection).expect("apply schema"); + connection + .execute( + "INSERT INTO summaries (request_id, created_at) VALUES ('request-1', '2026-08-04T12:00:00Z')", + [], + ) + .expect("seed request"); + + let error = connection + .execute( + "INSERT INTO webhook_deliveries \ + (delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, target_url, attempt_number) \ + VALUES ('delivery-1', 'request-1', 'completed', 99, '2026-08-04T12:00:00Z', 'configured_webhook', 0)", + [], + ) + .expect_err("reject status below 100"); + + assert!( + matches!(error, rusqlite::Error::SqliteFailure(_, Some(message)) if message.contains("terminal_status_code")) + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs b/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs new file mode 100644 index 0000000000..79419f934d --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs @@ -0,0 +1,42 @@ +use rusqlite::Connection; + +#[test] +fn deleting_maximum_rowid_reuses_it_without_autoincrement_but_not_with_autoincrement() { + // Given + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch( + "CREATE TABLE reusable (sequence INTEGER PRIMARY KEY); + CREATE TABLE monotonic (sequence INTEGER PRIMARY KEY AUTOINCREMENT); + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + DELETE FROM reusable WHERE sequence = 3; + DELETE FROM monotonic WHERE sequence = 3;", + ) + .expect("seed and delete maximum rowids"); + + // When + connection + .execute_batch( + "INSERT INTO reusable DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES;", + ) + .expect("insert replacement rows"); + + // Then + let reusable = connection + .query_row("SELECT MAX(sequence) FROM reusable", [], |row| { + row.get::<_, u64>(0) + }) + .expect("read reusable sequence"); + let monotonic = connection + .query_row("SELECT MAX(sequence) FROM monotonic", [], |row| { + row.get::<_, u64>(0) + }) + .expect("read monotonic sequence"); + assert_eq!((reusable, monotonic), (3, 4)); +} diff --git a/crates/mesh-llm-log-store/src/query/mod.rs b/crates/mesh-llm-log-store/src/query/mod.rs index 9aa9610d3b..4edabe0f66 100644 --- a/crates/mesh-llm-log-store/src/query/mod.rs +++ b/crates/mesh-llm-log-store/src/query/mod.rs @@ -4,9 +4,12 @@ //! parsing, active-registry merging, and artifact-content authorization belong //! to later host/API layers. +mod records; mod related; mod requests; +pub use records::{RequestRecord, RequestRecordWithCaller}; + use chrono::DateTime; use crate::LogStoreError; @@ -67,6 +70,8 @@ pub struct RequestQuery { pub from: Option, pub to: Option, pub route: Option, + pub exclude_route: Option, + pub exclude_route_prefix: Option, pub model: Option, pub provider: Option, pub engine: Option, @@ -79,7 +84,14 @@ impl RequestQuery { pub fn validate(&self) -> Result<(), LogStoreError> { validate_limit(self.limit)?; validate_time_range(self.from.as_deref(), self.to.as_deref())?; - for value in [&self.route, &self.model, &self.provider, &self.engine] { + for value in [ + &self.route, + &self.exclude_route, + &self.exclude_route_prefix, + &self.model, + &self.provider, + &self.engine, + ] { validate_filter(value.as_deref())?; } Ok(()) @@ -118,19 +130,6 @@ impl ProxyQuery { } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RequestRecord { - pub request_id: String, - pub outcome: String, - pub created_at: String, - pub terminal_at: Option, - pub route: Option, - pub model: Option, - pub provider: Option, - pub engine: Option, - pub status_code: Option, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub struct EventRecord { pub event_id: String, diff --git a/crates/mesh-llm-log-store/src/query/records.rs b/crates/mesh-llm-log-store/src/query/records.rs new file mode 100644 index 0000000000..46638be485 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/records.rs @@ -0,0 +1,27 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RequestRecord { + pub request_id: String, + pub outcome: String, + pub created_at: String, + pub terminal_at: Option, + pub route: Option, + pub model: Option, + pub provider: Option, + pub engine: Option, + pub status_code: Option, +} + +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RequestRecordWithCaller { + pub request: RequestRecord, + pub caller_endpoint_id: Option, + pub caller_addr: Option, + pub caller_path_type: Option, +} + +impl From for RequestRecord { + fn from(detailed: RequestRecordWithCaller) -> Self { + detailed.request + } +} diff --git a/crates/mesh-llm-log-store/src/query/related.rs b/crates/mesh-llm-log-store/src/query/related.rs index cb93f16876..54e91d57b1 100644 --- a/crates/mesh-llm-log-store/src/query/related.rs +++ b/crates/mesh-llm-log-store/src/query/related.rs @@ -2,13 +2,20 @@ use std::collections::BTreeMap; use rusqlite::types::Value; -use super::requests::{Conditions, CursorPosition, query_page, validate_cursor_position}; +mod projection; +mod selection; + +use super::requests::selection::{ + Conditions, CursorPosition, query_page, validate_cursor_position, +}; use super::{ ArtifactRecord, EventRecord, PageQuery, ProxyQuery, ProxyRecord, QueryPage, validate_identifier, }; use crate::cursor::decode_ordering_cursor; use crate::{LogStore, LogStoreError}; +use projection::{artifact_record, event_record, proxy_record}; + impl LogStore { /// Return lifecycle records only for the named durable request. pub fn query_events( @@ -176,130 +183,9 @@ impl LogStore { |record| (&record.occurred_at, &record.attempt_id), ) } - - fn query_related_page( - &self, - related: RelatedQuery<'_>, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - cursor_fields: impl Fn(&T) -> (&str, &str), - ) -> Result, LogStoreError> { - validate_identifier(related.request_id)?; - related.page.validate()?; - let conditions = Conditions { - sql: " WHERE request_id = ?".to_string(), - }; - let mut values = vec![Value::Text(related.request_id.to_string())]; - let connection = self.conn(); - if let Some(cursor) = &related.page.cursor { - let (timestamp, id) = decode_ordering_cursor(cursor)?; - validate_cursor_position( - &connection, - CursorPosition { - table: related.table, - timestamp_column: "occurred_at", - id_column: related.id_column, - timestamp: ×tamp, - id: &id, - conditions: &conditions, - values: &values, - }, - )?; - values.push(Value::Text(timestamp)); - values.push(Value::Text(id)); - } - let mut sql = format!("SELECT {} FROM {}", related.columns, related.table); - conditions.append_to_sql(&mut sql); - if related.page.cursor.is_some() { - sql.push_str(&format!( - " AND (occurred_at, {}) {} (?, ?)", - related.id_column, - related.page.sort.cursor_operator() - )); - } - sql.push_str(&format!( - " ORDER BY occurred_at {}, {} {} LIMIT ?", - related.page.sort.sql_order(), - related.id_column, - related.page.sort.sql_order() - )); - values.push(Value::Integer( - i64::try_from(related.page.limit + 1) - .map_err(|_| LogStoreError::InvalidQuery("limit is out of range".to_string()))?, - )); - query_page( - &connection, - sql, - values, - related.page.limit, - map, - cursor_fields, - ) - } - - fn query_related_for_requests( - &self, - request_ids: &[String], - per_request_limit: usize, - related: RelatedBatchQuery, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - request_id: impl Fn(&T) -> &str, - ) -> Result>, LogStoreError> { - if request_ids.is_empty() { - return Ok(BTreeMap::new()); - } - if request_ids.len() > super::MAX_QUERY_LIMIT { - return Err(LogStoreError::InvalidQuery(format!( - "request owner count must be at most {}", - super::MAX_QUERY_LIMIT - ))); - } - super::validate_limit(per_request_limit)?; - for owner in request_ids { - validate_identifier(owner)?; - } - - let placeholders = std::iter::repeat_n("?", request_ids.len()) - .collect::>() - .join(", "); - let sql = format!( - "SELECT {columns} FROM (\ - SELECT {columns}, ROW_NUMBER() OVER (\ - PARTITION BY request_id ORDER BY occurred_at ASC, {id_column} ASC\ - ) AS owner_row_number \ - FROM {table} WHERE request_id IN ({placeholders})\ - ) WHERE owner_row_number <= ? \ - ORDER BY request_id ASC, occurred_at ASC, {id_column} ASC", - columns = related.columns, - id_column = related.id_column, - table = related.table, - ); - let mut values = request_ids - .iter() - .cloned() - .map(Value::Text) - .collect::>(); - values.push(Value::Integer(i64::try_from(per_request_limit).map_err( - |_| LogStoreError::InvalidQuery("limit is out of range".to_string()), - )?)); - let connection = self.conn(); - let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let rows = statement - .query_map(rusqlite::params_from_iter(values), map) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - let mut grouped: BTreeMap> = BTreeMap::new(); - for row in rows { - grouped - .entry(request_id(&row).to_owned()) - .or_default() - .push(row); - } - Ok(grouped) - } } -struct RelatedQuery<'a> { +pub(super) struct RelatedQuery<'a> { table: &'static str, columns: &'static str, id_column: &'static str, @@ -307,49 +193,8 @@ struct RelatedQuery<'a> { page: &'a PageQuery, } -struct RelatedBatchQuery { +pub(super) struct RelatedBatchQuery { table: &'static str, columns: &'static str, id_column: &'static str, } - -fn event_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(EventRecord { - event_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - payload_json: row.get(3)?, - }) -} - -fn artifact_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(ArtifactRecord { - artifact_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - kind: row.get(3)?, - media_kind: row.get(4)?, - checksum: row.get(5)?, - bytes: row.get(6)?, - version: row.get(7)?, - redacted: row.get::<_, i32>(8)? != 0, - truncated: row.get::<_, i32>(9)? != 0, - missing: row.get::<_, i32>(10)? != 0, - corrupt: row.get::<_, i32>(11)? != 0, - unavailable_reason: row.get(12)?, - }) -} - -fn proxy_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(ProxyRecord { - attempt_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - target: row.get(3)?, - provider: row.get(4)?, - engine: row.get(5)?, - started_at: row.get(6)?, - completed_at: row.get(7)?, - status_code: row.get(8)?, - }) -} diff --git a/crates/mesh-llm-log-store/src/query/related/projection.rs b/crates/mesh-llm-log-store/src/query/related/projection.rs new file mode 100644 index 0000000000..83f54713d7 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/related/projection.rs @@ -0,0 +1,42 @@ +use super::super::{ArtifactRecord, EventRecord, ProxyRecord}; + +pub(super) fn event_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(EventRecord { + event_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + payload_json: row.get(3)?, + }) +} + +pub(super) fn artifact_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ArtifactRecord { + artifact_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + kind: row.get(3)?, + media_kind: row.get(4)?, + checksum: row.get(5)?, + bytes: row.get(6)?, + version: row.get(7)?, + redacted: row.get::<_, i32>(8)? != 0, + truncated: row.get::<_, i32>(9)? != 0, + missing: row.get::<_, i32>(10)? != 0, + corrupt: row.get::<_, i32>(11)? != 0, + unavailable_reason: row.get(12)?, + }) +} + +pub(super) fn proxy_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ProxyRecord { + attempt_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + target: row.get(3)?, + provider: row.get(4)?, + engine: row.get(5)?, + started_at: row.get(6)?, + completed_at: row.get(7)?, + status_code: row.get(8)?, + }) +} diff --git a/crates/mesh-llm-log-store/src/query/related/selection.rs b/crates/mesh-llm-log-store/src/query/related/selection.rs new file mode 100644 index 0000000000..9eb2b6e551 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/related/selection.rs @@ -0,0 +1,133 @@ +use std::collections::BTreeMap; + +use rusqlite::types::Value; + +use super::{RelatedBatchQuery, RelatedQuery}; +use crate::cursor::decode_ordering_cursor; +use crate::query::requests::selection::{ + Conditions, CursorPosition, query_page, validate_cursor_position, +}; +use crate::query::{QueryPage, validate_identifier}; +use crate::{LogStore, LogStoreError}; + +impl LogStore { + pub(super) fn query_related_page( + &self, + related: RelatedQuery<'_>, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + cursor_fields: impl Fn(&T) -> (&str, &str), + ) -> Result, LogStoreError> { + validate_identifier(related.request_id)?; + related.page.validate()?; + let conditions = Conditions { + sql: " WHERE request_id = ?".to_string(), + }; + let mut values = vec![Value::Text(related.request_id.to_string())]; + let connection = self.conn(); + if let Some(cursor) = &related.page.cursor { + let (timestamp, id) = decode_ordering_cursor(cursor)?; + validate_cursor_position( + &connection, + CursorPosition { + table: related.table, + timestamp_column: "occurred_at", + id_column: related.id_column, + timestamp: ×tamp, + id: &id, + conditions: &conditions, + values: &values, + }, + )?; + values.push(Value::Text(timestamp)); + values.push(Value::Text(id)); + } + let mut sql = format!("SELECT {} FROM {}", related.columns, related.table); + conditions.append_to_sql(&mut sql); + if related.page.cursor.is_some() { + sql.push_str(&format!( + " AND (occurred_at, {}) {} (?, ?)", + related.id_column, + related.page.sort.cursor_operator() + )); + } + sql.push_str(&format!( + " ORDER BY occurred_at {}, {} {} LIMIT ?", + related.page.sort.sql_order(), + related.id_column, + related.page.sort.sql_order() + )); + values.push(Value::Integer( + i64::try_from(related.page.limit + 1) + .map_err(|_| LogStoreError::InvalidQuery("limit is out of range".to_string()))?, + )); + query_page( + &connection, + sql, + values, + related.page.limit, + map, + cursor_fields, + ) + } + + pub(super) fn query_related_for_requests( + &self, + request_ids: &[String], + per_request_limit: usize, + related: RelatedBatchQuery, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + request_id: impl Fn(&T) -> &str, + ) -> Result>, LogStoreError> { + if request_ids.is_empty() { + return Ok(BTreeMap::new()); + } + if request_ids.len() > super::super::MAX_QUERY_LIMIT { + return Err(LogStoreError::InvalidQuery(format!( + "request owner count must be at most {}", + super::super::MAX_QUERY_LIMIT + ))); + } + super::super::validate_limit(per_request_limit)?; + for owner in request_ids { + validate_identifier(owner)?; + } + let placeholders = std::iter::repeat_n("?", request_ids.len()) + .collect::>() + .join(", "); + let sql = format!( + "SELECT {columns} FROM (\ + SELECT {columns}, ROW_NUMBER() OVER (\ + PARTITION BY request_id ORDER BY occurred_at ASC, {id_column} ASC\ + ) AS owner_row_number \ + FROM {table} WHERE request_id IN ({placeholders})\ + ) WHERE owner_row_number <= ? \ + ORDER BY request_id ASC, occurred_at ASC, {id_column} ASC", + columns = related.columns, + id_column = related.id_column, + table = related.table, + ); + let mut values = request_ids + .iter() + .cloned() + .map(Value::Text) + .collect::>(); + values.push(Value::Integer(i64::try_from(per_request_limit).map_err( + |_| LogStoreError::InvalidQuery("limit is out of range".to_string()), + )?)); + let connection = self.conn(); + let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let rows = statement + .query_map(rusqlite::params_from_iter(values), map) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + let mut grouped = BTreeMap::new(); + for row in rows { + grouped + .entry(request_id(&row).to_owned()) + .or_insert_with(Vec::new) + .push(row); + } + Ok(grouped) + } +} diff --git a/crates/mesh-llm-log-store/src/query/requests.rs b/crates/mesh-llm-log-store/src/query/requests.rs index bf3ab98e6a..b05c73f489 100644 --- a/crates/mesh-llm-log-store/src/query/requests.rs +++ b/crates/mesh-llm-log-store/src/query/requests.rs @@ -1,41 +1,51 @@ -use rusqlite::types::Value; +mod projection; +pub(super) mod selection; -use mesh_llm_events::logging::timestamp::canonical_logging_timestamp; +use rusqlite::types::Value; -use super::{QueryPage, RequestQuery, RequestRecord}; -use crate::cursor::{decode_ordering_cursor, encode_cursor}; +use super::{QueryPage, RequestQuery, RequestRecord, RequestRecordWithCaller}; +use crate::cursor::decode_ordering_cursor; use crate::{LogStore, LogStoreError}; -const REQUEST_COLUMNS: &str = - "request_id, state, created_at, terminal_at, route, model, provider, engine, status_code"; +use projection::{REQUEST_COLUMNS, request_record_with_caller}; +use selection::{CursorPosition, query_page, validate_cursor_position}; + const REQUEST_ID_BATCH_SIZE: usize = 100; impl LogStore { - /// Return one durable request summary without exposing internal identity or - /// error columns. Active-state merging is intentionally owned by the host. pub fn query_request(&self, request_id: &str) -> Result, LogStoreError> { + self.query_request_with_caller(request_id) + .map(|record| record.map(Into::into)) + } + + pub fn query_request_with_caller( + &self, + request_id: &str, + ) -> Result, LogStoreError> { let connection = self.conn(); let sql = format!("SELECT {REQUEST_COLUMNS} FROM summaries WHERE request_id = ?"); - match connection.query_row(&sql, [request_id], request_record) { + match connection.query_row(&sql, [request_id], request_record_with_caller) { Ok(record) => Ok(Some(record)), Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), } } - /// Return durable summaries for a bounded set of request IDs. - /// - /// The caller may supply more IDs than SQLite's conservative parameter - /// budget because this method splits them into fixed-size statements while - /// holding the connection only once. Missing IDs are omitted. pub fn query_requests_by_ids( &self, request_ids: &[String], ) -> Result, LogStoreError> { + self.query_requests_by_ids_with_caller(request_ids) + .map(|records| records.into_iter().map(Into::into).collect()) + } + + pub fn query_requests_by_ids_with_caller( + &self, + request_ids: &[String], + ) -> Result, LogStoreError> { if request_ids.is_empty() { return Ok(Vec::new()); } - let connection = self.conn(); let mut records = Vec::with_capacity(request_ids.len()); for request_ids in request_ids.chunks(REQUEST_ID_BATCH_SIZE) { @@ -47,7 +57,10 @@ impl LogStore { ); let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; let rows = statement - .query_map(rusqlite::params_from_iter(request_ids), request_record) + .query_map( + rusqlite::params_from_iter(request_ids), + request_record_with_caller, + ) .map_err(LogStoreError::Sqlite)?; records.extend( rows.collect::, _>>() @@ -57,17 +70,24 @@ impl LogStore { Ok(records) } - /// List durable summaries by a stable `(created_at, request_id)` keyset. - /// A valid cursor must name a row in the exact current filter scope, so a - /// forged pair and a cursor retained away both fail deterministically. pub fn query_requests( &self, query: &RequestQuery, ) -> Result, LogStoreError> { + self.query_requests_with_caller(query) + .map(|page| QueryPage { + items: page.items.into_iter().map(Into::into).collect(), + next_cursor: page.next_cursor, + }) + } + + pub fn query_requests_with_caller( + &self, + query: &RequestQuery, + ) -> Result, LogStoreError> { query.validate()?; - let (conditions, mut values) = request_conditions(query); + let (conditions, mut values) = selection::request_conditions(query); let connection = self.conn(); - if let Some(cursor) = &query.cursor { let (timestamp, request_id) = decode_ordering_cursor(cursor)?; validate_cursor_position( @@ -85,7 +105,6 @@ impl LogStore { values.push(Value::Text(timestamp)); values.push(Value::Text(request_id)); } - let mut sql = format!("SELECT {REQUEST_COLUMNS} FROM summaries"); conditions.append_to_sql(&mut sql); if query.cursor.is_some() { @@ -107,138 +126,8 @@ impl LogStore { sql, values, query.limit, - request_record, - |record| (&record.created_at, &record.request_id), + request_record_with_caller, + |record| (&record.request.created_at, &record.request.request_id), ) } } - -pub(super) struct Conditions { - pub(super) sql: String, -} - -impl Conditions { - pub(super) fn append_to_sql(&self, sql: &mut String) { - sql.push_str(&self.sql); - } -} - -fn request_conditions(query: &RequestQuery) -> (Conditions, Vec) { - let mut sql = String::new(); - let mut values = Vec::new(); - if let Some(from) = &query.from { - sql.push_str(" WHERE created_at >= ?"); - values.push(Value::Text(normalize_timestamp(from))); - } else { - sql.push_str(" WHERE 1 = 1"); - } - if let Some(to) = &query.to { - sql.push_str(" AND created_at <= ?"); - values.push(Value::Text(normalize_timestamp(to))); - } - for (column, value) in [ - ("route", &query.route), - ("model", &query.model), - ("provider", &query.provider), - ("engine", &query.engine), - ] { - if let Some(value) = value { - sql.push_str(&format!(" AND {column} = ?")); - values.push(Value::Text(value.clone())); - } - } - if let Some(status_code) = query.status_code { - sql.push_str(" AND status_code = ?"); - values.push(Value::Integer(i64::from(status_code))); - } - if let Some(outcome) = query.outcome { - sql.push_str(" AND state = ?"); - values.push(Value::Text(outcome.as_str().to_string())); - } - let conditions = Conditions { sql }; - (conditions, values) -} - -pub(super) struct CursorPosition<'a> { - pub(super) table: &'static str, - pub(super) timestamp_column: &'static str, - pub(super) id_column: &'static str, - pub(super) timestamp: &'a str, - pub(super) id: &'a str, - pub(super) conditions: &'a Conditions, - pub(super) values: &'a [Value], -} - -pub(super) fn validate_cursor_position( - connection: &rusqlite::Connection, - position: CursorPosition<'_>, -) -> Result<(), LogStoreError> { - let mut sql = format!( - "SELECT 1 FROM {} WHERE {} = ? AND {} = ?", - position.table, position.timestamp_column, position.id_column - ); - let mut parameters = vec![ - Value::Text(position.timestamp.to_string()), - Value::Text(position.id.to_string()), - ]; - if position.conditions.sql.starts_with(" WHERE ") { - sql.push_str(" AND "); - sql.push_str(&position.conditions.sql[7..]); - } - // `values` is supplied separately so callers cannot accidentally validate - // the cursor with a different set of filters than their page query. - parameters.extend(position.values.iter().cloned()); - match connection.query_row(&sql, rusqlite::params_from_iter(parameters.iter()), |_| { - Ok(()) - }) { - Ok(()) => Ok(()), - Err(rusqlite::Error::QueryReturnedNoRows) => Err(LogStoreError::CursorInvalid), - Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), - } -} - -pub(super) fn query_page( - connection: &rusqlite::Connection, - sql: String, - values: Vec, - limit: usize, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - cursor_fields: impl Fn(&T) -> (&str, &str), -) -> Result, LogStoreError> { - let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let rows = statement - .query_map(rusqlite::params_from_iter(values.iter()), map) - .map_err(LogStoreError::Sqlite)?; - let mut items = rows - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - let has_more = items.len() > limit; - items.truncate(limit); - let next_cursor = if has_more { - items.last().map(|item| { - let (timestamp, id) = cursor_fields(item); - encode_cursor(timestamp, id) - }) - } else { - None - }; - Ok(QueryPage { items, next_cursor }) -} - -fn normalize_timestamp(value: &str) -> String { - canonical_logging_timestamp(value).expect("RequestQuery::validate parses time bounds") -} - -fn request_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(RequestRecord { - request_id: row.get(0)?, - outcome: row.get(1)?, - created_at: row.get(2)?, - terminal_at: row.get(3)?, - route: row.get(4)?, - model: row.get(5)?, - provider: row.get(6)?, - engine: row.get(7)?, - status_code: row.get(8)?, - }) -} diff --git a/crates/mesh-llm-log-store/src/query/requests/projection.rs b/crates/mesh-llm-log-store/src/query/requests/projection.rs new file mode 100644 index 0000000000..e1b0ca6494 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/requests/projection.rs @@ -0,0 +1,25 @@ +use super::super::{RequestRecord, RequestRecordWithCaller}; + +pub(super) const REQUEST_COLUMNS: &str = "request_id, state, created_at, terminal_at, route, model, provider, engine, status_code, \ + caller_endpoint_id, caller_addr, caller_path_type"; + +pub(super) fn request_record_with_caller( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(RequestRecordWithCaller { + request: RequestRecord { + request_id: row.get(0)?, + outcome: row.get(1)?, + created_at: row.get(2)?, + terminal_at: row.get(3)?, + route: row.get(4)?, + model: row.get(5)?, + provider: row.get(6)?, + engine: row.get(7)?, + status_code: row.get(8)?, + }, + caller_endpoint_id: row.get(9)?, + caller_addr: row.get(10)?, + caller_path_type: row.get(11)?, + }) +} diff --git a/crates/mesh-llm-log-store/src/query/requests/selection.rs b/crates/mesh-llm-log-store/src/query/requests/selection.rs new file mode 100644 index 0000000000..44b4a34e46 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/requests/selection.rs @@ -0,0 +1,128 @@ +use mesh_llm_events::logging::timestamp::canonical_logging_timestamp; +use rusqlite::types::Value; + +use super::super::{QueryPage, RequestQuery}; +use crate::LogStoreError; +use crate::cursor::encode_cursor; + +pub(crate) struct Conditions { + pub(crate) sql: String, +} + +impl Conditions { + pub(crate) fn append_to_sql(&self, sql: &mut String) { + sql.push_str(&self.sql); + } +} + +pub(super) fn request_conditions(query: &RequestQuery) -> (Conditions, Vec) { + let mut sql = String::new(); + let mut values = Vec::new(); + if let Some(from) = &query.from { + sql.push_str(" WHERE created_at >= ?"); + values.push(Value::Text(normalize_timestamp(from))); + } else { + sql.push_str(" WHERE 1 = 1"); + } + if let Some(to) = &query.to { + sql.push_str(" AND created_at <= ?"); + values.push(Value::Text(normalize_timestamp(to))); + } + for (column, value) in [ + ("route", &query.route), + ("model", &query.model), + ("provider", &query.provider), + ("engine", &query.engine), + ] { + if let Some(value) = value { + sql.push_str(&format!(" AND {column} = ?")); + values.push(Value::Text(value.clone())); + } + } + if let Some(route) = &query.exclude_route { + sql.push_str(" AND (route IS NULL OR route != ?)"); + values.push(Value::Text(route.clone())); + } + if let Some(prefix) = &query.exclude_route_prefix { + sql.push_str(" AND (route IS NULL OR substr(route, 1, length(?)) != ?)"); + values.push(Value::Text(prefix.clone())); + values.push(Value::Text(prefix.clone())); + } + if let Some(status_code) = query.status_code { + sql.push_str(" AND status_code = ?"); + values.push(Value::Integer(i64::from(status_code))); + } + if let Some(outcome) = query.outcome { + sql.push_str(" AND state = ?"); + values.push(Value::Text(outcome.as_str().to_string())); + } + (Conditions { sql }, values) +} + +pub(crate) struct CursorPosition<'a> { + pub(crate) table: &'static str, + pub(crate) timestamp_column: &'static str, + pub(crate) id_column: &'static str, + pub(crate) timestamp: &'a str, + pub(crate) id: &'a str, + pub(crate) conditions: &'a Conditions, + pub(crate) values: &'a [Value], +} + +pub(crate) fn validate_cursor_position( + connection: &rusqlite::Connection, + position: CursorPosition<'_>, +) -> Result<(), LogStoreError> { + let mut sql = format!( + "SELECT 1 FROM {} WHERE {} = ? AND {} = ?", + position.table, position.timestamp_column, position.id_column + ); + let mut parameters = vec![ + Value::Text(position.timestamp.to_string()), + Value::Text(position.id.to_string()), + ]; + if position.conditions.sql.starts_with(" WHERE ") { + sql.push_str(" AND "); + sql.push_str(&position.conditions.sql[7..]); + } + parameters.extend(position.values.iter().cloned()); + match connection.query_row(&sql, rusqlite::params_from_iter(parameters.iter()), |_| { + Ok(()) + }) { + Ok(()) => Ok(()), + Err(rusqlite::Error::QueryReturnedNoRows) => Err(LogStoreError::CursorInvalid), + Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), + } +} + +pub(crate) fn query_page( + connection: &rusqlite::Connection, + sql: String, + values: Vec, + limit: usize, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + cursor_fields: impl Fn(&T) -> (&str, &str), +) -> Result, LogStoreError> { + let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let rows = statement + .query_map(rusqlite::params_from_iter(values.iter()), map) + .map_err(LogStoreError::Sqlite)?; + let mut items = rows + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + let has_more = items.len() > limit; + items.truncate(limit); + let next_cursor = if has_more { + items.last().map(|item| { + let (timestamp, id) = cursor_fields(item); + encode_cursor(timestamp, id) + }) + } else { + None + }; + Ok(QueryPage { items, next_cursor }) +} + +fn normalize_timestamp(value: &str) -> String { + canonical_logging_timestamp(value).expect("RequestQuery::validate parses time bounds") +} diff --git a/crates/mesh-llm-log-store/src/query_pagination_tests.rs b/crates/mesh-llm-log-store/src/query_pagination_tests.rs index 9220708cc8..da6cb27dd1 100644 --- a/crates/mesh-llm-log-store/src/query_pagination_tests.rs +++ b/crates/mesh-llm-log-store/src/query_pagination_tests.rs @@ -26,6 +26,8 @@ fn descending_page(cursor: Option) -> RequestQuery { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, diff --git a/crates/mesh-llm-log-store/src/query_tests.rs b/crates/mesh-llm-log-store/src/query_tests.rs index 34f4a090fd..d8b9201b5c 100644 --- a/crates/mesh-llm-log-store/src/query_tests.rs +++ b/crates/mesh-llm-log-store/src/query_tests.rs @@ -27,6 +27,8 @@ fn request_query() -> RequestQuery { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, @@ -83,6 +85,8 @@ fn request_query_applies_all_filters_and_normalizes_time_bounds() { from: Some("2026-08-02T20:00:00-04:00".to_string()), to: Some("2026-08-03T00:01:00Z".to_string()), route: Some("chat".to_string()), + exclude_route: None, + exclude_route_prefix: None, model: Some("model-a".to_string()), provider: Some("provider-a".to_string()), engine: Some("engine-a".to_string()), @@ -96,6 +100,527 @@ fn request_query_applies_all_filters_and_normalizes_time_bounds() { assert_eq!(page.items[0].request_id, "matching-request"); } +#[test] +fn request_route_exclusions_apply_before_pagination_and_preserve_null_routes() { + // Given + let (_root, store) = open_store(); + for (request_id, route, created_at) in [ + ("visible-null-route", None, "2026-08-03T00:00:01Z"), + ( + "visible-chat-route", + Some("chat_completions"), + "2026-08-03T00:00:02Z", + ), + ( + "hidden-management-route", + Some("management_get_status"), + "2026-08-03T00:00:03Z", + ), + ( + "hidden-models-route", + Some("models"), + "2026-08-03T00:00:04Z", + ), + ] { + store + .insert_summary( + request_id, None, route, None, None, created_at, None, None, None, + ) + .expect("insert request summary"); + } + + // When + let page = store + .query_requests(&RequestQuery { + limit: 2, + exclude_route: Some("models".to_string()), + exclude_route_prefix: Some("management_".to_string()), + ..request_query() + }) + .expect("query visible request summaries"); + + // Then + assert_eq!( + page.items + .iter() + .map(|record| record.request_id.as_str()) + .collect::>(), + ["visible-chat-route", "visible-null-route"] + ); + assert!(page.next_cursor.is_none()); +} + +#[test] +fn caller_identity_round_trips_without_replacing_principal_identity() { + let (_root, store) = open_store(); + let endpoint_id = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + store + .conn() + .execute( + "INSERT INTO summaries \ + (request_id, created_at, tenant_id, account_id, user_id, caller_endpoint_id, caller_addr, caller_path_type) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + rusqlite::params![ + "request-with-caller", + "2026-08-20T12:00:00Z", + "tenant-a", + "account-a", + "user-a", + endpoint_id, + "192.0.2.42:11204", + "remote_quic_http", + ], + ) + .expect("insert caller summary"); + + let repository_row = store + .get_summary("request-with-caller") + .expect("query repository summary") + .expect("repository summary"); + let query_row = store + .query_request_with_caller("request-with-caller") + .expect("query request record") + .expect("request record"); + + assert_eq!(repository_row.tenant_id.as_deref(), Some("tenant-a")); + assert_eq!(repository_row.account_id.as_deref(), Some("account-a")); + assert_eq!(repository_row.user_id.as_deref(), Some("user-a")); + assert_eq!(query_row.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(query_row.caller_addr.as_deref(), Some("192.0.2.42:11204")); + assert_eq!( + query_row.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + +#[test] +fn authenticated_relay_upsert_replaces_and_protects_the_complete_caller_tuple() { + let (_root, store) = open_store(); + let request_id = "request-with-authenticated-relay"; + let endpoint_id = "7272727272727272727272727272727272727272727272727272727272727272"; + + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + Some("127.0.0.1:40123"), + Some("local_http"), + "2026-08-20T12:00:00Z", + ) + .expect("persist provisional local caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + None, + Some("relay"), + "2026-08-20T12:00:01Z", + ) + .expect("replace with authenticated relay caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + None, + Some("127.0.0.1:49999"), + Some("local_http"), + "2026-08-20T12:00:02Z", + ) + .expect("ignore later provisional local caller"); + + let repository_row = store + .get_summary(request_id) + .expect("query summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(query_row.caller_addr, None); + assert_eq!(query_row.caller_path_type.as_deref(), Some("relay")); +} + +#[test] +fn authenticated_endpoint_only_caller_round_trips_through_public_query_api() { + let (_root, store) = open_store(); + let request_id = "request-with-endpoint-only-caller"; + let endpoint_id = "7373737373737373737373737373737373737373737373737373737373737373"; + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + Some(endpoint_id), + None, + None, + "2026-08-20T12:00:00Z", + ) + .expect("persist endpoint-only caller"); + + let direct = store + .query_request_with_caller(request_id) + .expect("query request") + .expect("request record"); + let page = store + .query_requests_with_caller(&request_query()) + .expect("query request page"); + let listed = page + .items + .iter() + .find(|record| record.request.request_id == request_id) + .expect("listed request"); + + for record in [&direct, listed] { + assert_eq!(record.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(record.caller_addr, None); + assert_eq!(record.caller_path_type, None); + } +} + +#[test] +fn endpoint_only_upsert_replaces_local_or_empty_and_protects_first_authenticated_caller() { + let (_root, store) = open_store(); + let endpoint_id = "7474747474747474747474747474747474747474747474747474747474747474"; + let later_endpoint_id = "7575757575757575757575757575757575757575757575757575757575757575"; + + for (request_id, caller_addr, caller_path_type) in [ + ("endpoint-only-replaces-empty", None, None), + ( + "endpoint-only-replaces-local", + Some("127.0.0.1:40123"), + Some("local_http"), + ), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("persist initial caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + None, + None, + "2026-08-20T12:00:01Z", + ) + .expect("replace with endpoint-only caller"); + + for (later_endpoint, later_addr, later_path) in [ + (None, Some("127.0.0.1:49999"), Some("local_http")), + ( + Some(later_endpoint_id), + Some("192.0.2.75:11204"), + Some("remote_quic_http"), + ), + (Some(later_endpoint_id), None, Some("relay")), + (Some(later_endpoint_id), None, None), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + later_endpoint, + later_addr, + later_path, + "2026-08-20T12:00:02Z", + ) + .expect("ignore later caller"); + } + + let record = store + .query_request_with_caller(request_id) + .expect("query request") + .expect("request record"); + assert_eq!(record.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(record.caller_addr, None); + assert_eq!(record.caller_path_type, None); + assert_eq!(record.request.model.as_deref(), Some("model-a")); + assert_eq!(record.request.route.as_deref(), Some("responses")); + assert_eq!(record.request.provider.as_deref(), Some("provider-a")); + assert_eq!(record.request.engine.as_deref(), Some("engine-a")); + } +} + +#[test] +fn unrecognized_stage_caller_does_not_replace_local_request_metadata() { + let (_root, store) = open_store(); + let request_id = "request-with-stage-second"; + let endpoint_id = "7575757575757575757575757575757575757575757575757575757575757575"; + + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + Some("127.0.0.1:40123"), + Some("local_http"), + "2026-08-20T12:00:00Z", + ) + .expect("persist provisional local caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + Some("192.0.2.75:11204"), + Some("remote_quic_stage"), + "2026-08-20T12:00:01Z", + ) + .expect("ignore unrecognized stage caller"); + + let repository_row = store + .get_summary(request_id) + .expect("query summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id, None); + assert_eq!(query_row.caller_addr.as_deref(), Some("127.0.0.1:40123")); + assert_eq!(query_row.caller_path_type.as_deref(), Some("local_http")); +} + +#[test] +fn unrecognized_stage_caller_does_not_enter_empty_summary_metadata() { + let (_root, store) = open_store(); + let endpoint_id = "7676767676767676767676767676767676767676767676767676767676767676"; + + for (request_id, path_type) in [ + ("request-with-stage-first", "remote_quic_stage"), + ("request-with-empty-path", ""), + ("request-with-partial-path", "remote_quic_http/"), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + Some(endpoint_id), + Some("192.0.2.76:11204"), + Some(path_type), + "2026-08-20T12:00:00Z", + ) + .expect("ignore unrecognized caller on empty summary"); + + let repository_row = store + .get_summary(request_id) + .expect("query repository summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id, None); + assert_eq!(query_row.caller_addr, None); + assert_eq!(query_row.caller_path_type, None); + } +} + +#[test] +fn partial_caller_tuples_do_not_enter_empty_summary_metadata() { + let (_root, store) = open_store(); + let endpoint_id = "7777777777777777777777777777777777777777777777777777777777777777"; + let partial_tuples = [ + ( + "request-local-without-address", + None, + None, + Some("local_http"), + ), + ( + "request-remote-without-endpoint", + None, + Some("192.0.2.77:11204"), + Some("remote_quic_http"), + ), + ("request-relay-without-endpoint", None, None, Some("relay")), + ( + "request-values-without-path", + Some(endpoint_id), + Some("192.0.2.77:11204"), + None, + ), + ( + "request-invalid-endpoint-only", + Some("not-an-authenticated-endpoint"), + None, + None, + ), + ( + "request-remote-invalid-endpoint", + Some("not-an-authenticated-endpoint"), + Some("192.0.2.77:11204"), + Some("remote_quic_http"), + ), + ( + "request-local-invalid-address", + None, + Some("not-a-socket-address"), + Some("local_http"), + ), + ]; + + for (request_id, caller_endpoint_id, caller_addr, caller_path_type) in partial_tuples { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + caller_endpoint_id, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("ignore partial caller tuple"); + + let row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(row.caller_endpoint_id, None); + assert_eq!(row.caller_addr, None); + assert_eq!(row.caller_path_type, None); + } +} + +#[test] +fn supported_caller_tuples_are_canonicalized_before_persistence() { + let (_root, store) = open_store(); + let endpoint_id = "7878787878787878787878787878787878787878787878787878787878787878"; + + for (request_id, caller_endpoint_id, caller_addr, caller_path_type) in [ + ( + "request-local-canonical", + Some("ignored-endpoint"), + Some("[2001:0db8:0:0::1]:40123"), + Some("local_http"), + ), + ( + "request-remote-canonical", + Some(endpoint_id), + Some("[2001:0db8:0:0::2]:11204"), + Some("remote_quic_http"), + ), + ( + "request-relay-clears-address", + Some(endpoint_id), + Some("192.0.2.78:11204"), + Some("relay"), + ), + ( + "request-remote-clears-invalid-address", + Some(endpoint_id), + Some("not-a-socket-address"), + Some("remote_quic_http"), + ), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + caller_endpoint_id, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("persist supported caller tuple"); + } + + let local = store + .query_request_with_caller("request-local-canonical") + .expect("query local caller") + .expect("local caller"); + assert_eq!(local.caller_endpoint_id, None); + assert_eq!(local.caller_addr.as_deref(), Some("[2001:db8::1]:40123")); + assert_eq!(local.caller_path_type.as_deref(), Some("local_http")); + + let remote = store + .query_request_with_caller("request-remote-canonical") + .expect("query remote caller") + .expect("remote caller"); + assert_eq!(remote.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(remote.caller_addr.as_deref(), Some("[2001:db8::2]:11204")); + assert_eq!(remote.caller_path_type.as_deref(), Some("remote_quic_http")); + + let relay = store + .query_request_with_caller("request-relay-clears-address") + .expect("query relay caller") + .expect("relay caller"); + assert_eq!(relay.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(relay.caller_addr, None); + assert_eq!(relay.caller_path_type.as_deref(), Some("relay")); + + let remote_without_addr = store + .query_request_with_caller("request-remote-clears-invalid-address") + .expect("query remote caller with invalid address") + .expect("remote caller with invalid address"); + assert_eq!( + remote_without_addr.caller_endpoint_id.as_deref(), + Some(endpoint_id) + ); + assert_eq!(remote_without_addr.caller_addr, None); + assert_eq!( + remote_without_addr.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + #[test] fn request_time_bounds_compare_instants_within_the_boundary_second() { let (_root, store) = open_store(); @@ -386,6 +911,13 @@ fn forged_or_scope_mismatched_request_cursor_is_rejected() { store.query_requests(&query), Err(LogStoreError::CursorInvalid) )); + + query.route = None; + query.exclude_route = Some("chat".to_string()); + assert!(matches!( + store.query_requests(&query), + Err(LogStoreError::CursorInvalid) + )); } #[test] diff --git a/crates/mesh-llm-log-store/src/repositories.rs b/crates/mesh-llm-log-store/src/repositories.rs index eac4a54fe4..c9cc06b6b8 100644 --- a/crates/mesh-llm-log-store/src/repositories.rs +++ b/crates/mesh-llm-log-store/src/repositories.rs @@ -4,15 +4,20 @@ use crate::cursor::{decode_ordering_cursor, encode_cursor}; use crate::error::LogStoreError; use crate::store::LogStore; use crate::timestamps::{ - canonical_comparison_timestamp, canonical_optional_persisted_timestamp, - canonical_persisted_timestamp, canonical_timestamp_metadata, + canonical_optional_persisted_timestamp, canonical_persisted_timestamp, + canonical_timestamp_metadata, }; -use rusqlite::{OptionalExtension, Row, Transaction, types::Value}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use rusqlite::{OptionalExtension, Row, Transaction}; +use serde::Serialize; +mod audit; +mod caller_metadata; mod cleanup; +pub use audit::{ + AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, + DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT, +}; pub use cleanup::{ RetentionCleanupResult, RetentionPolicy, RetentionTable, RetentionTablePolicy, RetentionTableResult, @@ -24,8 +29,6 @@ const MAX_WEBHOOK_TIMESTAMP_BYTES: usize = 64; const MIN_HTTP_STATUS_CODE: u16 = 100; const MAX_HTTP_STATUS_CODE: u16 = 599; const CONFIGURED_WEBHOOK_TARGET: &str = "configured_webhook"; -pub const DEFAULT_AUDIT_ENTRY_LIMIT: usize = 50; -pub const MAX_AUDIT_ENTRY_LIMIT: usize = 100; // ─── Row types returned by queries ────────────────────── @@ -63,236 +66,6 @@ pub struct LifecycleEventRow { pub occurred_at: String, } -/// Public, privacy-safe projection of a durable operational audit entry. -/// -/// `detail_json` remains private to durable audit storage. Only the versioned, -/// bounded fields below are projected from it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AuditEntryRow { - pub sequence: i64, - pub entry_id: String, - pub request_id: Option, - pub occurred_at: String, - pub source: String, - pub code: String, - pub severity: Option, - pub context_version: Option, - pub subject_kind: Option, - pub subject_id: Option, - pub operation_id: Option, - pub correlation_request_id: Option, - pub reason_code: Option, - pub outcome: Option, - pub duration_ms: Option, - pub numeric_summaries: BTreeMap, -} - -#[derive(Default, Deserialize)] -struct StoredAuditDetail { - severity: Option, - context_version: Option, - subject_kind: Option, - subject_id: Option, - operation_id: Option, - request_id: Option, - reason_code: Option, - outcome: Option, - duration_ms: Option, - #[serde(default)] - numeric_summaries: BTreeMap, -} - -impl StoredAuditDetail { - fn parse(raw: Option) -> Self { - raw.and_then(|raw| serde_json::from_str(&raw).ok()) - .unwrap_or_default() - } - - fn bounded(mut self) -> Self { - if self.context_version != Some(1) { - return Self { - severity: self.severity, - ..Self::default() - }; - } - self.subject_kind = self.subject_kind.filter(|value| { - matches!( - value.as_str(), - "runtime" | "model" | "runtime_instance" | "cli_command" - ) - }); - self.subject_id = bounded_audit_value(self.subject_id); - self.operation_id = bounded_audit_value(self.operation_id); - self.request_id = bounded_audit_value(self.request_id); - self.reason_code = bounded_audit_code(self.reason_code); - self.outcome = bounded_audit_code(self.outcome); - self.numeric_summaries = self - .numeric_summaries - .into_iter() - .filter(|(key, _)| bounded_code(key)) - .take(8) - .collect(); - self - } -} - -fn bounded_audit_value(value: Option) -> Option { - value.filter(|value| !value.is_empty() && value.chars().count() <= 256) -} - -fn bounded_audit_code(value: Option) -> Option { - value.filter(|value| bounded_code(value)) -} - -fn bounded_code(value: &str) -> bool { - !value.is_empty() - && value.len() <= 64 - && value - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') -} - -/// Finite source vocabulary accepted by durable audit queries. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AuditEntrySource { - LoggingService, - Runtime, - Mesh, - Cli, - LogsApi, -} - -impl AuditEntrySource { - const fn as_str(self) -> &'static str { - match self { - Self::LoggingService => "logging_service", - Self::Runtime => "runtime", - Self::Mesh => "mesh", - Self::Cli => "cli", - Self::LogsApi => "logs_api", - } - } -} - -/// Finite severity vocabulary accepted by durable audit queries. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AuditEntrySeverity { - Info, - Warning, - Error, -} - -impl AuditEntrySeverity { - const fn as_str(self) -> &'static str { - match self { - Self::Info => "info", - Self::Warning => "warning", - Self::Error => "error", - } - } - - fn parse(value: Option) -> Option { - match value.as_deref() { - Some("info") => Some(Self::Info), - Some("warning") => Some(Self::Warning), - Some("error") => Some(Self::Error), - _ => None, - } - } -} - -/// Typed allowlisted filters for the operational audit read model. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AuditEntryFilters { - pub source: Option, - pub severity: Option, - pub from: Option, - pub to: Option, -} - -fn validate_audit_entry_limit(limit: Option) -> Result { - let limit = limit.unwrap_or(DEFAULT_AUDIT_ENTRY_LIMIT); - if (1..=MAX_AUDIT_ENTRY_LIMIT).contains(&limit) { - Ok(limit) - } else { - Err(LogStoreError::InvalidQuery(format!( - "audit entry limit must be within 1..={MAX_AUDIT_ENTRY_LIMIT}" - ))) - } -} - -fn audit_entry_row(row: &Row<'_>) -> rusqlite::Result { - let detail = StoredAuditDetail::parse(row.get(6)?).bounded(); - Ok(AuditEntryRow { - sequence: row.get(0)?, - entry_id: row.get(1)?, - request_id: row.get(2)?, - occurred_at: row.get(3)?, - source: row.get(4)?, - code: row.get(5)?, - severity: AuditEntrySeverity::parse(detail.severity), - context_version: detail.context_version, - subject_kind: detail.subject_kind, - subject_id: detail.subject_id, - operation_id: detail.operation_id, - correlation_request_id: detail.request_id, - reason_code: detail.reason_code, - outcome: detail.outcome, - duration_ms: detail.duration_ms, - numeric_summaries: detail.numeric_summaries, - }) -} - -fn audit_entry_query_parts( - cursor: Option<&(String, String)>, - filters: AuditEntryFilters, -) -> Result<(String, Vec), LogStoreError> { - let mut clauses = Vec::new(); - let mut parameters = Vec::new(); - if let Some((occurred_at, entry_id)) = cursor { - clauses.push("(occurred_at, entry_id) < (?, ?)".to_string()); - parameters.push(Value::Text(occurred_at.clone())); - parameters.push(Value::Text(entry_id.clone())); - } - if let Some(source) = filters.source { - clauses.push("actor = ?".to_string()); - parameters.push(Value::Text(source.as_str().to_string())); - } - if let Some(severity) = filters.severity { - clauses.push("CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END = ?".to_string()); - parameters.push(Value::Text(severity.as_str().to_string())); - } - let from = filters - .from - .as_deref() - .map(canonical_comparison_timestamp) - .transpose()?; - let to = filters - .to - .as_deref() - .map(canonical_comparison_timestamp) - .transpose()?; - if from > to && to.is_some() { - return Err(LogStoreError::InvalidQuery( - "audit from must not be after to".to_string(), - )); - } - if let Some(from) = from { - clauses.push("occurred_at >= ?".to_string()); - parameters.push(Value::Text(from)); - } - if let Some(to) = to { - clauses.push("occurred_at <= ?".to_string()); - parameters.push(Value::Text(to)); - } - let where_clause = if clauses.is_empty() { - String::new() - } else { - format!("WHERE {}", clauses.join(" AND ")) - }; - Ok((where_clause, parameters)) -} - /// Closed terminal request classification safe to retain for webhook delivery. /// /// The fixed vocabulary excludes raw errors, endpoints, paths, prompts, @@ -992,35 +765,6 @@ impl LogStore { } } - /// Insert a summary when absent and otherwise fill only metadata fields - /// that have not yet been recorded. Lifecycle state, timestamps, and - /// identity fields remain untouched. - #[allow(clippy::too_many_arguments)] - pub fn upsert_summary_metadata( - &self, - request_id: &str, - model: Option<&str>, - route: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - occurred_at: &str, - ) -> Result<(), LogStoreError> { - let occurred_at = canonical_persisted_timestamp(occurred_at)?; - self.conn() - .execute( - "INSERT INTO summaries (request_id, created_at, model, route, provider, engine) \ - VALUES (?, ?, ?, ?, ?, ?) \ - ON CONFLICT(request_id) DO UPDATE SET \ - model = COALESCE(summaries.model, excluded.model), \ - route = COALESCE(summaries.route, excluded.route), \ - provider = COALESCE(summaries.provider, excluded.provider), \ - engine = COALESCE(summaries.engine, excluded.engine)", - rusqlite::params![request_id, occurred_at, model, route, provider, engine], - ) - .map(|_| ()) - .map_err(|error| LogStoreError::InsertFailed(error.to_string())) - } - /// Get a summary by request_id. Returns None if not found (no-op style). pub fn get_summary(&self, request_id: &str) -> Result, LogStoreError> { let conn = self.conn(); @@ -1361,106 +1105,6 @@ impl LogStore { } } - // ════════════════════════════ - // Audit Entries - // ════════════════════════════ - - pub fn insert_audit_entry( - &self, - entry_id: &str, - request_id: Option<&str>, - occurred_at: &str, - source: &str, - code: &str, - detail_json: Option<&str>, - ) -> Result<(), LogStoreError> { - let occurred_at = canonical_persisted_timestamp(occurred_at)?; - let conn = self.conn(); - match conn.execute( - "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?, ?, ?, ?, ?, ?)", - rusqlite::params![entry_id, request_id, occurred_at, source, code, detail_json], - ) { - Ok(_) => Ok(()), - Err(ref e) => match map_insert_constraint_error(e, format!("audit_entry {entry_id}")) { - Some(error) => Err(error), - None => Err(LogStoreError::InsertFailed(e.to_string())), - }, - } - } - - /// List privacy-safe operational audit rows with cursor pagination. - /// - /// The query mirrors lifecycle ordering on `(occurred_at, entry_id)` while - /// fetching one extra row to determine the next cursor. `detail_json` is - /// only used internally for the bounded severity projection. - pub fn list_audit_entries( - &self, - limit: Option, - after_cursor: Option<&str>, - filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - let limit = validate_audit_entry_limit(limit)?; - let cursor = after_cursor.map(decode_ordering_cursor).transpose()?; - let (where_clause, parameters) = audit_entry_query_parts(cursor.as_ref(), filters)?; - let sql = format!( - "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ - FROM audit_entries {where_clause} \ - ORDER BY occurred_at DESC, entry_id DESC LIMIT {}", - limit + 1 - ); - let conn = self.conn(); - let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let mut items: Vec = statement - .query_map(rusqlite::params_from_iter(parameters), audit_entry_row) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - - let has_more = items.len() > limit; - if has_more { - items.pop(); - } - let next_cursor = has_more.then(|| { - let last = &items[items.len() - 1]; - encode_cursor(&last.occurred_at, &last.entry_id) - }); - Ok(Page { items, next_cursor }) - } - - /// Read newly committed audit rows in database sequence order for the - /// trusted-local event stream. Unlike the public keyset listing, this is a - /// forward-only reconciliation cursor shared by every process writing the - /// durable store. - pub fn list_audit_entries_after_sequence( - &self, - after_sequence: u64, - limit: usize, - filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - let limit = validate_audit_entry_limit(Some(limit))?; - let after_sequence = i64::try_from(after_sequence).map_err(|_| { - LogStoreError::InvalidQuery("audit sequence is outside the supported range".to_string()) - })?; - let (where_clause, mut parameters) = audit_entry_query_parts(None, filters)?; - let predicate = if where_clause.is_empty() { - "WHERE sequence > ?".to_string() - } else { - format!("{where_clause} AND sequence > ?") - }; - parameters.push(Value::Integer(after_sequence)); - let sql = format!( - "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ - FROM audit_entries {predicate} ORDER BY sequence ASC LIMIT {limit}" - ); - let conn = self.conn(); - let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; - statement - .query_map(rusqlite::params_from_iter(parameters), audit_entry_row) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string())) - } - // ════════════════════════════ // Webhook Deliveries // ════════════════════════════ diff --git a/crates/mesh-llm-log-store/src/repositories/audit.rs b/crates/mesh-llm-log-store/src/repositories/audit.rs new file mode 100644 index 0000000000..73061f42fb --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit.rs @@ -0,0 +1,155 @@ +use mesh_llm_events::audit::{SanitizedAuditDetailJson, SanitizedAuditScalar}; +use rusqlite::types::Value; + +use super::{Page, map_insert_constraint_error}; +use crate::cursor::{decode_ordering_cursor, encode_cursor}; +use crate::error::LogStoreError; +use crate::store::LogStore; +use crate::timestamps::canonical_persisted_timestamp; + +mod detail; +mod model; +mod paging; +mod projection; + +pub use model::{ + AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, + DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT, +}; + +use crate::AuditEntryDetail; +use paging::{ + query_parts as audit_entry_query_parts, validate_limit as validate_audit_entry_limit, +}; +use projection::audit_entry_detail; + +impl LogStore { + pub fn insert_audit_entry( + &self, + entry_id: &str, + request_id: Option<&str>, + occurred_at: &str, + source: &str, + code: &str, + detail_json: Option<&str>, + ) -> Result<(), LogStoreError> { + let occurred_at = canonical_persisted_timestamp(occurred_at)?; + let source = SanitizedAuditScalar::sanitize(source); + let code = SanitizedAuditScalar::sanitize(code); + let detail_json = detail_json.map(SanitizedAuditDetailJson::sanitize); + let conn = self.conn(); + match conn.execute( + "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?, ?, ?, ?, ?, ?)", + rusqlite::params![ + entry_id, + request_id, + occurred_at, + source.as_str(), + code.as_str(), + detail_json.as_ref().map(SanitizedAuditDetailJson::as_str) + ], + ) { + Ok(_) => Ok(()), + Err(ref e) => match map_insert_constraint_error(e, format!("audit_entry {entry_id}")) { + Some(error) => Err(error), + None => Err(LogStoreError::InsertFailed(e.to_string())), + }, + } + } + + /// List privacy-safe operational audit rows with cursor pagination. + /// + /// The query mirrors lifecycle ordering on `(occurred_at, entry_id)` while + /// fetching one extra row to determine the next cursor. `detail_json` is + /// only used internally for the bounded severity projection. + pub fn list_audit_entries( + &self, + limit: Option, + after_cursor: Option<&str>, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + self.list_audit_entry_details(limit, after_cursor, filters) + .map(|page| Page { + items: page.items.into_iter().map(Into::into).collect(), + next_cursor: page.next_cursor, + }) + } + + pub fn list_audit_entry_details( + &self, + limit: Option, + after_cursor: Option<&str>, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + let limit = validate_audit_entry_limit(limit)?; + let cursor = after_cursor.map(decode_ordering_cursor).transpose()?; + let (where_clause, parameters) = audit_entry_query_parts(cursor.as_ref(), filters)?; + let sql = format!( + "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ + FROM audit_entries {where_clause} \ + ORDER BY occurred_at DESC, entry_id DESC LIMIT {}", + limit + 1 + ); + let conn = self.conn(); + let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let mut items: Vec = statement + .query_map(rusqlite::params_from_iter(parameters), audit_entry_detail) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + + let has_more = items.len() > limit; + if has_more { + items.pop(); + } + let next_cursor = has_more.then(|| { + let last = &items[items.len() - 1]; + encode_cursor(&last.entry.occurred_at, &last.entry.entry_id) + }); + Ok(Page { items, next_cursor }) + } + + /// Read newly committed audit rows in database sequence order for the + /// trusted-local event stream. Unlike the public keyset listing, this is a + /// forward-only reconciliation cursor shared by every process writing the + /// durable store. + pub fn list_audit_entries_after_sequence( + &self, + after_sequence: u64, + limit: usize, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + self.list_audit_entry_details_after_sequence(after_sequence, limit, filters) + .map(|entries| entries.into_iter().map(Into::into).collect()) + } + + pub fn list_audit_entry_details_after_sequence( + &self, + after_sequence: u64, + limit: usize, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + let limit = validate_audit_entry_limit(Some(limit))?; + let after_sequence = i64::try_from(after_sequence).map_err(|_| { + LogStoreError::InvalidQuery("audit sequence is outside the supported range".to_string()) + })?; + let (where_clause, mut parameters) = audit_entry_query_parts(None, filters)?; + let predicate = if where_clause.is_empty() { + "WHERE sequence > ?".to_string() + } else { + format!("{where_clause} AND sequence > ?") + }; + parameters.push(Value::Integer(after_sequence)); + let sql = format!( + "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ + FROM audit_entries {predicate} ORDER BY sequence ASC LIMIT {limit}" + ); + let conn = self.conn(); + let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; + statement + .query_map(rusqlite::params_from_iter(parameters), audit_entry_detail) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string())) + } +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/detail.rs b/crates/mesh-llm-log-store/src/repositories/audit/detail.rs new file mode 100644 index 0000000000..8e1e930cd4 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/detail.rs @@ -0,0 +1,90 @@ +use std::{collections::BTreeMap, net::SocketAddr}; + +use mesh_llm_events::CliCommandSummary; +use serde::Deserialize; + +#[derive(Default, Deserialize)] +pub(super) struct StoredAuditDetail { + pub(super) severity: Option, + pub(super) context_version: Option, + pub(super) subject_kind: Option, + pub(super) subject_id: Option, + pub(super) remote_addr: Option, + pub(super) path_type: Option, + pub(super) operation_id: Option, + pub(super) request_id: Option, + pub(super) reason_code: Option, + pub(super) outcome: Option, + pub(super) command_summary: Option, + pub(super) duration_ms: Option, + #[serde(default)] + pub(super) numeric_summaries: BTreeMap, +} + +impl StoredAuditDetail { + pub(super) fn parse(raw: Option) -> Self { + raw.and_then(|raw| serde_json::from_str(&raw).ok()) + .unwrap_or_default() + } + + pub(super) fn bounded(mut self) -> Self { + if self.context_version != Some(1) { + return Self { + severity: self.severity, + ..Self::default() + }; + } + self.subject_kind = self.subject_kind.filter(|value| { + matches!( + value.as_str(), + "runtime" | "model" | "runtime_instance" | "cli_command" | "mesh_peer" + ) + }); + self.subject_id = bounded_audit_value(self.subject_id); + self.path_type = self + .path_type + .filter(|value| matches!(value.as_str(), "direct" | "relay")); + self.remote_addr = match self.path_type.as_deref() { + Some("direct") => self + .remote_addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + Some("relay") | None => None, + Some(_) => None, + }; + self.operation_id = bounded_audit_value(self.operation_id); + self.request_id = bounded_audit_value(self.request_id); + self.reason_code = bounded_audit_code(self.reason_code); + self.outcome = bounded_audit_code(self.outcome); + self.command_summary = bounded_command_summary(self.command_summary); + self.numeric_summaries = self + .numeric_summaries + .into_iter() + .filter(|(key, _)| bounded_code(key)) + .take(8) + .collect(); + self + } +} + +fn bounded_audit_value(value: Option) -> Option { + value.filter(|value| !value.is_empty() && value.chars().count() <= 256) +} + +fn bounded_command_summary(value: Option) -> Option { + value + .and_then(|value| CliCommandSummary::sanitize(&value)) + .map(|summary| summary.as_str().to_owned()) +} + +fn bounded_audit_code(value: Option) -> Option { + value.filter(|value| bounded_code(value)) +} + +fn bounded_code(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/model.rs b/crates/mesh-llm-log-store/src/repositories/audit/model.rs new file mode 100644 index 0000000000..664c1a96fe --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/model.rs @@ -0,0 +1,79 @@ +use std::collections::BTreeMap; + +pub const DEFAULT_AUDIT_ENTRY_LIMIT: usize = 50; +pub const MAX_AUDIT_ENTRY_LIMIT: usize = 100; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditEntryRow { + pub sequence: i64, + pub entry_id: String, + pub request_id: Option, + pub occurred_at: String, + pub source: String, + pub code: String, + pub severity: Option, + pub context_version: Option, + pub subject_kind: Option, + pub subject_id: Option, + pub operation_id: Option, + pub correlation_request_id: Option, + pub reason_code: Option, + pub outcome: Option, + pub duration_ms: Option, + pub numeric_summaries: BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditEntrySource { + LoggingService, + Runtime, + Mesh, + Cli, + LogsApi, +} + +impl AuditEntrySource { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::LoggingService => "logging_service", + Self::Runtime => "runtime", + Self::Mesh => "mesh", + Self::Cli => "cli", + Self::LogsApi => "logs_api", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditEntrySeverity { + Info, + Warning, + Error, +} + +impl AuditEntrySeverity { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Info => "info", + Self::Warning => "warning", + Self::Error => "error", + } + } + + pub(super) fn parse(value: Option) -> Option { + match value.as_deref() { + Some("info") => Some(Self::Info), + Some("warning") => Some(Self::Warning), + Some("error") => Some(Self::Error), + _ => None, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditEntryFilters { + pub source: Option, + pub severity: Option, + pub from: Option, + pub to: Option, +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/paging.rs b/crates/mesh-llm-log-store/src/repositories/audit/paging.rs new file mode 100644 index 0000000000..adc264c1f8 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/paging.rs @@ -0,0 +1,66 @@ +use rusqlite::types::Value; + +use super::{AuditEntryFilters, DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT}; +use crate::error::LogStoreError; +use crate::timestamps::canonical_comparison_timestamp; + +pub(super) fn validate_limit(limit: Option) -> Result { + let limit = limit.unwrap_or(DEFAULT_AUDIT_ENTRY_LIMIT); + if (1..=MAX_AUDIT_ENTRY_LIMIT).contains(&limit) { + Ok(limit) + } else { + Err(LogStoreError::InvalidQuery(format!( + "audit entry limit must be within 1..={MAX_AUDIT_ENTRY_LIMIT}" + ))) + } +} + +pub(super) fn query_parts( + cursor: Option<&(String, String)>, + filters: AuditEntryFilters, +) -> Result<(String, Vec), LogStoreError> { + let mut clauses = Vec::new(); + let mut parameters = Vec::new(); + if let Some((occurred_at, entry_id)) = cursor { + clauses.push("(occurred_at, entry_id) < (?, ?)".to_string()); + parameters.push(Value::Text(occurred_at.clone())); + parameters.push(Value::Text(entry_id.clone())); + } + if let Some(source) = filters.source { + clauses.push("actor = ?".to_string()); + parameters.push(Value::Text(source.as_str().to_string())); + } + if let Some(severity) = filters.severity { + clauses.push("CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END = ?".to_string()); + parameters.push(Value::Text(severity.as_str().to_string())); + } + let from = filters + .from + .as_deref() + .map(canonical_comparison_timestamp) + .transpose()?; + let to = filters + .to + .as_deref() + .map(canonical_comparison_timestamp) + .transpose()?; + if from > to && to.is_some() { + return Err(LogStoreError::InvalidQuery( + "audit from must not be after to".to_string(), + )); + } + if let Some(from) = from { + clauses.push("occurred_at >= ?".to_string()); + parameters.push(Value::Text(from)); + } + if let Some(to) = to { + clauses.push("occurred_at <= ?".to_string()); + parameters.push(Value::Text(to)); + } + let where_clause = if clauses.is_empty() { + String::new() + } else { + format!("WHERE {}", clauses.join(" AND ")) + }; + Ok((where_clause, parameters)) +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/projection.rs b/crates/mesh-llm-log-store/src/repositories/audit/projection.rs new file mode 100644 index 0000000000..62caad00db --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/projection.rs @@ -0,0 +1,36 @@ +use mesh_llm_events::audit::SanitizedAuditScalar; + +use super::detail::StoredAuditDetail; +use super::{AuditEntryRow, AuditEntrySeverity}; +use crate::AuditEntryDetail; + +pub(super) fn audit_entry_detail(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let detail = StoredAuditDetail::parse(row.get(6)?).bounded(); + Ok(AuditEntryDetail { + entry: AuditEntryRow { + sequence: row.get(0)?, + entry_id: row.get(1)?, + request_id: row.get(2)?, + occurred_at: row.get(3)?, + source: SanitizedAuditScalar::sanitize(&row.get::<_, String>(4)?) + .as_str() + .to_owned(), + code: SanitizedAuditScalar::sanitize(&row.get::<_, String>(5)?) + .as_str() + .to_owned(), + severity: AuditEntrySeverity::parse(detail.severity), + context_version: detail.context_version, + subject_kind: detail.subject_kind, + subject_id: detail.subject_id, + operation_id: detail.operation_id, + correlation_request_id: detail.request_id, + reason_code: detail.reason_code, + outcome: detail.outcome, + duration_ms: detail.duration_ms, + numeric_summaries: detail.numeric_summaries, + }, + remote_addr: detail.remote_addr, + path_type: detail.path_type, + command_summary: detail.command_summary, + }) +} diff --git a/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs b/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs new file mode 100644 index 0000000000..c4a96c4b66 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs @@ -0,0 +1,162 @@ +use std::net::SocketAddr; + +use crate::error::LogStoreError; +use crate::store::LogStore; +use crate::timestamps::canonical_persisted_timestamp; + +#[derive(Default)] +pub(super) struct CallerMetadata<'a> { + pub(super) endpoint_id: Option<&'a str>, + pub(super) addr: Option, + pub(super) path_type: Option<&'static str>, +} + +pub(super) fn canonical_caller_metadata<'a>( + endpoint_id: Option<&'a str>, + addr: Option<&str>, + path_type: Option<&str>, +) -> CallerMetadata<'a> { + match path_type { + Some("local_http") => addr + .and_then(|value| value.parse::().ok()) + .map(|value| CallerMetadata { + endpoint_id: None, + addr: Some(value.to_string()), + path_type: Some("local_http"), + }) + .unwrap_or_default(), + Some("remote_quic_http") if endpoint_id.is_some_and(authenticated_endpoint_id) => { + CallerMetadata { + endpoint_id, + addr: addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + path_type: Some("remote_quic_http"), + } + } + Some("relay") if endpoint_id.is_some_and(authenticated_endpoint_id) => CallerMetadata { + endpoint_id, + addr: None, + path_type: Some("relay"), + }, + None if endpoint_id.is_some_and(authenticated_endpoint_id) && addr.is_none() => { + CallerMetadata { + endpoint_id, + addr: None, + path_type: None, + } + } + Some("remote_quic_http" | "relay") | Some(_) | None => CallerMetadata::default(), + } +} + +fn authenticated_endpoint_id(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +impl LogStore { + /// Insert a summary when absent and otherwise fill only metadata fields + /// that have not yet been recorded. + #[allow(clippy::too_many_arguments)] + pub fn upsert_summary_metadata( + &self, + request_id: &str, + model: Option<&str>, + route: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + occurred_at: &str, + ) -> Result<(), LogStoreError> { + self.upsert_summary_metadata_with_caller( + request_id, + model, + route, + provider, + engine, + None, + None, + None, + occurred_at, + ) + } + + /// Insert a summary when absent and otherwise fill only metadata fields + /// that have not yet been recorded. Lifecycle state, timestamps, and + /// identity fields remain untouched. + #[allow(clippy::too_many_arguments)] + pub fn upsert_summary_metadata_with_caller( + &self, + request_id: &str, + model: Option<&str>, + route: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + caller_endpoint_id: Option<&str>, + caller_addr: Option<&str>, + caller_path_type: Option<&str>, + occurred_at: &str, + ) -> Result<(), LogStoreError> { + let occurred_at = canonical_persisted_timestamp(occurred_at)?; + let caller = canonical_caller_metadata(caller_endpoint_id, caller_addr, caller_path_type); + self.conn() + .execute( + "INSERT INTO summaries \ + (request_id, created_at, model, route, provider, engine, caller_endpoint_id, caller_addr, caller_path_type) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(request_id) DO UPDATE SET \ + model = COALESCE(summaries.model, excluded.model), \ + route = COALESCE(summaries.route, excluded.route), \ + provider = COALESCE(summaries.provider, excluded.provider), \ + engine = COALESCE(summaries.engine, excluded.engine), \ + caller_endpoint_id = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_endpoint_id \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_endpoint_id \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_endpoint_id \ + ELSE excluded.caller_endpoint_id END, \ + caller_addr = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_addr \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_addr \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_addr \ + ELSE excluded.caller_addr END, \ + caller_path_type = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_path_type \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_path_type \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_path_type \ + ELSE excluded.caller_path_type END", + rusqlite::params![ + request_id, + occurred_at, + model, + route, + provider, + engine, + caller.endpoint_id, + caller.addr, + caller.path_type + ], + ) + .map(|_| ()) + .map_err(|error| LogStoreError::InsertFailed(error.to_string())) + } +} diff --git a/crates/mesh-llm-log-store/src/schema.rs b/crates/mesh-llm-log-store/src/schema.rs new file mode 100644 index 0000000000..2a8092473e --- /dev/null +++ b/crates/mesh-llm-log-store/src/schema.rs @@ -0,0 +1,159 @@ +use rusqlite::Connection; + +const INITIAL_SCHEMA: &str = r#" +CREATE TABLE summaries ( + request_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active' + CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), + created_at TEXT NOT NULL, terminal_at TEXT, route TEXT, model TEXT, provider TEXT, + engine TEXT, status_code INTEGER, error_msg TEXT, tenant_id TEXT, account_id TEXT, user_id TEXT, + caller_endpoint_id TEXT, caller_addr TEXT, caller_path_type TEXT +); +CREATE INDEX idx_summaries_created ON summaries (created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_state ON summaries (state); +CREATE INDEX idx_summaries_terminal_order +ON summaries (state, COALESCE(terminal_at, created_at), request_id); +CREATE INDEX idx_summaries_route_created ON summaries (route, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_model_created ON summaries (model, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_provider_created ON summaries (provider, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_engine_created ON summaries (engine, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_status_created ON summaries (status_code, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_state_created ON summaries (state, created_at DESC, request_id DESC); + +CREATE TABLE lifecycle_events ( + event_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}', + event_type TEXT NOT NULL DEFAULT 'unknown', is_terminal INTEGER NOT NULL DEFAULT 0 + CHECK (is_terminal IN (0, 1)), UNIQUE(request_id, event_id) +); +CREATE UNIQUE INDEX idx_terminal_event_one_per_request +ON lifecycle_events (request_id) WHERE is_terminal = 1; +CREATE INDEX idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); +CREATE INDEX idx_lifecycle_events_request ON lifecycle_events (request_id); +CREATE INDEX idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); +CREATE INDEX idx_lifecycle_events_request_occurred +ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); + +CREATE TABLE artifact_pointers ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, kind TEXT NOT NULL, metadata_json TEXT, media_kind TEXT, + checksum TEXT, bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), + version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), + redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), + truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), stored_at TEXT, + missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), + corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), + unavailable_reason TEXT CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( + 'streaming_response_not_assembled', 'response_body_not_bounded', + 'capture_content_limit_exceeded', 'capture_memory_budget_exceeded', + 'artifact_capture_disabled', 'artifact_capture_failed' + )), UNIQUE(request_id, artifact_id) +); +CREATE INDEX idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); +CREATE INDEX idx_artifact_pointers_request_occurred +ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); + +CREATE TABLE proxy_records ( + attempt_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, target TEXT NOT NULL, provider TEXT, engine TEXT, + started_at TEXT, completed_at TEXT, status_code INTEGER, error_msg TEXT, + UNIQUE(request_id, attempt_id) +); +CREATE INDEX idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_request_occurred +ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_provider_occurred +ON proxy_records (provider, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_engine_occurred +ON proxy_records (engine, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_status_occurred +ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); + +CREATE TABLE audit_entries ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), entry_id TEXT NOT NULL UNIQUE, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, action TEXT NOT NULL, detail_json TEXT, UNIQUE(request_id, entry_id) +); +CREATE INDEX idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); +CREATE INDEX idx_audit_entries_actor_occurred +ON audit_entries (actor, occurred_at DESC, entry_id DESC); +CREATE INDEX idx_audit_entries_severity_occurred ON audit_entries ( + CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, + occurred_at DESC, entry_id DESC +); + +CREATE TABLE webhook_deliveries ( + delivery_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + terminal_outcome TEXT NOT NULL + CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), + terminal_status_code INTEGER CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), + occurred_at TEXT NOT NULL, target_url TEXT NOT NULL, + attempt_number INTEGER NOT NULL DEFAULT 0 CHECK (attempt_number BETWEEN 0 AND 20), + status_code INTEGER CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), + response_body TEXT, error_msg TEXT, + state TEXT NOT NULL DEFAULT 'succeeded' + CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), + created_at TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT '', next_attempt_at TEXT, + lease_expires_at TEXT, claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), + max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), + last_error_code TEXT CHECK (last_error_code IS NULL OR last_error_code IN + ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), + UNIQUE(request_id, delivery_id) +); +CREATE INDEX idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); +CREATE INDEX idx_webhook_deliveries_eligible +ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); +CREATE INDEX idx_webhook_deliveries_ready +ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) +WHERE state IN ('pending', 'retry', 'manual_retry'); +CREATE INDEX idx_webhook_deliveries_expired_lease +ON webhook_deliveries (lease_expires_at, created_at, delivery_id) WHERE state = 'in_flight'; + +CREATE TABLE cleanup_runs ( + run_id TEXT PRIMARY KEY, occurred_at TEXT NOT NULL, policy_name TEXT NOT NULL, + cutoff_before TEXT NOT NULL, deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), + duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) +); +CREATE INDEX idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); + +CREATE TABLE maintenance_operations ( + operation_id TEXT PRIMARY KEY, action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), + cutoff_before TEXT NOT NULL, request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), + reason TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), + planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), + planned_events INTEGER NOT NULL CHECK (planned_events >= 0), + planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), + planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), + planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), + executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), + executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), + executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), + executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), + executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), + has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), created_at TEXT NOT NULL, + completed_at TEXT, selection_fingerprint TEXT NOT NULL DEFAULT '', + artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), + artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), + artifact_file_failure_class TEXT + CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), + preview_audit_id TEXT, execution_audit_id TEXT, cleanup_filters_json TEXT NOT NULL DEFAULT '{}' +); +CREATE TABLE maintenance_operation_targets ( + operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), request_id TEXT NOT NULL, + PRIMARY KEY (operation_id, request_id), UNIQUE (operation_id, ordinal) +); +CREATE INDEX idx_maintenance_operation_targets_operation +ON maintenance_operation_targets (operation_id, ordinal); +CREATE TABLE pending_artifact_deletions ( + artifact_id TEXT PRIMARY KEY, request_id TEXT NOT NULL +); +CREATE INDEX idx_pending_artifact_deletions_request +ON pending_artifact_deletions (request_id, artifact_id); +"#; + +pub(super) fn initialize(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch(INITIAL_SCHEMA) +} diff --git a/crates/mesh-llm-log-store/src/space_maintenance_tests.rs b/crates/mesh-llm-log-store/src/space_maintenance_tests.rs index a6e66b152d..071d224ccd 100644 --- a/crates/mesh-llm-log-store/src/space_maintenance_tests.rs +++ b/crates/mesh-llm-log-store/src/space_maintenance_tests.rs @@ -50,6 +50,10 @@ fn new_store_reports_zero_reclaimed_pages_when_nothing_can_be_reclaimed() { .expect("new store auto-vacuum mode"); assert_eq!(auto_vacuum, 2); + store + .conn() + .execute_batch("PRAGMA analysis_limit = 400; PRAGMA optimize;") + .expect("prepare optimizer metadata"); let before = page_count(&store); let maintenance = store .maintain_space_after_cleanup() diff --git a/crates/mesh-llm-log-store/src/store.rs b/crates/mesh-llm-log-store/src/store.rs index ba27a14967..a6ff898acd 100644 --- a/crates/mesh-llm-log-store/src/store.rs +++ b/crates/mesh-llm-log-store/src/store.rs @@ -59,7 +59,14 @@ impl LogStore { let conn = Connection::open(&db_path).map_err(|e| { LogStoreError::IoError(std::io::Error::other(format!("sqlite open: {}", e))) })?; - prepare_private_database_files(&db_path)?; + conn.busy_timeout(std::time::Duration::from_secs(30)) + .map_err(LogStoreError::Sqlite)?; + + if let Some((found, supported)) = crate::migrations::incompatible_schema(&conn) + .map_err(|error| LogStoreError::MigrationFailed(error.to_string()))? + { + return Err(LogStoreError::SchemaIncompatible { found, supported }); + } if new_database { // This only takes effect before schema allocation. Existing stores @@ -71,15 +78,9 @@ impl LogStore { let pragmas = " PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; - PRAGMA busy_timeout = 30000; "; conn.execute_batch(pragmas).map_err(LogStoreError::Sqlite)?; - if let Some((found, supported)) = crate::migrations::incompatible_schema(&conn) - .map_err(|error| LogStoreError::MigrationFailed(error.to_string()))? - { - return Err(LogStoreError::SchemaIncompatible { found, supported }); - } crate::migrations::apply_migrations(&conn) .map_err(|e| LogStoreError::MigrationFailed(e.to_string()))?; prepare_private_database_files(&db_path)?; diff --git a/crates/mesh-llm-log-store/tests/public_api_compat.rs b/crates/mesh-llm-log-store/tests/public_api_compat.rs new file mode 100644 index 0000000000..ee9f5598bc --- /dev/null +++ b/crates/mesh-llm-log-store/tests/public_api_compat.rs @@ -0,0 +1,82 @@ +use std::collections::BTreeMap; + +use mesh_llm_log_store::{ + AuditEntryFilters, AuditEntryRow, LogStore, LogStoreError, Page, QueryPage, RequestQuery, + RequestRecord, +}; + +type LegacyAuditPageMethod = fn( + &LogStore, + Option, + Option<&str>, + AuditEntryFilters, +) -> Result, LogStoreError>; +type LegacyAuditSequenceMethod = + fn(&LogStore, u64, usize, AuditEntryFilters) -> Result, LogStoreError>; +type LegacySummaryMetadataMethod = fn( + &LogStore, + &str, + Option<&str>, + Option<&str>, + Option<&str>, + Option<&str>, + &str, +) -> Result<(), LogStoreError>; + +#[test] +fn released_request_record_struct_literal_remains_source_compatible() { + let record = RequestRecord { + request_id: "released-request".to_owned(), + outcome: "completed".to_owned(), + created_at: "2026-08-22T12:00:00.000000000Z".to_owned(), + terminal_at: Some("2026-08-22T12:00:01.000000000Z".to_owned()), + route: Some("chat".to_owned()), + model: Some("released-model".to_owned()), + provider: Some("released-provider".to_owned()), + engine: Some("released-engine".to_owned()), + status_code: Some(200), + }; + + assert_eq!(record.request_id, "released-request"); +} + +#[test] +fn released_audit_entry_row_struct_literal_remains_source_compatible() { + let row = AuditEntryRow { + sequence: 7, + entry_id: "released-audit".to_owned(), + request_id: Some("released-request".to_owned()), + occurred_at: "2026-08-22T12:00:02.000000000Z".to_owned(), + source: "runtime".to_owned(), + code: "runtime_ready".to_owned(), + severity: None, + context_version: Some(1), + subject_kind: Some("runtime".to_owned()), + subject_id: Some("released-host".to_owned()), + operation_id: Some("released-operation".to_owned()), + correlation_request_id: Some("released-correlation".to_owned()), + reason_code: Some("released-reason".to_owned()), + outcome: Some("ready".to_owned()), + duration_ms: Some(11), + numeric_summaries: BTreeMap::from([("models".to_owned(), 2)]), + }; + + assert_eq!(row.entry_id, "released-audit"); +} + +#[test] +fn released_query_and_list_methods_keep_legacy_return_shapes() { + let _: fn(&LogStore, &str) -> Result, LogStoreError> = + LogStore::query_request; + let _: fn(&LogStore, &[String]) -> Result, LogStoreError> = + LogStore::query_requests_by_ids; + let _: fn(&LogStore, &RequestQuery) -> Result, LogStoreError> = + LogStore::query_requests; + let _: LegacyAuditPageMethod = LogStore::list_audit_entries; + let _: LegacyAuditSequenceMethod = LogStore::list_audit_entries_after_sequence; +} + +#[test] +fn released_summary_metadata_method_keeps_legacy_signature() { + let _: LegacySummaryMetadataMethod = LogStore::upsert_summary_metadata; +} diff --git a/crates/mesh-llm-log-store/tests/public_query_projections.rs b/crates/mesh-llm-log-store/tests/public_query_projections.rs new file mode 100644 index 0000000000..783fd99587 --- /dev/null +++ b/crates/mesh-llm-log-store/tests/public_query_projections.rs @@ -0,0 +1,387 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use mesh_llm_log_store::{ + AuditEntryDetail, AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, LogStore, QuerySort, + RealClock, RequestQuery, RequestRecord, RequestRecordWithCaller, +}; + +const CALLER_ENDPOINT_ID: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const CALLER_ADDR: &str = "192.0.2.42:11204"; +const CALLER_PATH_TYPE: &str = "remote_quic_http"; +const AUDIT_REMOTE_ADDR: &str = "198.51.100.27:443"; +const AUDIT_PATH_TYPE: &str = "direct"; +const AUDIT_COMMAND: &str = "mesh-llm runtime guardrails --mode metrics --port 41731"; + +fn open_store() -> (tempfile::TempDir, LogStore) { + let root = tempfile::tempdir().expect("create public projection store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open log store"); + (root, store) +} + +fn request_query(limit: usize, cursor: Option) -> RequestQuery { + RequestQuery { + limit, + cursor, + from: None, + to: None, + route: None, + exclude_route: None, + exclude_route_prefix: None, + model: None, + provider: None, + engine: None, + status_code: None, + outcome: None, + sort: QuerySort::Descending, + } +} + +fn insert_request_with_caller( + store: &LogStore, + request_id: &str, + created_at: &str, + caller_endpoint_id: &str, + caller_addr: &str, +) { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("caller-model"), + Some("responses"), + Some("caller-provider"), + Some("caller-engine"), + Some(caller_endpoint_id), + Some(caller_addr), + Some(CALLER_PATH_TYPE), + created_at, + ) + .expect("insert request caller metadata"); +} + +fn assert_released_request(record: &RequestRecord, request_id: &str) { + assert_eq!(record.request_id, request_id); + assert_eq!(record.outcome, "active"); + assert_eq!(record.route.as_deref(), Some("responses")); + assert_eq!(record.model.as_deref(), Some("caller-model")); + assert_eq!(record.provider.as_deref(), Some("caller-provider")); + assert_eq!(record.engine.as_deref(), Some("caller-engine")); +} + +fn assert_caller(detailed: &RequestRecordWithCaller, endpoint_id: &str, caller_addr: &str) { + assert_eq!(detailed.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(detailed.caller_addr.as_deref(), Some(caller_addr)); + assert_eq!(detailed.caller_path_type.as_deref(), Some(CALLER_PATH_TYPE)); +} + +#[test] +fn detailed_request_preserves_caller_while_legacy_query_keeps_released_projection() { + // Given + let (_root, store) = open_store(); + insert_request_with_caller( + &store, + "request-with-caller", + "2026-08-22T12:00:00Z", + CALLER_ENDPOINT_ID, + CALLER_ADDR, + ); + + // When + let detailed = store + .query_request_with_caller("request-with-caller") + .expect("query detailed request") + .expect("detailed request"); + let legacy = store + .query_request("request-with-caller") + .expect("query legacy request") + .expect("legacy request"); + + // Then + assert_released_request(&detailed.request, "request-with-caller"); + assert_caller(&detailed, CALLER_ENDPOINT_ID, CALLER_ADDR); + assert_released_request(&legacy, "request-with-caller"); +} + +#[test] +fn detailed_request_pagination_keeps_legacy_order_and_cursor() { + // Given + let (_root, store) = open_store(); + for (request_id, created_at, endpoint_digit, caller_addr) in [ + ("request-a", "2026-08-22T12:00:01Z", 'a', "192.0.2.10:11204"), + ("request-b", "2026-08-22T12:00:02Z", 'b', "192.0.2.11:11204"), + ("request-c", "2026-08-22T12:00:03Z", 'c', "192.0.2.12:11204"), + ] { + let endpoint_id = endpoint_digit.to_string().repeat(64); + insert_request_with_caller(&store, request_id, created_at, &endpoint_id, caller_addr); + } + + // When + let detailed_first = store + .query_requests_with_caller(&request_query(2, None)) + .expect("query first detailed page"); + let legacy_first = store + .query_requests(&request_query(2, None)) + .expect("query first legacy page"); + let detailed_second = store + .query_requests_with_caller(&request_query(2, detailed_first.next_cursor.clone())) + .expect("query second detailed page"); + + // Then + assert_eq!(detailed_first.next_cursor, legacy_first.next_cursor); + assert_eq!( + detailed_first + .items + .iter() + .map(|item| item.request.request_id.as_str()) + .collect::>(), + ["request-c", "request-b"] + ); + assert_eq!( + legacy_first + .items + .iter() + .map(|item| item.request_id.as_str()) + .collect::>(), + ["request-c", "request-b"] + ); + assert_eq!(detailed_second.items[0].request.request_id, "request-a"); + assert!(detailed_second.next_cursor.is_none()); +} + +#[test] +fn detailed_request_batch_omits_missing_ids_across_chunk_boundary() { + // Given + let (_root, store) = open_store(); + let mut requested_ids = Vec::new(); + for index in 0..=100_u8 { + let request_id = format!("request-{index:03}"); + let endpoint_id = format!("{index:064x}"); + let caller_addr = format!("192.0.2.{}:11204", index + 1); + insert_request_with_caller( + &store, + &request_id, + "2026-08-22T12:00:00Z", + &endpoint_id, + &caller_addr, + ); + requested_ids.push(request_id); + } + requested_ids.push("request-missing".to_owned()); + + // When + let detailed = store + .query_requests_by_ids_with_caller(&requested_ids) + .expect("query detailed request batch"); + let legacy = store + .query_requests_by_ids(&requested_ids) + .expect("query legacy request batch"); + + // Then + assert_eq!(detailed.len(), 101); + assert_eq!(legacy.len(), 101); + let detailed_by_id = detailed + .iter() + .map(|item| (item.request.request_id.as_str(), item)) + .collect::>(); + let legacy_ids = legacy + .iter() + .map(|item| item.request_id.as_str()) + .collect::>(); + assert!(!detailed_by_id.contains_key("request-missing")); + assert!(!legacy_ids.contains("request-missing")); + for index in [99_u8, 100_u8] { + let request_id = format!("request-{index:03}"); + let endpoint_id = format!("{index:064x}"); + let caller_addr = format!("192.0.2.{}:11204", index + 1); + assert_caller( + detailed_by_id + .get(request_id.as_str()) + .expect("detailed request at chunk boundary"), + &endpoint_id, + &caller_addr, + ); + } +} + +fn insert_detailed_audit( + store: &LogStore, + entry_id: &str, + occurred_at: &str, + remote_addr: &str, + command_summary: &str, +) { + let detail = serde_json::json!({ + "severity": "info", + "context_version": 1, + "subject_kind": "mesh_peer", + "subject_id": entry_id, + "remote_addr": remote_addr, + "path_type": AUDIT_PATH_TYPE, + "operation_id": format!("operation-{entry_id}"), + "request_id": format!("correlation-{entry_id}"), + "reason_code": "accepted", + "outcome": "ready", + "command_summary": command_summary, + "duration_ms": 17, + "numeric_summaries": {"peers": 3} + }); + store + .insert_audit_entry( + entry_id, + None, + occurred_at, + "cli", + "command_completed", + Some(&detail.to_string()), + ) + .expect("insert detailed audit entry"); +} + +fn assert_released_audit(row: &AuditEntryRow, entry_id: &str) { + assert_eq!(row.entry_id, entry_id); + assert_eq!(row.source, "cli"); + assert_eq!(row.code, "command_completed"); + assert_eq!(row.severity, Some(AuditEntrySeverity::Info)); + assert_eq!(row.context_version, Some(1)); + assert_eq!(row.subject_kind.as_deref(), Some("mesh_peer")); + assert_eq!(row.subject_id.as_deref(), Some(entry_id)); + assert_eq!(row.reason_code.as_deref(), Some("accepted")); + assert_eq!(row.outcome.as_deref(), Some("ready")); + assert_eq!(row.duration_ms, Some(17)); + assert_eq!(row.numeric_summaries.get("peers"), Some(&3)); +} + +fn assert_audit_detail( + detail: &AuditEntryDetail, + entry_id: &str, + remote_addr: &str, + command_summary: &str, +) { + assert_released_audit(&detail.entry, entry_id); + assert_eq!(detail.remote_addr.as_deref(), Some(remote_addr)); + assert_eq!(detail.path_type.as_deref(), Some(AUDIT_PATH_TYPE)); + assert_eq!(detail.command_summary.as_deref(), Some(command_summary)); +} + +#[test] +fn detailed_audit_preserves_path_remote_and_command_while_legacy_list_stays_released() { + // Given + let (_root, store) = open_store(); + insert_detailed_audit( + &store, + "audit-detailed", + "2026-08-22T12:00:00Z", + AUDIT_REMOTE_ADDR, + AUDIT_COMMAND, + ); + + // When + let detailed = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .expect("list detailed audit entries"); + let legacy = store + .list_audit_entries(Some(1), None, AuditEntryFilters::default()) + .expect("list legacy audit entries"); + + // Then + assert_audit_detail( + &detailed.items[0], + "audit-detailed", + AUDIT_REMOTE_ADDR, + AUDIT_COMMAND, + ); + assert_released_audit(&legacy.items[0], "audit-detailed"); +} + +#[test] +fn detailed_audit_pagination_keeps_legacy_order_and_cursor_boundary() { + // Given + let (_root, store) = open_store(); + for (entry_id, remote_addr, command) in [ + ("audit-a", "198.51.100.10:443", AUDIT_COMMAND), + ("audit-b", "198.51.100.11:443", AUDIT_COMMAND), + ("audit-c", "198.51.100.12:443", AUDIT_COMMAND), + ] { + insert_detailed_audit( + &store, + entry_id, + "2026-08-22T12:00:00Z", + remote_addr, + command, + ); + } + + // When + let detailed_first = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list first detailed audit page"); + let legacy_first = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .expect("list first legacy audit page"); + let detailed_second = store + .list_audit_entry_details( + Some(2), + detailed_first.next_cursor.as_deref(), + AuditEntryFilters::default(), + ) + .expect("list second detailed audit page"); + + // Then + assert_eq!(detailed_first.next_cursor, legacy_first.next_cursor); + assert_eq!( + detailed_first + .items + .iter() + .map(|item| item.entry.entry_id.as_str()) + .collect::>(), + ["audit-c", "audit-b"] + ); + assert_eq!(detailed_second.items[0].entry.entry_id, "audit-a"); + assert!(detailed_second.next_cursor.is_none()); +} + +#[test] +fn detailed_audit_sequence_reconciliation_resumes_after_exact_boundary() { + // Given + let (root, store) = open_store(); + let writer = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit writer"); + for (entry_id, remote_addr, command) in [ + ("audit-sequence-a", "203.0.113.10:443", AUDIT_COMMAND), + ("audit-sequence-b", "203.0.113.11:443", AUDIT_COMMAND), + ("audit-sequence-c", "203.0.113.12:443", AUDIT_COMMAND), + ] { + insert_detailed_audit( + &writer, + entry_id, + "2026-08-22T12:00:00Z", + remote_addr, + command, + ); + } + + // When + let first = store + .list_audit_entry_details_after_sequence(0, 2, AuditEntryFilters::default()) + .expect("reconcile first detailed audit batch"); + let boundary = u64::try_from(first[1].entry.sequence).expect("positive audit sequence"); + let second = store + .list_audit_entry_details_after_sequence(boundary, 2, AuditEntryFilters::default()) + .expect("reconcile second detailed audit batch"); + + // Then + assert_eq!( + first + .iter() + .map(|item| item.entry.entry_id.as_str()) + .collect::>(), + ["audit-sequence-a", "audit-sequence-b"] + ); + assert_eq!(second.len(), 1); + assert_audit_detail( + &second[0], + "audit-sequence-c", + "203.0.113.12:443", + AUDIT_COMMAND, + ); + assert!(second[0].entry.sequence > i64::try_from(boundary).expect("sequence fits i64")); +} diff --git a/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs b/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs new file mode 100644 index 0000000000..61d25b7c19 --- /dev/null +++ b/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs @@ -0,0 +1,60 @@ +use std::sync::Arc; + +use mesh_llm_log_store::{AuditEntryFilters, AuditEntrySeverity, LogStore, RealClock}; + +#[test] +fn audit_entry_reads_sanitize_raw_actor_and_action_scalars() { + // Given: rows written outside the typed repository with malformed and oversized scalars. + let temp = tempfile::tempdir().expect("create temporary log store"); + let store = LogStore::open(temp.path(), Arc::new(RealClock)).expect("open log store"); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let malformed_actor = format!("{home}/runtime\nactor"); + let secret_action = "Bearer raw-sql-action-secret"; + let oversized_actor = "a".repeat(1_100); + let oversized_action = "z".repeat(1_100); + store + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + rusqlite::params![ + "raw-audit-malformed", + "2026-08-22T12:00:01.000000000Z", + malformed_actor, + secret_action, + r#"{"severity":"warning"}"#, + ], + ) + .expect("insert malformed raw audit row"); + store + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, NULL)", + rusqlite::params![ + "raw-audit-oversized", + "2026-08-22T12:00:00.000000000Z", + oversized_actor, + oversized_action, + ], + ) + .expect("insert oversized raw audit row"); + + // When: the shared durable read projection maps both rows. + let page = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .expect("list raw audit rows"); + + // Then: ordering and detail projection survive while both scalar columns are canonicalized. + assert_eq!(page.items[0].entry_id, "raw-audit-malformed"); + assert_eq!(page.items[0].source, "~/runtimeactor"); + assert_eq!(page.items[0].code, "[REDACTED]"); + assert_eq!(page.items[0].severity, Some(AuditEntrySeverity::Warning)); + assert_eq!(page.items[1].entry_id, "raw-audit-oversized"); + assert_eq!(page.items[1].source.chars().count(), 1_024); + assert!(page.items[1].source.ends_with("... [TRUNCATED]")); + assert_eq!(page.items[1].code.chars().count(), 1_024); + assert!(page.items[1].code.ends_with("... [TRUNCATED]")); +} diff --git a/crates/mesh-llm-ui/e2e/a11y/logs-a11y.spec.ts b/crates/mesh-llm-ui/e2e/a11y/logs-a11y.spec.ts index f4ccb33a51..a29d95a9fa 100644 --- a/crates/mesh-llm-ui/e2e/a11y/logs-a11y.spec.ts +++ b/crates/mesh-llm-ui/e2e/a11y/logs-a11y.spec.ts @@ -175,29 +175,58 @@ test('fallback log polling toggle remains AA-compliant while paused', async ({ p (storageKey) => window.localStorage.setItem(storageKey, 'live'), 'mesh-llm-ui-preview:data-mode:v2' ) - await page.route('**/api/logs/requests*', (route) => route.fulfill({ json: { items: [], nextCursor: null } })) - await page.route('**/api/logs/audit*', (route) => route.fulfill({ json: { items: [], nextCursor: null } })) - // Hold the SSE connection open — see the comment on the previous test. - let releaseEventsStream: (() => void) | undefined + let resolveFallbackRequestsFulfillment: (() => void) | undefined + let resolveFallbackAuditFulfillment: (() => void) | undefined + await page.route('**/api/logs/requests*', async (route) => { + const resolveFulfillment = resolveFallbackRequestsFulfillment + resolveFallbackRequestsFulfillment = undefined + await route.fulfill({ json: { items: [], nextCursor: null } }) + resolveFulfillment?.() + }) + await page.route('**/api/logs/audit*', async (route) => { + const resolveFulfillment = resolveFallbackAuditFulfillment + resolveFallbackAuditFulfillment = undefined + await route.fulfill({ json: { items: [], nextCursor: null } }) + resolveFulfillment?.() + }) + // Hold both SSE connections open — see the comment on the previous test. + let releaseRequestsStream: (() => void) | undefined + let releaseAuditStream: (() => void) | undefined await page.route('**/api/logs/events*', async (route) => { + const isAuditStream = new URL(route.request().url()).searchParams.get('audit') === '1' await new Promise((resolve) => { - releaseEventsStream = resolve + if (isAuditStream) { + releaseAuditStream = resolve + } else { + releaseRequestsStream = resolve + } }) await route.fulfill({ contentType: 'text/event-stream', - body: 'retry: 600000\nid: v1:0.0.0\nevent: stream_error\ndata: {"code":"invalid_event"}\n\n' + body: 'retry: 600000\n\n' }) }) await page.goto('/logs') await expect(page.getByRole('heading', { level: 1, name: 'System logs' })).toBeVisible() - await expect.poll(() => releaseEventsStream).toBeDefined() + await expect.poll(() => releaseRequestsStream).toBeDefined() + await expect.poll(() => releaseAuditStream).toBeDefined() await page.clock.pauseAt(new Date(await page.evaluate(() => Date.now() + 50))) - releaseEventsStream?.() + releaseRequestsStream?.() + releaseAuditStream?.() await expect(page.getByText('Reconnecting', { exact: true })).toBeVisible() - // Step deliberately past FALLBACK_DELAY_MS (1s) into the `polling` state, - // where the toggle under test renders. + const fallbackRequestsFulfilled = new Promise((resolve) => { + resolveFallbackRequestsFulfillment = resolve + }) + const fallbackAuditFulfilled = new Promise((resolve) => { + resolveFallbackAuditFulfillment = resolve + }) + // Trigger fallback at exactly FALLBACK_DELAY_MS (1s). Active fallback stays + // `reconnecting`; await both hydrations and flush query notifications. await page.clock.runFor(1_000) + await Promise.all([fallbackRequestsFulfilled, fallbackAuditFulfilled]) + await page.clock.runFor(0) + await expect(page.getByText('Updating', { exact: true })).toHaveCount(0) const pollingToggle = page.getByRole('button', { name: 'Fallback log polling' }) await expect(pollingToggle).toHaveAttribute('aria-pressed', 'true') diff --git a/crates/mesh-llm-ui/e2e/logs/log-workflows.spec.ts b/crates/mesh-llm-ui/e2e/logs/log-workflows.spec.ts index 81df70bcd5..61b1107ff0 100644 --- a/crates/mesh-llm-ui/e2e/logs/log-workflows.spec.ts +++ b/crates/mesh-llm-ui/e2e/logs/log-workflows.spec.ts @@ -391,7 +391,7 @@ test('logs recovery uses the dedicated stream gap and bounded polling fallback', await browserPage.clock.pauseAt(new Date(await browserPage.evaluate(() => Date.now() + 2_000))) const pollingToggle = browserPage.getByRole('button', { name: 'Fallback log polling' }) await expect(pollingToggle).toHaveAttribute('aria-pressed', 'true') - await expect(pollingToggle).toContainText('Polling') + await expect(pollingToggle).toContainText('Reconnecting') backend.streamMode = 'event' const streamAttemptsBeforePause = backend.streamUrls.length @@ -408,7 +408,7 @@ test('logs recovery uses the dedicated stream gap and bounded polling fallback', await pollingToggle.click() await expect(pollingToggle).toHaveAttribute('aria-pressed', 'true') - await expect(pollingToggle).toContainText('Polling') + await expect(pollingToggle).toContainText('Reconnecting') expect(backend.listCalls).toBe(listCallsBeforePause) expect(backend.streamUrls).toHaveLength(streamAttemptsBeforeResume) await browserPage.clock.runFor(5_000) @@ -431,6 +431,7 @@ test('metadata-only export and previewed cleanup stay separated without dead-let }) => { const backend = await installLogsBackend(browserPage, { lifecycle: 'completed', streamMode: 'unavailable' }) + await browserPage.setViewportSize({ width: 1280, height: 900 }) await browserPage.goto('/logs') const infoBanner = browserPage.getByRole('region', { name: 'System logs' }) const ledgerControls = browserPage.getByRole('region', { name: 'Event log controls' }) @@ -458,6 +459,77 @@ test('metadata-only export and previewed cleanup stay separated without dead-let await expect( cleanupDialog.getByRole('button', { name: /System chart layer.*retained during cleanup/ }) ).toHaveAttribute('data-state', 'on') + const layerControls = cleanupDialog.getByRole('button', { name: /chart layer/ }) + const desktopLayerLayout = await layerControls.evaluateAll(([requests, system, quic]) => { + if (!(requests instanceof HTMLElement) || !(system instanceof HTMLElement) || !(quic instanceof HTMLElement)) { + return { controlsFit: false, twoColumns: false } + } + const requestsBounds = requests.getBoundingClientRect() + const systemBounds = system.getBoundingClientRect() + const quicBounds = quic.getBoundingClientRect() + return { + controlsFit: [system, quic].every((control) => control.scrollWidth <= control.clientWidth), + twoColumns: Math.abs(requestsBounds.top - systemBounds.top) <= 1 && quicBounds.top >= requestsBounds.bottom - 1 + } + }) + expect.soft(desktopLayerLayout).toEqual({ controlsFit: true, twoColumns: true }) + + await browserPage.setViewportSize({ width: 375, height: 900 }) + const mobileLayerLayout = await layerControls.evaluateAll(([requests, system, quic]) => { + if (!(requests instanceof HTMLElement) || !(system instanceof HTMLElement) || !(quic instanceof HTMLElement)) { + return { controlsFit: false, oneColumn: false } + } + const requestsBounds = requests.getBoundingClientRect() + const systemBounds = system.getBoundingClientRect() + const quicBounds = quic.getBoundingClientRect() + return { + controlsFit: [system, quic].every((control) => control.scrollWidth <= control.clientWidth), + oneColumn: + systemBounds.top >= requestsBounds.bottom - 1 && + quicBounds.top >= systemBounds.bottom - 1 && + Math.abs(requestsBounds.left - systemBounds.left) <= 1 && + Math.abs(systemBounds.left - quicBounds.left) <= 1 + } + }) + expect.soft(mobileLayerLayout).toEqual({ controlsFit: true, oneColumn: true }) + + const selectorPanelBottom = await cleanupDialog + .getByRole('slider', { name: 'Window start' }) + .evaluate( + (element) => element.parentElement?.parentElement?.getBoundingClientRect().bottom ?? Number.POSITIVE_INFINITY + ) + const helperTop = await cleanupDialog + .getByText('Drag either edge to narrow the loaded history. The server preview confirms what can be removed.', { + exact: true + }) + .evaluate((element) => element.getBoundingClientRect().top) + expect.soft(helperTop).toBeGreaterThanOrEqual(selectorPanelBottom) + + const requestSummaryGeometry = await cleanupDialog + .locator('p') + .filter({ + hasText: /loaded request events? in this window\. Server review identifies removable terminal request groups\./ + }) + .evaluate((explanation) => { + const summary = explanation.parentElement + const counter = summary?.querySelector(':scope > span') + if (!(summary instanceof HTMLElement) || !(counter instanceof HTMLElement)) return null + const counterBounds = counter.getBoundingClientRect() + const explanationBounds = explanation.getBoundingClientRect() + const counterStyle = getComputedStyle(counter) + return { + counter: counter.textContent?.trim() ?? '', + fontFamily: counterStyle.fontFamily, + fontVariantNumeric: counterStyle.fontVariantNumeric, + gap: explanationBounds.left - counterBounds.right + } + }) + expect.soft(requestSummaryGeometry?.counter ?? '').toMatch(/^\d+$/) + expect.soft(requestSummaryGeometry?.fontFamily ?? '').toMatch(/JetBrains Mono|ui-monospace|Menlo|monospace/i) + expect.soft(requestSummaryGeometry?.fontVariantNumeric ?? '').toContain('tabular-nums') + expect.soft(requestSummaryGeometry?.gap ?? 0).toBeGreaterThan(0) + + await browserPage.setViewportSize({ width: 1280, height: 900 }) await cleanupDialog.getByRole('button', { name: /System chart layer.*retained during cleanup/ }).click() await expect( cleanupDialog.getByRole('button', { name: /System chart layer.*retained during cleanup/ }) diff --git a/crates/mesh-llm-ui/e2e/logs/logs-chart-stability.spec.ts b/crates/mesh-llm-ui/e2e/logs/logs-chart-stability.spec.ts index f171dece7e..5f8546b6ba 100644 --- a/crates/mesh-llm-ui/e2e/logs/logs-chart-stability.spec.ts +++ b/crates/mesh-llm-ui/e2e/logs/logs-chart-stability.spec.ts @@ -264,13 +264,9 @@ test.describe('events over time chart stability', () => { test('renders a high-volume dataset without exceeding React update depth', async ({ page }) => { // 370 requests spanning [now-370min, now-1min] under frozen time. // - // mergeLogEventWindow caps the MERGED request+audit list at 64 rows - // (LOG_EVENT_WINDOW_LIMIT), newest first — the cap is not per-category. - // AUDIT_ROWS contributes 3 entries inside the newest 64 (system@20min, - // quic@30min, gossip@50min); the 4th (gossip@490min) falls outside the - // window. So the legend reports Requests61 + System1 + QUIC1 + Gossip1, - // summing to exactly 64. Asserting the total proves the full mock dataset - // reached the ledger and was windowed rather than silently dropped. + // With the ledger window cap removed, every loaded fixture row is + // expected in the legend: all 370 requests plus System3, QUIC2, and + // Gossip2 from the seven audit rows. // The bar count itself stays small, so the fidelity gate is "bars render // at all and no render loop fires". const manyRequests = Array.from({ length: 370 }, (_, i) => @@ -284,10 +280,10 @@ test.describe('events over time chart stability', () => { await page.waitForTimeout(2500) const legend = page.getByRole('list', { name: 'Visible event categories' }) - await expect(legend).toContainText('Requests61') - await expect(legend).toContainText('System1') - await expect(legend).toContainText('QUIC1') - await expect(legend).toContainText('Gossip1') + await expect(legend).toContainText('Requests370') + await expect(legend).toContainText('System3') + await expect(legend).toContainText('QUIC2') + await expect(legend).toContainText('Gossip2') const bars = await page.locator('path.recharts-rectangle').count() expect(bars).toBeGreaterThan(0) expect(capture.depthErrors(), `render loop detected:\n${capture.depthErrors().join('\n')}`).toEqual([]) diff --git a/crates/mesh-llm-ui/e2e/logs/request-inspector-fixtures.ts b/crates/mesh-llm-ui/e2e/logs/request-inspector-fixtures.ts index 16b52c1b8c..957556527d 100644 --- a/crates/mesh-llm-ui/e2e/logs/request-inspector-fixtures.ts +++ b/crates/mesh-llm-ui/e2e/logs/request-inspector-fixtures.ts @@ -4,7 +4,8 @@ export const REQUEST_INSPECTOR_IDS = { empty: '10000000-0000-4000-8000-000000000003', failed: '10000000-0000-4000-8000-000000000004', active: '10000000-0000-4000-8000-000000000005', - transient: '10000000-0000-4000-8000-000000000006' + transient: '10000000-0000-4000-8000-000000000006', + streaming: '10000000-0000-4000-8000-000000000007' } as const export const REQUEST_INSPECTOR_ARTIFACT_IDS = { @@ -15,7 +16,8 @@ export const REQUEST_INSPECTOR_ARTIFACT_IDS = { unavailable: '30000000-0000-4000-8000-000000000005', corrupt: '30000000-0000-4000-8000-000000000006', errorCorrupt: '30000000-0000-4000-8000-000000000007', - errorMissing: '30000000-0000-4000-8000-000000000008' + errorMissing: '30000000-0000-4000-8000-000000000008', + streamingResponse: '30000000-0000-4000-8000-000000000009' } as const export const REQUEST_INSPECTOR_SHELL_STATUS = { @@ -26,6 +28,8 @@ export const REQUEST_INSPECTOR_SHELL_STATUS = { gpus: [] } +const CALLER_ENDPOINT_ID = '9f0c4cbe8cb7a8d5d577c20e50ef03fd2f63a2e7fd9897c155823bcbb281bb04' + type EventFixture = readonly [ sequence: number, occurredAt: string, @@ -137,11 +141,23 @@ const failedArtifacts = [ artifact(REQUEST.failed, [ARTIFACT.errorCorrupt, 'error_diagnostic', 'corrupt', 2048, 4]), artifact(REQUEST.failed, [ARTIFACT.errorMissing, 'error_trace', 'missing']) ] as const +const streamingResponseArtifact = { + ...artifact(REQUEST.streaming, [ARTIFACT.streamingResponse, 'response_body', 'available']), + mediaKind: 'text/event-stream' +} + +export const REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT = '' export const REQUEST_INSPECTOR_SCENARIOS: Readonly> = { [REQUEST.completed]: { - summary: summary(REQUEST.completed, 'completed', 0), + summary: { + ...summary(REQUEST.completed, 'completed', 0), + callerEndpointId: CALLER_ENDPOINT_ID, + callerAddr: '203.0.113.24:48712', + callerPathType: 'remote_quic_http' + }, events: [ + event(REQUEST.completed, [0, timestamp(0, '00.500'), 'admitted']), event(REQUEST.completed, [1, timestamp(0, '01.200'), 'stream_started']), event(REQUEST.completed, [2, timestamp(0, '02'), 'stream_chunk']), event(REQUEST.completed, [3, timestamp(0, '03'), 'stream_completed', undefined, undefined, 42]) @@ -188,6 +204,12 @@ export const REQUEST_INSPECTOR_SCENARIOS: Readonly"') + }, + [ARTIFACT.streamingResponse]: { + ...streamingResponseArtifact, + contentBase64: encoded( + [ + 'event: delta', + 'id: stream-1', + 'data: {"delta":"hello"}', + '', + `data: ${REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT}`, + '', + 'event: done', + 'data: [DONE]', + '' + ].join('\n') + ) } } @@ -219,8 +257,8 @@ export function requestDeleteReceipt(requestId: string) { requestId, state: 'completed', selectionFingerprint: 'request-inspector-delete', - planned: { requests: 1, events: 3, artifacts: 5, proxyRecords: 1, databaseRows: 10 }, - executed: { requests: 1, events: 3, artifacts: 5, proxyRecords: 1, databaseRows: 10 }, + planned: { requests: 1, events: 4, artifacts: 5, proxyRecords: 1, databaseRows: 11 }, + executed: { requests: 1, events: 4, artifacts: 5, proxyRecords: 1, databaseRows: 11 }, artifactDeletion: { removed: 5, failed: 0 } } } diff --git a/crates/mesh-llm-ui/e2e/logs/request-inspector-footer-clearance.spec.ts b/crates/mesh-llm-ui/e2e/logs/request-inspector-footer-clearance.spec.ts new file mode 100644 index 0000000000..cfd1c1288a --- /dev/null +++ b/crates/mesh-llm-ui/e2e/logs/request-inspector-footer-clearance.spec.ts @@ -0,0 +1,49 @@ +import type { Page } from '@playwright/test' +import { openInspector } from './request-inspector-helpers' +import { installRequestInspectorRoutes, REQUEST_INSPECTOR_IDS } from './request-inspector-routes' +import { expect, test } from './request-inspector-test' + +const VIEWPORTS = [ + { label: 'mobile', width: 375, height: 520 }, + { label: 'desktop', width: 1280, height: 800 } +] as const + +async function footerClearance(page: Page) { + const inspector = await openInspector(page, REQUEST_INSPECTOR_IDS.completed) + const scrollBody = inspector.locator('[data-request-inspector-scroll="body"]') + const footer = inspector.getByRole('contentinfo', { name: 'Request inspector actions' }) + const overview = inspector.getByRole('region', { name: 'Request overview' }) + await expect.poll(() => scrollBody.evaluate((element) => element.scrollHeight > element.clientHeight)).toBe(true) + await scrollBody.evaluate((element) => { + element.scrollTop = element.scrollHeight + }) + await expect + .poll(() => scrollBody.evaluate((element) => element.scrollTop + element.clientHeight)) + .toBe(await scrollBody.evaluate((element) => element.scrollHeight)) + + const [contentBottom, footerTop, spacing] = await Promise.all([ + overview.evaluate((element) => element.lastElementChild?.getBoundingClientRect().bottom ?? null), + footer.evaluate((element) => element.getBoundingClientRect().top), + scrollBody.evaluate((element) => ({ + expected: Number.parseFloat(getComputedStyle(document.documentElement).getPropertyValue('--shell-normal')), + paddingBottom: Number.parseFloat(getComputedStyle(element).paddingBottom) + })) + ]) + if (contentBottom === null) throw new Error('Request overview final panel bounds missing') + return { contentBottom, footerTop, ...spacing } +} + +for (const viewport of VIEWPORTS) { + test(`keeps the final overview panel one normal shell space above the footer at ${viewport.label} width`, async ({ + page + }) => { + await installRequestInspectorRoutes(page) + await page.setViewportSize({ width: viewport.width, height: viewport.height }) + await page.goto('/logs') + + const { contentBottom, expected, footerTop, paddingBottom } = await footerClearance(page) + + expect(paddingBottom).toBeCloseTo(expected, 1) + expect(footerTop - contentBottom).toBeGreaterThanOrEqual(expected - 1) + }) +} diff --git a/crates/mesh-llm-ui/e2e/logs/request-inspector-overview.spec.ts b/crates/mesh-llm-ui/e2e/logs/request-inspector-overview.spec.ts index fce01b64a1..5c4c258162 100644 --- a/crates/mesh-llm-ui/e2e/logs/request-inspector-overview.spec.ts +++ b/crates/mesh-llm-ui/e2e/logs/request-inspector-overview.spec.ts @@ -46,9 +46,21 @@ test('keeps completed evidence and footer actions visible while only the inspect await expect(inspector).toHaveAttribute('data-request-inspector-shell', 'fixed') await expect(overview.getByText('1 attempt / 0 retries', { exact: true })).toBeVisible() - await expect(overview.getByRole('list', { name: 'Lifecycle events' })).toContainText( - /stream_started[\s\S]*stream_chunk[\s\S]*stream_completed/ - ) + const caller = overview.getByRole('region', { name: 'Caller' }) + await expect(caller).toContainText('9f0c…bb04') + await expect(caller).toContainText('203.0.113.24:48712') + await expect(caller).toContainText('Remote QUIC HTTP') + const callerCopy = caller.getByRole('button', { name: 'Copy caller endpoint ID' }) + await expect(callerCopy).toBeVisible() + await expect + .poll(() => callerCopy.evaluate((element) => element.getBoundingClientRect().height)) + .toBeGreaterThanOrEqual(44) + const lifecycle = overview.getByRole('list', { name: 'Lifecycle events' }) + await expect(lifecycle.locator('li[data-event-kind="stream_started"]')).toHaveCount(1) + await expect(lifecycle).toContainText('Stream started') + await overview.getByRole('button', { name: 'Later lifecycle events' }).click() + await expect(lifecycle.locator('li[data-event-kind="stream_completed"]')).toHaveCount(1) + await expect(lifecycle).toContainText('Stream done') await expect(overview.getByRole('list', { name: 'Routing attempts' })).toContainText('mesh-primary') const retention = overview.getByRole('region', { name: 'Artifact retention' }) await expect(retention).toContainText('2 available · 1 unavailable · 1 missing · 1 corrupt') diff --git a/crates/mesh-llm-ui/e2e/logs/request-inspector-payloads.spec.ts b/crates/mesh-llm-ui/e2e/logs/request-inspector-payloads.spec.ts index 80afe336c9..6565b90d8f 100644 --- a/crates/mesh-llm-ui/e2e/logs/request-inspector-payloads.spec.ts +++ b/crates/mesh-llm-ui/e2e/logs/request-inspector-payloads.spec.ts @@ -2,14 +2,76 @@ import { openInspector, selectInspectorTab } from './request-inspector-helpers' import { installRequestInspectorRoutes, REQUEST_INSPECTOR_ARTIFACT_IDS, - REQUEST_INSPECTOR_IDS + REQUEST_INSPECTOR_IDS, + REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT } from './request-inspector-routes' import { expect, test } from './request-inspector-test' +import type { Locator, Page } from '@playwright/test' + +type ScrollEndpoint = 'start' | 'end' + +async function expectCopyAnchoredAtEndpoint( + page: Page, + payloadViewport: Locator, + copy: Locator, + endpoint: ScrollEndpoint +) { + const expectedState = { + atEndpoint: true, + copyContained: true, + copyPainted: true, + horizontallyScrollable: true, + toolbarMatchesViewport: true + } + + await payloadViewport.evaluate((viewport, target) => { + viewport.scrollLeft = target === 'start' ? 0 : viewport.scrollWidth - viewport.clientWidth + }, endpoint) + await expect + .poll(() => + payloadViewport.evaluate((viewport, target) => { + const button = viewport.querySelector('button[aria-label="Copy JSON payload"]') + const buttonRect = button?.getBoundingClientRect() + const toolbarRect = button?.parentElement?.getBoundingClientRect() + const viewportRect = viewport.getBoundingClientRect() + const targetScrollLeft = target === 'start' ? 0 : viewport.scrollWidth - viewport.clientWidth + const hitTarget = + buttonRect === undefined + ? null + : document.elementFromPoint(buttonRect.left + buttonRect.width / 2, buttonRect.top + buttonRect.height / 2) + + return { + atEndpoint: Math.abs(viewport.scrollLeft - targetScrollLeft) <= 1, + copyContained: + buttonRect !== undefined && + buttonRect.left >= viewportRect.left - 1 && + buttonRect.right <= viewportRect.right + 1, + copyPainted: button !== null && hitTarget !== null && (hitTarget === button || button.contains(hitTarget)), + horizontallyScrollable: viewport.scrollWidth > viewport.clientWidth, + toolbarMatchesViewport: toolbarRect !== undefined && Math.abs(toolbarRect.width - viewportRect.width) <= 1 + } + }, endpoint) + ) + .toEqual(expectedState) + + await payloadViewport.focus() + await page.keyboard.press('Tab') + await expect(copy).toBeFocused() + await expect + .poll(() => + payloadViewport.evaluate((viewport, target) => { + const targetScrollLeft = target === 'start' ? 0 : viewport.scrollWidth - viewport.clientWidth + return Math.abs(viewport.scrollLeft - targetScrollLeft) <= 1 + }, endpoint) + ) + .toBe(true) +} test('loads only the selected payload and safely exposes format, lines, and copy controls', async ({ context, page }) => { + await page.setViewportSize({ width: 375, height: 812 }) await context.grantPermissions(['clipboard-read', 'clipboard-write']) const backend = await installRequestInspectorRoutes(page) await page.goto('/logs') @@ -21,25 +83,83 @@ test('loads only the selected payload and safely exposes format, lines, and copy const responsePane = inspector.getByRole('region', { name: 'Response', exact: true }) // Only the Request pane is visible by default (single-pane toggle) - await expect(inspector.getByRole('button', { name: 'Load payload' })).toHaveCount(1) for (const state of ['missing', 'unavailable', 'corrupt'] as const) { await expect(inspector.getByText(state, { exact: true }).first()).toBeVisible() } - expect(backend.artifactDetailCalls).toEqual([]) - - // Load request payload - await requestPane.getByRole('button', { name: 'Load payload' }).click() await expect.poll(() => backend.artifactDetailCalls).toEqual([REQUEST_INSPECTOR_ARTIFACT_IDS.request]) const requestJson = requestPane.getByRole('region', { name: 'Request JSON payload' }) + const paneHeader = requestPane.locator('header').first() + const payloadControl = paneHeader.getByRole('radiogroup', { name: 'Payload' }) + const displayToolbar = inspector.getByRole('toolbar', { name: 'Display' }) + const formatControl = displayToolbar.getByRole('radiogroup', { name: 'Display' }) + await expect(payloadControl.getByRole('radio', { name: 'Request' })).toBeChecked() + await expect(payloadControl.getByRole('radio', { name: 'Response' })).toBeVisible() + await expect(displayToolbar).toHaveCount(1) + await expect(formatControl).toHaveCount(1) + await expect(paneHeader.getByRole('radio', { name: 'Pretty' })).toHaveCount(0) + await expect(displayToolbar.getByRole('radio', { name: 'Request' })).toHaveCount(0) + await expect(requestJson.getByRole('radiogroup', { name: 'Display' })).toHaveCount(0) await expect(requestJson.locator('[data-json-token="key"]').first()).toBeVisible() await expect(requestJson.locator('[data-line-number]')).toHaveCount(5) - await expect(requestJson.getByRole('radio', { name: 'Pretty' })).toBeChecked() + await expect(formatControl.getByRole('radio', { name: 'Pretty' })).toBeChecked() + + const payloadViewport = requestPane.getByRole('region', { name: 'Request payload content' }) + const copy = requestJson.getByRole('button', { name: 'Copy JSON payload' }) + const scrollArea = payloadViewport.locator('..') + const horizontalScrollbar = scrollArea.locator('[data-orientation="horizontal"]') + const verticalScrollbar = scrollArea.locator('[data-orientation="vertical"]') + const horizontalThumb = horizontalScrollbar.locator(':scope > div').first() + const inspectorBody = inspector.locator('[data-request-inspector-scroll="body"]') + await expect(inspectorBody).toHaveCount(1) + await expect + .poll(() => + inspectorBody.evaluate((body) => ({ + overflowY: getComputedStyle(body).overflowY, + verticallyScrollable: body.scrollHeight > body.clientHeight + })) + ) + .toEqual({ overflowY: 'auto', verticallyScrollable: true }) + await expect(inspector.locator('[data-radix-scroll-area-viewport]')).toHaveCount(1) + await expect(payloadViewport.locator('[data-radix-scroll-area-viewport]')).toHaveCount(0) + await expect(horizontalScrollbar).toBeVisible() + await expect(horizontalThumb).toBeVisible() + await expect(verticalScrollbar).toHaveCount(0) + await expect(scrollArea).not.toHaveClass(/(?:^|\s)(?:h-64|h-80|sm:h-\[28rem\]|lg:h-\[32rem\])(?:\s|$)/) + await expect + .poll(() => + payloadViewport.evaluate((viewport) => ({ + overflowY: getComputedStyle(viewport).overflowY, + verticallyScrollable: viewport.scrollHeight > viewport.clientHeight + })) + ) + .toEqual({ overflowY: 'hidden', verticallyScrollable: false }) + const prettyPayloadHeight = await payloadViewport.evaluate((viewport) => viewport.getBoundingClientRect().height) + await expectCopyAnchoredAtEndpoint(page, payloadViewport, copy, 'start') + await expectCopyAnchoredAtEndpoint(page, payloadViewport, copy, 'end') + await expect + .poll(() => + horizontalScrollbar.evaluate((track) => { + const thumb = track.firstElementChild + const trackColor = getComputedStyle(track).backgroundColor + const thumbColor = thumb === null ? '' : getComputedStyle(thumb).backgroundColor + return { + colorsDiffer: trackColor !== thumbColor, + thumbOpaque: thumbColor !== '' && thumbColor !== 'rgba(0, 0, 0, 0)', + trackOpaque: trackColor !== 'rgba(0, 0, 0, 0)' + } + }) + ) + .toEqual({ colorsDiffer: true, thumbOpaque: true, trackOpaque: true }) // Format, lines, copy controls on request JSON - await requestJson.getByRole('radio', { name: 'Raw' }).click() - await expect(requestJson.getByRole('radio', { name: 'Raw' })).toBeChecked() + await formatControl.getByRole('radio', { name: 'Raw' }).click() + await expect(formatControl.getByRole('radio', { name: 'Raw' })).toBeChecked() await expect(requestJson.locator('[data-line-number]')).toHaveCount(1) - const copy = requestJson.getByRole('button', { name: 'Copy JSON payload' }) + await expect + .poll(() => payloadViewport.evaluate((viewport) => viewport.getBoundingClientRect().height)) + .toBeLessThan(prettyPayloadHeight) + await expectCopyAnchoredAtEndpoint(page, payloadViewport, copy, 'start') + await expectCopyAnchoredAtEndpoint(page, payloadViewport, copy, 'end') await expect(copy).toBeVisible() await copy.click() await expect(requestJson.getByRole('status')).toContainText('Raw JSON representation selected. JSON payload copied.') @@ -48,28 +168,26 @@ test('loads only the selected payload and safely exposes format, lines, and copy await expect(requestJson.locator('img')).toHaveCount(0) await expect(requestJson.locator('script')).toHaveCount(0) - // Toggle to Response, load response payload await inspector.getByRole('radio', { name: 'Response' }).click() - await responsePane.getByRole('button', { name: 'Load payload' }).click() await expect .poll(() => backend.artifactDetailCalls) .toEqual([REQUEST_INSPECTOR_ARTIFACT_IDS.request, REQUEST_INSPECTOR_ARTIFACT_IDS.response]) const responseJson = responsePane.getByRole('region', { name: 'Response JSON payload' }) await expect(responseJson.locator('[data-json-token="boolean"]').first()).toBeVisible() + await expect(formatControl.getByRole('radio', { name: 'Raw' })).toBeChecked() + await expect(responseJson.locator('[data-line-number]')).toHaveCount(1) + + await inspector.getByRole('radio', { name: 'Request' }).click() + await expect(formatControl.getByRole('radio', { name: 'Raw' })).toBeChecked() + await expect(requestJson.locator('[data-line-number]')).toHaveCount(1) // Tab round-trip — both panes cached, no new artifactDetailCalls - // Toggle resets to Request on remount after tab switch await selectInspectorTab(page, inspector, { id: 'overview', name: 'Overview' }) await selectInspectorTab(page, inspector, { id: 'payloads', name: 'Payloads' }) - // Request pane shows cached ready-to-view state (1 View payload button visible) - await expect(requestPane.getByRole('button', { name: 'View payload' })).toBeVisible() - await requestPane.getByRole('button', { name: 'View payload' }).click() await expect(requestJson).toBeVisible() - // Toggle to Response — also cached, no new fetches await inspector.getByRole('radio', { name: 'Response' }).click() - await responsePane.getByRole('button', { name: 'View payload' }).click() await expect(responseJson).toBeVisible() expect(backend.artifactDetailCalls).toEqual([ @@ -78,19 +196,103 @@ test('loads only the selected payload and safely exposes format, lines, and copy ]) }) -test('renders malformed retained JSON as inert plaintext only after explicit load', async ({ page }) => { +test('pages one multi-frame SSE response with numbered controls while preserving format and inert frame content', async ({ + page +}) => { + await page.setViewportSize({ width: 375, height: 812 }) + const backend = await installRequestInspectorRoutes(page) + await page.goto('/logs') + const inspector = await openInspector(page, REQUEST_INSPECTOR_IDS.streaming) + await selectInspectorTab(page, inspector, { id: 'payloads', name: 'Payloads' }) + + const displayToolbar = inspector.getByRole('toolbar', { name: 'Display' }) + const formatControl = displayToolbar.getByRole('radiogroup', { name: 'Display' }) + await expect(displayToolbar).toHaveCount(1) + await expect(formatControl).toHaveCount(1) + await formatControl.getByRole('radio', { name: 'Raw' }).click() + await inspector.getByRole('radio', { name: 'Response' }).click() + await expect.poll(() => backend.artifactDetailCalls).toEqual([REQUEST_INSPECTOR_ARTIFACT_IDS.streamingResponse]) + + const payloadViewport = inspector.getByRole('region', { name: 'Response payload content' }) + const firstFrame = inspector.getByRole('region', { name: 'Response event stream frame 1', exact: true }) + await expect(firstFrame).toContainText('Frame 1 of 3') + await expect(firstFrame).toContainText('delta') + await expect(firstFrame).toContainText('stream-1') + await expect(firstFrame.locator('[data-json-line]')).toHaveCount(1) + await expect(firstFrame.getByRole('button', { name: 'Copy JSON payload' })).toBeVisible() + await expect(inspector.getByRole('region', { name: /^Response event stream frame \d$/ })).toHaveCount(1) + await expect(inspector.getByRole('listitem')).toHaveCount(0) + await expect(payloadViewport.locator('[data-radix-scroll-area-viewport]')).toHaveCount(0) + const framePager = inspector.getByRole('radiogroup', { name: 'Response frames' }) + const frameNavigator = framePager.locator('..') + const frameChoices = framePager.getByRole('radio') + await expect(frameChoices).toHaveText(['1', '2', '3']) + const previous = inspector.getByRole('button', { name: 'Previous response frame' }) + const next = inspector.getByRole('button', { name: 'Next response frame' }) + await expect(previous).toBeDisabled() + await expect(next).toBeEnabled() + await expect(inspector.getByRole('status').filter({ hasText: 'Frame 1 of 3' })).toHaveCount(1) + await expect + .poll(() => + frameNavigator.evaluate((navigator) => { + const header = navigator.closest('header') + const context = header?.firstElementChild + if (!(header instanceof HTMLElement) || !(context instanceof HTMLElement)) { + return { fillsRow: false, followsContext: false } + } + const navigatorRect = navigator.getBoundingClientRect() + const headerRect = header.getBoundingClientRect() + const contextRect = context.getBoundingClientRect() + return { + fillsRow: navigatorRect.width >= headerRect.width - 26, + followsContext: navigatorRect.top >= contextRect.bottom + } + }) + ) + .toEqual({ fillsRow: true, followsContext: true }) + for (const target of [previous, ...(await frameChoices.all()), next]) { + await expect + .poll(() => + target.evaluate((control) => { + const rect = control.getBoundingClientRect() + return Math.min(rect.width, rect.height) + }) + ) + .toBeGreaterThanOrEqual(32) + } + + await framePager.getByRole('radio', { name: 'Response frame 2 of 3' }).click() + const secondFrame = inspector.getByRole('region', { name: 'Response event stream frame 2', exact: true }) + await expect(firstFrame).toHaveCount(0) + await expect(secondFrame).toContainText(REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT) + await expect(secondFrame.locator('img')).toHaveCount(0) + await expect(secondFrame.locator('script')).toHaveCount(0) + await expect(secondFrame.getByRole('button', { name: 'Copy JSON payload' })).toHaveCount(0) + await expect(inspector.getByRole('region', { name: /^Response event stream frame \d$/ })).toHaveCount(1) + await expect(formatControl.getByRole('radio', { name: 'Raw' })).toBeChecked() + + await framePager.getByRole('radio', { name: 'Response frame 2 of 3' }).focus() + await page.keyboard.press('ArrowRight') + const doneFrame = inspector.getByRole('region', { name: 'Response event stream frame 3', exact: true }) + await expect(secondFrame).toHaveCount(0) + await expect(doneFrame).toContainText('Frame 3 of 3') + await expect(doneFrame).toContainText('done') + await expect(doneFrame).toContainText('[DONE]') + await expect(doneFrame.getByRole('button', { name: 'Copy JSON payload' })).toHaveCount(0) + await expect(inspector.getByRole('region', { name: /^Response event stream frame \d$/ })).toHaveCount(1) + await expect(next).toBeDisabled() + await expect(previous).toBeEnabled() + await expect(inspector.getByRole('status').filter({ hasText: 'Frame 3 of 3' })).toHaveCount(1) + await expect(formatControl.getByRole('radio', { name: 'Raw' })).toBeChecked() + expect(backend.artifactDetailCalls).toEqual([REQUEST_INSPECTOR_ARTIFACT_IDS.streamingResponse]) +}) + +test('renders malformed retained JSON as inert plaintext when the selected payload loads', async ({ page }) => { const backend = await installRequestInspectorRoutes(page) await page.goto('/logs') const inspector = await openInspector(page, REQUEST_INSPECTOR_IDS.malformed) await selectInspectorTab(page, inspector, { id: 'payloads', name: 'Payloads' }) - expect(backend.artifactDetailCalls).toEqual([]) - await inspector - .getByRole('region', { name: 'Request', exact: true }) - .getByRole('button', { - name: 'Load payload' - }) - .click() await expect.poll(() => backend.artifactDetailCalls).toEqual([REQUEST_INSPECTOR_ARTIFACT_IDS.malformed]) await expect(inspector.getByText('Malformed JSON. Showing inert plaintext; no markup is interpreted.')).toBeVisible() await expect(inspector.getByRole('region', { name: 'Request malformed JSON plaintext' })).toContainText( diff --git a/crates/mesh-llm-ui/e2e/logs/request-inspector-routes.ts b/crates/mesh-llm-ui/e2e/logs/request-inspector-routes.ts index 9f351512b8..afb4dc660d 100644 --- a/crates/mesh-llm-ui/e2e/logs/request-inspector-routes.ts +++ b/crates/mesh-llm-ui/e2e/logs/request-inspector-routes.ts @@ -5,12 +5,13 @@ import { REQUEST_INSPECTOR_ARTIFACT_IDS, REQUEST_INSPECTOR_IDS, REQUEST_INSPECTOR_SCENARIOS, - REQUEST_INSPECTOR_SHELL_STATUS + REQUEST_INSPECTOR_SHELL_STATUS, + REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT } from './request-inspector-fixtures' const DATA_MODE_STORAGE_KEY = 'mesh-llm-ui-preview:data-mode:v2' -export { REQUEST_INSPECTOR_ARTIFACT_IDS, REQUEST_INSPECTOR_IDS } +export { REQUEST_INSPECTOR_ARTIFACT_IDS, REQUEST_INSPECTOR_IDS, REQUEST_INSPECTOR_STREAM_HOSTILE_TEXT } type RequestCapability = 'supported' | 'unsupported' | 'loading' diff --git a/crates/mesh-llm-ui/src/app/layout/RootLayout.test.tsx b/crates/mesh-llm-ui/src/app/layout/RootLayout.test.tsx index 78d08d839b..b15b679ee3 100644 --- a/crates/mesh-llm-ui/src/app/layout/RootLayout.test.tsx +++ b/crates/mesh-llm-ui/src/app/layout/RootLayout.test.tsx @@ -109,6 +109,12 @@ describe('RootLayout', () => { expect(useStatusStreamSpy).toHaveBeenCalledWith({ enabled: true }) }) + it('keeps the app shell width stable by reserving the gutter on its scroll container', () => { + renderRootLayout('harness') + + expect(document.querySelector('main')).toHaveClass('overflow-y-auto', '[scrollbar-gutter:stable]') + }) + it('selects the Logs tab for the logs route', () => { routerState.pathname = '/logs' featureFlagState.logsPage = true diff --git a/crates/mesh-llm-ui/src/app/layout/RootLayout.tsx b/crates/mesh-llm-ui/src/app/layout/RootLayout.tsx index 0601858fc9..5fa61831f7 100644 --- a/crates/mesh-llm-ui/src/app/layout/RootLayout.tsx +++ b/crates/mesh-llm-ui/src/app/layout/RootLayout.tsx @@ -212,7 +212,7 @@ export function RootLayout({ data = SHELL_HARNESS }: RootLayoutProps = {}) { /> ) : null} -
+
diff --git a/crates/mesh-llm-ui/src/components/ui/AccentIconFrame.tsx b/crates/mesh-llm-ui/src/components/ui/AccentIconFrame.tsx index 6fd09532a1..9098af2d48 100644 --- a/crates/mesh-llm-ui/src/components/ui/AccentIconFrame.tsx +++ b/crates/mesh-llm-ui/src/components/ui/AccentIconFrame.tsx @@ -1,13 +1,13 @@ import type { CSSProperties, ReactNode } from 'react' import { cn } from '@/lib/cn' -type AccentIconFrameTone = 'accent' | 'subtle' +type AccentIconFrameTone = 'accent' | 'subtle' | 'bad' type AccentIconFrameProps = { - children: ReactNode - className?: string - style?: CSSProperties - tone?: AccentIconFrameTone + readonly children: ReactNode + readonly className?: string + readonly style?: CSSProperties + readonly tone?: AccentIconFrameTone } const frameStyleByTone: Record = { @@ -19,6 +19,11 @@ const frameStyleByTone: Record = { background: 'color-mix(in oklab, var(--color-accent-soft) 42%, var(--color-panel-strong))', border: '1px solid color-mix(in oklab, var(--color-accent) 18%, var(--color-border))', color: 'color-mix(in oklab, var(--color-accent) 48%, var(--color-fg-dim))' + }, + bad: { + background: 'color-mix(in oklab, var(--color-bad) 18%, transparent)', + border: '1px solid color-mix(in oklab, var(--color-bad) 30%, var(--color-border))', + color: 'var(--color-bad-text)' } } diff --git a/crates/mesh-llm-ui/src/components/ui/InfoBanner.test.tsx b/crates/mesh-llm-ui/src/components/ui/InfoBanner.test.tsx new file mode 100644 index 0000000000..563d277518 --- /dev/null +++ b/crates/mesh-llm-ui/src/components/ui/InfoBanner.test.tsx @@ -0,0 +1,39 @@ +import '@testing-library/jest-dom/vitest' + +import { render, screen, within } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { InfoBanner } from '@/components/ui/InfoBanner' + +describe('InfoBanner', () => { + it('keeps the default banner as a named region and renders its annotation', () => { + render( + Window notice} + description="Current activity remains available." + title="System activity" + titleId="system-activity-title" + /> + ) + + const region = screen.getByRole('region', { name: 'System activity' }) + expect(within(region).getByText('Window notice')).toBeVisible() + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + }) + + it('announces the error variant once while keeping annotation details visible', () => { + render( + Request history: Unavailable} + description="Request history could not be loaded." + title="System logs" + titleId="system-logs-title" + variant="error" + /> + ) + + const alert = screen.getByRole('alert', { name: 'System logs' }) + expect(alert).toHaveTextContent('Request history could not be loaded.') + expect(within(alert).getByText('Request history: Unavailable')).toBeVisible() + expect(screen.getAllByRole('alert')).toHaveLength(1) + }) +}) diff --git a/crates/mesh-llm-ui/src/components/ui/InfoBanner.tsx b/crates/mesh-llm-ui/src/components/ui/InfoBanner.tsx index b344b3890e..96c0648764 100644 --- a/crates/mesh-llm-ui/src/components/ui/InfoBanner.tsx +++ b/crates/mesh-llm-ui/src/components/ui/InfoBanner.tsx @@ -1,21 +1,61 @@ import type { ReactNode } from 'react' +import { CircleAlert } from 'lucide-react' import { AccentIconFrame } from '@/components/ui/AccentIconFrame' import { cn } from '@/lib/cn' -type InfoBannerProps = { - title: ReactNode - description: ReactNode - action?: ReactNode - actionClassName?: string - className?: string - contentClassName?: string - descriptionClassName?: string - leadingIcon?: ReactNode - leadingIconClassName?: string - status?: ReactNode - titleClassName?: string - titleId?: string - titleLevel?: 'h1' | 'h2' | 'h3' +type InfoBannerVariant = 'default' | 'error' +type InfoBannerAnnotationTone = 'warn' | 'bad' + +export type InfoBannerProps = { + readonly title: ReactNode + readonly description: ReactNode + readonly action?: ReactNode + readonly actionClassName?: string + readonly annotation?: ReactNode + readonly className?: string + readonly contentClassName?: string + readonly descriptionClassName?: string + readonly leadingIcon?: ReactNode + readonly leadingIconClassName?: string + readonly status?: ReactNode + readonly titleClassName?: string + readonly titleId?: string + readonly titleLevel?: 'h1' | 'h2' | 'h3' + readonly variant?: InfoBannerVariant +} + +type InfoBannerAnnotationProps = { + readonly ariaLabel: string + readonly children: ReactNode + readonly className?: string + readonly tone?: InfoBannerAnnotationTone +} + +const bannerBackground: Record = { + default: + 'linear-gradient(90deg, color-mix(in oklab, var(--color-accent) 10%, var(--color-panel)) 0%, var(--color-panel) 60%)', + error: + 'linear-gradient(90deg, color-mix(in oklab, var(--color-bad) 10%, var(--color-panel)) 0%, var(--color-panel) 60%)' +} + +const annotationToneClass: Record = { + warn: 'text-warn', + bad: 'text-bad' +} + +export function InfoBannerAnnotation({ ariaLabel, children, className, tone = 'warn' }: InfoBannerAnnotationProps) { + return ( +
+
+ ) } export function InfoBanner({ @@ -31,7 +71,9 @@ export function InfoBanner({ status, titleClassName, titleId, - titleLevel = 'h2' + titleLevel = 'h2', + variant = 'default', + annotation }: InfoBannerProps) { const Heading = titleLevel @@ -40,14 +82,17 @@ export function InfoBanner({ aria-labelledby={titleId} className={cn( 'panel-shell flex items-center gap-5 rounded-[var(--radius-lg)] border border-border px-5 py-4', + variant === 'error' && 'border-bad/40', className )} - style={{ - background: - 'linear-gradient(90deg, color-mix(in oklab, var(--color-accent) 10%, var(--color-panel)) 0%, var(--color-panel) 60%)' - }} + role={variant === 'error' ? 'alert' : undefined} + style={{ background: bannerBackground[variant] }} > - {leadingIcon ? {leadingIcon} : null} + {leadingIcon ? ( + + {leadingIcon} + + ) : null}
{status}
: null}
{description}
+ {annotation} {action ? (
{action}
diff --git a/crates/mesh-llm-ui/src/components/ui/Pager.test.tsx b/crates/mesh-llm-ui/src/components/ui/Pager.test.tsx new file mode 100644 index 0000000000..7731f01885 --- /dev/null +++ b/crates/mesh-llm-ui/src/components/ui/Pager.test.tsx @@ -0,0 +1,353 @@ +// @vitest-environment jsdom +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { describe, expect, it, vi } from 'vitest' +import { Pager } from '@/components/ui/Pager' + +describe('Pager', () => { + it('renders nothing when a single page covers the content', () => { + // Given a pager with one page + const { container } = render() + + // Then no paging affordance is offered + expect(container).toBeEmptyDOMElement() + }) + + it('renders one dot per page and marks the active page', () => { + // Given a pager across four pages positioned on the second + render() + + // Then every page is reachable and the active one is checked + const dots = screen.getAllByRole('radio') + expect(dots).toHaveLength(4) + expect(dots[1]).toBeChecked() + expect(screen.getByRole('radiogroup', { name: 'Pages' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Page 3 of 4' })).toBeInTheDocument() + }) + + it('keeps radio hit targets separate from the visual dot sizes', () => { + // Given a pager with active and inactive page choices + render() + + // Then each radio has a 24px hit target and an aria-hidden visual child + const radios = screen.getAllByRole('radio') + expect(radios).toHaveLength(2) + for (const radio of radios) { + expect(radio).toHaveClass('size-6', 'p-0') + expect(radio.querySelector('[aria-hidden="true"]')).toBeInTheDocument() + } + expect(radios[0].querySelector('[aria-hidden="true"]')).toHaveClass('h-1.5', 'w-4') + expect(radios[1].querySelector('[aria-hidden="true"]')).toHaveClass('size-1.5') + }) + + it('renders visible page numbers only when the numbered variant is requested', () => { + // Given a numbered pager positioned on the second of four pages + render() + + // Then each radio shows its page number and the active page uses the selected control treatment + const radios = screen.getAllByRole('radio') + expect(radios).toHaveLength(4) + expect(radios.map((radio) => radio.textContent)).toEqual(['1', '2', '3', '4']) + expect(radios[0].querySelector('[aria-hidden="true"]')).toHaveClass('text-fg-dim') + expect(radios[1].querySelector('[aria-hidden="true"]')).toHaveClass('bg-accent', 'text-accent-ink') + }) + + it('gives numbered controls distinct operational geometry and surfaces', () => { + // Given a numbered pager on its first page + render() + + // Then the navigator uses a wrapping grid with deliberate spacing + const group = screen.getByRole('radiogroup', { name: 'Frames' }) + expect(group.parentElement).toHaveClass('grid', 'grid-cols-[auto_minmax(0,1fr)_auto]', 'gap-2') + expect(group).toHaveClass('flex-wrap', 'gap-1.5') + + // And every step and direct choice is a bordered 32px operational target + const previous = screen.getByRole('button', { name: 'Previous page' }) + const next = screen.getByRole('button', { name: 'Next page' }) + expect(previous).toHaveClass('size-10', 'border', 'border-border', 'bg-panel') + expect(previous).toHaveClass('disabled:opacity-50') + expect(previous).toBeDisabled() + expect(next).toHaveClass('size-10', 'border', 'border-border', 'bg-panel') + expect(next).toBeEnabled() + + const radios = screen.getAllByRole('radio') + expect(radios.map((radio) => radio.textContent)).toEqual(['1', '2', '3']) + for (const radio of radios) { + expect(radio).toHaveClass('size-10', 'border') + } + expect(radios[0]).toHaveClass('border-accent', 'bg-accent', 'text-accent-ink') + expect(radios[1]).toHaveClass('border-border', 'bg-panel', 'hover:border-border-strong', 'hover:bg-panel-strong') + }) + + it('supports direct and keyboard numbered movement with a domain status', async () => { + // Given a controlled numbered frame pager + const user = userEvent.setup() + const onValueChange = vi.fn() + const statusLabel = (index: number, count: number) => `Frame ${index + 1} of ${count}` + const { rerender } = render( + + ) + + // Then the first boundary and live status are explicit + expect(screen.getByRole('button', { name: 'Previous page' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Frame 1 of 3') + + // When a numbered choice is selected directly + await user.click(screen.getByRole('radio', { name: 'Page 3 of 3' })) + + // Then the direct logical index is reported + expect(onValueChange).toHaveBeenLastCalledWith(2) + + // When the controlled value moves to frame two and ArrowRight is pressed + rerender( + + ) + screen.getByRole('radio', { name: 'Page 2 of 3' }).focus() + await user.keyboard('{ArrowRight}') + + // Then one logical frame is requested + expect(onValueChange).toHaveBeenLastCalledWith(2) + + // When the controlled pager reaches the final frame + rerender( + + ) + + // Then the final boundary and domain status are explicit + expect(screen.getByRole('button', { name: 'Next page' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Previous page' })).toBeEnabled() + expect(screen.getByRole('status')).toHaveTextContent('Frame 3 of 3') + }) + + it('uses zero adjacent spacing so seven targets and arrows fit the narrow contract', () => { + // Given the maximum visible page window + render() + + // Then nine 24px controls occupy 216px before zero-width visual gaps + expect(screen.getByRole('radiogroup').parentElement).toHaveClass('gap-0') + expect(screen.getByRole('radiogroup')).toHaveClass('gap-0') + expect(screen.getByRole('button', { name: 'Previous page' })).toHaveClass('size-6') + expect(screen.getByRole('button', { name: 'Next page' })).toHaveClass('size-6') + expect(screen.getAllByRole('radio')).toHaveLength(7) + for (const radio of screen.getAllByRole('radio')) { + expect(radio).toHaveClass('size-6') + } + for (const gap of screen.getAllByTestId('pager-gap')) { + expect(gap).toHaveClass('w-0', 'text-foreground') + } + }) + + it('announces the current page once through a polite status', () => { + // Given a pager with multiple pages + const { rerender } = render() + + // Then the current page is available as one non-focus-moving live status + const status = screen.getByRole('status') + expect(status).toHaveTextContent('Page 1 of 4') + expect(status).toHaveAttribute('aria-live', 'polite') + expect(status).toHaveAttribute('aria-atomic', 'true') + + // When the controlled page changes + rerender() + + // Then only the status text changes + expect(screen.getByRole('status')).toHaveTextContent('Page 2 of 4') + }) + + it('disables the indicator transition under reduced motion', () => { + // Given a pager with active and inactive indicators + render() + + // Then indicator motion has an explicit reduced-motion override + expect(screen.getAllByRole('radio')[0].querySelector('[aria-hidden="true"]')).toHaveClass( + 'motion-reduce:transition-none' + ) + }) + + it('bounds large page selections while retaining the current, first, and last pages', async () => { + // Given a pager with 84 pages positioned in the middle + const user = userEvent.setup() + const onValueChange = vi.fn() + render( + `Page ${index + 1} of 84`} + value={41} + onValueChange={onValueChange} + /> + ) + + // Then the visible page choices remain bounded and retain navigation anchors + const radios = screen.getAllByRole('radio') + expect(radios.length).toBeLessThanOrEqual(7) + expect(screen.getByRole('radio', { name: 'Page 1 of 84' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Page 42 of 84' })).toBeChecked() + expect(screen.getByRole('radio', { name: 'Page 84 of 84' })).toBeInTheDocument() + + // When the reader uses previous and next navigation + await user.click(screen.getByRole('button', { name: 'Previous page' })) + await user.click(screen.getByRole('button', { name: 'Next page' })) + + // Then each control advances exactly one page + expect(onValueChange).toHaveBeenNthCalledWith(1, 40) + expect(onValueChange).toHaveBeenNthCalledWith(2, 42) + }) + + it('keeps visible large-page choices in Radix arrow-key order', async () => { + // Given a bounded page window focused on the current page + const user = userEvent.setup() + render() + const current = screen.getByRole('radio', { name: 'Page 42 of 84' }) + + // When the reader moves forward with the horizontal arrow key + current.focus() + await user.keyboard('{ArrowRight}') + + // Then focus advances to the next visible radio without adding controls + expect(screen.getByRole('radio', { name: 'Page 43 of 84' })).toHaveFocus() + expect(screen.getAllByRole('radio')).toHaveLength(7) + }) + + it('marks nonadjacent visible page choices with hidden visual gaps', () => { + // Given a bounded pager around page six of a large collection + render() + + // Then gaps separate nonconsecutive choices without becoming radio options or focus targets + const gaps = screen.getAllByTestId('pager-gap') + expect(gaps).toHaveLength(2) + for (const gap of gaps) { + expect(gap).toHaveAttribute('aria-hidden', 'true') + expect(gap).not.toHaveAttribute('role') + expect(gap).not.toHaveAttribute('tabindex') + } + expect(screen.getAllByRole('radio')).toHaveLength(7) + expect(screen.getByRole('radio', { name: 'Page 1 of 125' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Page 4 of 125' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Page 8 of 125' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Page 125 of 125' })).toBeInTheDocument() + }) + + it('moves one logical page per horizontal arrow and clamps at both ends', async () => { + // Given a large pager positioned on page six + const user = userEvent.setup() + const onValueChange = vi.fn() + const { rerender } = render() + const current = screen.getByRole('radio', { name: 'Page 6 of 125' }) + + // When the reader presses ArrowRight + current.focus() + await user.keyboard('{ArrowRight}') + + // Then exactly the next logical page is requested + expect(onValueChange).toHaveBeenLastCalledWith(6) + + // When the controlled pager recomputes around page seven + rerender() + + // Then the selected page remains in the visible window + expect(screen.getByRole('radio', { name: 'Page 7 of 125' })).toBeChecked() + + // When the pager is at the first page and ArrowLeft is pressed + rerender() + screen.getByRole('radio', { name: 'Page 1 of 125' }).focus() + await user.keyboard('{ArrowLeft}') + expect(onValueChange).toHaveBeenLastCalledWith(0) + + // When the pager is at the last page and ArrowRight is pressed + rerender() + screen.getByRole('radio', { name: 'Page 125 of 125' }).focus() + await user.keyboard('{ArrowRight}') + expect(onValueChange).toHaveBeenLastCalledWith(124) + }) + + it('moves one logical page for vertical arrow keys', async () => { + // Given a large pager positioned on page six + const user = userEvent.setup() + const onValueChange = vi.fn() + render() + const current = screen.getByRole('radio', { name: 'Page 6 of 125' }) + current.focus() + + // When the reader presses ArrowDown and ArrowUp + await user.keyboard('{ArrowDown}') + await user.keyboard('{ArrowUp}') + + // Then each key requests one adjacent logical page + expect(onValueChange).toHaveBeenNthCalledWith(1, 6) + expect(onValueChange).toHaveBeenNthCalledWith(2, 4) + }) + + it('steps through pages and clamps at both ends', async () => { + // Given a pager on the first of three pages + const user = userEvent.setup() + const onValueChange = vi.fn() + const { rerender } = render() + + // Then the backwards step is unavailable and the forwards step advances + expect(screen.getByRole('button', { name: 'Previous page' })).toBeDisabled() + await user.click(screen.getByRole('button', { name: 'Next page' })) + expect(onValueChange).toHaveBeenCalledWith(1) + + // When the pager reaches the final page + rerender() + + // Then the forwards step is unavailable + expect(screen.getByRole('button', { name: 'Next page' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Previous page' })).toBeEnabled() + }) + + it('selects a page directly from its dot', async () => { + // Given a pager across three pages + const user = userEvent.setup() + const onValueChange = vi.fn() + render() + + // When the last dot is chosen + await user.click(screen.getByRole('radio', { name: 'Page 3 of 3' })) + + // Then the pager reports that page + expect(onValueChange).toHaveBeenCalledWith(2) + }) + + it('uses caller-supplied labels', () => { + // Given a pager with domain labels + render( + `Segment ${index + 1}`} + previousLabel="Earlier events" + value={0} + onValueChange={vi.fn()} + /> + ) + + // Then those labels reach assistive technology + expect(screen.getByRole('button', { name: 'Later events' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Earlier events' })).toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Segment 2' })).toBeInTheDocument() + }) +}) diff --git a/crates/mesh-llm-ui/src/components/ui/Pager.tsx b/crates/mesh-llm-ui/src/components/ui/Pager.tsx new file mode 100644 index 0000000000..bad40be969 --- /dev/null +++ b/crates/mesh-llm-ui/src/components/ui/Pager.tsx @@ -0,0 +1,169 @@ +import * as RadioGroup from '@radix-ui/react-radio-group' +import { ChevronLeft, ChevronRight } from 'lucide-react' +import { Fragment } from 'react' +import { Button } from '@/components/ui/button' +import { cn } from '@/lib/cn' + +type PagerProps = { + readonly ariaLabel: string + readonly className?: string + readonly count: number + readonly nextLabel?: string + readonly pageLabel?: (index: number) => string + readonly previousLabel?: string + readonly statusLabel?: (index: number, count: number) => string + readonly value: number + readonly variant?: 'dots' | 'numbered' + readonly onValueChange: (value: number) => void +} + +const dotStepClassName = 'size-6 rounded-full text-fg-dim hover:text-foreground disabled:opacity-40' + +const dotRadioClassName = + 'inline-grid size-6 shrink-0 place-items-center rounded-full p-0 outline-none focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent' + +const dotClassName = + 'rounded-full transition-[width,background-color] duration-150 ease-out motion-reduce:transition-none' + +const numberedStepClassName = + 'size-10 rounded-[var(--radius-control)] border border-border bg-panel text-fg-dim transition-colors hover:border-border-strong hover:bg-panel-strong hover:text-foreground disabled:pointer-events-none disabled:border-border-soft disabled:bg-panel disabled:text-fg-faint disabled:opacity-50' + +const numberedRadioClassName = + 'inline-grid size-10 shrink-0 place-items-center rounded-[var(--radius-control)] border p-0 font-mono type-caption tabular-nums outline-none transition-colors duration-150 ease-out focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent motion-reduce:transition-none' + +const numberClassName = + 'grid size-full place-items-center rounded-[var(--radius-control)] font-mono type-caption tabular-nums transition-colors duration-150 ease-out motion-reduce:transition-none' + +const MAX_VISIBLE_PAGE_ITEMS = 7 + +const defaultStatusLabel = (index: number, count: number) => `Page ${index + 1} of ${count}` + +function visiblePageIndexes(count: number, current: number): readonly number[] { + if (count <= MAX_VISIBLE_PAGE_ITEMS) return Array.from({ length: count }, (_, index) => index) + + const windowSize = MAX_VISIBLE_PAGE_ITEMS - 2 + const windowStart = Math.max(1, Math.min(current - Math.floor(windowSize / 2), count - windowSize - 1)) + return [0, ...Array.from({ length: windowSize }, (_, offset) => windowStart + offset), count - 1] +} + +export function Pager({ + ariaLabel, + className, + count, + nextLabel = 'Next page', + pageLabel, + previousLabel = 'Previous page', + statusLabel = defaultStatusLabel, + value, + variant = 'dots', + onValueChange +}: PagerProps) { + if (count < 2) return null + const clamped = (next: number) => Math.min(count - 1, Math.max(0, next)) + const currentPage = clamped(value) + const pageIndexes = visiblePageIndexes(count, currentPage) + const numbered = variant === 'numbered' + + return ( +
+ + { + const direction = + event.key === 'ArrowLeft' || event.key === 'ArrowUp' + ? -1 + : event.key === 'ArrowRight' || event.key === 'ArrowDown' + ? 1 + : 0 + if (direction === 0) return + event.preventDefault() + onValueChange(clamped(currentPage + direction)) + }} + onValueChange={(next) => onValueChange(clamped(Number(next)))} + orientation="horizontal" + value={String(currentPage)} + > + {pageIndexes.map((index, position) => { + const hasGap = position < pageIndexes.length - 1 && pageIndexes[position + 1] !== index + 1 + return ( + + + + + {hasGap ? ( + + ) : null} + + ) + })} + + + {statusLabel(currentPage, count)} + + +
+ ) +} diff --git a/crates/mesh-llm-ui/src/components/ui/SegmentedControl.test.tsx b/crates/mesh-llm-ui/src/components/ui/SegmentedControl.test.tsx index 830d9c5e58..ae4da70eee 100644 --- a/crates/mesh-llm-ui/src/components/ui/SegmentedControl.test.tsx +++ b/crates/mesh-llm-ui/src/components/ui/SegmentedControl.test.tsx @@ -9,6 +9,34 @@ const options = [ ] describe('SegmentedControl', () => { + it.each(['buttons', 'pill'] as const)('uses a semantic focus ring for the %s variant', (variant) => { + // Given + render( + + ) + + // When + const selectedItem = screen.getByRole('radio', { name: 'On' }) + + // Then + expect(selectedItem).toHaveClass( + 'focus-visible:outline-none', + 'focus-visible:!ring-2', + 'focus-visible:!ring-accent-contrast', + 'focus-visible:!ring-offset-1', + 'focus-visible:!ring-offset-background' + ) + expect(selectedItem).not.toHaveClass('focus-visible:!ring-accent') + expect(selectedItem).not.toHaveClass('focus-visible:outline', 'focus-visible:outline-2') + }) + it('renders radio group with the given options', () => { render( export function SegmentedControl({ diff --git a/crates/mesh-llm-ui/src/components/ui/data-table.test.tsx b/crates/mesh-llm-ui/src/components/ui/data-table.test.tsx index d5f6e1f07d..2ab3c0e3ee 100644 --- a/crates/mesh-llm-ui/src/components/ui/data-table.test.tsx +++ b/crates/mesh-llm-ui/src/components/ui/data-table.test.tsx @@ -29,6 +29,7 @@ describe('DataTable', () => { } render() + await user.click(screen.getByRole('button', { name: /Go to next page/ })) await user.click(screen.getByRole('button', { name: /Name/i })) await user.click(await screen.findByRole('menuitem', { name: 'Asc' })) @@ -38,15 +39,22 @@ describe('DataTable', () => { expect(renders).toBe(settledRenders) expect(renders).toBeLessThan(20) expect(screen.getByRole('button', { name: 'Name, sorted ascending' })).toBeInTheDocument() + expect(screen.getByText('row-10')).toBeInTheDocument() }) it('settles after a page change instead of re-rendering in a loop', async () => { const user = userEvent.setup() - render() + const { rerender } = render() await user.click(screen.getByRole('button', { name: /Go to next page/ })) expect(screen.getByText('row-10')).toBeInTheDocument() expect(screen.queryByText('row-0')).not.toBeInTheDocument() + + const refreshedRows = rows.map((row) => ({ ...row, name: `${row.name}-refreshed` })) + rerender() + + expect(screen.getByText('row-10-refreshed')).toBeInTheDocument() + expect(screen.queryByText('row-0-refreshed')).not.toBeInTheDocument() }) it('settles while typing a filter instead of re-rendering in a loop', async () => { @@ -67,6 +75,141 @@ describe('DataTable', () => { expect(renders).toBeLessThan(20) }) + it('clamps to the final page when refreshed data shrinks', async () => { + const user = userEvent.setup() + const { rerender } = render() + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + expect(screen.getByText('row-20')).toBeInTheDocument() + + rerender() + + expect(screen.getByText('row-10')).toBeInTheDocument() + expect(screen.queryByText('row-0')).not.toBeInTheDocument() + }) + + it('never commits an impossible page while refreshed data shrinks', async () => { + const user = userEvent.setup() + const snapshots: Array<{ pageIndex: number; pageCount: number; rowCount: number }> = [] + const { rerender } = render( + + {(table) => { + snapshots.push({ + pageIndex: table.state.pagination.pageIndex, + pageCount: table.getPageCount(), + rowCount: table.getRowModel().rows.length + }) + return null + }} + + ) + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + rerender( + + {(table) => { + snapshots.push({ + pageIndex: table.state.pagination.pageIndex, + pageCount: table.getPageCount(), + rowCount: table.getRowModel().rows.length + }) + return null + }} + + ) + + expect(snapshots).not.toContainEqual({ pageIndex: 2, pageCount: 2, rowCount: 0 }) + expect(snapshots.at(-1)).toEqual({ pageIndex: 1, pageCount: 2, rowCount: 5 }) + }) + + it('clamps to the final page when filtering reduces the page count', async () => { + const user = userEvent.setup() + render() + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + expect(screen.getByText('row-20')).toBeInTheDocument() + + await user.type(screen.getByLabelText('Filter...'), 'row-1') + + expect(screen.getByText('row-19')).toBeInTheDocument() + expect(screen.queryByText('row-0')).not.toBeInTheDocument() + }) + + it('never commits an impossible page while filtering reduces the page count', async () => { + const user = userEvent.setup() + const snapshots: Array<{ pageIndex: number; pageCount: number; rowCount: number }> = [] + render( + + {(table) => { + snapshots.push({ + pageIndex: table.state.pagination.pageIndex, + pageCount: table.getPageCount(), + rowCount: table.getRowModel().rows.length + }) + return null + }} + + ) + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.type(screen.getByLabelText('Filter...'), 'row-1') + + expect(snapshots).not.toContainEqual({ pageIndex: 2, pageCount: 2, rowCount: 0 }) + expect(snapshots.at(-1)).toEqual({ pageIndex: 1, pageCount: 2, rowCount: 1 }) + }) + + it('renders the valid empty page immediately when all data is removed', async () => { + const user = userEvent.setup() + const snapshots: Array<{ pageIndex: number; pageCount: number; rowCount: number }> = [] + const { rerender } = render( + + {(table) => { + snapshots.push({ + pageIndex: table.state.pagination.pageIndex, + pageCount: table.getPageCount(), + rowCount: table.getRowModel().rows.length + }) + return null + }} + + ) + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + rerender( + + {(table) => { + snapshots.push({ + pageIndex: table.state.pagination.pageIndex, + pageCount: table.getPageCount(), + rowCount: table.getRowModel().rows.length + }) + return null + }} + + ) + + expect(snapshots.at(-1)).toEqual({ pageIndex: 0, pageCount: 0, rowCount: 0 }) + expect(screen.getByText('Page 0 of 0')).toBeInTheDocument() + expect(screen.getByText('No results.')).toBeInTheDocument() + }) + + it('clamps the current page when a larger page size reduces page count', async () => { + const user = userEvent.setup() + render() + + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.click(screen.getByRole('button', { name: /Go to next page/ })) + await user.selectOptions(screen.getByRole('combobox', { name: 'Rows per page' }), '25') + + expect(screen.getByText('Page 1 of 1')).toBeInTheDocument() + expect(screen.getByText('row-0')).toBeInTheDocument() + }) + it('reflects column visibility changes when the Columns menu is reopened', async () => { const user = userEvent.setup() render( diff --git a/crates/mesh-llm-ui/src/components/ui/data-table.tsx b/crates/mesh-llm-ui/src/components/ui/data-table.tsx index 964b65f1bc..478d9c7245 100644 --- a/crates/mesh-llm-ui/src/components/ui/data-table.tsx +++ b/crates/mesh-llm-ui/src/components/ui/data-table.tsx @@ -1,4 +1,4 @@ -import { useMemo, useState, type ComponentPropsWithoutRef, type ReactNode } from 'react' +import { startTransition, useEffect, useMemo, useState, type ComponentPropsWithoutRef, type ReactNode } from 'react' import { type ColumnFiltersState, type ColumnVisibilityState, @@ -78,11 +78,30 @@ export function DataTable({ onColumnFiltersChange: setColumnFilters, onColumnVisibilityChange: setColumnVisibility, onPaginationChange: setPagination, + autoResetPageIndex: false, manualPagination: !enablePagination }), [columnFilters, columnVisibility, columns, data, enablePagination, getRowId, pagination, sorting] ) - const table = useTable(tableOptions) + const rowModelTable = useTable(tableOptions) + const filteredRowCount = rowModelTable.getFilteredRowModel().rows.length + const lastPageIndex = Math.max(Math.ceil(filteredRowCount / pagination.pageSize) - 1, 0) + const effectiveTableOptions = useMemo(() => { + if (pagination.pageIndex <= lastPageIndex) return tableOptions + const effectivePagination = { ...pagination, pageIndex: lastPageIndex } + return { + ...tableOptions, + state: { ...tableOptions.state, pagination: effectivePagination } + } + }, [lastPageIndex, pagination, tableOptions]) + const table = useTable(effectiveTableOptions) + + useEffect(() => { + const nextPageIndex = Math.max(Math.ceil(filteredRowCount / pagination.pageSize) - 1, 0) + if (pagination.pageIndex <= nextPageIndex) return + + startTransition(() => setPagination((current) => ({ ...current, pageIndex: nextPageIndex }))) + }, [filteredRowCount, pagination.pageIndex, pagination.pageSize]) const filterValue = filterColumnId ? ((table.getColumn(filterColumnId)?.getFilterValue() as string) ?? '') : undefined diff --git a/crates/mesh-llm-ui/src/components/ui/scroll-area.test.tsx b/crates/mesh-llm-ui/src/components/ui/scroll-area.test.tsx new file mode 100644 index 0000000000..8a21dc16c2 --- /dev/null +++ b/crates/mesh-llm-ui/src/components/ui/scroll-area.test.tsx @@ -0,0 +1,41 @@ +// @vitest-environment jsdom + +import '@testing-library/jest-dom/vitest' + +import { render, screen } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { ScrollArea } from '@/components/ui/scroll-area' + +describe('ScrollArea', () => { + it('renders the vertical scrollbar by default', () => { + // Given / When + render( + + Payload + + ) + + // Then + const viewport = screen.getByRole('region', { name: 'Default content' }) + const scrollArea = viewport.parentElement + expect(scrollArea?.querySelector('[data-orientation="vertical"]')).toBeInTheDocument() + expect(scrollArea?.querySelector('[data-orientation="horizontal"]')).not.toBeInTheDocument() + }) + + it('suppresses only the vertical scrollbar when horizontal scrolling is enabled', () => { + // Given / When + render( + + Payload + + ) + + // Then + const viewport = screen.getByRole('region', { name: 'Horizontal payload' }) + const scrollArea = viewport.parentElement + expect(viewport).toHaveAttribute('tabindex', '0') + expect(viewport).not.toHaveClass('overflow-x-hidden') + expect(scrollArea?.querySelector('[data-orientation="vertical"]')).not.toBeInTheDocument() + expect(scrollArea?.querySelector('[data-orientation="horizontal"]')).toBeInTheDocument() + }) +}) diff --git a/crates/mesh-llm-ui/src/components/ui/scroll-area.tsx b/crates/mesh-llm-ui/src/components/ui/scroll-area.tsx index f00a0a987d..4197d6d976 100644 --- a/crates/mesh-llm-ui/src/components/ui/scroll-area.tsx +++ b/crates/mesh-llm-ui/src/components/ui/scroll-area.tsx @@ -7,19 +7,21 @@ const ScrollArea = React.forwardRef< React.ElementRef, React.ComponentPropsWithoutRef & { readonly horizontal?: boolean + readonly vertical?: boolean + readonly viewportClassName?: string readonly viewportLabel?: string } ->(({ className, children, horizontal = false, viewportLabel, ...props }, ref) => ( +>(({ className, children, horizontal = false, vertical = true, viewportClassName, viewportLabel, ...props }, ref) => ( {children} - + {vertical ? : null} {horizontal ? : null} diff --git a/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.test.ts b/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.test.ts new file mode 100644 index 0000000000..d8a2c7e55a --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from 'vitest' +import { + beginAuditTerminalRecovery, + completeAuditTerminalEof, + completeAuditTerminalHydration, + type AuditTerminalRecovery +} from './audit-terminal-recovery' + +type AuditSource = { readonly id: string } +type PendingHydrationPhase = 'hydrating' | 'awaiting_hydration' +type SettledHydrationPhase = 'awaiting_eof' | 'failed' | 'replaced' +type TerminalRecoveryPhase = 'failed' | 'replaced' + +const ORIGINAL_SOURCE: AuditSource = { id: 'original' } +const REPLACEMENT_SOURCE: AuditSource = { id: 'replacement' } +const PENDING_HYDRATION_PHASES: readonly PendingHydrationPhase[] = ['hydrating', 'awaiting_hydration'] +const SETTLED_HYDRATION_PHASES: readonly SettledHydrationPhase[] = ['awaiting_eof', 'failed', 'replaced'] +const TERMINAL_RECOVERY_PHASES: readonly TerminalRecoveryPhase[] = ['failed', 'replaced'] +const RECOVERIES: readonly AuditTerminalRecovery[] = [ + { phase: 'hydrating', source: ORIGINAL_SOURCE }, + { phase: 'awaiting_eof', source: ORIGINAL_SOURCE }, + { phase: 'awaiting_hydration', source: ORIGINAL_SOURCE }, + { phase: 'failed', source: ORIGINAL_SOURCE }, + { phase: 'replaced', source: ORIGINAL_SOURCE } +] + +describe('beginAuditTerminalRecovery', () => { + it('starts hydration when no terminal recovery exists', () => { + const transition = beginAuditTerminalRecovery(undefined, ORIGINAL_SOURCE) + + expect(transition).toEqual({ + recovery: { phase: 'hydrating', source: ORIGINAL_SOURCE }, + shouldReconnect: false, + shouldMarkStale: false + }) + }) + + it('starts hydration for a different source', () => { + const transition = beginAuditTerminalRecovery( + { phase: 'awaiting_eof', source: ORIGINAL_SOURCE }, + REPLACEMENT_SOURCE + ) + + expect(transition).toEqual({ + recovery: { phase: 'hydrating', source: REPLACEMENT_SOURCE }, + shouldReconnect: false, + shouldMarkStale: false + }) + }) + + it.each(RECOVERIES)('ignores a duplicate begin for the same source from $phase', (recovery) => { + const transition = beginAuditTerminalRecovery(recovery, ORIGINAL_SOURCE) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) +}) + +describe('completeAuditTerminalHydration', () => { + it('waits for EOF when hydration succeeds first', () => { + const transition = completeAuditTerminalHydration({ phase: 'hydrating', source: ORIGINAL_SOURCE }, true) + + expect(transition).toEqual({ + recovery: { phase: 'awaiting_eof', source: ORIGINAL_SOURCE }, + shouldReconnect: false, + shouldMarkStale: false + }) + }) + + it('replaces the source when hydration succeeds after EOF', () => { + const transition = completeAuditTerminalHydration({ phase: 'awaiting_hydration', source: ORIGINAL_SOURCE }, true) + + expect(transition).toEqual({ + recovery: { phase: 'replaced', source: ORIGINAL_SOURCE }, + shouldReconnect: true, + shouldMarkStale: false + }) + }) + + it('preserves an awaiting EOF recovery after duplicate hydration success', () => { + const recovery: AuditTerminalRecovery = { phase: 'awaiting_eof', source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalHydration(recovery, true) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) + + it.each(PENDING_HYDRATION_PHASES)('marks hydration failure stale from %s', (phase) => { + const recovery: AuditTerminalRecovery = { phase, source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalHydration(recovery, false) + + expect(transition).toEqual({ + recovery: { phase: 'failed', source: ORIGINAL_SOURCE }, + shouldReconnect: false, + shouldMarkStale: true + }) + }) + + it.each(SETTLED_HYDRATION_PHASES)('ignores late hydration failure from %s', (phase) => { + const recovery: AuditTerminalRecovery = { phase, source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalHydration(recovery, false) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) + + it.each(TERMINAL_RECOVERY_PHASES)('ignores late hydration success from %s', (phase) => { + const recovery: AuditTerminalRecovery = { phase, source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalHydration(recovery, true) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) +}) + +describe('completeAuditTerminalEof', () => { + it('waits for hydration when EOF arrives first', () => { + const transition = completeAuditTerminalEof({ phase: 'hydrating', source: ORIGINAL_SOURCE }) + + expect(transition).toEqual({ + recovery: { phase: 'awaiting_hydration', source: ORIGINAL_SOURCE }, + shouldReconnect: false, + shouldMarkStale: false + }) + }) + + it('replaces the source when EOF arrives after hydration', () => { + const transition = completeAuditTerminalEof({ phase: 'awaiting_eof', source: ORIGINAL_SOURCE }) + + expect(transition).toEqual({ + recovery: { phase: 'replaced', source: ORIGINAL_SOURCE }, + shouldReconnect: true, + shouldMarkStale: false + }) + }) + + it('preserves recovery when EOF repeats while hydration is pending', () => { + const recovery: AuditTerminalRecovery = { + phase: 'awaiting_hydration', + source: ORIGINAL_SOURCE + } + + const transition = completeAuditTerminalEof(recovery) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) + + it('keeps a failed recovery stale after late EOF', () => { + const recovery: AuditTerminalRecovery = { phase: 'failed', source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalEof(recovery) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: true }) + expect(transition.recovery).toBe(recovery) + }) + + it('ignores late EOF after the source was replaced', () => { + const recovery: AuditTerminalRecovery = { phase: 'replaced', source: ORIGINAL_SOURCE } + + const transition = completeAuditTerminalEof(recovery) + + expect(transition).toEqual({ recovery, shouldReconnect: false, shouldMarkStale: false }) + expect(transition.recovery).toBe(recovery) + }) +}) diff --git a/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.ts b/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.ts new file mode 100644 index 0000000000..063026fbf5 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/audit-terminal-recovery.ts @@ -0,0 +1,89 @@ +type AuditTerminalRecoveryPhase = 'hydrating' | 'awaiting_eof' | 'awaiting_hydration' | 'failed' | 'replaced' + +export type AuditTerminalRecovery = { + readonly phase: AuditTerminalRecoveryPhase + readonly source: Source +} + +export type AuditTerminalRecoveryTransition = { + readonly recovery: AuditTerminalRecovery + readonly shouldReconnect: boolean + readonly shouldMarkStale: boolean +} + +function remain( + recovery: AuditTerminalRecovery, + shouldMarkStale = false +): AuditTerminalRecoveryTransition { + return { recovery, shouldReconnect: false, shouldMarkStale } +} + +function advance( + recovery: AuditTerminalRecovery, + phase: AuditTerminalRecoveryPhase, + shouldReconnect = false +): AuditTerminalRecoveryTransition { + return { + recovery: { phase, source: recovery.source }, + shouldReconnect, + shouldMarkStale: false + } +} + +function fail(recovery: AuditTerminalRecovery): AuditTerminalRecoveryTransition { + return { recovery: { phase: 'failed', source: recovery.source }, shouldReconnect: false, shouldMarkStale: true } +} + +function unreachablePhase(phase: never): never { + throw new RangeError(`Unknown audit terminal recovery phase: ${phase}`) +} + +export function beginAuditTerminalRecovery( + recovery: AuditTerminalRecovery | undefined, + source: Source +): AuditTerminalRecoveryTransition { + if (recovery?.source === source) return remain(recovery) + return { + recovery: { phase: 'hydrating', source }, + shouldReconnect: false, + shouldMarkStale: false + } +} + +export function completeAuditTerminalHydration( + recovery: AuditTerminalRecovery, + succeeded: boolean +): AuditTerminalRecoveryTransition { + const phase = recovery.phase + switch (phase) { + case 'hydrating': + return succeeded ? advance(recovery, 'awaiting_eof') : fail(recovery) + case 'awaiting_hydration': + return succeeded ? advance(recovery, 'replaced', true) : fail(recovery) + case 'awaiting_eof': + case 'failed': + case 'replaced': + return remain(recovery) + default: + return unreachablePhase(phase) + } +} + +export function completeAuditTerminalEof( + recovery: AuditTerminalRecovery +): AuditTerminalRecoveryTransition { + const phase = recovery.phase + switch (phase) { + case 'hydrating': + return advance(recovery, 'awaiting_hydration') + case 'awaiting_eof': + return advance(recovery, 'replaced', true) + case 'failed': + return remain(recovery, true) + case 'awaiting_hydration': + case 'replaced': + return remain(recovery) + default: + return unreachablePhase(phase) + } +} diff --git a/crates/mesh-llm-ui/src/features/logs/api/client-info-schemas.test.ts b/crates/mesh-llm-ui/src/features/logs/api/client-info-schemas.test.ts new file mode 100644 index 0000000000..cc85cd88d2 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/client-info-schemas.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' +import { parseAuditEntry, parseLogRequest } from '@/features/logs/api/schemas' + +const ENDPOINT_ID = '9f0c4cbe8cb7a8d5d577c20e50ef03fd2f63a2e7fd9897c155823bcbb281bb04' +const TIMESTAMP = '2026-02-20T14:22:12.944Z' + +function requestDto(requestId: string) { + return { + requestId, + outcome: 'completed', + createdAt: TIMESTAMP, + terminalAt: TIMESTAMP, + route: 'chat_completions', + model: 'Qwen3-30B-A3B-Q4_K_M.gguf', + provider: 'openai_frontend', + engine: 'skippy', + statusCode: 200, + source: 'durable' + } +} + +describe('client information schemas', () => { + it('retains optional peer fields on mesh audit entries', () => { + const parsed = parseAuditEntry({ + entryId: 'audit-gossip-1', + occurredAt: '2026-02-20T14:22:08.301Z', + sequence: 18, + severity: 'info', + code: 'gossip_peer_discovered', + source: 'mesh', + subjectKind: 'mesh_peer', + subjectId: ENDPOINT_ID, + remoteAddr: '203.0.113.24:48712', + pathType: 'direct' + }) + + expect(parsed).toMatchObject({ + subjectKind: 'mesh_peer', + subjectId: ENDPOINT_ID, + remoteAddr: '203.0.113.24:48712', + pathType: 'direct' + }) + }) + + it('accepts relay peer entries and legacy audit entries without client fields', () => { + const relay = parseAuditEntry({ + entryId: 'audit-quic-1', + occurredAt: '2026-02-20T14:22:10.944Z', + sequence: 19, + severity: 'warning', + code: 'quic_path_degraded', + source: 'mesh', + subjectKind: 'mesh_peer', + subjectId: ENDPOINT_ID, + pathType: 'relay' + }) + const legacy = parseAuditEntry({ + entryId: 'audit-legacy-1', + occurredAt: '2026-02-20T14:22:11.944Z', + sequence: 20, + severity: 'info', + code: 'auto_join_started', + source: 'mesh' + }) + + expect(relay).toMatchObject({ subjectKind: 'mesh_peer', subjectId: ENDPOINT_ID, pathType: 'relay' }) + expect(legacy).not.toHaveProperty('subjectKind') + expect(legacy).not.toHaveProperty('remoteAddr') + expect(legacy).not.toHaveProperty('pathType') + }) + + it('retains optional caller fields while accepting legacy requests', () => { + const direct = parseLogRequest({ + ...requestDto('00000000-0000-4000-8000-000000000101'), + callerEndpointId: ENDPOINT_ID, + callerAddr: '203.0.113.24:48712', + callerPathType: 'remote_quic_http' + }) + const relay = parseLogRequest({ + ...requestDto('00000000-0000-4000-8000-000000000102'), + callerEndpointId: ENDPOINT_ID, + callerPathType: 'relay' + }) + const legacy = parseLogRequest(requestDto('00000000-0000-4000-8000-000000000103')) + + expect(direct).toMatchObject({ + callerEndpointId: ENDPOINT_ID, + callerAddr: '203.0.113.24:48712', + callerPathType: 'remote_quic_http' + }) + expect(relay).toMatchObject({ callerEndpointId: ENDPOINT_ID, callerPathType: 'relay' }) + expect(legacy).not.toHaveProperty('callerEndpointId') + expect(legacy).not.toHaveProperty('callerAddr') + expect(legacy).not.toHaveProperty('callerPathType') + }) + + it('rejects stage transport as a top-level request caller', () => { + expect(() => + parseLogRequest({ + ...requestDto('00000000-0000-4000-8000-000000000104'), + callerEndpointId: ENDPOINT_ID, + callerAddr: '203.0.113.24:48712', + callerPathType: 'remote_quic_stage' + }) + ).toThrow() + }) +}) diff --git a/crates/mesh-llm-ui/src/features/logs/api/client.test.ts b/crates/mesh-llm-ui/src/features/logs/api/client.test.ts index 182b2d8f60..324b9ea54e 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/client.test.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/client.test.ts @@ -103,7 +103,7 @@ describe('LogsApiClient', () => { error: { code: 'logging_schema_incompatible', message: 'the local log database schema is incompatible with this MeshLLM version', - details: { schema_version: 14, supported_schema_version: 11 } + details: { schema_version: 2, supported_schema_version: 1 } } }, 503 @@ -117,7 +117,7 @@ describe('LogsApiClient', () => { status: 503, code: 'logging_schema_incompatible', message: 'the local log database schema is incompatible with this MeshLLM version', - details: { schemaVersion: 14, supportedSchemaVersion: 11 } + details: { schemaVersion: 2, supportedSchemaVersion: 1 } }) }) @@ -281,6 +281,20 @@ describe('LogsApiClient', () => { expect(fetchMock).toHaveBeenCalledWith('/api/logs/requests?cursor=opaque+cursor%2B%2F%3D') }) + it('serializes exact and prefix route exclusions as singular request keys', async () => { + // Given + const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ items: [], nextCursor: null })) + + // When + await new LogsApiClient(fetchMock).listRequests({ + excludeRoute: 'models', + excludeRoutePrefix: 'management_' + }) + + // Then + expect(fetchMock).toHaveBeenCalledWith('/api/logs/requests?exclude_route=models&exclude_route_prefix=management_') + }) + it('uses strict POST bodies for bounded export, cleanup, and request deletion', async () => { const operationId = LogOperationId.parse('00000000-0000-4000-8000-000000000002') const fetchMock = vi @@ -558,6 +572,28 @@ describe('LogsApiClient', () => { expect(fetchMock).not.toHaveBeenCalled() }) + it('filters harness route exclusions before applying the page limit', async () => { + // Given + const fetchMock = vi.fn() + const client = new LogsApiClient(fetchMock) + + // When + const result = await client.listRequests( + { limit: 2, excludeRoute: 'models', excludeRoutePrefix: 'management_' }, + 'harness' + ) + + // Then + expect(result).toMatchObject({ state: 'supported', value: { items: expect.any(Array) } }) + if (result.state === 'supported') { + expect(result.value.items).toHaveLength(2) + expect( + result.value.items.every((item) => item.route !== 'models' && !item.route?.startsWith('management_')) + ).toBe(true) + } + expect(fetchMock).not.toHaveBeenCalled() + }) + it('returns a typed harness not-found error for an unknown request', async () => { const fetchMock = vi.fn() const client = new LogsApiClient(fetchMock) diff --git a/crates/mesh-llm-ui/src/features/logs/api/client.ts b/crates/mesh-llm-ui/src/features/logs/api/client.ts index e664cb4859..d62e9b34d6 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/client.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/client.ts @@ -63,6 +63,8 @@ export type LogsRequestQuery = { readonly from?: string readonly to?: string readonly route?: string + readonly excludeRoute?: string + readonly excludeRoutePrefix?: string readonly model?: string readonly provider?: string readonly engine?: string @@ -152,6 +154,8 @@ function serializeRequestQuery(input: LogsRequestQuery) { setQueryValue(query, 'from', input.from) setQueryValue(query, 'to', input.to) setQueryValue(query, 'route', input.route) + setQueryValue(query, 'exclude_route', input.excludeRoute) + setQueryValue(query, 'exclude_route_prefix', input.excludeRoutePrefix) setQueryValue(query, 'model', input.model) setQueryValue(query, 'provider', input.provider) setQueryValue(query, 'engine', input.engine) @@ -243,6 +247,8 @@ function filterHarnessRequests(items: readonly LogRequest[], query: LogsRequestQ if (query.from && compareLogInstants(item.createdAt, query.from) < 0) return false if (query.to && compareLogInstants(item.createdAt, query.to) > 0) return false if (query.route && item.route !== query.route) return false + if (query.excludeRoute && item.route === query.excludeRoute) return false + if (query.excludeRoutePrefix && item.route?.startsWith(query.excludeRoutePrefix)) return false if (query.model && item.model !== query.model) return false if (query.provider && item.provider !== query.provider) return false if (query.engine && item.engine !== query.engine) return false diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-options.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-options.ts new file mode 100644 index 0000000000..ca32a62cb2 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-options.ts @@ -0,0 +1,93 @@ +import type { SummaryDescriptor, SummaryRawKind } from './command-summary-descriptor-types' + +export const NONE = [] as const +export const JSON_FLAGS = ['--json'] as const +export const YES_JSON = ['--yes', '--json'] as const +export const SETUP_FLAGS = [ + '--yes', + '--no-interactive', + '--service', + '--no-service', + '--skip-runtime', + '--verbose' +] as const +export const UNINSTALL_FLAGS = [ + '--dry-run', + '--yes', + '--keep-cache', + '--keep-service-files', + '--purge-config', + '--keep-config', + '--json', + '--verbose' +] as const +export const DRAFT = ['--draft'] as const +export const DISCOVER_FLAGS = ['--auto'] as const +export const MODEL_PACKAGE_FLAGS = [ + '--experimental', + '--dry-run', + '--confirm', + '--follow', + '--list', + '--update-script', + '--json' +] as const +export const MODEL_PREPARE_FLAGS = [ + '--dry-run', + '--confirm', + '--follow', + '--json', + '--list', + '--update-script' +] as const +export const RUNTIME_LIST_FLAGS = ['--available', '--installed', '--json'] as const +export const RUNTIME_PRUNE_FLAGS = ['--active-only', '--json'] as const +export const AUTH_INIT_FLAGS = ['--force', '--no-passphrase', '--keychain'] as const +export const AUTH_ROTATE_NODE_FLAGS = ['--revoke-current'] as const +export const TUNE_FLAGS = [ + '--json', + '--no-speculative-tune', + '--apply', + '--replace-existing', + '--launch-args', + '--debug-telemetry' +] as const +export const SEARCH_FLAGS = ['--gguf', '--mlx', '--catalog', '--json'] as const +export const MODEL_CERTIFY_FLAGS = ['--json', '--package-only'] as const +export const SETUP_CONFLICTS = [['--service', '--no-service']] as const +export const UNINSTALL_CONFLICTS = [['--purge-config', '--keep-config']] as const +export const AUTH_INIT_CONFLICTS = [['--no-passphrase', '--keychain']] as const +export const UPDATE_CONFLICTS = [['--flavor', '--detect-flavor']] as const +export const PLUGIN_INSTALL_CONFLICTS = [['reference', '--archive']] as const +export const SKILLS_INSTALL_CONFLICTS = [['--agent', '--all']] as const +export const MODEL_SEARCH_CONFLICTS = [['--gguf', '--mlx']] as const +export const TUNE_CONFLICTS = [ + ['--model', '--models'], + ['--no-speculative-tune', '--speculative-types'], + ['--no-speculative-tune', '--spec-draft-models'], + ['--no-speculative-tune', '--spec-draft-max-tokens'], + ['--no-speculative-tune', '--spec-draft-min-tokens'], + ['--no-speculative-tune', '--spec-draft-acceptance-threshold'], + ['--no-speculative-tune', '--spec-draft-split-probability'], + ['--no-speculative-tune', '--spec-ngram-min'], + ['--no-speculative-tune', '--spec-ngram-max'] +] as const +export const RUNTIME_LIST_CONFLICTS = [['--available', '--installed']] as const +export const REMOTE_MODEL_CONFLICTS = [['--model', '--instance-id']] as const + +export const REDACTED_NAME = ['name'] as const +export const REDACTED_MODEL = ['model'] as const +export const REDACTED_REMOTE = ['--endpoint'] as const +export const REDACTED_REMOTE_MODEL = ['--endpoint', '--model', '--profile'] as const +export const REDACTED_APPLY_CONFIG = ['--endpoint', '--expected-revision', '--config'] as const + +const REDACTED_NONE = [] as const + +export const descriptor = ( + path: readonly string[], + booleans: readonly string[] = NONE, + redacted: readonly string[] = REDACTED_NONE, + hasPort = false, + raw: SummaryRawKind = 'none', + conflicts: readonly (readonly string[])[] = NONE +): SummaryDescriptor => ({ path, booleans, redacted, conflicts, hasPort, raw }) diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-types.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-types.ts new file mode 100644 index 0000000000..42c6fced17 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptor-types.ts @@ -0,0 +1,10 @@ +export type SummaryRawKind = 'backend' | 'mode' | 'none' + +export type SummaryDescriptor = { + readonly path: readonly string[] + readonly booleans: readonly string[] + readonly redacted: readonly string[] + readonly conflicts: readonly (readonly string[])[] + readonly hasPort: boolean + readonly raw: SummaryRawKind +} diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-auth.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-auth.ts new file mode 100644 index 0000000000..991a47bb35 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-auth.ts @@ -0,0 +1,51 @@ +import { + AUTH_INIT_CONFLICTS, + AUTH_INIT_FLAGS, + AUTH_ROTATE_NODE_FLAGS, + NONE, + descriptor +} from './command-summary-descriptor-options' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export const AUTH_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor(['mesh-llm', 'auth', 'init'], AUTH_INIT_FLAGS, ['--owner-key'], false, 'none', AUTH_INIT_CONFLICTS), + descriptor(['mesh-llm', 'auth', 'status'], NONE, ['--owner-key', '--node-key', '--node-ownership', '--trust-store']), + descriptor(['mesh-llm', 'auth', 'sign-node'], NONE, [ + '--owner-key', + '--node-key', + '--out', + '--hostname-hint', + '--node-label', + '--expires-in-hours' + ]), + descriptor(['mesh-llm', 'auth', 'renew-node'], NONE, [ + '--owner-key', + '--node-key', + '--out', + '--hostname-hint', + '--node-label', + '--expires-in-hours' + ]), + descriptor(['mesh-llm', 'auth', 'verify-node'], NONE, [ + '--file', + '--node-id', + '--trust-store', + '--verify-trust-policy' + ]), + descriptor(['mesh-llm', 'auth', 'rotate-node'], AUTH_ROTATE_NODE_FLAGS, [ + '--owner-key', + '--node-key', + '--out', + '--hostname-hint', + '--node-label', + '--expires-in-hours', + '--reason', + '--trust-store' + ]), + descriptor(['mesh-llm', 'auth', 'revoke-owner'], NONE, ['owner_id', '--reason', '--trust-store']), + descriptor(['mesh-llm', 'auth', 'revoke-node'], NONE, ['--cert-id', '--node-id', '--reason', '--trust-store']), + descriptor(['mesh-llm', 'auth', 'rotate-owner'], ['--no-passphrase', '--force'], ['--owner-key']), + descriptor(['mesh-llm', 'auth', 'trust', 'add'], NONE, ['owner_id', '--label', '--trust-store']), + descriptor(['mesh-llm', 'auth', 'trust', 'remove'], NONE, ['owner_id', '--trust-store']), + descriptor(['mesh-llm', 'auth', 'trust', 'list'], NONE, ['--trust-store']) +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-models.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-models.ts new file mode 100644 index 0000000000..f4b5ef9346 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-models.ts @@ -0,0 +1,50 @@ +import { + JSON_FLAGS, + MODEL_CERTIFY_FLAGS, + MODEL_PACKAGE_FLAGS, + MODEL_SEARCH_CONFLICTS, + REDACTED_MODEL, + SEARCH_FLAGS, + YES_JSON, + descriptor +} from './command-summary-descriptor-options' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export const MODEL_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor(['mesh-llm', 'models', 'package'], MODEL_PACKAGE_FLAGS, [ + 'source_repo', + '--quant', + '--target', + '--model-id', + '--flavor', + '--timeout', + '--mesh-llm-ref', + '--status', + '--logs', + '--cancel' + ]), + descriptor(['mesh-llm', 'models', 'recommended'], JSON_FLAGS), + descriptor(['mesh-llm', 'models', 'installed'], JSON_FLAGS), + descriptor(['mesh-llm', 'models', 'cleanup'], YES_JSON, ['--unused-since']), + descriptor(['mesh-llm', 'models', 'prune'], YES_JSON), + descriptor(['mesh-llm', 'models', 'certify'], MODEL_CERTIFY_FLAGS, [ + 'model', + '--report-out', + '--api-base', + '--prompt', + '--max-tokens' + ]), + descriptor(['mesh-llm', 'models', 'list'], JSON_FLAGS), + descriptor( + ['mesh-llm', 'models', 'search'], + SEARCH_FLAGS, + ['query', '--limit', '--sort'], + false, + 'none', + MODEL_SEARCH_CONFLICTS + ), + descriptor(['mesh-llm', 'models', 'show'], JSON_FLAGS, REDACTED_MODEL), + descriptor(['mesh-llm', 'models', 'download'], ['--draft', '--direct', '--json'], REDACTED_MODEL), + descriptor(['mesh-llm', 'models', 'updates'], ['--all', '--check', '--json'], ['repo']), + descriptor(['mesh-llm', 'models', 'delete'], YES_JSON, REDACTED_MODEL) +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-plugins-benchmark.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-plugins-benchmark.ts new file mode 100644 index 0000000000..cb14178639 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-plugins-benchmark.ts @@ -0,0 +1,61 @@ +import { + NONE, + PLUGIN_INSTALL_CONFLICTS, + REDACTED_NAME, + TUNE_CONFLICTS, + TUNE_FLAGS, + descriptor +} from './command-summary-descriptor-options' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export const PLUGIN_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor( + ['mesh-llm', 'plugins', 'install'], + NONE, + ['reference', '--archive', '--name', '--version'], + false, + 'none', + PLUGIN_INSTALL_CONFLICTS + ), + descriptor(['mesh-llm', 'plugins', 'update'], NONE, REDACTED_NAME), + descriptor(['mesh-llm', 'plugins', 'enable'], NONE, REDACTED_NAME), + descriptor(['mesh-llm', 'plugins', 'disable'], NONE, REDACTED_NAME), + descriptor(['mesh-llm', 'plugins', 'delete'], NONE, REDACTED_NAME), + descriptor(['mesh-llm', 'plugins', 'info'], NONE, REDACTED_NAME), + descriptor(['mesh-llm', 'plugins', 'search'], NONE, ['query']), + descriptor(['mesh-llm', 'plugins', 'list']) +] + +export const BENCHMARK_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor( + ['mesh-llm', 'benchmark', 'tune'], + TUNE_FLAGS, + [ + '--model', + '--models', + '--ctx-sizes', + '--batch-sizes', + '--ubatch-sizes', + '--mmap-values', + '--mlock-values', + '--flash-attention', + '--speculative-types', + '--spec-draft-models', + '--spec-draft-max-tokens', + '--spec-draft-min-tokens', + '--spec-ngram-min', + '--spec-ngram-max', + '--spec-draft-acceptance-threshold', + '--spec-draft-split-probability', + '--throughput-tolerance-pct', + '--max-tokens', + '--startup-timeout-secs', + '--request-timeout-secs', + '--prompt' + ], + false, + 'none', + TUNE_CONFLICTS + ), + descriptor(['mesh-llm', 'benchmark', 'import-prompts'], NONE, ['--source', '--limit', '--max-tokens', '--output']) +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-runtime.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-runtime.ts new file mode 100644 index 0000000000..7af9532b7e --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-runtime.ts @@ -0,0 +1,49 @@ +import { + JSON_FLAGS, + NONE, + REDACTED_APPLY_CONFIG, + REDACTED_NAME, + REDACTED_REMOTE, + REDACTED_REMOTE_MODEL, + REMOTE_MODEL_CONFLICTS, + RUNTIME_LIST_CONFLICTS, + RUNTIME_LIST_FLAGS, + RUNTIME_PRUNE_FLAGS, + descriptor +} from './command-summary-descriptor-options' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export const RUNTIME_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor(['mesh-llm', 'runtime', 'status'], NONE, NONE, true), + descriptor(['mesh-llm', 'runtime']), + descriptor(['mesh-llm', 'runtime', 'load'], NONE, REDACTED_NAME, true), + descriptor(['mesh-llm', 'runtime', 'unload'], NONE, REDACTED_NAME, true), + descriptor(['mesh-llm', 'runtime', 'guardrails'], JSON_FLAGS, NONE, true, 'mode'), + descriptor(['mesh-llm', 'runtime', 'bootstrap'], JSON_FLAGS, NONE, true), + descriptor( + ['mesh-llm', 'runtime', 'list'], + RUNTIME_LIST_FLAGS, + ['--manifest', '--bundle-dir', '--cache-dir'], + false, + 'none', + RUNTIME_LIST_CONFLICTS + ), + descriptor(['mesh-llm', 'runtime', 'install'], JSON_FLAGS, [ + 'runtime_ref', + '--manifest', + '--bundle-dir', + '--cache-dir' + ]), + descriptor(['mesh-llm', 'runtime', 'remove'], JSON_FLAGS, ['native_runtime_id', '--mesh-version', '--cache-dir']), + descriptor(['mesh-llm', 'runtime', 'prune'], RUNTIME_PRUNE_FLAGS, ['--mesh-version', '--cache-dir']), + descriptor(['mesh-llm', 'runtime', 'remote'], JSON_FLAGS, REDACTED_REMOTE, true), + descriptor( + ['mesh-llm', 'runtime', 'remote-model'], + JSON_FLAGS, + [...REDACTED_REMOTE_MODEL, '--instance-id'], + true, + 'none', + REMOTE_MODEL_CONFLICTS + ), + descriptor(['mesh-llm', 'runtime', 'apply-config'], JSON_FLAGS, REDACTED_APPLY_CONFIG, true) +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-top-level.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-top-level.ts new file mode 100644 index 0000000000..1ef37e5a45 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors-top-level.ts @@ -0,0 +1,60 @@ +import { + DISCOVER_FLAGS, + DRAFT, + JSON_FLAGS, + MODEL_PREPARE_FLAGS, + NONE, + REDACTED_NAME, + SETUP_CONFLICTS, + SETUP_FLAGS, + SKILLS_INSTALL_CONFLICTS, + UNINSTALL_CONFLICTS, + UNINSTALL_FLAGS, + UPDATE_CONFLICTS, + descriptor +} from './command-summary-descriptor-options' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export const TOP_LEVEL_DESCRIPTORS: readonly SummaryDescriptor[] = [ + descriptor(['mesh-llm', 'setup'], SETUP_FLAGS, NONE, false, 'none', SETUP_CONFLICTS), + descriptor(['mesh-llm', 'uninstall'], UNINSTALL_FLAGS, ['--binary-path'], false, 'none', UNINSTALL_CONFLICTS), + descriptor(['mesh-llm', 'download'], DRAFT, REDACTED_NAME), + descriptor(['mesh-llm', 'update'], ['--detect-flavor'], ['--version', '--flavor'], false, 'none', UPDATE_CONFLICTS), + descriptor(['mesh-llm', 'status'], NONE, NONE, true), + descriptor(['mesh-llm', 'load'], NONE, REDACTED_NAME, true), + descriptor(['mesh-llm', 'unload'], NONE, REDACTED_NAME, true), + descriptor(['mesh-llm', 'discover'], DISCOVER_FLAGS, ['--name', '--model', '--min-vram', '--region', '--relay']), + descriptor(['mesh-llm', 'rotate-key']), + descriptor(['mesh-llm', 'goose'], NONE, ['--model'], true), + descriptor(['mesh-llm', 'claude'], NONE, ['--model'], true), + descriptor(['mesh-llm', 'pi'], ['--write'], ['--model', '--host']), + descriptor(['mesh-llm', 'opencode'], ['--write'], ['--model', '--host']), + descriptor(['mesh-llm', 'stop']), + descriptor(['mesh-llm', 'external-plugin'], NONE, ['argv']), + descriptor(['mesh-llm', 'model-prepare'], MODEL_PREPARE_FLAGS, [ + 'source_repo', + '--quant', + '--target', + '--model-id', + '--flavor', + '--timeout', + '--mesh-llm-ref', + '--status', + '--logs', + '--cancel' + ]), + descriptor(['mesh-llm', 'gpus'], JSON_FLAGS), + descriptor(['mesh-llm', 'gpus', 'detect'], JSON_FLAGS), + descriptor(['mesh-llm', 'gpus', 'run-benchmark'], JSON_FLAGS, NONE, false, 'backend'), + descriptor(['mesh-llm', 'config', 'validate'], JSON_FLAGS, ['--config-path']), + descriptor(['mesh-llm', 'doctor'], JSON_FLAGS), + descriptor(['mesh-llm', 'doctor', 'split'], JSON_FLAGS, ['--model-ref', '--output-dir'], true), + descriptor( + ['mesh-llm', 'skills', 'install'], + ['--all', '--dry-run', '--force'], + ['--agent'], + false, + 'none', + SKILLS_INSTALL_CONFLICTS + ) +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors.ts new file mode 100644 index 0000000000..ed6a3f2efa --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-descriptors.ts @@ -0,0 +1,17 @@ +import { AUTH_DESCRIPTORS } from './command-summary-descriptors-auth' +import { MODEL_DESCRIPTORS } from './command-summary-descriptors-models' +import { BENCHMARK_DESCRIPTORS, PLUGIN_DESCRIPTORS } from './command-summary-descriptors-plugins-benchmark' +import { RUNTIME_DESCRIPTORS } from './command-summary-descriptors-runtime' +import { TOP_LEVEL_DESCRIPTORS } from './command-summary-descriptors-top-level' +import type { SummaryDescriptor } from './command-summary-descriptor-types' + +export type { SummaryDescriptor, SummaryRawKind } from './command-summary-descriptor-types' + +export const SUMMARY_DESCRIPTORS: readonly SummaryDescriptor[] = [ + ...TOP_LEVEL_DESCRIPTORS, + ...PLUGIN_DESCRIPTORS, + ...MODEL_DESCRIPTORS, + ...BENCHMARK_DESCRIPTORS, + ...RUNTIME_DESCRIPTORS, + ...AUTH_DESCRIPTORS +] diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary-vocabulary.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary-vocabulary.ts new file mode 100644 index 0000000000..68e98d65c9 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary-vocabulary.ts @@ -0,0 +1,196 @@ +export const STATIC_SUMMARY_TOKENS: ReadonlySet = new Set([ + 'mesh-llm', + 'load', + 'unload', + 'status', + 'discover', + 'rotate-key', + 'goose', + 'claude', + 'pi', + 'opencode', + 'stop', + 'external-plugin', + 'setup', + 'uninstall', + 'gpus', + 'detect', + 'run-benchmark', + 'config', + 'validate', + 'doctor', + 'split', + 'skills', + 'install', + 'plugins', + 'update', + 'enable', + 'disable', + 'delete', + 'info', + 'search', + 'list', + 'models', + 'package', + 'recommended', + 'installed', + 'cleanup', + 'prune', + 'certify', + 'show', + 'download', + 'updates', + 'benchmark', + 'tune', + 'import-prompts', + 'model-prepare', + 'runtime', + 'guardrails', + 'bootstrap', + 'remove', + 'remote', + 'remote-model', + 'apply-config', + 'auth', + 'init', + 'sign-node', + 'renew-node', + 'verify-node', + 'rotate-node', + 'revoke-owner', + 'revoke-node', + 'rotate-owner', + 'trust', + 'add' +]) + +export const BOOLEAN_SUMMARY_TOKENS: ReadonlySet = new Set([ + '--draft', + '--detect-flavor', + '--yes', + '--no-interactive', + '--service', + '--no-service', + '--skip-runtime', + '--verbose', + '--dry-run', + '--keep-cache', + '--keep-config', + '--keep-service-files', + '--json', + '--auto', + '--write', + '--all', + '--force', + '--experimental', + '--confirm', + '--follow', + '--list', + '--update-script', + '--gguf', + '--mlx', + '--catalog', + '--direct', + '--check', + '--no-speculative-tune', + '--apply', + '--replace-existing', + '--launch-args', + '--debug-telemetry', + '--available', + '--installed', + '--active-only', + '--keychain', + '--no-passphrase', + '--revoke-current', + '--package-only', + '--purge-config' +]) + +export const REDACTED_SUMMARY_TOKENS: ReadonlySet = new Set([ + 'argv', + 'model', + 'name', + 'native_runtime_id', + 'owner_id', + 'query', + 'reference', + 'repo', + 'runtime_ref', + 'source_repo', + '--version', + '--flavor', + '--binary-path', + '--name', + '--model', + '--min-vram', + '--region', + '--relay', + '--root-relay', + '--join', + '--relay-auth', + '--sort', + '--host', + '--config-path', + '--model-ref', + '--output-dir', + '--agent', + '--archive', + '--quant', + '--target', + '--model-id', + '--timeout', + '--mesh-llm-ref', + '--status', + '--logs', + '--cancel', + '--unused-since', + '--report-out', + '--api-base', + '--prompt', + '--max-tokens', + '--limit', + '--models', + '--ctx-sizes', + '--batch-sizes', + '--ubatch-sizes', + '--mmap-values', + '--mlock-values', + '--flash-attention', + '--speculative-types', + '--spec-draft-models', + '--spec-draft-max-tokens', + '--spec-draft-min-tokens', + '--spec-ngram-min', + '--spec-ngram-max', + '--spec-draft-acceptance-threshold', + '--spec-draft-split-probability', + '--throughput-tolerance-pct', + '--startup-timeout-secs', + '--request-timeout-secs', + '--source', + '--output', + '--manifest', + '--bundle-dir', + '--cache-dir', + '--mesh-version', + '--endpoint', + '--profile', + '--instance-id', + '--expected-revision', + '--config', + '--owner-key', + '--node-key', + '--node-ownership', + '--trust-store', + '--out', + '--hostname-hint', + '--node-label', + '--expires-in-hours', + '--file', + '--node-id', + '--verify-trust-policy', + '--reason', + '--cert-id', + '--label' +]) diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary.test.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary.test.ts new file mode 100644 index 0000000000..2c97978de6 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from 'vitest' +import { safeParse } from 'valibot' +import { commandSummarySchema, isSafeCommandSummary } from './command-summary' +import { SUMMARY_DESCRIPTORS } from './command-summary-descriptors' + +describe('command summary grammar', () => { + it('accepts every parsed producer raw-option context', () => { + const validSummaries = [ + 'mesh-llm status --port 41731', + 'mesh-llm load --port 41731 name [REDACTED]', + 'mesh-llm goose --port 41731 --model [REDACTED]', + 'mesh-llm doctor split --json --port 41731 --model-ref [REDACTED]', + 'mesh-llm gpus run-benchmark --backend cuda --json', + 'mesh-llm runtime guardrails --mode metrics --json --port 41731', + 'mesh-llm runtime bootstrap --json --port 41731', + 'mesh-llm runtime remote --json --port 41731 --endpoint [REDACTED]', + 'mesh-llm runtime remote-model --json --port 41731 --endpoint [REDACTED] --model [REDACTED]', + 'mesh-llm runtime apply-config --json --port 41731 --endpoint [REDACTED] --expected-revision [REDACTED] --config [REDACTED]' + ] + + for (const summary of validSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(true) + expect(safeParse(commandSummarySchema, summary).success, summary).toBe(true) + } + }) + + it('accepts ordinary static and redacted summaries', () => { + expect(isSafeCommandSummary('mesh-llm runtime load name [REDACTED]')).toBe(true) + expect(isSafeCommandSummary('mesh-llm models list --json')).toBe(true) + }) + + it('rejects private values, controls, bounds, and deep malformed prefixes', () => { + const malformedSummaries = [ + 'mesh-llm load private-value', + `mesh-llm load\u0001name [REDACTED]`, + `mesh-llm ${new Array(32).fill('load').join(' ')}`, + 'mesh-llm load unload status discover rotate-key setup --port 1234', + 'mesh-llm gpus run-benchmark --backend rocm', + 'mesh-llm runtime guardrails --mode strict', + 'mesh-llm load name [REDACTED] --port nope' + ] + + for (const summary of malformedSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + expect(safeParse(commandSummarySchema, summary).success, summary).toBe(false) + } + }) + + it('rejects inserted safe tokens and impossible raw-option ordering', () => { + const malformedSummaries = [ + 'mesh-llm gpus --draft run-benchmark --backend cuda', + 'mesh-llm gpus run-benchmark model [REDACTED] --backend cuda', + 'mesh-llm gpus --json run-benchmark --backend cuda', + 'mesh-llm doctor --json split --port 41731', + 'mesh-llm runtime guardrails --mode metrics --port 41731 --json', + 'mesh-llm runtime bootstrap --port 41731 --json', + 'mesh-llm runtime remote --port 41731 --json --endpoint [REDACTED]' + ] + + for (const summary of malformedSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + } + }) + + it('rejects non-canonical whole-command shapes', () => { + const malformedSummaries = [ + ' ', + 'mesh-llm models list --json --json', + 'mesh-llm models --json list', + 'mesh-llm load name [REDACTED] name [REDACTED]', + 'mesh-llm gpus run-benchmark --backend cuda --json --json', + 'mesh-llm load --port 41731 --port 41732', + 'mesh-llm load --port 41731 name [REDACTED] --json', + 'mesh-llm load name [REDACTED] status', + 'mesh-llm models nonsense', + 'mesh-llm load --json name [REDACTED]', + 'mesh-llm runtime status name [REDACTED]', + 'mesh-llm models list name [REDACTED] --json' + ] + + for (const summary of malformedSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + expect(safeParse(commandSummarySchema, summary).success, summary).toBe(false) + } + }) + + it('rejects non-canonical ASCII whitespace', () => { + const malformedSummaries = [ + ' mesh-llm models list', + 'mesh-llm models list ', + 'mesh-llm models list', + 'mesh-llm\tmodels list', + 'mesh-llm models\nlist' + ] + + for (const summary of malformedSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + } + }) + + it('rejects conflicting boolean pairs', () => { + const malformedSummaries = [ + 'mesh-llm setup --service --no-service', + 'mesh-llm setup --no-service --service', + 'mesh-llm uninstall --purge-config --keep-config', + 'mesh-llm uninstall --keep-config --purge-config', + 'mesh-llm auth init --no-passphrase --keychain', + 'mesh-llm auth init --keychain --no-passphrase' + ] + + for (const summary of malformedSummaries) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + } + }) + + it('rejects each speculative option when benchmark speculative tuning is disabled', () => { + const speculativeOptions = [ + '--speculative-types', + '--spec-draft-models', + '--spec-draft-max-tokens', + '--spec-draft-min-tokens', + '--spec-draft-acceptance-threshold', + '--spec-draft-split-probability', + '--spec-ngram-min', + '--spec-ngram-max' + ] as const + + for (const option of speculativeOptions) { + const summary = `mesh-llm benchmark tune --no-speculative-tune ${option} [REDACTED]` + expect(isSafeCommandSummary(summary), option).toBe(false) + } + }) + + it('accepts only ASCII decimal u16 port values', () => { + for (const port of ['0', '1', '65535']) { + expect(isSafeCommandSummary(`mesh-llm status --port ${port}`), port).toBe(true) + } + + for (const port of ['+1', '-1', '65536', '1.0', '١']) { + expect(isSafeCommandSummary(`mesh-llm status --port ${port}`), port).toBe(false) + } + }) + + it('accepts only the redacted global relay suffix shape', () => { + expect(isSafeCommandSummary('mesh-llm load name [REDACTED] --root-relay [REDACTED]')).toBe(true) + for (const summary of [ + 'mesh-llm load name [REDACTED] --relay private-relay', + 'mesh-llm load name [REDACTED] --root-relay [REDACTED] value', + 'mesh-llm load name [REDACTED] --relay-auth private-token', + 'mesh-llm load --root-relay [REDACTED] name [REDACTED]', + 'mesh-llm load name [REDACTED] --relay-auth [REDACTED] --root-relay [REDACTED]' + ]) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + } + }) + + it('accepts every descriptor option set in canonical phase order', () => { + for (const descriptor of SUMMARY_DESCRIPTORS) { + const tokens = [...descriptor.path] + if (descriptor.raw === 'backend') tokens.push('--backend', 'cuda') + if (descriptor.raw === 'mode') tokens.push('--mode', 'metrics') + tokens.push(...descriptor.booleans) + if (descriptor.hasPort) tokens.push('--port', '41731') + for (const marker of descriptor.redacted) tokens.push(marker, '[REDACTED]') + const summary = tokens.join(' ') + if (descriptor.conflicts.some((pair) => pair.every((flag) => tokens.includes(flag)))) { + expect(isSafeCommandSummary(summary), summary).toBe(false) + continue + } + if (tokens.length <= 32 && Array.from(summary).length <= 256) { + expect(isSafeCommandSummary(summary), summary).toBe(true) + } else { + for (const marker of [...descriptor.booleans, ...descriptor.redacted]) { + const single = [...descriptor.path] + if (descriptor.raw === 'backend') single.push('--backend', 'cuda') + if (descriptor.raw === 'mode') single.push('--mode', 'metrics') + single.push(marker) + if (descriptor.redacted.includes(marker)) single.push('[REDACTED]') + expect(isSafeCommandSummary(single.join(' ')), marker).toBe(true) + } + } + } + }) +}) diff --git a/crates/mesh-llm-ui/src/features/logs/api/command-summary.ts b/crates/mesh-llm-ui/src/features/logs/api/command-summary.ts new file mode 100644 index 0000000000..2bd39a98c7 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/command-summary.ts @@ -0,0 +1,116 @@ +import * as v from 'valibot' +import { BOOLEAN_SUMMARY_TOKENS, REDACTED_SUMMARY_TOKENS, STATIC_SUMMARY_TOKENS } from './command-summary-vocabulary' +import { SUMMARY_DESCRIPTORS, type SummaryDescriptor } from './command-summary-descriptors' + +const CONTROL_CHARACTER = /\p{Cc}/u + +function hasValidPort(value: string): boolean { + if (!/^[0-9]+$/.test(value)) return false + const port = Number(value) + return port >= 0 && port <= 65535 +} + +const BACKEND_VALUES = ['metal', 'cuda', 'hip', 'intel'] as const +const MODE_VALUES = ['disabled', 'metrics', 'enforce'] as const +const GLOBAL_REDACTED_TOKENS = ['--join', '--root-relay', '--relay-auth'] as const + +function isBackendValue(value: string): boolean { + return BACKEND_VALUES.some((candidate) => candidate === value) +} + +function isModeValue(value: string): boolean { + return MODE_VALUES.some((candidate) => candidate === value) +} + +function isAllowedRedactedToken(descriptor: SummaryDescriptor, token: string): boolean { + return descriptor.redacted.includes(token) || GLOBAL_REDACTED_TOKENS.some((candidate) => candidate === token) +} + +function matchesPath(tokens: readonly string[], descriptor: SummaryDescriptor): boolean { + return ( + descriptor.path.length <= tokens.length && + descriptor.path.every((token, index) => { + return STATIC_SUMMARY_TOKENS.has(token) && tokens[index] === token + }) + ) +} + +function hasDuplicate(seen: readonly string[], token: string): boolean { + return seen.includes(token) +} + +function validateDescriptor(tokens: readonly string[], descriptor: SummaryDescriptor): boolean { + if (!matchesPath(tokens, descriptor)) return false + + let index = descriptor.path.length + if (descriptor.raw === 'backend' || descriptor.raw === 'mode') { + const rawOption = descriptor.raw === 'backend' ? '--backend' : '--mode' + const rawValue = tokens[index + 1] + if (tokens[index] !== rawOption || rawValue === undefined) return false + if (descriptor.raw === 'backend' && !isBackendValue(rawValue)) return false + if (descriptor.raw === 'mode' && !isModeValue(rawValue)) return false + index += 2 + } + + let phase: 'booleans' | 'port' | 'redacted' = 'booleans' + let portSeen = false + const seenBooleans: string[] = [] + const seenRedacted: string[] = [] + const seenTokens: string[] = [] + let globalPhase = false + let lastGlobalRank = 0 + while (index < tokens.length) { + const token = tokens[index] + if (token === undefined) return false + if (BOOLEAN_SUMMARY_TOKENS.has(token)) { + if (phase !== 'booleans' || !descriptor.booleans.includes(token) || hasDuplicate(seenBooleans, token)) + return false + seenBooleans.push(token) + seenTokens.push(token) + index += 1 + continue + } + if (token === '--port') { + if (phase === 'redacted' || portSeen || !descriptor.hasPort) return false + const rawValue = tokens[index + 1] + if (rawValue === undefined || !hasValidPort(rawValue)) return false + portSeen = true + phase = 'port' + index += 2 + continue + } + if (REDACTED_SUMMARY_TOKENS.has(token)) { + const isGlobal = GLOBAL_REDACTED_TOKENS.some((candidate) => candidate === token) + if (globalPhase && !isGlobal) return false + if (isGlobal) { + const globalRank = GLOBAL_REDACTED_TOKENS.findIndex((candidate) => candidate === token) + 1 + if (globalRank <= lastGlobalRank) return false + globalPhase = true + lastGlobalRank = globalRank + } + if (!isAllowedRedactedToken(descriptor, token) || hasDuplicate(seenRedacted, token)) return false + if (tokens[index + 1] !== '[REDACTED]') return false + seenRedacted.push(token) + seenTokens.push(token) + phase = 'redacted' + index += 2 + continue + } + return false + } + return !descriptor.conflicts.some((pair) => pair.every((flag) => seenTokens.includes(flag))) +} + +export function isSafeCommandSummary(value: string): boolean { + const tokens = value.split(' ') + if ( + Array.from(value).length === 0 || + Array.from(value).length > 256 || + CONTROL_CHARACTER.test(value) || + tokens.some((token) => token.length === 0 || /\s/u.test(token)) + ) + return false + return tokens.length <= 32 && SUMMARY_DESCRIPTORS.some((descriptor) => validateDescriptor(tokens, descriptor)) +} + +export const commandSummarySchema = v.pipe(v.string(), v.minLength(1), v.maxLength(256), v.check(isSafeCommandSummary)) diff --git a/crates/mesh-llm-ui/src/features/logs/api/schemas.test.ts b/crates/mesh-llm-ui/src/features/logs/api/schemas.test.ts index 07cab4c40d..c792dffd11 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/schemas.test.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/schemas.test.ts @@ -395,12 +395,57 @@ describe('dedicated logs SSE frame parser', () => { reasonCode: 'model_loaded', outcome: 'ready', durationMs: 412, - numericSummaries: { layers: 36 } + numericSummaries: { layers: 36 }, + commandSummary: 'mesh-llm load name [REDACTED] --root-relay [REDACTED]' } const page = parseLogAuditPage({ items: [oldEntry, typedEntry], nextCursor: null }) expect(page.items[0]).toEqual(oldEntry) expect(page.items[1]).toEqual(typedEntry) + expect(page.items[0]?.commandSummary).toBeUndefined() + expect(page.items[1]?.commandSummary).toBe('mesh-llm load name [REDACTED] --root-relay [REDACTED]') + + const sse = parseLogsSseFrame({ + event: 'audit_entry', + lastEventId: 'a1:2', + data: JSON.stringify(typedEntry) + }) + expect(sse).toMatchObject({ + type: 'audit_entry', + entry: { commandSummary: 'mesh-llm load name [REDACTED] --root-relay [REDACTED]' } + }) + }) + + it('rejects malformed command summaries at REST and SSE boundaries', () => { + const malformedSummaries = [ + 'mesh-llm load private-value', + 'mesh-llm models list --json --json', + 'mesh-llm models --json list', + 'mesh-llm load name [REDACTED] name [REDACTED]', + 'mesh-llm gpus run-benchmark --backend cuda --json --json', + 'mesh-llm load name [REDACTED] --relay private-relay', + 'mesh-llm models list --json' + ] + + for (const [index, commandSummary] of malformedSummaries.entries()) { + const malformedEntry = { + entryId: `audit-malformed-summary-${index}`, + occurredAt: TIMESTAMP, + source: 'cli', + code: 'command_completed', + sequence: index + 3, + commandSummary + } + + expect(() => parseLogAuditPage({ items: [malformedEntry], nextCursor: null })).toThrow(LogsDtoError) + expect(() => + parseLogsSseFrame({ + event: 'audit_entry', + lastEventId: `a1:${index + 3}`, + data: JSON.stringify(malformedEntry) + }) + ).toThrow(LogsDtoError) + } }) it('parses lifecycle, gap, and typed stream-error frames', () => { @@ -470,14 +515,89 @@ describe('dedicated logs SSE frame parser', () => { ) }) - it('parses audit stream errors with the audit cursor family', () => { - expect( - parseLogsSseFrame({ - event: 'stream_error', - lastEventId: 'a1:42', - data: JSON.stringify({ code: 'invalid_event' }) - }) - ).toEqual({ type: 'stream_error', cursor: LogAuditCursor.parse('a1:42'), code: 'invalid_event' }) + it('parses invalid-event stream errors with either valid cursor family', () => { + // Given + const auditInput = { + event: 'stream_error', + lastEventId: 'a1:42', + data: JSON.stringify({ code: 'invalid_event' }) + } + const lifecycleInput = { + event: 'stream_error', + lastEventId: 'v1:2.0.0', + data: JSON.stringify({ code: 'invalid_event' }) + } + + // When + const auditFrame = parseLogsSseFrame(auditInput) + const lifecycleFrame = parseLogsSseFrame(lifecycleInput) + + // Then + expect(auditFrame).toEqual({ type: 'stream_error', cursor: LogAuditCursor.parse('a1:42'), code: 'invalid_event' }) + expect(lifecycleFrame).toEqual({ + type: 'stream_error', + cursor: LogReplayCursor.parse('v1:2.0.0'), + code: 'invalid_event' + }) + }) + + it('parses audit reconciliation failures only with a valid audit cursor', () => { + // Given + const input = { + event: 'stream_error', + lastEventId: 'a1:43', + data: JSON.stringify({ code: 'audit_reconcile_failed' }) + } + + // When + const frame = parseLogsSseFrame(input) + + // Then + expect(frame).toEqual({ + type: 'stream_error', + cursor: LogAuditCursor.parse('a1:43'), + code: 'audit_reconcile_failed' + }) + }) + + it('rejects an audit reconciliation failure paired with a lifecycle cursor', () => { + // Given + const input = { + event: 'stream_error', + lastEventId: 'v1:2.0.0', + data: JSON.stringify({ code: 'audit_reconcile_failed' }) + } + + // When / Then + expect(() => parseLogsSseFrame(input)).toThrow() + }) + + it.each([ + ['audit invalid-event cursor', 'a1:not-a-sequence', 'invalid_event'], + ['lifecycle invalid-event cursor', 'v1:2.0', 'invalid_event'], + ['audit reconciliation cursor', 'a1:not-a-sequence', 'audit_reconcile_failed'] + ])('rejects a malformed %s', (_label, lastEventId, code) => { + // Given + const input = { + event: 'stream_error', + lastEventId, + data: JSON.stringify({ code }) + } + + // When / Then + expect(() => parseLogsSseFrame(input)).toThrow() + }) + + it('rejects unknown audit stream-error codes', () => { + // Given + const input = { + event: 'stream_error', + lastEventId: 'a1:44', + data: JSON.stringify({ code: 'future_error' }) + } + + // When / Then + expect(() => parseLogsSseFrame(input)).toThrow(LogsDtoError) }) it('parses audit replay gaps from the shared replay_gap event name', () => { diff --git a/crates/mesh-llm-ui/src/features/logs/api/schemas.ts b/crates/mesh-llm-ui/src/features/logs/api/schemas.ts index d7213213fa..f015bb4cfc 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/schemas.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/schemas.ts @@ -1,4 +1,5 @@ import * as v from 'valibot' +import { commandSummarySchema } from './command-summary' import { LogArtifactId, LogAuditId, @@ -22,6 +23,7 @@ import type { LogRequest, LogsPage } from './schemas/types' + type LogArtifactBase = { readonly artifactId: LogArtifactId readonly requestId: LogRequestId @@ -34,6 +36,7 @@ type LogArtifactBase = { readonly redacted: boolean readonly truncated: boolean } + export { LogsDtoError } from './schemas/types' export type { LogArtifact, @@ -46,6 +49,7 @@ export type { LogCleanupOutcome, LogCleanupReceipt, LogCleanupScope, + LogCallerPathType, LogDeleteReceipt, LogEventKind, LogExport, @@ -53,6 +57,7 @@ export type { LogLifecycleEvent, LogMaintenanceCounts, LogOutcome, + LogPeerPathType, LogProxyAttempt, LogRequest, LogSource, @@ -84,6 +89,8 @@ const eventKindSchema = v.picklist([ const channelSchema = v.picklist(['requests', 'operations', 'system']) const auditSourceSchema = v.picklist(['logging_service', 'logs_api', 'runtime', 'mesh', 'cli']) const auditSeveritySchema = v.picklist(['info', 'warning', 'error']) +const peerPathTypeSchema = v.picklist(['direct', 'relay']) +const callerPathTypeSchema = v.picklist(['local_http', 'remote_quic_http', 'relay']) const artifactUnavailableReasonSchema = v.picklist([ 'streaming_response_not_assembled', 'response_body_not_bounded', @@ -151,7 +158,10 @@ const requestSchema = v.object({ provider: v.nullable(v.string()), engine: v.nullable(v.string()), statusCode: v.nullable(statusCodeSchema), - source: sourceSchema + source: sourceSchema, + callerEndpointId: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), + callerAddr: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), + callerPathType: v.optional(callerPathTypeSchema) }) const lifecycleEventSchema = v.object({ @@ -290,15 +300,24 @@ const auditEntrySchema = v.object({ sequence: v.pipe(nonNegativeIntegerSchema, v.minValue(1)), contextVersion: v.optional(v.literal(1)), subjectKind: v.optional( - v.union([v.literal('runtime'), v.literal('model'), v.literal('runtime_instance'), v.literal('cli_command')]) + v.union([ + v.literal('runtime'), + v.literal('model'), + v.literal('runtime_instance'), + v.literal('cli_command'), + v.literal('mesh_peer') + ]) ), subjectId: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), + remoteAddr: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), + pathType: v.optional(peerPathTypeSchema), operationId: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), requestId: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(256))), reasonCode: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(64))), outcome: v.optional(v.pipe(v.string(), v.minLength(1), v.maxLength(64))), durationMs: v.optional(nonNegativeIntegerSchema), - numericSummaries: v.optional(v.record(v.string(), nonNegativeIntegerSchema)) + numericSummaries: v.optional(v.record(v.string(), nonNegativeIntegerSchema)), + commandSummary: v.optional(commandSummarySchema) }) const auditGapSchema = v.object({ diff --git a/crates/mesh-llm-ui/src/features/logs/api/schemas/types.ts b/crates/mesh-llm-ui/src/features/logs/api/schemas/types.ts index 3fc5fbd76e..6e6d0a95cd 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/schemas/types.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/schemas/types.ts @@ -11,6 +11,8 @@ export type LogOutcome = 'active' | 'completed' | 'failed' | 'rejected' | 'cance export type LogSource = 'active' | 'durable' export type LogAuditSource = 'logging_service' | 'logs_api' | 'runtime' | 'mesh' | 'cli' export type LogAuditSeverity = 'info' | 'warning' | 'error' +export type LogPeerPathType = 'direct' | 'relay' +export type LogCallerPathType = 'local_http' | 'remote_quic_http' | 'relay' export type LogArtifactUnavailableReason = | 'streaming_response_not_assembled' | 'response_body_not_bounded' @@ -48,6 +50,9 @@ export type LogRequest = { readonly engine: string | undefined readonly statusCode: number | undefined readonly source: LogSource + readonly callerEndpointId?: string + readonly callerAddr?: string + readonly callerPathType?: LogCallerPathType } export type LogAuditEntry = { @@ -58,14 +63,17 @@ export type LogAuditEntry = { readonly severity?: LogAuditSeverity readonly sequence: number readonly contextVersion?: 1 - readonly subjectKind?: 'runtime' | 'model' | 'runtime_instance' | 'cli_command' + readonly subjectKind?: 'runtime' | 'model' | 'runtime_instance' | 'cli_command' | 'mesh_peer' readonly subjectId?: string + readonly remoteAddr?: string + readonly pathType?: LogPeerPathType readonly operationId?: string readonly requestId?: string readonly reasonCode?: string readonly outcome?: string readonly durationMs?: number readonly numericSummaries?: Readonly> + readonly commandSummary?: string } export type LogLifecycleEvent = { diff --git a/crates/mesh-llm-ui/src/features/logs/api/sse.ts b/crates/mesh-llm-ui/src/features/logs/api/sse.ts index a68483e03a..5774a6a556 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/sse.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/sse.ts @@ -12,6 +12,20 @@ import { export type LogsSseFilterKey = 'from' | 'to' | 'route' | 'model' | 'provider' | 'engine' | 'outcome' +export type LogsStreamErrorFrame = + | { + readonly type: 'stream_error' + readonly cursor: LogReplayCursor | LogAuditCursor + readonly code: 'invalid_event' + } + | { + readonly type: 'stream_error' + readonly cursor: LogAuditCursor + readonly code: 'audit_reconcile_failed' + } + +export type LogsStreamErrorCode = LogsStreamErrorFrame['code'] + export type LogsSseFilter = { readonly key: LogsSseFilterKey readonly value: string @@ -32,11 +46,7 @@ export type LogsSseSubscription = { export type LogsSseFrame = | { readonly type: 'log_event'; readonly cursor: LogReplayCursor; readonly event: ParsedReplayEvent } | { readonly type: 'replay_gap'; readonly cursor: LogReplayCursor; readonly gap: ParsedReplayGap } - | { - readonly type: 'stream_error' - readonly cursor: LogReplayCursor | LogAuditCursor - readonly code: 'invalid_event' - } + | LogsStreamErrorFrame | { readonly type: 'audit_entry' readonly cursor: LogAuditCursor @@ -68,11 +78,26 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' } -function parseStreamError(input: unknown): 'invalid_event' { - if (isRecord(input) && input['code'] === 'invalid_event') { - return 'invalid_event' +function parseStreamError(input: unknown, lastEventId: string): LogsStreamErrorFrame { + if (!isRecord(input)) throw new LogsDtoError() + + const code = input['code'] + switch (code) { + case 'invalid_event': { + switch (lastEventId.slice(0, 3)) { + case 'a1:': + return { type: 'stream_error', cursor: LogAuditCursor.parse(lastEventId), code } + case 'v1:': + return { type: 'stream_error', cursor: LogReplayCursor.parse(lastEventId), code } + default: + throw new LogsDtoError() + } + } + case 'audit_reconcile_failed': + return { type: 'stream_error', cursor: LogAuditCursor.parse(lastEventId), code } + default: + throw new LogsDtoError() } - throw new LogsDtoError() } export function parseLogsSseFrame(input: LogsSseFrameInput): LogsSseFrame { @@ -99,12 +124,8 @@ export function parseLogsSseFrame(input: LogsSseFrameInput): LogsSseFrame { const cursor = LogReplayCursor.parse(input.lastEventId) return { type: 'replay_gap', cursor, gap: parseReplayGap(data) } } - case 'stream_error': { - const cursor = input.lastEventId.startsWith('a1:') - ? LogAuditCursor.parse(input.lastEventId) - : LogReplayCursor.parse(input.lastEventId) - return { type: 'stream_error', cursor, code: parseStreamError(data) } - } + case 'stream_error': + return parseStreamError(data, input.lastEventId) case 'audit_entry': { const cursor = LogAuditCursor.parse(input.lastEventId) const entry = parseAuditEntry(data) diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-audit-live-recovery.ts b/crates/mesh-llm-ui/src/features/logs/api/use-audit-live-recovery.ts new file mode 100644 index 0000000000..8b5a7e9fb3 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/use-audit-live-recovery.ts @@ -0,0 +1,242 @@ +import { useEffect, useRef, useState } from 'react' +import { LogsApiClient } from '@/features/logs/api/client' +import { LogAuditCursor } from '@/features/logs/api/ids' +import type { LogAuditEntry } from '@/features/logs/api/schemas' +import { parseLogsSseFrame } from '@/features/logs/api/sse' +import { sortByOccurredAt } from '@/features/logs/lib/log-instant' +import * as auditTerminal from './audit-terminal-recovery' +import type { LogsEventSourceFactory, LogsLiveConnectionState } from './use-logs-live-recovery' + +const POLL_INTERVAL_MS = 5_000 +const FALLBACK_DELAY_MS = 1_000 +/** + * A stable empty value prevents the disabled audit stream from invalidating + * consumers that memoize the returned entry list by identity. + */ +const EMPTY_AUDIT_ENTRIES: readonly LogAuditEntry[] = [] + +type AuditEventSource = ReturnType +type AuditHydrationRequest = { readonly kind: 'standard'; readonly clearGap: boolean } | { readonly kind: 'terminal' } +type AuditLiveRecoveryOptions = { + readonly enabled: boolean + readonly hydrate: () => Promise + readonly cursor: LogAuditCursor | undefined + readonly pollingEnabledRef: { readonly current: boolean } + readonly eventSourceFactory: LogsEventSourceFactory +} +function mergeAuditEntries(current: readonly LogAuditEntry[], next: LogAuditEntry): LogAuditEntry[] { + return sortByOccurredAt([...current.filter((entry) => entry.entryId !== next.entryId), next]).slice(-64) +} + +export function useAuditLiveRecovery({ + enabled, + hydrate, + cursor, + pollingEnabledRef, + eventSourceFactory +}: AuditLiveRecoveryOptions) { + const [state, setState] = useState('reconnecting') + const [liveEntries, setLiveEntries] = useState([]) + const [fallbackPollingActive, setFallbackPollingActive] = useState(false) + const latestCursorRef = useRef(undefined) + const sequenceRef = useRef(0n) + const hydrateInFlightRef = useRef(false) + const hydratePendingRequestRef = useRef(undefined) + const hydrateRef = useRef(hydrate) + + useEffect(() => { + hydrateRef.current = hydrate + }, [hydrate]) + + useEffect(() => { + if (cursor && (!latestCursorRef.current || cursor.sequence() > latestCursorRef.current.sequence())) { + latestCursorRef.current = cursor + sequenceRef.current = cursor.sequence() + } + }, [cursor]) + + useEffect(() => { + if (!enabled) return + + let disposed = false + let source: AuditEventSource | undefined + let reconciliationTimer: number | undefined + let fallbackTimer: number | undefined + let terminalRecovery: auditTerminal.AuditTerminalRecovery | undefined + + const clearRecoveryTimers = () => { + if (fallbackTimer !== undefined) window.clearTimeout(fallbackTimer) + fallbackTimer = undefined + if (reconciliationTimer !== undefined) window.clearInterval(reconciliationTimer) + reconciliationTimer = undefined + setFallbackPollingActive(false) + } + const hydrateAuthoritatively = (request: AuditHydrationRequest) => { + if (disposed) return + if (hydrateInFlightRef.current) { + const pending = hydratePendingRequestRef.current + if (request.kind === 'terminal' || pending === undefined) { + hydratePendingRequestRef.current = request + } else if (pending.kind === 'standard') { + hydratePendingRequestRef.current = { + kind: 'standard', + clearGap: pending.clearGap || request.clearGap + } + } + return + } + hydrateInFlightRef.current = true + void Promise.resolve(hydrateRef.current()) + .then(() => { + if (disposed) return + if (request.kind === 'terminal') { + finishTerminalHydration(true) + } else if (request.clearGap && terminalRecovery === undefined) { + setState(source ? 'connected' : 'polling') + } + }) + .catch(() => { + if (disposed) return + setState('stale') + if (request.kind === 'terminal') finishTerminalHydration(false) + }) + .finally(() => { + if (disposed) return + hydrateInFlightRef.current = false + const pending = hydratePendingRequestRef.current + hydratePendingRequestRef.current = undefined + if (pending) hydrateAuthoritatively(pending) + }) + } + const startPolling = () => { + setState(source ? 'reconnecting' : 'polling') + if (reconciliationTimer !== undefined) return + if (pollingEnabledRef.current) hydrateAuthoritatively({ kind: 'standard', clearGap: false }) + reconciliationTimer = window.setInterval(() => { + if (pollingEnabledRef.current) hydrateAuthoritatively({ kind: 'standard', clearGap: false }) + }, POLL_INTERVAL_MS) + setFallbackPollingActive(true) + } + + function applyTerminalTransition(transition: auditTerminal.AuditTerminalRecoveryTransition) { + terminalRecovery = transition.recovery + if (transition.shouldMarkStale) setState('stale') + if (transition.shouldReconnect) { + setState('reconnecting') + connectAuditSource() + return + } + if (transition.recovery.phase === 'failed' && source === undefined) startPolling() + } + + function finishTerminalHydration(succeeded: boolean) { + if (!terminalRecovery) return + applyTerminalTransition(auditTerminal.completeAuditTerminalHydration(terminalRecovery, succeeded)) + } + + const markForReconciliation = (nextState: 'gap' | 'stale') => { + setState(nextState) + hydrateAuthoritatively({ kind: 'standard', clearGap: true }) + } + + const acceptAuditEvent = (connectedSource: AuditEventSource, event: MessageEvent) => { + if (disposed || source !== connectedSource) return + try { + const frame = parseLogsSseFrame({ event: event.type, lastEventId: event.lastEventId, data: event.data }) + if (!(frame.cursor instanceof LogAuditCursor)) { + markForReconciliation('stale') + return + } + latestCursorRef.current = frame.cursor + if (frame.type === 'audit_gap') { + markForReconciliation('gap') + return + } + if (frame.type === 'stream_error') { + setState('stale') + if (frame.code === 'audit_reconcile_failed') { + const transition = auditTerminal.beginAuditTerminalRecovery(terminalRecovery, connectedSource) + if (transition.recovery === terminalRecovery) return + clearRecoveryTimers() + applyTerminalTransition(transition) + hydrateAuthoritatively({ kind: 'terminal' }) + } else { + hydrateAuthoritatively({ kind: 'standard', clearGap: true }) + } + return + } + if (frame.type !== 'audit_entry') { + markForReconciliation('stale') + return + } + const sequence = BigInt(frame.entry.sequence) + if (sequence <= sequenceRef.current) return + sequenceRef.current = sequence + setLiveEntries((current) => mergeAuditEntries(current, frame.entry)) + } catch { + markForReconciliation('stale') + } + } + + function connectAuditSource(): void { + const url = new LogsApiClient().logsEventSourceUrl({ + channels: [], + audit: { cursor: latestCursorRef.current } + }) + try { + const connectedSource = eventSourceFactory(url) + source = connectedSource + connectedSource.onopen = () => { + if (disposed || source !== connectedSource) return + clearRecoveryTimers() + if (terminalRecovery?.phase === 'replaced') terminalRecovery = undefined + setState('connected') + } + connectedSource.onerror = () => { + if (disposed || source !== connectedSource) return + const recovery = terminalRecovery + if (!recovery || recovery.source !== connectedSource) { + if (fallbackTimer !== undefined) return + setState('reconnecting') + fallbackTimer = window.setTimeout(() => { + fallbackTimer = undefined + startPolling() + }, FALLBACK_DELAY_MS) + return + } + connectedSource.onopen = null + connectedSource.onerror = null + connectedSource.close() + source = undefined + applyTerminalTransition(auditTerminal.completeAuditTerminalEof(recovery)) + } + const acceptConnectedAuditEvent = (event: MessageEvent) => acceptAuditEvent(connectedSource, event) + connectedSource.addEventListener('audit_entry', acceptConnectedAuditEvent) + connectedSource.addEventListener('replay_gap', acceptConnectedAuditEvent) + connectedSource.addEventListener('stream_error', acceptConnectedAuditEvent) + } catch { + startPolling() + } + } + connectAuditSource() + + return () => { + disposed = true + hydrateInFlightRef.current = false + hydratePendingRequestRef.current = undefined + clearRecoveryTimers() + if (source) { + source.onopen = null + source.onerror = null + source.close() + source = undefined + } + } + }, [enabled, eventSourceFactory, pollingEnabledRef]) + + return { + state, + entries: enabled ? liveEntries : EMPTY_AUDIT_ENTRIES, + fallbackPollingActive: enabled && fallbackPollingActive + } +} diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.test.tsx b/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.test.tsx index 9b07edf781..ce54d81966 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.test.tsx +++ b/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.test.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { act, renderHook, waitFor } from '@testing-library/react' +import { renderHook, waitFor } from '@testing-library/react' import type { ReactNode } from 'react' import { afterEach, describe, expect, it, vi } from 'vitest' import { LogArtifactId, LogRequestId } from '@/features/logs/api/ids' @@ -38,37 +38,25 @@ afterEach(() => { }) describe('useLogArtifactContentQuery', () => { - it('waits for an explicit refetch before using the audited artifact endpoint', async () => { + it('uses the audited artifact endpoint when the selected payload mounts', async () => { // Given api.getArtifact.mockResolvedValue({ ...ARTIFACT, contentBase64: btoa('{}') }) // When - const { result } = renderHook(() => useLogArtifactContentQuery(ARTIFACT), { wrapper: createWrapper() }) - await waitFor(() => expect(result.current.fetchStatus).toBe('idle')) + renderHook(() => useLogArtifactContentQuery(ARTIFACT), { wrapper: createWrapper() }) // Then - expect(api.getArtifact).not.toHaveBeenCalled() - - // When - await act(async () => { - await result.current.refetch() - }) - - // Then - expect(api.getArtifact).toHaveBeenCalledOnce() + await waitFor(() => expect(api.getArtifact).toHaveBeenCalledOnce()) expect(api.getArtifact).toHaveBeenCalledWith(ARTIFACT.artifactId, 'live') }) - it('keeps explicit artifact reads in harness mode', async () => { + it('keeps automatic artifact reads in harness mode', async () => { api.getArtifact.mockResolvedValue({ ...ARTIFACT, contentBase64: btoa('{}') }) - const { result } = renderHook(() => useLogArtifactContentQuery(ARTIFACT), { + renderHook(() => useLogArtifactContentQuery(ARTIFACT), { wrapper: createWrapper('harness') }) - await act(async () => { - await result.current.refetch() - }) - + await waitFor(() => expect(api.getArtifact).toHaveBeenCalledOnce()) expect(api.getArtifact).toHaveBeenCalledWith(ARTIFACT.artifactId, 'harness') }) }) diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.ts b/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.ts index 6c39e9bd0b..2be08066d9 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-log-artifact-content-query.ts @@ -14,7 +14,7 @@ export function useLogArtifactContentQuery(artifact: AvailableLogArtifact) { return useQuery({ queryKey: logArtifactContentKeys.detail(artifact, dataMode.mode), queryFn: () => new LogsApiClient().getArtifact(artifact.artifactId, dataMode.mode), - enabled: false, + enabled: true, retry: false, staleTime: 10_000 }) diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-log-request-details-query.ts b/crates/mesh-llm-ui/src/features/logs/api/use-log-request-details-query.ts index 845b98af50..3a0ce8b7dd 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-log-request-details-query.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-log-request-details-query.ts @@ -2,7 +2,7 @@ import { useQuery } from '@tanstack/react-query' import { useDataMode, type DataMode } from '@/lib/data-mode' import { LogsApiClient } from '@/features/logs/api/client' import { LogPageCursor, type LogRequestId } from '@/features/logs/api/ids' -import type { LogsPage } from '@/features/logs/api/schemas' +import type { LogRequest, LogsPage } from '@/features/logs/api/schemas' const DETAIL_PAGE_SIZE = 50 export const DETAIL_ITEM_LIMIT = 250 @@ -63,11 +63,19 @@ export const logRequestDetailsKeys = { ] } -export function useLogRequestSummaryQuery(requestId: LogRequestId) { +/** + * Read one request summary, optionally seeded with the ledger row the caller + * already holds. The seed is treated as immediately stale, so the inspector + * paints real data on the first frame and the authoritative record still + * arrives from a background refetch. + */ +export function useLogRequestSummaryQuery(requestId: LogRequestId, knownRequest?: LogRequest) { const dataMode = useDataMode() return useQuery({ queryKey: logRequestDetailsKeys.summary(requestId, dataMode.mode), queryFn: () => new LogsApiClient().getRequest(requestId, dataMode.mode as DataMode), + initialData: knownRequest, + initialDataUpdatedAt: 0, staleTime: 10_000 }) } diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-log-request-summary-query.test.tsx b/crates/mesh-llm-ui/src/features/logs/api/use-log-request-summary-query.test.tsx new file mode 100644 index 0000000000..67b95b54af --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/use-log-request-summary-query.test.tsx @@ -0,0 +1,76 @@ +// @vitest-environment jsdom + +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { LogsApiClient } from '@/features/logs/api/client' +import { LogRequestId } from '@/features/logs/api/ids' +import type { LogRequest } from '@/features/logs/api/schemas' +import { useLogRequestSummaryQuery } from '@/features/logs/api/use-log-request-details-query' +import { DataModeProvider } from '@/lib/data-mode' + +const REQUEST_ID = LogRequestId.parse('00000000-0000-4000-8000-000000000001') + +const LEDGER_ROW: LogRequest = { + requestId: REQUEST_ID, + outcome: 'active', + createdAt: '2026-08-08T12:00:00Z', + terminalAt: undefined, + statusCode: undefined, + route: 'chat_completions', + model: 'Qwen3', + provider: 'mesh', + engine: 'skippy', + source: 'durable' +} + +const SERVER_RECORD: LogRequest = { + ...LEDGER_ROW, + outcome: 'completed', + terminalAt: '2026-08-08T12:00:01Z', + statusCode: 200 +} + +afterEach(() => vi.restoreAllMocks()) + +describe('useLogRequestSummaryQuery', () => { + it('paints the ledger row the caller already holds instead of a loading pass', async () => { + const getRequest = vi.spyOn(LogsApiClient.prototype, 'getRequest').mockResolvedValue(SERVER_RECORD) + + const { result } = renderHook(() => useLogRequestSummaryQuery(REQUEST_ID, LEDGER_ROW), { + wrapper: createWrapper() + }) + + expect(result.current.isLoading).toBe(false) + expect(result.current.data).toEqual(LEDGER_ROW) + + await waitFor(() => expect(result.current.data).toEqual(SERVER_RECORD)) + expect(getRequest).toHaveBeenCalledTimes(1) + }) + + it('reports a loading pass when no ledger row is available to seed the summary', async () => { + vi.spyOn(LogsApiClient.prototype, 'getRequest').mockResolvedValue(SERVER_RECORD) + + const { result } = renderHook(() => useLogRequestSummaryQuery(REQUEST_ID), { wrapper: createWrapper() }) + + expect(result.current.isLoading).toBe(true) + expect(result.current.data).toBeUndefined() + + await waitFor(() => expect(result.current.data).toEqual(SERVER_RECORD)) + }) +}) + +function createWrapper() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + return function Wrapper({ children }: { readonly children: ReactNode }) { + return ( + + + {children} + + + ) + } +} diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-audit-query.ts b/crates/mesh-llm-ui/src/features/logs/api/use-logs-audit-query.ts index 027e6b3af7..f83d87081a 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-audit-query.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-audit-query.ts @@ -1,4 +1,5 @@ import { useQuery } from '@tanstack/react-query' +import { useEffect, useRef } from 'react' import { LogsApiClient, type LogAuditQuery, type LogsCapability } from '@/features/logs/api/client' import type { LogAuditEntry, LogAuditPage } from '@/features/logs/api/schemas' import { logsKeys } from '@/features/logs/api/use-logs-ledger-query' @@ -52,9 +53,17 @@ export async function loadCompleteAudits(query: LogAuditQuery, mode: DataMode): export function useLogsAuditQuery(query: LogAuditQuery = {}) { const dataMode = useDataMode() - return useQuery({ + const retainedSuccessfulData = useRef | undefined>(undefined) + const result = useQuery({ queryKey: logsKeys.audit(query, dataMode.mode), queryFn: () => loadCompleteAudits(query, dataMode.mode), + placeholderData: (previousData) => previousData ?? retainedSuccessfulData.current, staleTime: 10_000 }) + + useEffect(() => { + if (result.data !== undefined && !result.isPlaceholderData) retainedSuccessfulData.current = result.data + }, [result.data, result.isPlaceholderData]) + + return result } diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.test.ts b/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.test.ts index 9a87079e50..674bc00097 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.test.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.test.ts @@ -34,6 +34,12 @@ describe('logsKeys.ledger', () => { expect(logsKeys.ledger(filteredQuery, 'live')).not.toEqual(logsKeys.ledger(REQUEST_QUERY, 'live')) }) + + it('includes the ledger route exclusions in the stable request cache key', () => { + expect(logsKeys.ledger(REQUEST_QUERY, 'live')).toContainEqual( + expect.objectContaining({ excludeRoute: 'models', excludeRoutePrefix: 'management_' }) + ) + }) }) describe('loadCompleteLedger', () => { @@ -60,6 +66,8 @@ describe('loadCompleteLedger', () => { from: '2026-08-01T00:00:00Z', to: '2026-08-02T00:00:00Z', model: 'Qwen3', + excludeRoute: 'models', + excludeRoutePrefix: 'management_', cursor: undefined, limit: LEDGER_PAGE_SIZE }, @@ -71,6 +79,8 @@ describe('loadCompleteLedger', () => { from: '2026-08-01T00:00:00Z', to: '2026-08-02T00:00:00Z', model: 'Qwen3', + excludeRoute: 'models', + excludeRoutePrefix: 'management_', cursor: LogPageCursor.parse('page-2'), limit: LEDGER_PAGE_SIZE }, @@ -98,6 +108,9 @@ describe('loadCompleteLedger', () => { }) if (result.state === 'supported') expect(result.value.nextCursor?.toString()).toBe('10') expect(listRequests).toHaveBeenCalledTimes(10) + for (const [query] of listRequests.mock.calls) { + expect(query).toMatchObject({ excludeRoute: 'models', excludeRoutePrefix: 'management_' }) + } }) it('stops safely when an empty page advertises a continuation cursor', async () => { diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.ts b/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.ts index 3aabb3d1d0..4f568d9dd6 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-ledger-query.ts @@ -1,4 +1,5 @@ import { useQuery } from '@tanstack/react-query' +import { useEffect, useRef } from 'react' import { useDataMode, type DataMode } from '@/lib/data-mode' import { LogsApiClient } from '@/features/logs/api/client' import type { LogAuditQuery, LogsCapability, LogsRequestQuery } from '@/features/logs/api/client' @@ -7,15 +8,25 @@ import type { LogsPage, LogRequest } from '@/features/logs/api/schemas' export const LEDGER_PAGE_SIZE = 100 export const LEDGER_MAX_RECORDS = 1_000 +const LEDGER_ROUTE_EXCLUSIONS = { + excludeRoute: 'models', + excludeRoutePrefix: 'management_' +} as const satisfies LogsRequestQuery + +function withLedgerRouteExclusions(query: LogsRequestQuery): LogsRequestQuery { + return { ...query, ...LEDGER_ROUTE_EXCLUSIONS } +} + export async function loadCompleteLedger( query: LogsRequestQuery, mode: DataMode ): Promise>> { const client = new LogsApiClient() + const scopedQuery = withLedgerRouteExclusions(query) const items: LogRequest[] = [] - let cursor = query.cursor + let cursor = scopedQuery.cursor while (items.length < LEDGER_MAX_RECORDS) { - const result = await client.listRequests({ ...query, cursor, limit: LEDGER_PAGE_SIZE }, mode) + const result = await client.listRequests({ ...scopedQuery, cursor, limit: LEDGER_PAGE_SIZE }, mode) if (result.state === 'unsupported') return result const remaining = LEDGER_MAX_RECORDS - items.length items.push(...result.value.items.slice(0, remaining)) @@ -38,6 +49,8 @@ function requestQueryKey(query: LogsRequestQuery) { from: query.from, to: query.to, route: query.route, + excludeRoute: query.excludeRoute, + excludeRoutePrefix: query.excludeRoutePrefix, model: query.model, provider: query.provider, engine: query.engine, @@ -50,7 +63,12 @@ function requestQueryKey(query: LogsRequestQuery) { export const logsKeys = { all: ['logs'], - ledger: (query: LogsRequestQuery, mode: DataMode) => [...logsKeys.all, 'ledger', requestQueryKey(query), mode], + ledger: (query: LogsRequestQuery, mode: DataMode) => [ + ...logsKeys.all, + 'ledger', + requestQueryKey(withLedgerRouteExclusions(query)), + mode + ], audit: (query: LogAuditQuery, mode: DataMode) => [ ...logsKeys.all, 'audit', @@ -61,9 +79,17 @@ export const logsKeys = { export function useLogsLedgerQuery(query: LogsRequestQuery) { const dataMode = useDataMode() - return useQuery({ + const retainedSuccessfulData = useRef> | undefined>(undefined) + const result = useQuery({ queryKey: logsKeys.ledger(query, dataMode.mode), queryFn: () => loadCompleteLedger(query, dataMode.mode as DataMode), + placeholderData: (previousData) => previousData ?? retainedSuccessfulData.current, staleTime: 10_000 }) + + useEffect(() => { + if (result.data !== undefined && !result.isPlaceholderData) retainedSuccessfulData.current = result.data + }, [result.data, result.isPlaceholderData]) + + return result } diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test-fixtures.tsx b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test-fixtures.tsx index 86802db2bf..3f956f4ed0 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test-fixtures.tsx +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test-fixtures.tsx @@ -11,10 +11,37 @@ export const unsupportedEventSourceFactory: LogsEventSourceFactory = () => { export type Listener = (event: MessageEvent) => void +export class DeferredHydration { + readonly promise: Promise + readonly #resolve: () => void + readonly #reject: (error: Error) => void + + constructor() { + let resolveHydration: () => void = () => undefined + let rejectHydration: (error: Error) => void = () => undefined + this.promise = new Promise((resolve, reject) => { + resolveHydration = () => resolve() + rejectHydration = (error) => reject(error) + }) + this.#resolve = resolveHydration + this.#reject = rejectHydration + } + + resolve() { + this.#resolve() + } + + reject(error: Error) { + this.#reject(error) + } +} + export class FakeEventSource { readonly listeners = new Map() readonly url: string closed = false + closeCalls = 0 + serverClosed = false onopen: ((event: Event) => void) | null = null onerror: ((event: Event) => void) | null = null @@ -28,6 +55,7 @@ export class FakeEventSource { close() { this.closed = true + this.closeCalls += 1 } open() { @@ -38,6 +66,11 @@ export class FakeEventSource { this.onerror?.(new Event('error')) } + serverClose() { + this.serverClosed = true + this.onerror?.(new Event('error')) + } + emit(type: string, data: string, lastEventId: string) { const event = new MessageEvent(type, { data }) Object.defineProperty(event, 'lastEventId', { value: lastEventId }) diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test.tsx b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test.tsx index b13fa227f4..e8caf0afff 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test.tsx +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.test.tsx @@ -1,12 +1,13 @@ // @vitest-environment jsdom import { act, renderHook } from '@testing-library/react' -import { describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { useLogsLiveRecovery, type LogsEventSourceFactory } from '@/features/logs/api/use-logs-live-recovery' import type { LogsLedgerSearch } from '@/features/logs/lib/log-search' import { REQUEST_A, REQUEST_B, + DeferredHydration, FakeEventSource, auditData, eventData, @@ -17,6 +18,11 @@ import { } from './use-logs-live-recovery.test-fixtures' describe('useLogsLiveRecovery', () => { + afterEach(() => { + if (vi.isFakeTimers()) vi.clearAllTimers() + vi.useRealTimers() + }) + it('preserves lifecycle gap recovery when the initial hydration is still in flight', async () => { let resolveInitial: (() => void) | undefined let calls = 0 @@ -101,6 +107,76 @@ describe('useLogsLiveRecovery', () => { expect(result.current.state).toBe('connected') }) + it('isolates audit hydration ownership across effect reruns', async () => { + // Given + const hydrationA = new DeferredHydration() + const hydrationB = new DeferredHydration() + const queuedHydrationB = new DeferredHydration() + const hydrate = vi.fn(async () => undefined) + const hydrateAuditA = vi.fn(() => hydrationA.promise) + let hydrationBCalls = 0 + const hydrateAuditB = vi.fn(() => { + hydrationBCalls += 1 + return hydrationBCalls === 1 ? hydrationB.promise : queuedHydrationB.promise + }) + const sourcesA: FakeEventSource[] = [] + const sourcesB: FakeEventSource[] = [] + const factoryA: LogsEventSourceFactory = (url) => { + const source = new FakeEventSource(url) + sourcesA.push(source) + return source + } + const factoryB: LogsEventSourceFactory = (url) => { + const source = new FakeEventSource(url) + sourcesB.push(source) + return source + } + const { rerender } = renderHook( + ({ hydrateAudit, eventSourceFactory }) => + useLogsLiveRecovery({ + enabled: false, + search: {}, + hydrate, + auditEnabled: true, + hydrateAudit, + eventSourceFactory + }), + { initialProps: { hydrateAudit: hydrateAuditA, eventSourceFactory: factoryA } } + ) + await flush() + act(() => sourcesA[0]?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:1')) + expect(hydrateAuditA).toHaveBeenCalledTimes(1) + + // When + rerender({ hydrateAudit: hydrateAuditB, eventSourceFactory: factoryB }) + act(() => sourcesB[0]?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:2')) + + // Then + expect(hydrateAuditB).toHaveBeenCalledTimes(1) + + // When + await act(async () => { + hydrationA.resolve() + await hydrationA.promise + }) + await flush() + act(() => sourcesB[0]?.emit('stream_error', JSON.stringify({ code: 'invalid_event' }), 'a1:3')) + + // Then + expect(hydrateAuditB).toHaveBeenCalledTimes(1) + + // When + await act(async () => { + hydrationB.resolve() + await hydrationB.promise + }) + await flush() + + // Then + expect(hydrateAuditB).toHaveBeenCalledTimes(2) + expect(hydrate).not.toHaveBeenCalled() + }) + it('accepts server-reconciled cross-process audit rows without browser polling', async () => { vi.useFakeTimers() const { hydrateAudit, result, sources } = renderLive({ enabled: false, auditEnabled: true }) @@ -238,6 +314,179 @@ describe('useLogsLiveRecovery', () => { expect(hydrateAudit).toHaveBeenCalledTimes(2) }) + it('waits for terminal hydration after EOF before replacing the audit source exactly once', async () => { + // Given + vi.useFakeTimers() + const hydration = new DeferredHydration() + const hydrateAudit = vi.fn(() => hydration.promise) + const { hydrate, result, sources } = renderLive({ enabled: false, auditEnabled: true, hydrateAudit }) + await flush() + const terminalSource = sources[0] + act(() => { + terminalSource?.open() + terminalSource?.emit('audit_entry', auditData(1), 'a1:1') + }) + expect(result.current.state).toBe('connected') + + // When + act(() => terminalSource?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:2')) + + // Then + expect(result.current.state).toBe('stale') + expect(hydrateAudit).toHaveBeenCalledTimes(1) + expect(hydrate).not.toHaveBeenCalled() + expect(sources).toHaveLength(1) + + // When + act(() => terminalSource?.serverClose()) + + // Then + expect(terminalSource?.serverClosed).toBe(true) + expect(terminalSource?.closed).toBe(true) + expect(terminalSource?.closeCalls).toBe(1) + expect(sources).toHaveLength(1) + expect(result.current.state).toBe('stale') + + act(() => terminalSource?.error()) + expect(sources).toHaveLength(1) + + // When + await act(async () => { + hydration.resolve() + await hydration.promise + }) + await flush() + + // Then + expect(sources).toHaveLength(2) + expect(sources[1]?.url).toBe('/api/logs/events?audit=1&cursor=a1%3A2') + act(() => { + sources[1]?.open() + terminalSource?.error() + terminalSource?.serverClose() + }) + expect(result.current.state).toBe('connected') + expect(sources).toHaveLength(2) + expect(terminalSource?.closeCalls).toBe(1) + + act(() => vi.advanceTimersByTime(5_000)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(1) + expect(hydrate).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + }) + + it('waits for terminal EOF when authoritative hydration succeeds first', async () => { + // Given + const hydration = new DeferredHydration() + const hydrateAudit = vi.fn(() => hydration.promise) + const { result, sources } = renderLive({ enabled: false, auditEnabled: true, hydrateAudit }) + await flush() + const terminalSource = sources[0] + act(() => terminalSource?.open()) + act(() => terminalSource?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:2')) + + // When + await act(async () => { + hydration.resolve() + await hydration.promise + }) + await flush() + + // Then + expect(hydrateAudit).toHaveBeenCalledTimes(1) + expect(result.current.state).toBe('stale') + expect(sources).toHaveLength(1) + + act(() => terminalSource?.serverClose()) + expect(terminalSource?.closed).toBe(true) + expect(terminalSource?.closeCalls).toBe(1) + expect(sources).toHaveLength(2) + expect(sources[1]?.url).toBe('/api/logs/events?audit=1&cursor=a1%3A2') + + act(() => terminalSource?.error()) + expect(sources).toHaveLength(2) + expect(terminalSource?.closeCalls).toBe(1) + }) + + it('falls back to polling when terminal audit hydration rejects before EOF', async () => { + // Given + vi.useFakeTimers() + const terminalHydration = new DeferredHydration() + let hydrationCalls = 0 + const hydrateAudit = vi.fn(() => { + hydrationCalls += 1 + return hydrationCalls === 1 ? terminalHydration.promise : Promise.resolve() + }) + const { result, sources } = renderLive({ enabled: false, auditEnabled: true, hydrateAudit }) + await flush() + const terminalSource = sources[0] + act(() => terminalSource?.open()) + act(() => terminalSource?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:2')) + expect(hydrateAudit).toHaveBeenCalledTimes(1) + + // When + act(() => terminalHydration.reject(new Error('authoritative audit hydration failed'))) + await flush() + expect(result.current.state).toBe('stale') + act(() => terminalSource?.serverClose()) + await flush() + await flush() + + // Then + expect(terminalSource?.closed).toBe(true) + expect(terminalSource?.closeCalls).toBe(1) + expect(result.current.state).toBe('polling') + expect(hydrateAudit).toHaveBeenCalledTimes(2) + + act(() => vi.advanceTimersByTime(4_999)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(2) + act(() => vi.advanceTimersByTime(1)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(3) + }) + + it('falls back to polling when terminal audit EOF precedes hydration rejection', async () => { + // Given + vi.useFakeTimers() + const terminalHydration = new DeferredHydration() + let hydrationCalls = 0 + const hydrateAudit = vi.fn(() => { + hydrationCalls += 1 + return hydrationCalls === 1 ? terminalHydration.promise : Promise.resolve() + }) + const { result, sources } = renderLive({ enabled: false, auditEnabled: true, hydrateAudit }) + await flush() + const terminalSource = sources[0] + act(() => terminalSource?.open()) + act(() => terminalSource?.emit('stream_error', JSON.stringify({ code: 'audit_reconcile_failed' }), 'a1:2')) + expect(hydrateAudit).toHaveBeenCalledTimes(1) + + // When + act(() => terminalSource?.serverClose()) + expect(terminalSource?.closed).toBe(true) + expect(terminalSource?.closeCalls).toBe(1) + act(() => terminalHydration.reject(new Error('authoritative audit hydration failed'))) + await flush() + await flush() + + // Then + expect(result.current.state).toBe('polling') + expect(hydrateAudit).toHaveBeenCalledTimes(2) + + act(() => vi.advanceTimersByTime(4_999)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(2) + act(() => vi.advanceTimersByTime(1)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(3) + }) + + it('starts the test after fake-timer coverage with real timers', () => { + expect(vi.isFakeTimers()).toBe(false) + }) + it('does not re-hydrate when the audit stream fails to reconnect a second time while already polling', async () => { vi.useFakeTimers() const { hydrateAudit, result, sources } = renderLive({ enabled: false, auditEnabled: true }) @@ -248,18 +497,21 @@ describe('useLogsLiveRecovery', () => { expect(result.current.state).toBe('reconnecting') act(() => vi.advanceTimersByTime(1_000)) await flush() - expect(result.current.state).toBe('polling') + expect(result.current.state).toBe('reconnecting') expect(hydrateAudit).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(1) // Native EventSource retries on its own schedule and calls onerror again on // every failed attempt. A second failure while already polling must not // re-enter startPolling and fire a duplicate hydrate — the reconciliation // interval from the first entry is still live and owns future refreshes. act(() => source?.error()) + expect(result.current.state).toBe('reconnecting') act(() => vi.advanceTimersByTime(1_000)) await flush() - expect(result.current.state).toBe('polling') + expect(result.current.state).toBe('reconnecting') expect(hydrateAudit).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(1) // The reconciliation interval from the first entry must still be the one // driving refreshes — the second failure should not have restarted or @@ -267,6 +519,41 @@ describe('useLogsLiveRecovery', () => { act(() => vi.advanceTimersByTime(5_000)) await flush() expect(hydrateAudit).toHaveBeenCalledTimes(2) + + act(() => source?.open()) + expect(result.current.state).toBe('connected') + expect(vi.getTimerCount()).toBe(0) + act(() => vi.advanceTimersByTime(5_000)) + await flush() + expect(hydrateAudit).toHaveBeenCalledTimes(2) + expect(sources).toHaveLength(1) + expect(source?.closed).toBe(false) + }) + + it('reports combined fallback polling until both lifecycle and audit timers reconnect', async () => { + vi.useFakeTimers() + const { result, sources } = renderLive({ auditEnabled: true }) + await flush() + act(() => { + sources[0]?.open() + sources[1]?.open() + sources[0]?.error() + sources[1]?.error() + }) + + expect(result.current.state).toBe('reconnecting') + expect(result.current.fallbackPollingActive).toBe(false) + + act(() => vi.advanceTimersByTime(1_000)) + await flush() + expect(result.current.fallbackPollingActive).toBe(true) + + act(() => sources[0]?.open()) + expect(result.current.fallbackPollingActive).toBe(true) + + act(() => sources[1]?.open()) + expect(result.current.state).toBe('connected') + expect(result.current.fallbackPollingActive).toBe(false) }) it('serializes route and reconnects while source remains unsupported', async () => { @@ -495,13 +782,34 @@ describe('useLogsLiveRecovery', () => { const source = sources[0] act(() => source?.error()) expect(result.current.state).toBe('reconnecting') + expect(result.current.fallbackPollingActive).toBe(false) act(() => vi.advanceTimersByTime(1_000)) - expect(result.current.state).toBe('polling') - act(() => vi.advanceTimersByTime(15_000)) + await flush() + expect(result.current.state).toBe('reconnecting') + expect(result.current.fallbackPollingActive).toBe(true) + expect(hydrate).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(1) + + act(() => source?.error()) + expect(result.current.state).toBe('reconnecting') + act(() => vi.advanceTimersByTime(1_000)) + await flush() + expect(result.current.state).toBe('reconnecting') + expect(hydrate).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(1) + + act(() => vi.advanceTimersByTime(5_000)) + await flush() + expect(hydrate).toHaveBeenCalledTimes(2) expect(sources).toHaveLength(1) act(() => source?.open()) expect(result.current.state).toBe('connected') - expect(hydrate.mock.calls.length).toBeLessThanOrEqual(2) + expect(result.current.fallbackPollingActive).toBe(false) + expect(vi.getTimerCount()).toBe(0) + act(() => vi.advanceTimersByTime(5_000)) + await flush() + expect(hydrate).toHaveBeenCalledTimes(2) + expect(source?.closed).toBe(false) }) it('pauses only future fallback interval hydrations without replacing the source or timer', async () => { diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.ts b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.ts index 5e1bbf7871..26cf429473 100644 --- a/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.ts +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-live-recovery.ts @@ -5,18 +5,11 @@ import { parseLogsSseFrame, type LogsSseFilter } from '@/features/logs/api/sse' import type { LogAuditEntry, LogRequest } from '@/features/logs/api/schemas' import { sortByOccurredAt } from '@/features/logs/lib/log-instant' import { resolveRelativeTime, type LogsLedgerSearch } from '@/features/logs/lib/log-search' +import { useAuditLiveRecovery } from './use-audit-live-recovery' const POLL_INTERVAL_MS = 5_000 const FALLBACK_DELAY_MS = 1_000 const DEFAULT_CHANNELS: readonly LogReplayChannel[] = ['requests', 'operations'] -/** - * Returned when audit streaming is disabled. Must be a shared module-level - * value, not a fresh `[]` literal: a new identity per render invalidates the - * ledger memo chain, which hands the events chart a new `data` array on every - * render and drives recharts into a synchronous re-dispatch loop until React's - * nested-update ceiling trips the `/logs` error boundary. - */ -const EMPTY_AUDIT_ENTRIES: readonly LogAuditEntry[] = [] export type LogsLiveConnectionState = 'connected' | 'reconnecting' | 'polling' | 'gap' | 'stale' @@ -47,6 +40,7 @@ export type LogsLiveRecovery = { readonly requestUpdates: readonly LogRequest[] readonly excludedRequestIds: readonly string[] readonly auditEntries: readonly LogAuditEntry[] + readonly fallbackPollingActive: boolean readonly pollingEnabled: boolean readonly togglePolling: () => void } @@ -135,10 +129,6 @@ function mergeLiveRequests(current: readonly LiveRequest[], next: LiveRequest): return sortByOccurredAt([...current.filter((entry) => entry.requestId !== next.requestId), next]).slice(-32) } -function mergeAuditEntries(current: readonly LogAuditEntry[], next: LogAuditEntry): LogAuditEntry[] { - return sortByOccurredAt([...current.filter((entry) => entry.entryId !== next.entryId), next]).slice(-64) -} - function requestMatchesSearch(request: LogRequest, search: LogsLedgerSearch) { const bounds = search.timeRange ? resolveRelativeTime(search.timeRange) : { from: search.from, to: search.to } const createdAt = Date.parse(request.createdAt) @@ -184,9 +174,8 @@ export function useLogsLiveRecovery({ eventSourceFactory: createEventSource = eventSourceFactory }: LogsLiveRecoveryOptions): LogsLiveRecovery { const [lifecycleState, setLifecycleState] = useState('reconnecting') - const [auditState, setAuditState] = useState('reconnecting') const [liveRequests, setLiveRequests] = useState({ subscriptionKey: '', entries: [] }) - const [liveAuditEntries, setLiveAuditEntries] = useState([]) + const [lifecycleFallbackPollingActive, setLifecycleFallbackPollingActive] = useState(false) const [pollingEnabled, setPollingEnabled] = useState(true) const pollingEnabledRef = useRef(true) const sequenceByChannelRef = useRef(new Map()) @@ -197,20 +186,10 @@ export function useLogsLiveRecovery({ const hydratePendingRef = useRef(false) const hydratePendingClearGapRef = useRef(false) const latestCursorRef = useRef(undefined) - const latestAuditCursorRef = useRef(undefined) - const auditSequenceRef = useRef(0n) - const auditHydrateInFlightRef = useRef(false) - const auditHydratePendingRef = useRef(false) - const auditHydratePendingClearGapRef = useRef(false) - const hydrateAuditRef = useRef(hydrateAudit) const searchRef = useRef(search) const restoredCursorValueRef = useRef(undefined) const previousAuthoritativeSnapshotRef = useRef(authoritativeSnapshot) - useEffect(() => { - hydrateAuditRef.current = hydrateAudit - }, [hydrateAudit]) - useEffect(() => { searchRef.current = search }, [search]) @@ -226,16 +205,6 @@ export function useLogsLiveRecovery({ })) }, [authoritativeSnapshot]) - useEffect(() => { - if ( - auditCursor && - (!latestAuditCursorRef.current || auditCursor.sequence() > latestAuditCursorRef.current.sequence()) - ) { - latestAuditCursorRef.current = auditCursor - auditSequenceRef.current = auditCursor.sequence() - } - }, [auditCursor]) - const togglePolling = useCallback(() => { setPollingEnabled((current) => { const next = !current @@ -316,9 +285,9 @@ export function useLogsLiveRecovery({ } const clearPolling = () => { - if (pollingTimer === undefined) return - window.clearInterval(pollingTimer) + if (pollingTimer !== undefined) window.clearInterval(pollingTimer) pollingTimer = undefined + setLifecycleFallbackPollingActive(false) } const clearFallback = () => { @@ -372,11 +341,12 @@ export function useLogsLiveRecovery({ const startPolling = () => { if (pollingTimer !== undefined) return - setLifecycleState('polling') + setLifecycleState(source ? 'reconnecting' : 'polling') if (pollingEnabledRef.current) hydrateAuthoritatively(false) pollingTimer = window.setInterval(() => { if (pollingEnabledRef.current) hydrateAuthoritatively(false) }, POLL_INTERVAL_MS) + setLifecycleFallbackPollingActive(true) } const queuePollingFallback = () => { @@ -482,145 +452,23 @@ export function useLogsLiveRecovery({ clearPolling() closeSource() } - }, [channels, createEventSource, enabled, filterScope, hydrate, key, search.replayCursor, subscriptionFilters]) - - useEffect(() => { - if (!auditEnabled) return - - let disposed = false - let source: LogsEventSource | undefined - let reconciliationTimer: number | undefined - let fallbackTimer: number | undefined - - const clearReconciliation = () => { - if (reconciliationTimer === undefined) return - window.clearInterval(reconciliationTimer) - reconciliationTimer = undefined - } - const clearFallback = () => { - if (fallbackTimer === undefined) return - window.clearTimeout(fallbackTimer) - fallbackTimer = undefined - } - const closeSource = () => { - if (!source) return - source.onopen = null - source.onerror = null - source.close() - source = undefined - } - const hydrateAuditAuthoritatively = (clearGap: boolean) => { - if (disposed) return - if (auditHydrateInFlightRef.current) { - auditHydratePendingRef.current = true - auditHydratePendingClearGapRef.current ||= clearGap - return - } - auditHydrateInFlightRef.current = true - void Promise.resolve(hydrateAuditRef.current()) - .then(() => { - if (!disposed && clearGap) setAuditState(source ? 'connected' : 'polling') - }) - .catch(() => { - if (!disposed) setAuditState('stale') - }) - .finally(() => { - auditHydrateInFlightRef.current = false - if (!disposed && auditHydratePendingRef.current) { - auditHydratePendingRef.current = false - const pendingClearGap = auditHydratePendingClearGapRef.current - auditHydratePendingClearGapRef.current = false - hydrateAuditAuthoritatively(pendingClearGap) - } - }) - } - const startReconciliation = () => { - if (reconciliationTimer !== undefined) return - reconciliationTimer = window.setInterval(() => { - if (pollingEnabledRef.current) hydrateAuditAuthoritatively(false) - }, POLL_INTERVAL_MS) - } - const startPolling = () => { - setAuditState('polling') - if (reconciliationTimer !== undefined) return - if (pollingEnabledRef.current) hydrateAuditAuthoritatively(false) - startReconciliation() - } - const queuePollingFallback = () => { - if (fallbackTimer !== undefined) return - setAuditState('reconnecting') - fallbackTimer = window.setTimeout(() => { - fallbackTimer = undefined - startPolling() - }, FALLBACK_DELAY_MS) - } - const acceptAuditEvent = (event: MessageEvent) => { - if (disposed) return - try { - const frame = parseLogsSseFrame({ event: event.type, lastEventId: event.lastEventId, data: event.data }) - if (!(frame.cursor instanceof LogAuditCursor)) { - setAuditState('stale') - hydrateAuditAuthoritatively(true) - return - } - latestAuditCursorRef.current = frame.cursor - if (frame.type === 'audit_gap') { - setAuditState('gap') - hydrateAuditAuthoritatively(true) - return - } - if (frame.type === 'stream_error') { - setAuditState('stale') - hydrateAuditAuthoritatively(true) - return - } - if (frame.type !== 'audit_entry') { - setAuditState('stale') - hydrateAuditAuthoritatively(true) - return - } - const sequence = BigInt(frame.entry.sequence) - if (sequence <= auditSequenceRef.current) return - auditSequenceRef.current = sequence - setLiveAuditEntries((current) => mergeAuditEntries(current, frame.entry)) - } catch { - setAuditState('stale') - hydrateAuditAuthoritatively(true) - } - } - - const url = new LogsApiClient().logsEventSourceUrl({ - channels: [], - audit: { cursor: latestAuditCursorRef.current } - }) - try { - const connectedSource = createEventSource(url) - source = connectedSource - connectedSource.onopen = () => { - if (disposed) return - clearFallback() - clearReconciliation() - setAuditState('connected') - } - connectedSource.onerror = () => { - if (!disposed) queuePollingFallback() - } - connectedSource.addEventListener('audit_entry', acceptAuditEvent) - connectedSource.addEventListener('replay_gap', acceptAuditEvent) - connectedSource.addEventListener('stream_error', acceptAuditEvent) - } catch { - startPolling() - } - - return () => { - disposed = true - clearFallback() - clearReconciliation() - closeSource() - } - }, [auditEnabled, createEventSource]) + }, [channels, createEventSource, enabled, hydrate, key, search.replayCursor, subscriptionFilters]) + + const { + state: auditState, + entries: auditEntries, + fallbackPollingActive: auditFallbackPollingActive + } = useAuditLiveRecovery({ + enabled: auditEnabled, + hydrate: hydrateAudit, + cursor: auditCursor, + pollingEnabledRef, + eventSourceFactory: createEventSource + }) const state = combinedConnectionState(lifecycleState, auditState, enabled, auditEnabled) + const fallbackPollingActive = + (enabled && lifecycleFallbackPollingActive) || (auditEnabled && auditFallbackPollingActive) const activeLiveRequests = useMemo( () => (enabled && liveRequests.subscriptionKey === key ? liveRequests.entries : []), [enabled, key, liveRequests] @@ -640,7 +488,8 @@ export function useLogsLiveRecovery({ liveRequestIds, requestUpdates, excludedRequestIds, - auditEntries: auditEnabled ? liveAuditEntries : EMPTY_AUDIT_ENTRIES, + auditEntries, + fallbackPollingActive, pollingEnabled, togglePolling } diff --git a/crates/mesh-llm-ui/src/features/logs/api/use-logs-query-retention.test.tsx b/crates/mesh-llm-ui/src/features/logs/api/use-logs-query-retention.test.tsx new file mode 100644 index 0000000000..78ec83bfa7 --- /dev/null +++ b/crates/mesh-llm-ui/src/features/logs/api/use-logs-query-retention.test.tsx @@ -0,0 +1,166 @@ +// @vitest-environment jsdom + +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { LogsApiClient, type LogsCapability } from '@/features/logs/api/client' +import { LogRequestId } from '@/features/logs/api/ids' +import type { LogAuditEntry, LogAuditPage, LogRequest, LogsPage } from '@/features/logs/api/schemas' +import { useLogsAuditQuery } from '@/features/logs/api/use-logs-audit-query' +import { useLogsLedgerQuery } from '@/features/logs/api/use-logs-ledger-query' +import { DataModeProvider } from '@/lib/data-mode' + +type RequestResult = LogsCapability> +type AuditResult = LogsCapability + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('logs query retention', () => { + it('keeps request and operational data visible through staggered chained key changes', async () => { + const firstRequest = createDeferred(requestResult(requestFixture('10000000-0000-4000-8000-000000000001'))) + const secondRequest = createDeferred(requestResult(requestFixture('10000000-0000-4000-8000-000000000002'))) + const thirdRequest = createDeferred(requestResult(requestFixture('10000000-0000-4000-8000-000000000003'))) + const firstAudit = createDeferred(auditResult(auditFixture('audit-1', 1))) + const secondAudit = createDeferred(auditResult(auditFixture('audit-2', 2))) + const thirdAudit = createDeferred(auditResult(auditFixture('audit-3', 3))) + + const listRequests = vi + .spyOn(LogsApiClient.prototype, 'listRequests') + .mockImplementationOnce(() => firstRequest.promise) + .mockImplementationOnce(() => secondRequest.promise) + .mockImplementationOnce(() => thirdRequest.promise) + const listAudits = vi + .spyOn(LogsApiClient.prototype, 'listAudits') + .mockImplementationOnce(() => firstAudit.promise) + .mockImplementationOnce(() => secondAudit.promise) + .mockImplementationOnce(() => thirdAudit.promise) + + const firstBounds = { from: '2026-08-01T00:00:00Z', to: '2026-08-01T00:01:00Z' } + const secondBounds = { from: '2026-08-01T00:01:00Z', to: '2026-08-01T00:02:00Z' } + const thirdBounds = { from: '2026-08-01T00:02:00Z', to: '2026-08-01T00:03:00Z' } + const { rerender, result } = renderHook( + ({ requestBounds, auditBounds }) => ({ + request: useLogsLedgerQuery(requestBounds), + audit: useLogsAuditQuery(auditBounds) + }), + { + initialProps: { requestBounds: firstBounds, auditBounds: firstBounds }, + wrapper: createWrapper() + } + ) + + await waitFor(() => { + expect(listRequests).toHaveBeenCalledTimes(1) + expect(listAudits).toHaveBeenCalledTimes(1) + }) + await act(async () => { + firstRequest.resolve() + firstAudit.resolve() + }) + await waitFor(() => { + expect(visibleRequestId(result.current.request.data)).toBe('10000000-0000-4000-8000-000000000001') + expect(visibleAuditId(result.current.audit.data)).toBe('audit-1') + }) + + rerender({ requestBounds: secondBounds, auditBounds: firstBounds }) + await waitFor(() => expect(listRequests).toHaveBeenCalledTimes(2)) + rerender({ requestBounds: secondBounds, auditBounds: secondBounds }) + await waitFor(() => expect(listAudits).toHaveBeenCalledTimes(2)) + await act(async () => { + secondRequest.resolve() + }) + await waitFor(() => { + expect(visibleRequestId(result.current.request.data)).toBe('10000000-0000-4000-8000-000000000002') + }) + + rerender({ requestBounds: thirdBounds, auditBounds: secondBounds }) + await waitFor(() => expect(listRequests).toHaveBeenCalledTimes(3)) + rerender({ requestBounds: thirdBounds, auditBounds: thirdBounds }) + await waitFor(() => expect(listAudits).toHaveBeenCalledTimes(3)) + expect({ + requestId: visibleRequestId(result.current.request.data), + auditId: visibleAuditId(result.current.audit.data) + }).toEqual({ requestId: '10000000-0000-4000-8000-000000000002', auditId: 'audit-1' }) + + await act(async () => { + thirdRequest.resolve() + }) + await waitFor(() => { + expect(visibleRequestId(result.current.request.data)).toBe('10000000-0000-4000-8000-000000000003') + expect(visibleAuditId(result.current.audit.data)).toBe('audit-1') + }) + + await act(async () => { + secondAudit.resolve() + thirdAudit.resolve() + }) + await waitFor(() => expect(visibleAuditId(result.current.audit.data)).toBe('audit-3')) + }) +}) + +function createDeferred(value: T) { + const gate = new AbortController() + const promise = new Promise((resolve) => { + gate.signal.addEventListener('abort', () => resolve(value), { once: true }) + }) + return { promise, resolve: () => gate.abort() } +} + +function requestFixture(requestId: string): LogRequest { + return { + requestId: LogRequestId.parse(requestId), + outcome: 'completed', + createdAt: '2026-08-01T00:00:00Z', + terminalAt: '2026-08-01T00:00:01Z', + route: 'chat_completions', + model: 'test-model', + provider: 'test-provider', + engine: 'test-engine', + statusCode: 200, + source: 'durable' + } +} + +function auditFixture(entryId: string, sequence: number): LogAuditEntry { + return { + entryId, + occurredAt: '2026-08-01T00:00:00Z', + source: 'runtime', + code: 'runtime_ready', + severity: 'info', + sequence + } +} + +function requestResult(item: LogRequest): RequestResult { + return { state: 'supported', value: { items: [item], nextCursor: undefined } } +} + +function auditResult(item: LogAuditEntry): AuditResult { + return { state: 'supported', value: { items: [item], nextCursor: undefined } } +} + +function visibleRequestId(data: RequestResult | undefined): string | undefined { + return data?.state === 'supported' ? data.value.items[0]?.requestId.toString() : undefined +} + +function visibleAuditId(data: AuditResult | undefined): string | undefined { + return data?.state === 'supported' ? data.value.items[0]?.entryId : undefined +} + +function createWrapper() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + return function Wrapper({ children }: { readonly children: ReactNode }) { + return ( + + + {children} + + + ) + } +} diff --git a/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.test.tsx b/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.test.tsx index c0d0a883aa..6a81886343 100644 --- a/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.test.tsx +++ b/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.test.tsx @@ -1,6 +1,6 @@ import '@testing-library/jest-dom/vitest' -import { act, render, renderHook, screen, within } from '@testing-library/react' +import { act, render, renderHook, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ChartTooltipPayloadItem } from '@/components/ui/chart' @@ -137,10 +137,44 @@ describe('EventsOverTimeChart', () => { const bucketSelect = screen.getByLabelText('Bucket interval') as HTMLSelectElement const rangeSelect = screen.getByLabelText('Chart time range') as HTMLSelectElement - expect(bucketSelect.value).toBe('5m') + expect(bucketSelect.value).toBe('15m') expect(rangeSelect.value).toBe('12h') }) + it('pairs the bucket interval with the selected time range', async () => { + const user = userEvent.setup() + render() + + const bucketSelect = screen.getByLabelText('Bucket interval') as HTMLSelectElement + const rangeSelect = screen.getByLabelText('Chart time range') + + await user.selectOptions(rangeSelect, '1h') + expect(bucketSelect.value).toBe('1m') + + await user.selectOptions(rangeSelect, '24h') + expect(bucketSelect.value).toBe('30m') + + await user.selectOptions(rangeSelect, 'all') + expect(bucketSelect.value).toBe('1h') + }) + + it('keeps an explicit bucket interval for the range it was chosen in', async () => { + const user = userEvent.setup() + render() + + const bucketSelect = screen.getByLabelText('Bucket interval') as HTMLSelectElement + const rangeSelect = screen.getByLabelText('Chart time range') + + await user.selectOptions(bucketSelect, '1m') + expect(bucketSelect.value).toBe('1m') + + await user.selectOptions(rangeSelect, '24h') + expect(bucketSelect.value).toBe('30m') + + await user.selectOptions(rangeSelect, '12h') + expect(bucketSelect.value).toBe('1m') + }) + it('reports time-range changes to an owning page', async () => { const user = userEvent.setup() const onSelectedRangeChange = vi.fn() @@ -208,6 +242,72 @@ describe('EventsOverTimeChart', () => { expect(legend).toHaveTextContent('Gossip0') }) + it('selects the active bucket with the keyboard', async () => { + const user = userEvent.setup() + const onBucketSelect = vi.fn() + render( + + ) + + const chart = screen.getByRole('img', { name: /Events over time stacked bar chart/ }) + chart.focus() + await user.keyboard('{Enter}') + + expect(onBucketSelect).toHaveBeenCalledOnce() + expect(onBucketSelect).toHaveBeenCalledWith({ + from: expect.any(String), + to: expect.any(String) + }) + }) + + it('renders a full-bucket band for a point-sized current table page window', async () => { + // Given + const occurredAt = NOW - 5 * 60_000 + + // When + const { container } = render( + + ) + + // Then + const band = await waitFor(() => { + const element = container.querySelector('.recharts-reference-area-rect') + expect(element).toBeInTheDocument() + return element + }) + expect(Number(band?.getAttribute('width'))).toBeGreaterThan(0) + expect(screen.getByText(/Accent band marks current table page:/i)).toBeVisible() + }) + + it('omits the current table page band when its window is outside the chart range', () => { + // Given + const occurredAt = NOW - 5 * 60_000 + + // When + const { container } = render( + + ) + + // Then + expect(container.querySelector('.recharts-reference-area')).not.toBeInTheDocument() + expect(screen.queryByText(/Accent band marks current table page:/i)).not.toBeInTheDocument() + }) + it('uses a stable, differentiated series palette and marker shapes', () => { const rows = LOG_EVENT_CATEGORIES.map((category, index) => eventAt(category, iso(NOW - index * 5 * 60_000), index + 1) @@ -277,4 +377,52 @@ describe('EventsOverTimeChart', () => { expect(screen.getByText(/Auto-bucketed to/)).toBeInTheDocument() expect(screen.getByLabelText(/Events over time stacked bar chart/)).toBeInTheDocument() }) + + it('offers a window clear action only while a clicked window is the active filter', async () => { + const user = userEvent.setup() + const onClearBucketSelection = vi.fn() + const { rerender } = render( + + ) + + expect(screen.queryByRole('button', { name: 'Clear window' })).not.toBeInTheDocument() + + rerender( + + ) + + await user.click(screen.getByRole('button', { name: 'Clear window' })) + + expect(onClearBucketSelection).toHaveBeenCalledTimes(1) + }) + + it('reports a loading window in a reserved slot that keeps the range controls in place', () => { + const { rerender } = render( + + ) + + const slot = screen.getByText('Loading system logs').parentElement + expect(slot).toHaveAttribute('aria-live', 'polite') + const controls = slot?.parentElement + expect(controls?.firstElementChild).toBe(slot) + expect(controls).toContainElement(screen.getByLabelText('Chart time range')) + + rerender() + + expect(controls?.firstElementChild).toBe(slot) + expect(slot).toBeEmptyDOMElement() + }) }) diff --git a/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.tsx b/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.tsx index 4e12ebb8b3..b690fc9613 100644 --- a/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.tsx +++ b/crates/mesh-llm-ui/src/features/logs/components/EventsOverTimeChart.tsx @@ -1,14 +1,17 @@ -import { useCallback, useMemo, useState } from 'react' +import { useCallback, useMemo, useState, type KeyboardEvent } from 'react' import { Bar, BarChart, CartesianGrid, Cell, + ReferenceArea, XAxis, YAxis, type MouseHandlerDataParam, type TooltipContentProps } from 'recharts' +import { Loader2, X } from 'lucide-react' +import { Button } from '@/components/ui/button' import { Card } from '@/components/ui/card' import { ChartContainer, @@ -18,6 +21,11 @@ import { type ChartTooltipPayloadItem } from '@/components/ui/chart' import { NativeSelect } from '@/components/ui/NativeSelect' +import { + LOG_EVENT_CATEGORY_COLORS, + LOG_EVENT_CATEGORY_LABELS, + LOG_EVENT_CATEGORY_MARKER_CLASS +} from '@/features/logs/lib/log-event-category-style' import { LOG_EVENT_CATEGORIES, type LogEventCategory, @@ -27,9 +35,11 @@ import { BUCKET_INTERVALS, VOLUME_TIME_RANGES, buildEventVolumeBuckets, + defaultBucketIntervalKey, effectiveEventVolumeIntervalMs, formatBucketInterval, formatBucketRange, + formatBucketTick, type BucketIntervalKey, type VolumeTimeRangeKey } from '@/features/logs/lib/log-volume' @@ -39,67 +49,60 @@ import { useAdvancingChartClock } from '@/features/logs/lib/use-advancing-chart- type EventsOverTimeChartProps = { readonly rows: readonly LogEventLedgerRow[] readonly selectedCategories: ReadonlySet + readonly currentPageTimeWindow?: { readonly from: number; readonly to: number } /** The ledger filter window; selecting a new ledger range resets the chart to the same window. */ readonly selectedRange?: VolumeTimeRangeKey /** Exact duration for a custom ledger window represented by `selected`. */ readonly selectedRangeMs?: number /** Promotes the chart selector to the owning page's time-range control. */ readonly onSelectedRangeChange?: (range: VolumeTimeRangeKey) => void + /** Narrows the owning ledger to a bucket the operator clicked. */ + readonly onBucketSelect?: (window: { readonly from: string; readonly to: string }) => void + /** Restores the ledger when a clicked bucket window is the active filter. */ + readonly onClearBucketSelection?: () => void + /** Marks the window as still loading; shown in a reserved slot so the controls never move. */ + readonly loading?: boolean /** Test seam: overrides the wall clock used to anchor the time window. */ readonly now?: number } const chartConfig = { - requests: { - label: 'Requests', - color: 'var(--color-log-requests)' - }, - system: { - label: 'System', - color: 'var(--color-log-system)' - }, - quic: { - label: 'QUIC', - color: 'var(--color-log-quic)' - }, - gossip: { - label: 'Gossip', - color: 'var(--color-log-gossip)' - }, - iroh: { - label: 'Iroh', - color: 'var(--color-log-iroh)' - } + requests: { label: LOG_EVENT_CATEGORY_LABELS.requests, color: LOG_EVENT_CATEGORY_COLORS.requests }, + system: { label: LOG_EVENT_CATEGORY_LABELS.system, color: LOG_EVENT_CATEGORY_COLORS.system }, + quic: { label: LOG_EVENT_CATEGORY_LABELS.quic, color: LOG_EVENT_CATEGORY_COLORS.quic }, + gossip: { label: LOG_EVENT_CATEGORY_LABELS.gossip, color: LOG_EVENT_CATEGORY_COLORS.gossip }, + iroh: { label: LOG_EVENT_CATEGORY_LABELS.iroh, color: LOG_EVENT_CATEGORY_COLORS.iroh } } satisfies ChartConfig -const chartMarkerClassNames: Record = { - requests: 'rounded-[2px]', - system: 'rounded-full', - quic: 'rounded-[1px] rotate-45', - gossip: 'h-1.5 w-2.5 rounded-[1px]', - iroh: 'h-1.5 w-2.5 rounded-full' -} - export function EventsOverTimeChart({ rows, selectedCategories, + currentPageTimeWindow, now, selectedRange, selectedRangeMs, - onSelectedRangeChange + onSelectedRangeChange, + onBucketSelect, + onClearBucketSelection, + loading = false }: EventsOverTimeChartProps) { - const [intervalKey, setIntervalKey] = useState('5m') const [rangeSelection, setRangeSelection] = useState<{ readonly filter: VolumeTimeRangeKey | undefined readonly value: VolumeTimeRangeKey }>({ filter: selectedRange, value: selectedRange ?? '12h' }) const rangeKey = rangeSelection.filter === selectedRange ? rangeSelection.value : (selectedRange ?? '12h') - - const intervalMs = BUCKET_INTERVALS.find((option) => option.value === intervalKey)?.ms ?? 300_000 const rangeMs = rangeKey === 'selected' ? (selectedRangeMs ?? Number.POSITIVE_INFINITY) : (VOLUME_TIME_RANGES.find((option) => option.value === rangeKey)?.ms ?? 43_200_000) + + const [intervalSelection, setIntervalSelection] = useState<{ + readonly range: VolumeTimeRangeKey + readonly value: BucketIntervalKey + }>() + const intervalKey = + intervalSelection?.range === rangeKey ? intervalSelection.value : defaultBucketIntervalKey(rangeMs) + const intervalMs = BUCKET_INTERVALS.find((option) => option.value === intervalKey)?.ms ?? 300_000 const liveCurrent = useAdvancingChartClock(now === undefined) const current = now ?? liveCurrent const activeCategories = useMemo( @@ -121,19 +124,54 @@ export function EventsOverTimeChart({ ) const effectiveIntervalMs = effectiveEventVolumeIntervalMs(data, intervalMs) const wasAutoBucketed = effectiveIntervalMs > intervalMs + const currentPageBucketWindow = useMemo( + () => overlappingBucketWindow(data, currentPageTimeWindow), + [currentPageTimeWindow, data] + ) const [activeIndex, setActiveIndex] = useState(undefined) - const handleChartMouseMove = useCallback( + const bucketIndexOf = useCallback( (nextState: MouseHandlerDataParam) => { const rawIndex = nextState.activeTooltipIndex const index = typeof rawIndex === 'number' ? rawIndex : rawIndex === null || rawIndex === '' ? undefined : Number(rawIndex) - setActiveIndex(index !== undefined && Number.isInteger(index) && data[index]?.total ? index : undefined) + return index !== undefined && Number.isInteger(index) && data[index]?.total ? index : undefined }, [data] ) + const handleChartMouseMove = useCallback( + (nextState: MouseHandlerDataParam) => setActiveIndex(bucketIndexOf(nextState)), + [bucketIndexOf] + ) const handleChartMouseLeave = useCallback(() => setActiveIndex(undefined), []) const activeBucket = activeIndex === undefined ? undefined : data[activeIndex] + const selectBucket = useCallback( + (index: number | undefined) => { + const bucket = index === undefined ? undefined : data[index] + if (!onBucketSelect || bucket === undefined) return + onBucketSelect({ + from: new Date(bucket.bucketStart).toISOString(), + to: new Date(bucket.bucketEnd).toISOString() + }) + }, + [data, onBucketSelect] + ) + const handleChartClick = useCallback( + (nextState: MouseHandlerDataParam) => selectBucket(bucketIndexOf(nextState) ?? activeIndex), + [activeIndex, bucketIndexOf, selectBucket] + ) + const handleChartFocus = useCallback(() => { + setActiveIndex((currentIndex) => currentIndex ?? data.findIndex((bucket) => bucket.total > 0)) + }, [data]) + const handleChartKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.key !== 'Enter' && event.key !== ' ') return + if (activeIndex === undefined || activeIndex < 0) return + event.preventDefault() + selectBucket(activeIndex) + }, + [activeIndex, selectBucket] + ) const renderTooltip = useCallback((tooltipProps: TooltipContentProps) => { if (!hasVisibleEventVolumeTooltip(tooltipProps.payload as readonly ChartTooltipPayloadItem[] | undefined)) @@ -155,6 +193,7 @@ export function EventsOverTimeChart({ return ( Events Over Time -

+

Loaded event volume by category and time bucket {wasAutoBucketed ? ` · Auto-bucketed to ${formatBucketInterval(effectiveIntervalMs)}` : ''} + {currentPageBucketWindow ? ( + + {' · '}Accent band marks current table page:{' '} + {formatBucketRange(currentPageBucketWindow.from, currentPageBucketWindow.to)}. + + ) : null}

+ + {loading ? ( + <> + + {onClearBucketSelection && selectedRange === 'selected' ? ( + + ) : null} setIntervalKey(value as BucketIntervalKey)} + onValueChange={(value) => setIntervalSelection({ range: rangeKey, value: value as BucketIntervalKey })} options={BUCKET_INTERVALS.map(({ value, label }) => ({ value, label }))} value={intervalKey} /> { const range = value as VolumeTimeRangeKey @@ -197,12 +256,12 @@ export function EventsOverTimeChart({
{activeCategories.length > 0 ? ( -
    +
      {activeCategories.map((category) => (