diff --git a/Cargo.lock b/Cargo.lock index 07f3cf69bf..4f5171f8a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3797,6 +3797,7 @@ version = "0.76.0-rc7" dependencies = [ "anyhow", "chrono", + "clap", "dirs", "hex", "iroh", diff --git a/crates/mesh-llm-cli/src/parser/commands.rs b/crates/mesh-llm-cli/src/parser/commands.rs index 9efaa36db4..1842542fe0 100644 --- a/crates/mesh-llm-cli/src/parser/commands.rs +++ b/crates/mesh-llm-cli/src/parser/commands.rs @@ -1453,6 +1453,8 @@ mod tests { ("--spec-draft-models", "/models/draft.gguf"), ("--spec-draft-max-tokens", "8"), ("--spec-draft-min-tokens", "2"), + ("--spec-draft-acceptance-threshold", "0.8"), + ("--spec-draft-split-probability", "0.5"), ("--spec-ngram-min", "2"), ("--spec-ngram-max", "4"), ] { diff --git a/crates/mesh-llm-commands/Cargo.toml b/crates/mesh-llm-commands/Cargo.toml index a1a511a80d..6bcdbe0133 100644 --- a/crates/mesh-llm-commands/Cargo.toml +++ b/crates/mesh-llm-commands/Cargo.toml @@ -51,6 +51,7 @@ url = "2" zeroize = { version = "1", features = ["derive"] } [dev-dependencies] +clap.workspace = true rand = "0.10" serial_test = "3" tempfile = "3" diff --git a/crates/mesh-llm-commands/src/operational_logging.rs b/crates/mesh-llm-commands/src/operational_logging.rs index 67e36498e5..e70ae72ae0 100644 --- a/crates/mesh-llm-commands/src/operational_logging.rs +++ b/crates/mesh-llm-commands/src/operational_logging.rs @@ -1,21 +1,20 @@ //! Bounded CLI command dispatch events. -//! -//! This boundary starts after `clap` has parsed a [`Command`] and finishes at -//! the top-level command dispatcher. It deliberately does not inspect command -//! arguments or error text. Parse failures happen before a `Command` exists -//! and remain outside this boundary. + +mod command_summary; use anyhow::Result; +use command_summary::command_summary; use mesh_llm_cli::Command; #[cfg(test)] use mesh_llm_events::OutputEvent; -use mesh_llm_events::{CliCommandFamily, CliCommandOutcome, emit_cli_command_event}; +use mesh_llm_events::{ + CliCommandFamily, CliCommandOutcome, CliCommandSummary, emit_cli_command_event, +}; use std::fmt; use std::sync::{Arc, LazyLock, RwLock}; /// Marker a handler can retain in its error chain when it explicitly rejects a -/// parsed command. Other handler or dispatcher errors are classified as -/// failures without reading their details. +/// parsed command. #[derive(Debug)] pub struct CommandDispatchRejected; @@ -27,35 +26,25 @@ impl fmt::Display for CommandDispatchRejected { impl std::error::Error for CommandDispatchRejected {} -/// Fail-open durable-audit bridge invoked after each command boundary -/// emission. The caller installs it only after logging runtime -/// initialization, so no durable record is produced before initialization. -/// The bridge receives only the static command family and outcome; it never -/// sees command arguments, argv, or error text. -pub type CliOperationalAuditBridge = Arc; +/// Fail-open durable-audit bridge invoked after each command boundary emission. +pub type CliOperationalAuditBridge = + Arc) + Send + Sync>; static CLI_OPERATIONAL_AUDIT_BRIDGE: LazyLock>> = LazyLock::new(|| RwLock::new(None)); -/// Install the durable command audit bridge. Callers invoke this only after -/// the logging runtime has been initialized so command behavior is unchanged -/// before initialization. Replacing the bridge is allowed for tests and -/// embedded hosts. pub fn install_cli_operational_audit_bridge(bridge: CliOperationalAuditBridge) { *CLI_OPERATIONAL_AUDIT_BRIDGE .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(bridge); } -/// Remove the command audit bridge. Used by tests and by callers that must -/// stop persisting command outcomes. pub fn clear_cli_operational_audit_bridge() { *CLI_OPERATIONAL_AUDIT_BRIDGE .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = None; } -/// Return the installed bridge, if any. pub fn cli_operational_audit_bridge() -> Option { CLI_OPERATIONAL_AUDIT_BRIDGE .read() @@ -63,42 +52,71 @@ pub fn cli_operational_audit_bridge() -> Option { .clone() } -/// Process-local lifecycle emitter for one parsed command dispatch. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct CommandDispatchBoundary { family: CliCommandFamily, + summary: Option, } impl CommandDispatchBoundary { - /// Start the command boundary after parsing succeeds. pub fn start(command: &Command) -> Self { - Self::start_family(command_family(command)) + Self::start_with_family(command_family(command), command) + } + + pub fn start_with_family(family: CliCommandFamily, command: &Command) -> Self { + Self::start_with_summary(family, command_summary(command)) + } + + pub fn start_with_cli(cli: &mesh_llm_cli::Cli, family: CliCommandFamily) -> Self { + let Some(command) = cli.command.as_ref() else { + return Self::start_family(family); + }; + let mut summary = command_summary(command).map(|value| value.as_str().to_owned()); + if let Some(summary_value) = summary.as_mut() { + for (present, marker) in [ + (!cli.join.is_empty(), " --join [REDACTED]"), + (!cli.relay.is_empty(), " --root-relay [REDACTED]"), + (!cli.relay_auth.is_empty(), " --relay-auth [REDACTED]"), + ] { + if present && !summary_value.contains(marker) { + summary_value.push_str(marker); + } + } + } + Self::start_with_summary( + family, + summary.and_then(|value| CliCommandSummary::sanitize(&value)), + ) + } + + fn start_with_summary(family: CliCommandFamily, summary: Option) -> Self { + let boundary = Self { family, summary }; + boundary.emit(CliCommandOutcome::Started); + boundary } - /// Start a command boundary whose bounded family was resolved by the - /// dispatcher (for example, external plugin versus truly unknown). pub fn start_family(family: CliCommandFamily) -> Self { - let boundary = Self { family }; + let boundary = Self { + family, + summary: None, + }; boundary.emit(CliCommandOutcome::Started); boundary } - /// Emit exactly one terminal result for the command dispatcher outcome. pub fn finish(self, result: &Result<()>) { self.emit(command_outcome(result)); } - fn emit(self, outcome: CliCommandOutcome) { - // Command results must retain their existing return and output behavior - // even if the process-local presentation sink is unavailable. + fn emit(&self, outcome: CliCommandOutcome) { let _ = emit_cli_command_event(self.family, outcome); if let Some(bridge) = cli_operational_audit_bridge() { - bridge(self.family, outcome); + bridge(self.family, outcome, self.summary.clone()); } } #[cfg(test)] - fn event(self, outcome: CliCommandOutcome) -> OutputEvent { + fn event(&self, outcome: CliCommandOutcome) -> OutputEvent { OutputEvent::CliCommandLifecycle { family: self.family, outcome, @@ -106,14 +124,14 @@ impl CommandDispatchBoundary { } } -/// Emit a process lifecycle outcome that occurs before a parsed [`Command`] -/// exists, while retaining the same presentation and durable-audit bridge. pub fn emit_cli_process_event(family: CliCommandFamily, outcome: CliCommandOutcome) { - let boundary = CommandDispatchBoundary { family }; + let boundary = CommandDispatchBoundary { + family, + summary: None, + }; boundary.emit(outcome); } -/// Map a parsed command to a stable, argument-free event family. pub fn command_family(command: &Command) -> CliCommandFamily { match command { Command::Models { .. } | Command::Download { .. } | Command::ModelPrepare { .. } => { @@ -156,6 +174,7 @@ fn command_outcome(result: &Result<()>) -> CliCommandOutcome { mod tests { use super::*; use anyhow::{Error, anyhow}; + use clap::Parser; use mesh_llm_events::{clear_output_sink, set_cli_command_event_verbose, set_output_sink}; use std::io; @@ -211,14 +230,12 @@ mod tests { impl Drop for CommandEventVerboseResetGuard { fn drop(&mut self) { - mesh_llm_events::set_cli_command_event_verbose(false); + set_cli_command_event_verbose(false); } } fn lifecycle_events(command: &Command, result: &Result<()>) -> [OutputEvent; 2] { - let boundary = CommandDispatchBoundary { - family: command_family(command), - }; + let boundary = CommandDispatchBoundary::start(command); [ boundary.event(CliCommandOutcome::Started), boundary.event(command_outcome(result)), @@ -226,42 +243,33 @@ mod tests { } #[test] + #[serial_test::serial] fn command_dispatch_orders_started_before_completed_without_command_arguments() { let command = Command::Load { name: "private-model.gguf?token=private-token".to_string(), port: 41731, }; - let result = Ok(()); - - let events = lifecycle_events(&command, &result); - + let events = lifecycle_events(&command, &Ok(())); assert_eq!( - events, - [ - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }, - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }, - ] + events[0].clone(), + OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Started + } + ); + assert_eq!( + events[1].clone(), + OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Completed + } ); let serialized = format!("{events:?}"); - for raw_value in [ - "private-model.gguf?token=private-token", - "41731", - "private-token", - ] { - assert!( - !serialized.contains(raw_value), - "command metadata must not enter lifecycle events" - ); - } + assert!(!serialized.contains("private-token")); } #[test] + #[serial_test::serial] fn explicit_handler_rejection_maps_to_rejected_without_error_detail() { let command = Command::Discover { name: Some("private-mesh".to_string()), @@ -273,31 +281,19 @@ mod tests { }; let result: Result<()> = Err(Error::new(CommandDispatchRejected) .context("private command rejection detail with private-token")); - let events = lifecycle_events(&command, &result); - assert_eq!( - events[1], + events[1].clone(), OutputEvent::CliCommandLifecycle { family: CliCommandFamily::Discovery, - outcome: CliCommandOutcome::Rejected, + outcome: CliCommandOutcome::Rejected } ); - let serialized = format!("{events:?}"); - for raw_value in [ - "private-mesh", - "private-region", - "wss://relay.private.example/?token=private-token", - "private command rejection detail with private-token", - ] { - assert!( - !serialized.contains(raw_value), - "rejection detail must not enter lifecycle events" - ); - } + assert!(!format!("{events:?}").contains("private-token")); } #[test] + #[serial_test::serial] fn unmarked_dispatch_failure_maps_to_failed_without_error_detail() { let command = Command::Download { name: Some("private/model?token=private-token".to_string()), @@ -306,18 +302,15 @@ mod tests { let result: Result<()> = Err(anyhow!( "download failed for https://private.example/model?token=private-token" )); - let events = lifecycle_events(&command, &result); - assert_eq!( - events[1], + events[1].clone(), OutputEvent::CliCommandLifecycle { family: CliCommandFamily::Models, - outcome: CliCommandOutcome::Failed, + outcome: CliCommandOutcome::Failed } ); - let serialized = format!("{events:?}"); - assert!(!serialized.contains("https://private.example/model?token=private-token")); + assert!(!format!("{events:?}").contains("private-token")); } #[test] @@ -331,42 +324,25 @@ mod tests { let recording = Arc::new(RecordingBridge::default()); let _bridge_guard = BridgeResetGuard; let bridge_recording = recording.clone(); - install_cli_operational_audit_bridge(Arc::new(move |family, outcome| { + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, _summary| { bridge_recording .calls .lock() .expect("recording bridge mutex poisoned") .push((family, outcome)); })); - - let command = Command::Load { + let boundary = CommandDispatchBoundary::start(&Command::Load { name: "model.gguf".to_string(), port: 9337, - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Ok(()); - boundary.finish(&result); - - assert_eq!( - sink.take_events(), - vec![ - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }, - OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }, - ] - ); + }); + boundary.finish(&Ok(())); + assert_eq!(sink.take_events().len(), 2); assert_eq!( recording.take_calls(), vec![ (CliCommandFamily::Runtime, CliCommandOutcome::Started), - (CliCommandFamily::Runtime, CliCommandOutcome::Completed), - ], - "installed bridge fires exactly once per outcome" + (CliCommandFamily::Runtime, CliCommandOutcome::Completed) + ] ); } @@ -374,18 +350,12 @@ mod tests { #[serial_test::serial] fn command_dispatch_without_bridge_stays_fail_open() { clear_cli_operational_audit_bridge(); - let command = Command::Load { + let boundary = CommandDispatchBoundary::start(&Command::Load { name: "private-model.gguf".to_string(), port: 41731, - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Err(anyhow!("private dispatch failure")); - boundary.finish(&result); - - assert!( - cli_operational_audit_bridge().is_none(), - "uninstalled bridge must remain absent after dispatch" - ); + }); + boundary.finish(&Err(anyhow!("private dispatch failure"))); + assert!(cli_operational_audit_bridge().is_none()); } #[test] @@ -394,46 +364,156 @@ mod tests { let recording = Arc::new(RecordingBridge::default()); let _bridge_guard = BridgeResetGuard; let bridge_recording = recording.clone(); - install_cli_operational_audit_bridge(Arc::new(move |family, outcome| { + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, _summary| { bridge_recording .calls .lock() .expect("recording bridge mutex poisoned") .push((family, outcome)); })); - - let command = Command::Discover { + let boundary = CommandDispatchBoundary::start(&Command::Discover { name: Some("private-mesh".to_string()), model: None, min_vram: None, region: Some("private-region".to_string()), auto: false, relay: vec!["wss://relay.private.example/?token=private-token".to_string()], - }; - let boundary = CommandDispatchBoundary::start(&command); - let result: Result<()> = Err(Error::new(CommandDispatchRejected) - .context("private command rejection detail with private-token")); - boundary.finish(&result); - - let calls = recording.take_calls(); + }); + boundary.finish(&Err(Error::new(CommandDispatchRejected) + .context("private command rejection detail with private-token"))); assert_eq!( - calls, + recording.take_calls(), vec![ (CliCommandFamily::Discovery, CliCommandOutcome::Started), - (CliCommandFamily::Discovery, CliCommandOutcome::Rejected), + (CliCommandFamily::Discovery, CliCommandOutcome::Rejected) ] ); - let serialized = format!("{calls:?}"); - for raw_value in [ - "private-mesh", - "private-region", - "wss://relay.private.example/?token=private-token", - "private command rejection detail with private-token", - ] { - assert!( - !serialized.contains(raw_value), - "command metadata must not cross the audit bridge" - ); - } + } + + #[test] + #[serial_test::serial] + fn dispatch_summary_is_shared_by_started_and_terminal_bridge_records() { + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let expected = CliCommandSummary::sanitize("mesh-llm load --port 9337 name [REDACTED]") + .expect("bounded summary"); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |_, _, summary| { + calls.lock().expect("bridge mutex").push(summary) + })); + let boundary = CommandDispatchBoundary::start(&Command::Load { + name: "model.gguf".to_owned(), + port: 9337, + }); + boundary.finish(&Ok(())); + assert_eq!( + recording.lock().expect("bridge mutex").as_slice(), + &[Some(expected.clone()), Some(expected)] + ); + } + + #[test] + #[serial_test::serial] + fn start_with_cli_redacts_global_credentials_in_both_bridge_records() { + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay", + "wss://relay.example/?token=private-relay-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "load", + "private/model", + ]); + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, summary| { + calls + .lock() + .expect("bridge mutex") + .push((family, outcome, summary)); + })); + + let boundary = CommandDispatchBoundary::start_with_cli(&cli, CliCommandFamily::Runtime); + boundary.finish(&Ok(())); + + let calls = recording.lock().expect("bridge mutex"); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, CliCommandFamily::Runtime); + assert_eq!(calls[1].0, CliCommandFamily::Runtime); + assert_eq!(calls[0].1, CliCommandOutcome::Started); + assert_eq!(calls[1].1, CliCommandOutcome::Completed); + assert_eq!(calls[0].2, calls[1].2); + let summary = calls[0].2.as_ref().expect("CLI summary").as_str(); + assert!(summary.contains("--join [REDACTED]")); + assert!(summary.contains("--root-relay [REDACTED]")); + assert!(summary.contains("--relay-auth [REDACTED]")); + assert!(CliCommandSummary::sanitize(summary).is_some()); + assert!(!summary.contains("private-invite-token")); + assert!(!summary.contains("private-relay-token")); + assert!(!summary.contains("private-token")); + } + + #[test] + #[serial_test::serial] + fn start_with_cli_does_not_fabricate_global_markers_without_base_summary() { + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "benchmark", + "tune", + "--model", + "private-model", + "--json", + "--ctx-sizes", + "1", + "--batch-sizes", + "1", + "--ubatch-sizes", + "1", + "--mmap-values", + "auto", + "--mlock-values", + "enabled", + "--flash-attention", + "on", + "--speculative-types", + "auto", + "--spec-draft-models", + "private-draft.gguf", + "--spec-draft-max-tokens", + "1", + "--spec-draft-min-tokens", + "1", + "--spec-ngram-min", + "1", + "--spec-ngram-max", + "1", + "--apply", + "--launch-args", + "--debug-telemetry", + ]); + let recording = Arc::new(std::sync::Mutex::new(Vec::new())); + let _bridge_guard = BridgeResetGuard; + let calls = recording.clone(); + install_cli_operational_audit_bridge(Arc::new(move |family, outcome, summary| { + calls + .lock() + .expect("bridge mutex") + .push((family, outcome, summary)); + })); + + let boundary = CommandDispatchBoundary::start_with_cli(&cli, CliCommandFamily::Benchmark); + boundary.finish(&Ok(())); + + let calls = recording.lock().expect("bridge mutex"); + assert_eq!(calls.len(), 2); + assert!(calls[0].2.is_none()); + assert!(calls[1].2.is_none()); } } diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary.rs new file mode 100644 index 0000000000..799720e1e9 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary.rs @@ -0,0 +1,97 @@ +mod administration; +mod auth; +mod benchmark; +mod dispatch; +mod models; +mod runtime; + +use mesh_llm_cli::Command; +use mesh_llm_events::CliCommandSummary; + +const DEFAULT_LOCAL_PORT: u16 = 3131; +const DEFAULT_AGENT_PORT: u16 = 9337; + +#[derive(Default)] +struct SummaryAssembly { + command: String, + flags: Vec, + values: Vec, + redacted: Vec<&'static str>, +} + +struct ModelPrepareSummary<'a> { + source_repo: &'a Option, + quant: &'a Option, + target: &'a Option, + model_id: &'a Option, + flavor: &'a str, + timeout: &'a str, + mesh_llm_ref: &'a str, + dry_run: bool, + confirm: bool, + follow: bool, + json: bool, + status: &'a Option, + logs: &'a Option, + cancel: &'a Option, + list: bool, + update_script: bool, +} + +impl SummaryAssembly { + fn new(command: &str) -> Self { + Self { + command: command.to_owned(), + ..Self::default() + } + } + + fn flag(&mut self, name: &'static str, present: bool) { + if present { + let marker = format!("--{name}"); + if !self.flags.contains(&marker) { + self.flags.push(marker); + } + } + } + + fn redact(&mut self, name: &'static str, present: bool) { + if present { + self.redacted.push(name); + } + } + + fn port(&mut self, port: u16, default: u16) { + if port != default { + self.values.push(format!("--port {port}")); + } + } + + fn finish(self) -> Option { + let mut value = self.command; + for part in self.flags.into_iter().chain(self.values) { + value.push(' '); + value.push_str(&part); + } + for name in self.redacted { + value.push(' '); + value.push_str(name); + value.push_str(" [REDACTED]"); + } + CliCommandSummary::sanitize(&value) + } +} + +pub(super) fn command_summary(command: &Command) -> Option { + let mut assembly = SummaryAssembly::new("mesh-llm"); + dispatch::format_command(command, &mut assembly); + assembly.finish() +} + +#[cfg(test)] +#[path = "command_summary_tests.rs"] +mod command_summary_tests; + +#[cfg(test)] +#[path = "command_summary_context_tests.rs"] +mod command_summary_context_tests; diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs new file mode 100644 index 0000000000..9d973cafa7 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/administration.rs @@ -0,0 +1,102 @@ +use super::{DEFAULT_LOCAL_PORT, SummaryAssembly}; + +pub(super) fn format_gpus( + json: bool, + command: Option<&mesh_llm_cli::GpuCommand>, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" gpus"); + let child_json = match command { + None => false, + Some(mesh_llm_cli::GpuCommand::Detect { json }) => { + assembly.command.push_str(" detect"); + *json + } + Some(mesh_llm_cli::GpuCommand::RunBenchmark { backend }) => { + assembly.command.push_str(" run-benchmark --backend "); + assembly + .command + .push_str(&format!("{backend:?}").to_lowercase()); + false + } + }; + assembly.flag("json", json || child_json); +} + +pub(super) fn format_config(command: &mesh_llm_cli::ConfigCommand, assembly: &mut SummaryAssembly) { + let mesh_llm_cli::ConfigCommand::Validate { config_path, json } = command; + assembly.command.push_str(" config validate"); + assembly.redact("--config-path", config_path.is_some()); + assembly.flag("json", *json); +} + +pub(super) fn format_doctor( + command: Option<&mesh_llm_cli::DoctorCommand>, + json: bool, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" doctor"); + let mut child_json = false; + if let Some(mesh_llm_cli::DoctorCommand::Split { + model_ref, + port, + json: split_json, + output_dir, + }) = command + { + assembly.command.push_str(" split"); + assembly.redact("--model-ref", !model_ref.is_empty()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + child_json = *split_json; + assembly.redact("--output-dir", output_dir.is_some()); + } + assembly.flag("json", json || child_json); +} + +pub(super) fn format_skills(command: &mesh_llm_cli::SkillCommand, assembly: &mut SummaryAssembly) { + let mesh_llm_cli::SkillCommand::Install { + agent, + all, + dry_run, + force, + } = command; + assembly.command.push_str(" skills install"); + assembly.redact("--agent", !agent.is_empty()); + assembly.flag("all", *all); + assembly.flag("dry-run", *dry_run); + assembly.flag("force", *force); +} + +pub(super) fn format_plugin(command: &mesh_llm_cli::PluginCommand, assembly: &mut SummaryAssembly) { + use mesh_llm_cli::PluginCommand; + match command { + PluginCommand::Install { + reference, + archive, + name, + version, + } => { + assembly.command.push_str(" plugins install"); + assembly.redact("reference", reference.is_some()); + assembly.redact("--archive", archive.is_some()); + assembly.redact("--name", name.is_some()); + assembly.redact("--version", version.is_some()); + } + PluginCommand::Update { .. } => format_plugin_name("update", assembly), + PluginCommand::Enable { .. } => format_plugin_name("enable", assembly), + PluginCommand::Disable { .. } => format_plugin_name("disable", assembly), + PluginCommand::Delete { .. } => format_plugin_name("delete", assembly), + PluginCommand::Info { .. } => format_plugin_name("info", assembly), + PluginCommand::Search { query } => { + assembly.command.push_str(" plugins search"); + assembly.redact("query", query.is_some()); + } + PluginCommand::List => assembly.command.push_str(" plugins list"), + } +} + +fn format_plugin_name(name: &str, assembly: &mut SummaryAssembly) { + assembly.command.push_str(" plugins "); + assembly.command.push_str(name); + assembly.redact("name", true); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs new file mode 100644 index 0000000000..8df09d0417 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/auth.rs @@ -0,0 +1,146 @@ +use super::SummaryAssembly; + +pub(super) fn format_auth(command: &mesh_llm_cli::AuthCommand, assembly: &mut SummaryAssembly) { + use mesh_llm_cli::{AuthCommand, TrustCommand}; + match command { + AuthCommand::Init { + owner_key, + force, + no_passphrase, + keychain, + } => { + assembly.command.push_str(" auth init"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.flag("force", *force); + assembly.flag("no-passphrase", *no_passphrase); + assembly.flag("keychain", *keychain); + } + AuthCommand::Status { + owner_key, + node_key, + node_ownership, + trust_store, + } => { + assembly.command.push_str(" auth status"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--node-ownership", node_ownership.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::SignNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + } + | AuthCommand::RenewNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + } => { + assembly + .command + .push_str(if matches!(command, AuthCommand::SignNode { .. }) { + " auth sign-node" + } else { + " auth renew-node" + }); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--out", out.is_some()); + assembly.redact("--hostname-hint", hostname_hint.is_some()); + assembly.redact("--node-label", node_label.is_some()); + assembly.redact("--expires-in-hours", *expires_in_hours != 168); + } + AuthCommand::VerifyNode { + file, + node_id, + trust_store, + trust_policy, + } => { + assembly.command.push_str(" auth verify-node"); + assembly.redact("--file", file.is_some()); + assembly.redact("--node-id", node_id.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + assembly.redact("--verify-trust-policy", trust_policy.is_some()); + } + AuthCommand::RotateNode { + owner_key, + node_key, + out, + hostname_hint, + node_label, + expires_in_hours, + revoke_current, + reason, + trust_store, + } => { + assembly.command.push_str(" auth rotate-node"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.redact("--node-key", node_key.is_some()); + assembly.redact("--out", out.is_some()); + assembly.redact("--hostname-hint", hostname_hint.is_some()); + assembly.redact("--node-label", node_label.is_some()); + assembly.redact("--expires-in-hours", *expires_in_hours != 168); + assembly.flag("revoke-current", *revoke_current); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RevokeOwner { + reason, + trust_store, + .. + } => { + assembly.command.push_str(" auth revoke-owner"); + assembly.redact("owner_id", true); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RevokeNode { + cert_id, + node_id, + reason, + trust_store, + } => { + assembly.command.push_str(" auth revoke-node"); + assembly.redact("--cert-id", cert_id.is_some()); + assembly.redact("--node-id", node_id.is_some()); + assembly.redact("--reason", reason.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + AuthCommand::RotateOwner { + owner_key, + no_passphrase, + force, + } => { + assembly.command.push_str(" auth rotate-owner"); + assembly.redact("--owner-key", owner_key.is_some()); + assembly.flag("no-passphrase", *no_passphrase); + assembly.flag("force", *force); + } + AuthCommand::Trust { command } => match command { + TrustCommand::Add { + label, trust_store, .. + } => { + assembly.command.push_str(" auth trust add"); + assembly.redact("owner_id", true); + assembly.redact("--label", label.is_some()); + assembly.redact("--trust-store", trust_store.is_some()); + } + TrustCommand::Remove { trust_store, .. } => { + assembly.command.push_str(" auth trust remove"); + assembly.redact("owner_id", true); + assembly.redact("--trust-store", trust_store.is_some()); + } + TrustCommand::List { trust_store } => { + assembly.command.push_str(" auth trust list"); + assembly.redact("--trust-store", trust_store.is_some()); + } + }, + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs new file mode 100644 index 0000000000..8471100bd5 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/benchmark.rs @@ -0,0 +1,70 @@ +use super::SummaryAssembly; + +pub(super) fn format_benchmark( + command: &mesh_llm_cli::benchmark::BenchmarkCommand, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::benchmark::BenchmarkCommand; + match command { + BenchmarkCommand::Tune(tune) => { + assembly.command.push_str(" benchmark tune"); + assembly.redact("--model", tune.model.is_some()); + assembly.redact("--models", !tune.models.is_empty()); + assembly.flag("json", tune.json); + assembly.redact("--ctx-sizes", !tune.ctx_sizes.is_empty()); + assembly.redact("--batch-sizes", !tune.batch_sizes.is_empty()); + assembly.redact("--ubatch-sizes", !tune.ubatch_sizes.is_empty()); + assembly.redact("--mmap-values", !tune.mmap_values.is_empty()); + assembly.redact("--mlock-values", !tune.mlock_values.is_empty()); + assembly.redact("--flash-attention", !tune.flash_attention.is_empty()); + assembly.redact("--speculative-types", !tune.speculative_types.is_empty()); + assembly.flag("no-speculative-tune", tune.no_speculative_tune); + assembly.redact("--spec-draft-models", !tune.spec_draft_models.is_empty()); + assembly.redact( + "--spec-draft-max-tokens", + !tune.spec_draft_max_tokens.is_empty(), + ); + assembly.redact( + "--spec-draft-min-tokens", + !tune.spec_draft_min_tokens.is_empty(), + ); + assembly.redact("--spec-ngram-min", !tune.spec_ngram_min.is_empty()); + assembly.redact("--spec-ngram-max", !tune.spec_ngram_max.is_empty()); + assembly.redact( + "--spec-draft-acceptance-threshold", + !tune.spec_draft_acceptance_threshold.is_empty(), + ); + assembly.redact( + "--spec-draft-split-probability", + !tune.spec_draft_split_probability.is_empty(), + ); + assembly.flag("apply", tune.apply); + assembly.flag("replace-existing", tune.replace_existing); + assembly.flag("launch-args", tune.launch_args); + assembly.redact( + "--throughput-tolerance-pct", + tune.throughput_tolerance_pct != 10.0, + ); + assembly.redact("--max-tokens", tune.max_tokens != 128); + assembly.redact("--startup-timeout-secs", tune.startup_timeout_secs != 600); + assembly.redact("--request-timeout-secs", tune.request_timeout_secs != 600); + assembly.flag("debug-telemetry", tune.debug_telemetry); + assembly.redact( + "--prompt", + tune.prompt != "Write a concise paragraph about distributed GPU inference.", + ); + } + BenchmarkCommand::ImportPrompts { + source: _, + limit, + max_tokens, + output: _, + } => { + assembly.command.push_str(" benchmark import-prompts"); + assembly.redact("--source", true); + assembly.redact("--limit", *limit != 20); + assembly.redact("--max-tokens", max_tokens.is_some()); + assembly.redact("--output", true); + } + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs new file mode 100644 index 0000000000..e931654fa4 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/dispatch.rs @@ -0,0 +1,170 @@ +use mesh_llm_cli::Command; + +use super::{ + DEFAULT_AGENT_PORT, DEFAULT_LOCAL_PORT, ModelPrepareSummary, SummaryAssembly, administration, + auth, benchmark, models, runtime, +}; + +pub(super) fn format_command(command: &Command, assembly: &mut SummaryAssembly) { + match command { + Command::Models { command } => models::format_models(command, assembly), + Command::Runtime { command } => runtime::format_runtime(command.as_ref(), assembly), + Command::Plugin { command } => administration::format_plugin(command, assembly), + Command::Auth { command } => auth::format_auth(command, assembly), + Command::Benchmark { command } => benchmark::format_benchmark(command, assembly), + Command::Config { command } => administration::format_config(command, assembly), + Command::Doctor { command, json } => { + administration::format_doctor(command.as_ref(), *json, assembly); + } + Command::Skills { command } => administration::format_skills(command, assembly), + Command::Download { name, draft } => { + assembly.command.push_str(" download"); + assembly.redact("name", name.is_some()); + assembly.flag("draft", *draft); + } + Command::Update { + version, + flavor, + detect_flavor, + } => { + assembly.command.push_str(" update"); + assembly.redact("--version", version.is_some()); + assembly.redact("--flavor", flavor.is_some()); + assembly.flag("detect-flavor", *detect_flavor); + } + Command::Gpus { json, command } => { + administration::format_gpus(*json, command.as_ref(), assembly); + } + Command::Setup { + yes, + no_interactive, + service, + no_service, + skip_runtime, + verbose, + } => { + assembly.command.push_str(" setup"); + assembly.flag("yes", *yes); + assembly.flag("no-interactive", *no_interactive); + assembly.flag("service", *service); + assembly.flag("no-service", *no_service); + assembly.flag("skip-runtime", *skip_runtime); + assembly.flag("verbose", *verbose); + } + Command::Uninstall { + dry_run, + yes, + keep_cache, + keep_service_files, + purge_config, + keep_config, + binary_path, + json, + verbose, + } => { + assembly.command.push_str(" uninstall"); + assembly.flag("dry-run", *dry_run); + assembly.flag("yes", *yes); + assembly.flag("keep-cache", *keep_cache); + assembly.flag("keep-service-files", *keep_service_files); + assembly.flag("purge-config", *purge_config); + assembly.flag("keep-config", *keep_config); + assembly.redact("--binary-path", binary_path.is_some()); + assembly.flag("json", *json); + assembly.flag("verbose", *verbose); + } + Command::Load { port, .. } => runtime::format_local_model("load", *port, assembly), + Command::Unload { port, .. } => runtime::format_local_model("unload", *port, assembly), + Command::Status { port } => { + assembly.command.push_str(" status"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + } + Command::Discover { + name, + model, + min_vram, + region, + auto, + relay, + } => { + assembly.command.push_str(" discover"); + assembly.redact("--name", name.is_some()); + assembly.redact("--model", model.is_some()); + assembly.redact("--min-vram", min_vram.is_some()); + assembly.redact("--region", region.is_some()); + assembly.flag("auto", *auto); + assembly.redact("--relay", !relay.is_empty()); + } + Command::RotateKey => assembly.command.push_str(" rotate-key"), + Command::Goose { model, port } => format_agent("goose", model, *port, assembly), + Command::Claude { model, port } => format_agent("claude", model, *port, assembly), + Command::Pi { model, host, write } => format_client("pi", model, host, *write, assembly), + Command::Opencode { model, host, write } => { + format_client("opencode", model, host, *write, assembly); + } + Command::Stop => assembly.command.push_str(" stop"), + Command::ModelPrepare { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + dry_run, + confirm, + follow, + json, + status, + logs, + cancel, + list, + update_script, + } => models::format_model_prepare( + ModelPrepareSummary { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + dry_run: *dry_run, + confirm: *confirm, + follow: *follow, + json: *json, + status, + logs, + cancel, + list: *list, + update_script: *update_script, + }, + assembly, + ), + Command::ExternalPlugin(args) => { + assembly.command.push_str(" external-plugin"); + assembly.redact("argv", !args.is_empty()); + } + } +} + +fn format_agent(name: &str, model: &Option, port: u16, assembly: &mut SummaryAssembly) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.redact("--model", model.is_some()); + assembly.port(port, DEFAULT_AGENT_PORT); +} + +fn format_client( + name: &str, + model: &Option, + host: &str, + write: bool, + assembly: &mut SummaryAssembly, +) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.redact("--model", model.is_some()); + assembly.redact("--host", host != "127.0.0.1:9337"); + assembly.flag("write", write); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs new file mode 100644 index 0000000000..5f3da370f5 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/models.rs @@ -0,0 +1,169 @@ +use super::{ModelPrepareSummary, SummaryAssembly}; + +pub(super) fn format_models( + command: &mesh_llm_cli::models::ModelsCommand, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::models::ModelsCommand; + match command { + ModelsCommand::Package { + source_repo, + quant, + target, + model_id, + flavor, + timeout, + mesh_llm_ref, + experimental, + dry_run, + confirm, + follow, + status, + logs, + cancel, + list, + update_script, + json, + } => { + assembly.command.push_str(" models package"); + assembly.redact("source_repo", source_repo.is_some()); + assembly.redact("--quant", quant.is_some()); + assembly.redact("--target", target.is_some()); + assembly.redact("--model-id", model_id.is_some()); + assembly.redact("--flavor", flavor != "auto"); + assembly.redact("--timeout", timeout != "1h"); + assembly.redact("--mesh-llm-ref", mesh_llm_ref != "main"); + assembly.flag("experimental", *experimental); + assembly.flag("dry-run", *dry_run); + assembly.flag("confirm", *confirm); + assembly.flag("follow", *follow); + assembly.redact("--status", status.is_some()); + assembly.redact("--logs", logs.is_some()); + assembly.redact("--cancel", cancel.is_some()); + assembly.flag("list", *list); + assembly.flag("update-script", *update_script); + assembly.flag("json", *json); + } + ModelsCommand::Recommended { json } => format_model_json("recommended", *json, assembly), + ModelsCommand::Installed { json } => format_model_json("installed", *json, assembly), + ModelsCommand::Cleanup { + unused_since, + yes, + json, + } => { + assembly.command.push_str(" models cleanup"); + assembly.redact("--unused-since", unused_since.is_some()); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + ModelsCommand::Prune { yes, json } => { + assembly.command.push_str(" models prune"); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + ModelsCommand::Certify { + model, + report_out, + json, + package_only, + api_base, + prompt, + max_tokens, + } => { + assembly.command.push_str(" models certify"); + assembly.redact("model", !model.is_empty()); + assembly.redact("--report-out", report_out.is_some()); + assembly.flag("json", *json); + assembly.flag("package-only", *package_only); + assembly.redact("--api-base", api_base.is_some()); + assembly.redact("--prompt", prompt != "Say ok."); + assembly.redact("--max-tokens", *max_tokens != 2); + } + ModelsCommand::List { json } => format_model_json("list", *json, assembly), + ModelsCommand::Search { + query, + gguf, + mlx, + catalog, + limit, + sort, + json, + } => { + assembly.command.push_str(" models search"); + assembly.redact("query", !query.is_empty()); + assembly.flag("gguf", *gguf); + assembly.flag("mlx", *mlx); + assembly.flag("catalog", *catalog); + assembly.redact("--limit", *limit != 20); + assembly.redact( + "--sort", + *sort != mesh_llm_cli::models::ModelSearchSort::Trending, + ); + assembly.flag("json", *json); + } + ModelsCommand::Show { model, json } => { + assembly.command.push_str(" models show"); + assembly.redact("model", !model.is_empty()); + assembly.flag("json", *json); + } + ModelsCommand::Download { + model, + draft, + direct, + json, + } => { + assembly.command.push_str(" models download"); + assembly.redact("model", !model.is_empty()); + assembly.flag("draft", *draft); + assembly.flag("direct", *direct); + assembly.flag("json", *json); + } + ModelsCommand::Updates { + repo, + all, + check, + json, + } => { + assembly.command.push_str(" models updates"); + assembly.redact("repo", repo.is_some()); + assembly.flag("all", *all); + assembly.flag("check", *check); + assembly.flag("json", *json); + } + ModelsCommand::Delete { model, yes, json } => { + assembly.command.push_str(" models delete"); + assembly.redact("model", !model.is_empty()); + assembly.flag("yes", *yes); + assembly.flag("json", *json); + } + } +} + +fn format_model_json(name: &str, json: bool, assembly: &mut SummaryAssembly) { + assembly.command.push_str(" models "); + assembly.command.push_str(name); + assembly.flag("json", json); +} + +pub(super) fn format_model_prepare( + options: ModelPrepareSummary<'_>, + assembly: &mut SummaryAssembly, +) { + assembly.command.push_str(" model-prepare"); + assembly.redact("source_repo", options.source_repo.is_some()); + assembly.redact("--quant", options.quant.is_some()); + assembly.redact("--target", options.target.is_some()); + assembly.redact("--model-id", options.model_id.is_some()); + assembly.redact("--flavor", options.flavor != "auto"); + assembly.redact("--timeout", options.timeout != "1h"); + assembly.redact("--mesh-llm-ref", options.mesh_llm_ref != "main"); + assembly.flag("dry-run", options.dry_run); + assembly.flag("confirm", options.confirm); + assembly.flag("follow", options.follow); + assembly.flag("json", options.json); + assembly.redact("--status", options.status.is_some()); + assembly.redact("--logs", options.logs.is_some()); + assembly.redact("--cancel", options.cancel.is_some()); + assembly.flag("list", options.list); + assembly.flag("update-script", options.update_script); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs new file mode 100644 index 0000000000..0555bd13c6 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary/runtime.rs @@ -0,0 +1,146 @@ +use super::{DEFAULT_LOCAL_PORT, SummaryAssembly}; + +pub(super) fn format_local_model(name: &str, port: u16, assembly: &mut SummaryAssembly) { + assembly.command.push(' '); + assembly.command.push_str(name); + assembly.port(port, DEFAULT_LOCAL_PORT); + assembly.redact("name", true); +} + +pub(super) fn format_runtime( + command: Option<&mesh_llm_cli::runtime::RuntimeCommand>, + assembly: &mut SummaryAssembly, +) { + use mesh_llm_cli::runtime::RuntimeCommand; + assembly.command.push_str(" runtime"); + let Some(command) = command else { return }; + match command { + RuntimeCommand::Status { port } => { + assembly.command.push_str(" status"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + } + RuntimeCommand::Load { port, .. } => format_local_model("load", *port, assembly), + RuntimeCommand::Unload { port, .. } => format_local_model("unload", *port, assembly), + RuntimeCommand::Guardrails { mode, port, json } => { + assembly.command.push_str(" guardrails --mode "); + assembly.command.push_str(mode.as_str()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::Bootstrap { port, json } => { + assembly.command.push_str(" bootstrap"); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::List { + available, + installed, + manifest, + bundle_dirs, + cache_dir, + json, + } => { + assembly.command.push_str(" list"); + assembly.flag("available", *available); + assembly.flag("installed", *installed); + assembly.redact("--manifest", manifest.is_some()); + assembly.redact("--bundle-dir", !bundle_dirs.is_empty()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Install { + runtime, + manifest, + bundle_dirs, + cache_dir, + json, + } => { + assembly.command.push_str(" install"); + assembly.redact("runtime_ref", runtime.is_some()); + assembly.redact("--manifest", manifest.is_some()); + assembly.redact("--bundle-dir", !bundle_dirs.is_empty()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Remove { + mesh_version, + cache_dir, + json, + .. + } => { + assembly.command.push_str(" remove"); + assembly.redact("native_runtime_id", true); + assembly.redact("--mesh-version", mesh_version.is_some()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::Prune { + active_only, + mesh_version, + cache_dir, + json, + } => { + assembly.command.push_str(" prune"); + assembly.flag("active-only", *active_only); + assembly.redact("--mesh-version", mesh_version.is_some()); + assembly.redact("--cache-dir", cache_dir.is_some()); + assembly.flag("json", *json); + } + RuntimeCommand::GetConfig { port, json, .. } + | RuntimeCommand::ScanRefresh { port, json, .. } + | RuntimeCommand::RefreshInventory { port, json, .. } => { + assembly.command.push_str(" remote"); + assembly.redact("--endpoint", true); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::LoadModel { + profile, + port, + json, + .. + } + | RuntimeCommand::EnsureModel { + profile, + port, + json, + .. + } => { + assembly.command.push_str(" remote-model"); + assembly.redact("--endpoint", true); + assembly.redact("--model", true); + assembly.redact("--profile", profile.is_some()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::UnloadModel { + model, + instance_id, + port, + json, + .. + } + | RuntimeCommand::DrainModel { + model, + instance_id, + port, + json, + .. + } => { + assembly.command.push_str(" remote-model"); + assembly.redact("--endpoint", true); + assembly.redact("--model", model.is_some()); + assembly.redact("--instance-id", instance_id.is_some()); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + RuntimeCommand::ApplyConfig { port, json, .. } => { + assembly.command.push_str(" apply-config"); + assembly.redact("--endpoint", true); + assembly.redact("--expected-revision", true); + assembly.redact("--config", true); + assembly.port(*port, DEFAULT_LOCAL_PORT); + assembly.flag("json", *json); + } + } +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs new file mode 100644 index 0000000000..770f6c41b4 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary_context_tests.rs @@ -0,0 +1,257 @@ +use super::command_summary; +use clap::Parser; +use mesh_llm_events::CliCommandSummary; + +fn assert_parsed_summaries(cases: &[(&[&str], &str)]) { + for (args, marker) in cases { + let cli = mesh_llm_cli::Cli::parse_from(*args); + let command = cli.command.as_ref().expect("parsed command"); + let summary = command_summary(command) + .map(|summary| summary.as_str().to_owned()) + .unwrap_or_else(|| panic!("producer summary for {args:?}")); + assert!(summary.contains(marker), "missing {marker} in {summary}"); + assert!( + CliCommandSummary::sanitize(&summary).is_some(), + "rejected: {summary}" + ); + } +} + +#[test] +fn parsed_commands_emit_local_raw_options() { + assert_parsed_summaries(&[ + (&["mesh-llm", "status", "--port", "41731"], "--port 41731"), + ( + &["mesh-llm", "load", "private/model", "--port", "41731"], + "--port 41731", + ), + ( + &["mesh-llm", "unload", "private/model", "--port", "41731"], + "--port 41731", + ), + (&["mesh-llm", "goose", "--port", "41731"], "--port 41731"), + (&["mesh-llm", "claude", "--port", "41731"], "--port 41731"), + ( + &[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "doctor", + "--json", + "split", + "--model-ref", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &["mesh-llm", "gpus", "run-benchmark", "--backend", "cuda"], + "--backend cuda", + ), + ( + &[ + "mesh-llm", + "gpus", + "--json", + "run-benchmark", + "--backend", + "cuda", + ], + "--backend cuda", + ), + ]); +} + +#[test] +fn parsed_commands_emit_runtime_raw_options() { + assert_parsed_summaries(&[ + ( + &[ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--port", + "41731", + ], + "--mode metrics", + ), + ( + &[ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--json", + "--port", + "41731", + ], + "--mode metrics", + ), + ( + &["mesh-llm", "runtime", "status", "--port", "41731"], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "unload", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &["mesh-llm", "runtime", "bootstrap", "--port", "41731"], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "bootstrap", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ]); +} + +#[test] +fn parsed_commands_emit_remote_raw_options() { + assert_parsed_summaries(&[ + ( + &[ + "mesh-llm", + "runtime", + "get-config", + "--endpoint", + "private/endpoint", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "get-config", + "--endpoint", + "private/endpoint", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load-model", + "--endpoint", + "private/endpoint", + "--model", + "private/model", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "load-model", + "--endpoint", + "private/endpoint", + "--model", + "private/model", + "--profile", + "profile-a", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "unload-model", + "--endpoint", + "private/endpoint", + "--instance-id", + "instance-a", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "apply-config", + "--endpoint", + "private/endpoint", + "--expected-revision", + "7", + "--config", + "/private/config.toml", + "--port", + "41731", + ], + "--port 41731", + ), + ( + &[ + "mesh-llm", + "runtime", + "apply-config", + "--endpoint", + "private/endpoint", + "--expected-revision", + "7", + "--config", + "/private/config.toml", + "--json", + "--port", + "41731", + ], + "--port 41731", + ), + ]); +} diff --git a/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs b/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs new file mode 100644 index 0000000000..fb50b3f720 --- /dev/null +++ b/crates/mesh-llm-commands/src/operational_logging/command_summary_tests.rs @@ -0,0 +1,329 @@ +use super::{SummaryAssembly, command_summary}; +use clap::Parser; +use mesh_llm_events::CliCommandSummary; + +fn parsed_summary(args: &[&str]) -> String { + let cli = mesh_llm_cli::Cli::parse_from(args); + let Some(command) = cli.command.as_ref() else { + return String::new(); + }; + match command_summary(command) { + Some(summary) => summary.as_str().to_owned(), + None => String::new(), + } +} + +#[test] +fn representative_command_summaries_survive_strict_sanitization() { + let commands: &[&[&str]] = &[ + &["mesh-llm", "setup"], + &["mesh-llm", "uninstall", "--yes"], + &["mesh-llm", "gpus", "detect"], + &["mesh-llm", "discover", "--auto"], + &["mesh-llm", "goose"], + &["mesh-llm", "plugins", "list"], + &["mesh-llm", "config", "validate"], + &[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + ], + &["mesh-llm", "models", "recommended"], + &["mesh-llm", "runtime", "bootstrap"], + &[ + "mesh-llm", + "benchmark", + "import-prompts", + "--source", + "mt-bench", + "--output", + "private.jsonl", + ], + &["mesh-llm", "model-prepare", "--list"], + &["mesh-llm", "auth", "status"], + ]; + + for args in commands { + let summary = parsed_summary(args); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer emitted unsanitized summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); + } +} + +#[test] +fn over_limit_producer_summary_is_omitted_without_fallback() { + let mut assembly = SummaryAssembly::new("mesh-llm"); + for _ in 0..32 { + assembly.redact("name", true); + } + + assert!(assembly.finish().is_none()); +} + +#[test] +fn uninstall_boolean_flags_survive_producer_sanitization() { + let summaries = [ + parsed_summary(&[ + "mesh-llm", + "uninstall", + "--dry-run", + "--yes", + "--keep-cache", + "--keep-service-files", + "--keep-config", + "--json", + "--verbose", + ]), + parsed_summary(&["mesh-llm", "uninstall", "--purge-config"]), + ]; + + for summary in summaries { + assert!(!summary.is_empty()); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); + } +} + +#[test] +fn command_summary_retains_static_prefix_nested_names_and_redacts_plugin_values() { + let summary = parsed_summary(&[ + "mesh-llm", + "plugins", + "install", + "https://user:secret@example.test/plugin?token=query-secret", + ]); + assert_eq!(summary, "mesh-llm plugins install reference [REDACTED]"); + assert!(!summary.contains("secret")); + assert!(!summary.contains("query-secret")); +} + +#[test] +fn command_summary_covers_plugin_config_and_doctor_values_without_defaults() { + assert_eq!( + parsed_summary(&["mesh-llm", "plugins", "update", "private-plugin"]), + "mesh-llm plugins update name [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "config", + "validate", + "--config-path", + "/private/config.toml" + ]), + "mesh-llm config validate --config-path [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "doctor", + "split", + "--model-ref", + "private/model", + "--output-dir", + "/private/report" + ]), + "mesh-llm doctor split --model-ref [REDACTED] --output-dir [REDACTED]" + ); +} + +#[test] +fn command_summary_covers_auth_trust_and_nested_command_families() { + let auth = parsed_summary(&[ + "mesh-llm", + "auth", + "rotate-node", + "--owner-key", + "/private/owner.json", + "--node-key", + "/private/node.key", + "--reason", + "private reason", + "--trust-store", + "/private/trust.json", + ]); + for marker in [ + "mesh-llm auth rotate-node", + "--owner-key [REDACTED]", + "--node-key [REDACTED]", + "--reason [REDACTED]", + "--trust-store [REDACTED]", + ] { + assert!(auth.contains(marker)); + } + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "auth", + "trust", + "add", + "private-owner-id", + "--trust-store", + "/private/trust.json" + ]), + "mesh-llm auth trust add owner_id [REDACTED] --trust-store [REDACTED]" + ); + assert!( + parsed_summary(&["mesh-llm", "models", "show", "private/model"]) + .contains("mesh-llm models show model [REDACTED]") + ); + assert!( + parsed_summary(&["mesh-llm", "runtime", "load", "private/model"]) + .contains("mesh-llm runtime load name [REDACTED]") + ); + assert!( + parsed_summary(&["mesh-llm", "benchmark", "tune", "--model", "private/model"]) + .contains("mesh-llm benchmark tune --model [REDACTED]") + ); +} + +#[test] +fn command_summary_retains_explicit_runtime_install_without_runtime_value() { + let summary = parsed_summary(&["mesh-llm", "runtime", "install", "private-runtime-value"]); + + assert_eq!(summary, "mesh-llm runtime install runtime_ref [REDACTED]"); + assert!(!summary.contains("private-runtime-value")); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); +} + +#[test] +fn command_summary_omits_default_ports_and_retains_explicit_safe_port() { + assert_eq!( + parsed_summary(&["mesh-llm", "load", "private/model"]), + "mesh-llm load name [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "load", "private/model", "--port", "41731"]), + "mesh-llm load --port 41731 name [REDACTED]" + ); +} + +#[test] +fn command_summary_uses_guardrail_mode_contract_tokens() { + for mode in ["disabled", "metrics", "enforce"] { + assert_eq!( + parsed_summary(&["mesh-llm", "runtime", "guardrails", "--mode", mode]), + format!("mesh-llm runtime guardrails --mode {mode}") + ); + } +} + +#[test] +fn command_summary_deduplicates_parent_and_child_json_flags() { + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "doctor", + "--json", + "split", + "--model-ref", + "private/model", + "--json", + ]), + "mesh-llm doctor split --json --model-ref [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "gpus", "--json", "detect", "--json"]), + "mesh-llm gpus detect --json" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "gpus", + "--json", + "run-benchmark", + "--backend", + "cuda", + ]), + "mesh-llm gpus run-benchmark --backend cuda --json" + ); +} + +#[test] +fn summary_assembly_deduplicates_repeated_flags() { + let mut assembly = SummaryAssembly::new("mesh-llm models list"); + assembly.flag("json", true); + assembly.flag("json", true); + + assert_eq!( + assembly.finish().map(|summary| summary.as_str().to_owned()), + Some("mesh-llm models list --json".to_owned()) + ); +} + +#[test] +fn command_summary_redacts_non_default_model_search_sort() { + let summary = parsed_summary(&[ + "mesh-llm", + "models", + "search", + "private-query", + "--sort", + "downloads", + ]); + assert_eq!( + summary, + "mesh-llm models search query [REDACTED] --sort [REDACTED]" + ); + let sanitized = CliCommandSummary::sanitize(&summary) + .unwrap_or_else(|| panic!("producer omitted valid summary: {summary}")); + assert_eq!(sanitized.as_str(), summary); +} + +#[test] +fn command_summary_redacts_spec_draft_tune_values() { + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "benchmark", + "tune", + "--spec-draft-acceptance-threshold", + "0.8125,0.9375", + ]), + "mesh-llm benchmark tune --spec-draft-acceptance-threshold [REDACTED]" + ); + assert_eq!( + parsed_summary(&[ + "mesh-llm", + "benchmark", + "tune", + "--spec-draft-split-probability", + "0.125,0.375", + ]), + "mesh-llm benchmark tune --spec-draft-split-probability [REDACTED]" + ); + assert_eq!( + parsed_summary(&["mesh-llm", "benchmark", "tune"]), + "mesh-llm benchmark tune" + ); +} + +#[test] +fn command_summary_redacts_external_argv_without_global_cli_context() { + assert_eq!( + parsed_summary(&["mesh-llm", "private-plugin", "--api-key", "private-key"]), + "mesh-llm external-plugin argv [REDACTED]" + ); + let cli = mesh_llm_cli::Cli::parse_from([ + "mesh-llm", + "--join", + "private-invite-token", + "--relay-auth", + "https://relay.example/?token=private-token=padding", + "load", + "private/model", + ]); + let Some(command) = cli.command.as_ref() else { + return; + }; + let summary = command_summary(command) + .map(|value| value.as_str().to_owned()) + .unwrap_or_default(); + assert_eq!(summary, "mesh-llm load name [REDACTED]"); +} diff --git a/crates/mesh-llm-events/src/audit.rs b/crates/mesh-llm-events/src/audit.rs index f59f83dc83..95fd76948b 100644 --- a/crates/mesh-llm-events/src/audit.rs +++ b/crates/mesh-llm-events/src/audit.rs @@ -16,6 +16,7 @@ use uuid::Uuid; mod sanitization; use sanitization::redact_secrets; +pub use sanitization::{SanitizedAuditDetailJson, SanitizedAuditScalar}; /// Audit log format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, clap::ValueEnum)] diff --git a/crates/mesh-llm-events/src/audit/sanitization.rs b/crates/mesh-llm-events/src/audit/sanitization.rs index 06193cafb3..51625be869 100644 --- a/crates/mesh-llm-events/src/audit/sanitization.rs +++ b/crates/mesh-llm-events/src/audit/sanitization.rs @@ -1,7 +1,27 @@ //! Audit-event privacy and size-boundary enforcement. use super::AuditEvent; -use serde_json::Value; + +mod detail; + +pub use detail::SanitizedAuditDetailJson; + +/// Free-form audit scalar that has passed the shared privacy and size policy. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SanitizedAuditScalar(String); + +impl SanitizedAuditScalar { + pub fn sanitize(value: &str) -> Self { + Self(sanitize_audit_text(value, true)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[cfg(test)] +mod tests; const SECRET_PATTERNS: &[&str] = &[ "token", @@ -19,91 +39,34 @@ const SECRET_PATTERNS: &[&str] = &[ "certificate", ]; -const MAX_AUDIT_TEXT_LEN: usize = 1024; -const MAX_AUDIT_METADATA_NODES: usize = 256; -const MAX_AUDIT_METADATA_DEPTH: usize = 8; +pub(super) const MAX_AUDIT_TEXT_LEN: usize = 1024; +pub(super) const MAX_AUDIT_DETAIL_RAW_BYTES: usize = 256 * 1024; +pub(super) const MAX_AUDIT_METADATA_NODES: usize = 256; +pub(super) const MAX_AUDIT_METADATA_DEPTH: usize = 8; +const AUDIT_TRUNCATION_SUFFIX: &str = "... [TRUNCATED]"; /// Redact sensitive fields and bound untrusted audit metadata before emission. pub(super) fn redact_secrets(event: AuditEvent) -> AuditEvent { let mut redacted = event; - - // Bound the complete metadata tree as well as each individual string. This - // keeps one hostile event from bypassing rotation with a wide/deep value. - let metadata = std::mem::take(&mut redacted.metadata); - let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; - for (key, mut value) in metadata { - if remaining_nodes == 0 { - break; - } - remaining_nodes -= 1; - redact_json_value(Some(&key), &mut value, 0, &mut remaining_nodes); - redacted.metadata.insert(key, value); - } - + redacted.metadata = detail::sanitize_metadata(std::mem::take(&mut redacted.metadata)); redacted.action = redact_audit_text(&redacted.action); redacted.resource = redacted.resource.as_deref().map(redact_audit_text); redacted.actor = redacted.actor.as_deref().map(redact_audit_text); redacted.error = redacted.error.as_deref().map(redact_audit_text); - redacted } -fn redact_json_value( - key: Option<&str>, - value: &mut Value, - depth: usize, - remaining_nodes: &mut usize, -) { - if let Value::String(token) = value - && key.is_some_and(is_invite_token_key) - { - *token = sanitize_audit_text(token, false); - return; +pub(super) fn sanitize_audit_key(key: &str) -> String { + if is_invite_token_key(key) { + return redact_audit_text(key); } - if key.is_some_and(is_sensitive_key) { - *value = Value::String("[REDACTED]".to_string()); - return; - } - if depth >= MAX_AUDIT_METADATA_DEPTH && matches!(value, Value::Object(_) | Value::Array(_)) { - *value = Value::String("[TRUNCATED]".to_string()); - return; - } - match value { - Value::Object(object) => { - let original = std::mem::take(object); - for (nested_key, mut nested_value) in original { - if *remaining_nodes == 0 { - break; - } - *remaining_nodes -= 1; - redact_json_value( - Some(&nested_key), - &mut nested_value, - depth + 1, - remaining_nodes, - ); - object.insert(nested_key, nested_value); - } - } - Value::Array(values) => { - let original = std::mem::take(values); - for mut nested_value in original { - if *remaining_nodes == 0 { - break; - } - *remaining_nodes -= 1; - redact_json_value(None, &mut nested_value, depth + 1, remaining_nodes); - values.push(nested_value); - } - } - Value::String(string) => { - *string = sanitize_audit_text(string, true); - } - _ => {} + if is_sensitive_key(key) { + return "[REDACTED]".to_owned(); } + redact_audit_text(key) } -fn is_invite_token_key(key: &str) -> bool { +pub(super) fn is_invite_token_key(key: &str) -> bool { let compact = key .chars() .filter(|character| character.is_ascii_alphanumeric()) @@ -112,7 +75,7 @@ fn is_invite_token_key(key: &str) -> bool { compact.ends_with("invitetoken") } -fn is_sensitive_key(key: &str) -> bool { +pub(super) fn is_sensitive_key(key: &str) -> bool { let key = key.to_lowercase(); SECRET_PATTERNS.iter().any(|pattern| key.contains(pattern)) } @@ -121,7 +84,7 @@ fn redact_audit_text(value: &str) -> String { sanitize_audit_text(value, true) } -fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { +pub(super) fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { let mut sanitized = value.to_string(); for variable in ["HOME", "USERPROFILE"] { if let Ok(home) = std::env::var(variable) @@ -130,14 +93,15 @@ fn sanitize_audit_text(value: &str, redact_secret_values: bool) -> String { sanitized = sanitized.replace(&home, "~"); } } + sanitized.retain(|character| !character.is_control()); if redact_secret_values && contains_secret_value(&sanitized) { - return "[REDACTED]".to_string(); + return "[REDACTED]".to_owned(); } - if sanitized.chars().count() > MAX_AUDIT_TEXT_LEN { - let prefix: String = sanitized.chars().take(MAX_AUDIT_TEXT_LEN).collect(); - return format!("{prefix}... [TRUNCATED]"); + let prefix_len = MAX_AUDIT_TEXT_LEN - AUDIT_TRUNCATION_SUFFIX.chars().count(); + let prefix = sanitized.chars().take(prefix_len).collect::(); + return format!("{prefix}{AUDIT_TRUNCATION_SUFFIX}"); } sanitized } @@ -179,146 +143,3 @@ fn contains_marker_at_token_boundary(value: &str, marker: &str) -> bool { .is_none_or(|character| !character.is_ascii_alphanumeric() && character != '_') }) } - -#[cfg(test)] -mod tests { - use super::*; - use crate::audit::{AuditCategory, AuditOutcome}; - - fn event() -> AuditEvent { - AuditEvent::new(AuditCategory::System, "event", AuditOutcome::Success) - } - - #[test] - fn nested_metadata_is_recursively_redacted() { - let event = event().with_metadata( - "changes", - serde_json::json!({ - "old_value": {"api_token": "secret-one"}, - "new_value": [{"PASSWORD": "secret-two"}, "Bearer secret-three"] - }), - ); - let serialized = redact_secrets(event).to_json_line().unwrap(); - assert!(!serialized.contains("secret-one")); - assert!(!serialized.contains("secret-two")); - assert!(!serialized.contains("secret-three")); - } - - #[test] - fn invite_token_metadata_is_preserved() { - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!("mesh-llm-invite-token-abc123"), - )); - - assert_eq!( - redacted.metadata["invite_token"], - "mesh-llm-invite-token-abc123" - ); - } - - #[test] - fn invite_token_metadata_is_path_sanitized_and_bounded() { - let home = ["HOME", "USERPROFILE"] - .into_iter() - .find_map(|variable| std::env::var(variable).ok()) - .filter(|home| !home.is_empty()) - .expect("HOME or USERPROFILE should be available to the test"); - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!(format!("{home}/{}", "x".repeat(MAX_AUDIT_TEXT_LEN + 1))), - )); - let token = redacted.metadata["invite_token"] - .as_str() - .expect("invite token should remain a string"); - - assert!(!token.contains(&home)); - assert!(token.starts_with("~/")); - assert!(token.ends_with("... [TRUNCATED]")); - assert!(token.chars().count() <= MAX_AUDIT_TEXT_LEN + "... [TRUNCATED]".len()); - } - - #[test] - fn metadata_strings_are_recursively_secret_path_and_length_safe() { - let home = ["HOME", "USERPROFILE"] - .into_iter() - .find_map(|variable| std::env::var(variable).ok()) - .filter(|home| !home.is_empty()) - .expect("HOME or USERPROFILE should be available to the test"); - let redacted = redact_secrets( - event() - .with_metadata("refresh", serde_json::json!("refresh_token=secret-value")) - .with_metadata( - "nested", - serde_json::json!({ - "note": format!("{} {}", home, "x".repeat(MAX_AUDIT_TEXT_LEN + 1)), - "credential": "private_key=secret-key" - }), - ), - ); - - assert_eq!(redacted.metadata["refresh"], "[REDACTED]"); - assert_eq!(redacted.metadata["nested"]["credential"], "[REDACTED]"); - let note = redacted.metadata["nested"]["note"] - .as_str() - .expect("note should remain a string"); - assert!(!note.contains(&home)); - assert!(note.ends_with("... [TRUNCATED]")); - assert!(note.chars().count() <= MAX_AUDIT_TEXT_LEN + "... [TRUNCATED]".len()); - } - - #[test] - fn metadata_collections_are_bounded_before_serialization() { - let values = (0..1_000) - .map(|index| serde_json::json!({"index": index, "note": "x".repeat(2_048)})) - .collect::>(); - let redacted = redact_secrets(event().with_metadata("items", Value::Array(values))); - let items = redacted.metadata["items"] - .as_array() - .expect("items should remain an array"); - - assert!(items.len() <= MAX_AUDIT_METADATA_NODES); - assert!(redacted.to_json_line().unwrap().len() < 300_000); - } - - #[test] - fn invite_token_exception_does_not_bypass_nested_sanitization() { - let redacted = redact_secrets(event().with_metadata( - "invite_token", - serde_json::json!({"password": "must-not-survive"}), - )); - - assert_eq!(redacted.metadata["invite_token"], "[REDACTED]"); - } - - #[test] - fn free_form_audit_fields_are_redacted_and_bounded() { - let event = AuditEvent::new( - AuditCategory::System, - "restart failed: Bearer audit-secret".to_string(), - AuditOutcome::Failure, - ) - .with_resource("model sk-live-example") - .with_actor("ghp_exampletoken") - .with_error("password=example-password") - .with_metadata("note", serde_json::json!("Basic YWxpY2U6cGFzcw==")); - let redacted = redact_secrets(event); - - assert_eq!(redacted.action, "[REDACTED]"); - assert_eq!(redacted.resource.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.actor.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.error.as_deref(), Some("[REDACTED]")); - assert_eq!(redacted.metadata["note"], "[REDACTED]"); - - let long_action = AuditEvent::new( - AuditCategory::System, - "x".repeat(MAX_AUDIT_TEXT_LEN + 1), - AuditOutcome::Success, - ); - assert!( - redact_secrets(long_action) - .action - .ends_with("... [TRUNCATED]") - ); - } -} diff --git a/crates/mesh-llm-events/src/audit/sanitization/detail.rs b/crates/mesh-llm-events/src/audit/sanitization/detail.rs new file mode 100644 index 0000000000..e08a2b49ce --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/detail.rs @@ -0,0 +1,152 @@ +use super::{ + MAX_AUDIT_DETAIL_RAW_BYTES, MAX_AUDIT_METADATA_DEPTH, MAX_AUDIT_METADATA_NODES, + MAX_AUDIT_TEXT_LEN, is_invite_token_key, is_sensitive_key, sanitize_audit_key, + sanitize_audit_text, +}; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, BTreeSet}; + +/// JSON audit detail that has passed the shared privacy and size policy. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SanitizedAuditDetailJson(String); + +impl SanitizedAuditDetailJson { + /// Bound raw input, sanitize valid JSON recursively, and replace malformed input completely. + pub fn sanitize(raw: &str) -> Self { + if raw.len() > MAX_AUDIT_DETAIL_RAW_BYTES { + return Self(Value::String("[TRUNCATED]".to_owned()).to_string()); + } + let Ok(mut value) = serde_json::from_str(raw) else { + return Self(Value::String("[REDACTED]".to_owned()).to_string()); + }; + sanitize_root(&mut value); + Self(value.to_string()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +pub(super) fn sanitize_metadata(metadata: BTreeMap) -> BTreeMap { + let mut object = metadata.into_iter().collect::>(); + let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; + sanitize_object(&mut object, 0, &mut remaining_nodes); + object.into_iter().collect() +} + +fn sanitize_root(value: &mut Value) { + let mut remaining_nodes = MAX_AUDIT_METADATA_NODES; + sanitize_json_value(None, value, 0, &mut remaining_nodes); +} + +fn sanitize_json_value( + key: Option<&str>, + value: &mut Value, + depth: usize, + remaining_nodes: &mut usize, +) { + if let Value::String(token) = value + && key.is_some_and(is_invite_token_key) + { + *token = sanitize_audit_text(token, false); + return; + } + if key.is_some_and(is_sensitive_key) { + *value = Value::String("[REDACTED]".to_owned()); + return; + } + if depth >= MAX_AUDIT_METADATA_DEPTH && matches!(value, Value::Object(_) | Value::Array(_)) { + *value = Value::String("[TRUNCATED]".to_owned()); + return; + } + match value { + Value::Object(object) => sanitize_object(object, depth, remaining_nodes), + Value::Array(values) => sanitize_array(values, depth, remaining_nodes), + Value::String(string) => *string = sanitize_audit_text(string, true), + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + +fn sanitize_object(object: &mut Map, depth: usize, remaining_nodes: &mut usize) { + let mut original = std::mem::take(object).into_iter().collect::>(); + original.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + let mut entries = Vec::new(); + for (original_key, mut nested_value) in original { + if *remaining_nodes == 0 { + break; + } + *remaining_nodes -= 1; + sanitize_json_value( + Some(&original_key), + &mut nested_value, + depth + 1, + remaining_nodes, + ); + let canonical_key = sanitize_audit_key(&original_key); + entries.push((original_key, canonical_key, nested_value)); + } + insert_collision_safe(object, entries); +} + +fn sanitize_array(values: &mut Vec, depth: usize, remaining_nodes: &mut usize) { + let original = std::mem::take(values); + for mut nested_value in original { + if *remaining_nodes == 0 { + break; + } + *remaining_nodes -= 1; + sanitize_json_value(None, &mut nested_value, depth + 1, remaining_nodes); + values.push(nested_value); + } +} + +fn insert_collision_safe(object: &mut Map, entries: Vec<(String, String, Value)>) { + let reserved_keys = entries + .iter() + .filter(|(original, canonical, _)| original == canonical) + .map(|(_, canonical, _)| canonical.clone()) + .collect::>(); + let mut used_keys = BTreeSet::new(); + let mut collision_indices = BTreeMap::new(); + for (original_key, canonical_key, value) in entries { + let owns_canonical = original_key == canonical_key; + let final_key = if (owns_canonical || !reserved_keys.contains(&canonical_key)) + && used_keys.insert(canonical_key.clone()) + { + canonical_key + } else { + next_collision_key( + &canonical_key, + &reserved_keys, + &mut used_keys, + &mut collision_indices, + ) + }; + object.insert(final_key, value); + } +} + +fn next_collision_key( + canonical_key: &str, + reserved_keys: &BTreeSet, + used_keys: &mut BTreeSet, + collision_indices: &mut BTreeMap, +) -> String { + let next_index = collision_indices + .entry(canonical_key.to_owned()) + .or_insert(2); + loop { + let suffix = format!("#{next_index}"); + let prefix_len = MAX_AUDIT_TEXT_LEN.saturating_sub(suffix.chars().count()); + let prefix = canonical_key.chars().take(prefix_len).collect::(); + let candidate = format!("{prefix}{suffix}"); + *next_index += 1; + if !reserved_keys.contains(&candidate) && used_keys.insert(candidate.clone()) { + return candidate; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs b/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs new file mode 100644 index 0000000000..1b9a036e09 --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/detail/tests.rs @@ -0,0 +1,171 @@ +use super::*; + +fn parse_detail(raw: &str) -> Value { + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON") +} + +fn home_directory() -> String { + ["HOME", "USERPROFILE"] + .into_iter() + .find_map(|variable| std::env::var(variable).ok()) + .filter(|home| !home.is_empty()) + .expect("HOME or USERPROFILE should be available to the test") +} + +#[test] +fn audit_detail_sanitization_preserves_safe_json_structure() { + let raw = r#"{"safe":{"outcome":"completed","count":3},"nested":{"password":"detail-secret"}}"#; + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + let value: Value = + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON"); + assert_eq!(value["safe"]["outcome"], "completed"); + assert_eq!(value["safe"]["count"], 3); + assert!(!sanitized.as_str().contains("detail-secret")); +} + +#[test] +fn audit_detail_sanitization_replaces_malformed_sensitive_input() { + let raw = r#"{"note":"Bearer malformed-credential""#; + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + assert_eq!(sanitized.as_str(), r#""[REDACTED]""#); + assert!(!sanitized.as_str().contains("malformed-credential")); +} + +#[test] +fn audit_detail_sanitization_truncates_oversized_raw_input_before_parsing() { + let raw = serde_json::json!({ + "sentinel": "OVERSIZED-RAW-SENTINEL", + "padding": "x".repeat(256 * 1024), + }) + .to_string(); + let sanitized = SanitizedAuditDetailJson::sanitize(&raw); + assert_eq!(sanitized.as_str(), r#""[TRUNCATED]""#); + assert!(!sanitized.as_str().contains("OVERSIZED-RAW-SENTINEL")); +} + +#[test] +fn audit_detail_sanitization_bounds_object_keys() { + let oversized_key = "k".repeat(MAX_AUDIT_TEXT_LEN + 1); + let raw = serde_json::json!({ oversized_key: "safe-value" }).to_string(); + let value = parse_detail(&raw); + let object = value + .as_object() + .expect("sanitized detail remains an object"); + let (key, stored_value) = object.iter().next().expect("bounded entry remains present"); + assert!(key.chars().count() <= MAX_AUDIT_TEXT_LEN); + assert_eq!(stored_value, "safe-value"); +} + +#[test] +fn audit_detail_sanitization_removes_private_and_credential_key_text() { + let home = home_directory(); + let raw = serde_json::json!({ + format!("{home}/private/audit\npath"): "safe-path-value", + "Bearer object-key-credential": "sensitive-field-value", + }) + .to_string(); + let value = parse_detail(&raw); + let keys = value + .as_object() + .expect("sanitized detail remains an object") + .keys() + .cloned() + .collect::>(); + assert!(keys.iter().all(|key| !key.contains(&home))); + assert!(keys.iter().all(|key| !key.contains("Bearer"))); + assert!( + keys.iter() + .all(|key| !key.contains("object-key-credential")) + ); + assert!(keys.iter().all(|key| !key.contains('\n'))); +} + +#[test] +fn audit_detail_sanitization_preserves_values_at_deterministic_collision_keys() { + let home = home_directory(); + let private_key = format!("{home}/collision"); + let canonical_key = "~/collision".to_string(); + let raw = serde_json::json!({ + private_key: "private-path-value", + canonical_key: "canonical-value", + }) + .to_string(); + let value = parse_detail(&raw); + assert_eq!(value["~/collision"], "canonical-value"); + assert_eq!(value["~/collision#2"], "private-path-value"); +} + +#[test] +fn sensitive_key_contract_redacts_nested_bare_key_names() { + // Given: nested detail containing every bare sensitive-key spelling. + let raw = serde_json::json!({ + "nested": { + "password": "password-value", + "credential": "credential-value", + "api_key": "api-key-value", + "private_key": "private-key-value", + "authorization": "authorization-value", + } + }) + .to_string(); + + // When: the durable detail sanitizer processes the document. + let sanitized = SanitizedAuditDetailJson::sanitize(&raw); + + // Then: neither sensitive names nor their values persist. + for sensitive in [ + "password", + "credential", + "api_key", + "private_key", + "authorization", + ] { + assert!(!sanitized.as_str().contains(sensitive)); + } + let value: Value = + serde_json::from_str(sanitized.as_str()).expect("sanitized detail is valid JSON"); + assert_eq!(value["nested"].as_object().map(Map::len), Some(5)); +} + +#[test] +fn sensitive_key_contract_preserves_invite_token_exception() { + // Given: a string value under the invite-token exception key. + let raw = r#"{"invite_token":"mesh-llm-invite-token-abc123"}"#; + + // When: the durable detail sanitizer processes the document. + let sanitized = SanitizedAuditDetailJson::sanitize(raw); + + // Then: the key and bounded string value remain available. + assert_eq!(sanitized.as_str(), raw); +} + +#[test] +fn sensitive_key_contract_reserves_canonical_collision_chain() { + // Given: canonical keys that reserve the first two collision positions. + let raw = r#"{"[REDACTED]":"canonical","[REDACTED]#2":"reserved","authorization":"auth-value","password":"password-value"}"#; + + // When: sensitive key names normalize in sorted original-key order. + let value = parse_detail(raw); + + // Then: every entry remains and later collisions use unreserved suffixes. + assert_eq!(value["[REDACTED]"], "canonical"); + assert_eq!(value["[REDACTED]#2"], "reserved"); + assert_eq!(value["[REDACTED]#3"], "[REDACTED]"); + assert_eq!(value["[REDACTED]#4"], "[REDACTED]"); + assert_eq!(value.as_object().map(Map::len), Some(4)); +} + +#[test] +fn sensitive_key_contract_is_byte_idempotent() { + // Given: detail with sensitive-key and canonical-key collisions. + let raw = + r#"{"[REDACTED]":"canonical","credential":"credential-value","password":"password-value"}"#; + + // When: the sanitizer processes its own output. + let first = SanitizedAuditDetailJson::sanitize(raw); + let second = SanitizedAuditDetailJson::sanitize(first.as_str()); + + // Then: the serialized bytes do not change. + assert_eq!(second.as_str(), first.as_str()); +} diff --git a/crates/mesh-llm-events/src/audit/sanitization/tests.rs b/crates/mesh-llm-events/src/audit/sanitization/tests.rs new file mode 100644 index 0000000000..29a01b7d90 --- /dev/null +++ b/crates/mesh-llm-events/src/audit/sanitization/tests.rs @@ -0,0 +1,178 @@ +use super::*; +use crate::audit::{AuditCategory, AuditOutcome}; +use serde_json::Value; +use std::collections::BTreeMap; + +fn event() -> AuditEvent { + AuditEvent::new(AuditCategory::System, "event", AuditOutcome::Success) +} + +fn home_directory() -> String { + ["HOME", "USERPROFILE"] + .into_iter() + .find_map(|variable| std::env::var(variable).ok()) + .filter(|home| !home.is_empty()) + .expect("HOME or USERPROFILE should be available to the test") +} + +#[test] +fn audit_scalar_sanitizes_private_paths_controls_credentials_and_bounds() { + let home = home_directory(); + let private_path = SanitizedAuditScalar::sanitize(&format!("{home}/private\naudit")); + let credential = SanitizedAuditScalar::sanitize("Bearer audit-secret"); + let oversized = SanitizedAuditScalar::sanitize(&"x".repeat(MAX_AUDIT_TEXT_LEN + 1)); + + assert_eq!(private_path.as_str(), "~/privateaudit"); + assert_eq!(credential.as_str(), "[REDACTED]"); + assert!(oversized.as_str().ends_with("... [TRUNCATED]")); + assert!(oversized.as_str().chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn audit_scalar_preserves_safe_and_fixed_values() { + for value in ["runtime", "runtime_ready", "[REDACTED]", "[TRUNCATED]"] { + assert_eq!(SanitizedAuditScalar::sanitize(value).as_str(), value); + } +} + +#[test] +fn nested_metadata_is_recursively_redacted() { + let event = event().with_metadata( + "changes", + serde_json::json!({ + "old_value": {"api_token": "secret-one"}, + "new_value": [{"PASSWORD": "secret-two"}, "Bearer secret-three"] + }), + ); + let serialized = redact_secrets(event).to_json_line().unwrap(); + assert!(!serialized.contains("secret-one")); + assert!(!serialized.contains("secret-two")); + assert!(!serialized.contains("secret-three")); +} + +#[test] +fn invite_token_metadata_is_preserved() { + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!("mesh-llm-invite-token-abc123"), + )); + assert_eq!( + redacted.metadata["invite_token"], + "mesh-llm-invite-token-abc123" + ); +} + +#[test] +fn invite_token_metadata_is_path_sanitized_and_bounded() { + let home = home_directory(); + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!(format!("{home}/{}", "x".repeat(MAX_AUDIT_TEXT_LEN + 1))), + )); + let token = redacted.metadata["invite_token"] + .as_str() + .expect("invite token should remain a string"); + assert!(!token.contains(&home)); + assert!(token.starts_with("~/")); + assert!(token.ends_with("... [TRUNCATED]")); + assert!(token.chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn metadata_strings_are_recursively_secret_path_and_length_safe() { + let home = home_directory(); + let redacted = redact_secrets( + event() + .with_metadata("refresh", serde_json::json!("refresh_token=secret-value")) + .with_metadata( + "nested", + serde_json::json!({ + "note": format!("{} {}", home, "x".repeat(MAX_AUDIT_TEXT_LEN + 1)), + "credential": "private_key=secret-key" + }), + ), + ); + assert_eq!(redacted.metadata["refresh"], "[REDACTED]"); + assert_eq!(redacted.metadata["nested"]["[REDACTED]"], "[REDACTED]"); + let note = redacted.metadata["nested"]["note"] + .as_str() + .expect("note should remain a string"); + assert!(!note.contains(&home)); + assert!(note.ends_with("... [TRUNCATED]")); + assert!(note.chars().count() <= MAX_AUDIT_TEXT_LEN); +} + +#[test] +fn metadata_collections_are_bounded_before_serialization() { + let values = (0..1_000) + .map(|index| serde_json::json!({"index": index, "note": "x".repeat(2_048)})) + .collect::>(); + let redacted = redact_secrets(event().with_metadata("items", Value::Array(values))); + let items = redacted.metadata["items"] + .as_array() + .expect("items should remain an array"); + assert!(items.len() <= MAX_AUDIT_METADATA_NODES); + assert!(redacted.to_json_line().unwrap().len() < 300_000); +} + +#[test] +fn invite_token_exception_does_not_bypass_nested_sanitization() { + let redacted = redact_secrets(event().with_metadata( + "invite_token", + serde_json::json!({"password": "must-not-survive"}), + )); + assert_eq!(redacted.metadata["invite_token"], "[REDACTED]"); +} + +#[test] +fn free_form_audit_fields_are_redacted_and_bounded() { + let event = AuditEvent::new( + AuditCategory::System, + "restart failed: Bearer audit-secret".to_string(), + AuditOutcome::Failure, + ) + .with_resource("model sk-live-example") + .with_actor("ghp_exampletoken") + .with_error("password=example-password") + .with_metadata("note", serde_json::json!("Basic YWxpY2U6cGFzcw==")); + let redacted = redact_secrets(event); + assert_eq!(redacted.action, "[REDACTED]"); + assert_eq!(redacted.resource.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.actor.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.error.as_deref(), Some("[REDACTED]")); + assert_eq!(redacted.metadata["note"], "[REDACTED]"); + + let long_action = AuditEvent::new( + AuditCategory::System, + "x".repeat(MAX_AUDIT_TEXT_LEN + 1), + AuditOutcome::Success, + ); + assert!( + redact_secrets(long_action) + .action + .ends_with("... [TRUNCATED]") + ); +} + +#[test] +fn sensitive_key_contract_matches_top_level_audit_metadata() { + // Given: equivalent detail JSON and top-level audit metadata. + let raw = r#"{"[REDACTED]":"canonical","credential":"credential-value","invite_token":"mesh-llm-invite-token-abc123","password":"password-value"}"#; + let metadata: BTreeMap = + serde_json::from_str(raw).expect("fixture is a JSON object"); + let mut audit_event = event(); + audit_event.metadata = metadata; + + // When: both audit sanitization paths process the same keys. + let detail = SanitizedAuditDetailJson::sanitize(raw); + let redacted_event = redact_secrets(audit_event); + + // Then: their normalized key and value behavior is identical. + let detail_value: Value = + serde_json::from_str(detail.as_str()).expect("sanitized detail is valid JSON"); + let event_value = + serde_json::to_value(redacted_event.metadata).expect("sanitized metadata is serializable"); + assert_eq!(event_value, detail_value); + assert!(!event_value.to_string().contains("credential")); + assert!(!event_value.to_string().contains("password")); +} diff --git a/crates/mesh-llm-events/src/command_lifecycle.rs b/crates/mesh-llm-events/src/command_lifecycle.rs index 395a80bb00..a9d4da525d 100644 --- a/crates/mesh-llm-events/src/command_lifecycle.rs +++ b/crates/mesh-llm-events/src/command_lifecycle.rs @@ -9,6 +9,9 @@ use super::{LogFormat, OutputEvent, OutputLevel, OutputSink, output_sink}; use std::io::{self, Write}; use std::sync::atomic::{AtomicBool, Ordering}; +#[path = "command_summary_grammar.rs"] +mod command_summary_grammar; + /// Stable family for a parsed command. The enum intentionally groups commands /// by responsibility instead of carrying user-supplied subcommand names or /// arguments. @@ -30,6 +33,44 @@ pub enum CliCommandFamily { Unknown, } +/// A grammar-validated summary of a parsed CLI command. +/// +/// External callers must use [`CliCommandSummary::sanitize`] to construct a +/// summary. +/// +/// ```compile_fail +/// use mesh_llm_events::CliCommandSummary; +/// +/// let _summary = CliCommandSummary::new("mesh-llm status"); +/// ``` +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CliCommandSummary(String); + +impl CliCommandSummary { + fn new(value: &str) -> Option { + let token_count = value.split_whitespace().count(); + (!value.is_empty() + && token_count > 0 + && token_count <= 32 + && value.chars().count() <= 256 + && !value.chars().any(char::is_control)) + .then(|| Self(value.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn sanitize(value: &str) -> Option { + let summary = Self::new(value)?; + is_safe_summary(&summary.0).then_some(summary) + } +} + +fn is_safe_summary(value: &str) -> bool { + command_summary_grammar::is_safe_summary(value) +} + impl CliCommandFamily { pub const fn as_str(self) -> &'static str { match self { @@ -164,7 +205,10 @@ fn write_cli_command_event_to_stderr( stderr: &mut W, ) -> io::Result<()> { let OutputEvent::CliCommandLifecycle { family, outcome } = event else { - unreachable!("CLI command event adapter received a different event variant"); + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "CLI command event adapter received a different event variant", + )); }; writeln!( @@ -178,161 +222,5 @@ fn write_cli_command_event_to_stderr( } #[cfg(test)] -mod tests { - use super::*; - use crate::{clear_output_sink, set_output_sink}; - use std::sync::{Arc, Mutex}; - - struct RecordingSink { - mode: LogFormat, - events: Mutex>, - } - - impl RecordingSink { - fn new(mode: LogFormat) -> Self { - Self { - mode, - events: Mutex::new(Vec::new()), - } - } - } - - impl OutputSink for RecordingSink { - fn emit_event(&self, event: OutputEvent) -> io::Result<()> { - self.events.lock().expect("recording sink lock").push(event); - Ok(()) - } - - fn mode(&self) -> LogFormat { - self.mode - } - } - - struct VerboseResetGuard; - - impl Drop for VerboseResetGuard { - fn drop(&mut self) { - set_cli_command_event_verbose(false); - } - } - - struct OutputSinkResetGuard; - - impl Drop for OutputSinkResetGuard { - fn drop(&mut self) { - clear_output_sink(); - } - } - - #[test] - fn public_emit_is_silent_unless_verbose_enabled() { - let sink = Arc::new(RecordingSink::new(LogFormat::Pretty)); - let _sink_guard = OutputSinkResetGuard; - let _verbose_guard = VerboseResetGuard; - set_output_sink(sink.clone()); - - set_cli_command_event_verbose(false); - emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Started) - .expect("silent emission must succeed"); - assert!( - sink.events.lock().expect("recording sink lock").is_empty(), - "without --debug the command event must not be presented" - ); - - set_cli_command_event_verbose(true); - emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Completed) - .expect("verbose emission must succeed"); - assert_eq!( - *sink.events.lock().expect("recording sink lock"), - vec![OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Completed, - }], - "with --debug the command event must reach the pretty sink" - ); - } - - #[test] - fn pretty_sink_receives_typed_command_event() { - let sink = RecordingSink::new(LogFormat::Pretty); - let mut stderr = Vec::new(); - let event = OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Runtime, - outcome: CliCommandOutcome::Started, - }; - - emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) - .expect("pretty sink should receive command event"); - - assert_eq!( - *sink.events.lock().expect("recording sink lock"), - vec![event] - ); - assert!(stderr.is_empty()); - } - - #[test] - fn json_sink_keeps_command_event_off_stdout() { - let sink = RecordingSink::new(LogFormat::Json); - let mut stderr = Vec::new(); - let event = OutputEvent::CliCommandLifecycle { - family: CliCommandFamily::Models, - outcome: CliCommandOutcome::Completed, - }; - - emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) - .expect("stderr fallback should write command event"); - - assert!(sink.events.lock().expect("recording sink lock").is_empty()); - assert_eq!( - String::from_utf8(stderr).expect("stderr must be utf-8"), - "mesh-llm command event: family=models code=cli_command_completed outcome=completed\n" - ); - } - - #[test] - fn command_event_vocabulary_is_bounded_and_static() { - let families = [ - CliCommandFamily::Agent, - CliCommandFamily::Benchmark, - CliCommandFamily::Configuration, - CliCommandFamily::Diagnostics, - CliCommandFamily::Discovery, - CliCommandFamily::Hardware, - CliCommandFamily::Identity, - CliCommandFamily::Installation, - CliCommandFamily::Models, - CliCommandFamily::Plugin, - CliCommandFamily::Process, - CliCommandFamily::Runtime, - CliCommandFamily::Skills, - CliCommandFamily::Unknown, - ]; - let outcomes = [ - CliCommandOutcome::Started, - CliCommandOutcome::Completed, - CliCommandOutcome::Failed, - CliCommandOutcome::Rejected, - CliCommandOutcome::ParseFailed, - ]; - - for family in families { - assert!(family.as_str().len() <= 24); - assert!( - family - .as_str() - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte == b'_') - ); - } - for outcome in outcomes { - assert!(outcome.code().len() <= 48); - assert!( - outcome - .code() - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte == b'_') - ); - } - } -} +#[path = "command_lifecycle/tests.rs"] +mod tests; diff --git a/crates/mesh-llm-events/src/command_lifecycle/tests.rs b/crates/mesh-llm-events/src/command_lifecycle/tests.rs new file mode 100644 index 0000000000..bb7562b763 --- /dev/null +++ b/crates/mesh-llm-events/src/command_lifecycle/tests.rs @@ -0,0 +1,359 @@ +use super::*; +use crate::{clear_output_sink, set_output_sink}; +use std::sync::{Arc, Mutex}; + +struct RecordingSink { + mode: LogFormat, + events: Mutex>, +} + +impl RecordingSink { + fn new(mode: LogFormat) -> Self { + Self { + mode, + events: Mutex::new(Vec::new()), + } + } +} + +impl OutputSink for RecordingSink { + fn emit_event(&self, event: OutputEvent) -> io::Result<()> { + self.events.lock().expect("recording sink lock").push(event); + Ok(()) + } + + fn mode(&self) -> LogFormat { + self.mode + } +} + +struct VerboseResetGuard; + +impl Drop for VerboseResetGuard { + fn drop(&mut self) { + set_cli_command_event_verbose(false); + } +} + +struct OutputSinkResetGuard; + +impl Drop for OutputSinkResetGuard { + fn drop(&mut self) { + clear_output_sink(); + } +} + +#[test] +fn public_emit_is_silent_unless_verbose_enabled() { + let sink = Arc::new(RecordingSink::new(LogFormat::Pretty)); + let _sink_guard = OutputSinkResetGuard; + let _verbose_guard = VerboseResetGuard; + set_output_sink(sink.clone()); + + set_cli_command_event_verbose(false); + emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Started) + .expect("silent emission must succeed"); + assert!( + sink.events.lock().expect("recording sink lock").is_empty(), + "without --debug the command event must not be presented" + ); + + set_cli_command_event_verbose(true); + emit_cli_command_event(CliCommandFamily::Runtime, CliCommandOutcome::Completed) + .expect("verbose emission must succeed"); + assert_eq!( + *sink.events.lock().expect("recording sink lock"), + vec![OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Completed, + }], + "with --debug the command event must reach the pretty sink" + ); +} + +#[test] +fn pretty_sink_receives_typed_command_event() { + let sink = RecordingSink::new(LogFormat::Pretty); + let mut stderr = Vec::new(); + let event = OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Runtime, + outcome: CliCommandOutcome::Started, + }; + + emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) + .expect("pretty sink should receive command event"); + + assert_eq!( + *sink.events.lock().expect("recording sink lock"), + vec![event] + ); + assert!(stderr.is_empty()); +} + +#[test] +fn json_sink_keeps_command_event_off_stdout() { + let sink = RecordingSink::new(LogFormat::Json); + let mut stderr = Vec::new(); + let event = OutputEvent::CliCommandLifecycle { + family: CliCommandFamily::Models, + outcome: CliCommandOutcome::Completed, + }; + + emit_cli_command_event_with_sink(&event, Some(&sink), &mut stderr) + .expect("stderr fallback should write command event"); + + assert!(sink.events.lock().expect("recording sink lock").is_empty()); + assert_eq!( + String::from_utf8(stderr).expect("stderr must be utf-8"), + "mesh-llm command event: family=models code=cli_command_completed outcome=completed\n" + ); +} + +#[test] +fn command_event_vocabulary_is_bounded_and_static() { + let families = [ + CliCommandFamily::Agent, + CliCommandFamily::Benchmark, + CliCommandFamily::Configuration, + CliCommandFamily::Diagnostics, + CliCommandFamily::Discovery, + CliCommandFamily::Hardware, + CliCommandFamily::Identity, + CliCommandFamily::Installation, + CliCommandFamily::Models, + CliCommandFamily::Plugin, + CliCommandFamily::Process, + CliCommandFamily::Runtime, + CliCommandFamily::Skills, + CliCommandFamily::Unknown, + ]; + let outcomes = [ + CliCommandOutcome::Started, + CliCommandOutcome::Completed, + CliCommandOutcome::Failed, + CliCommandOutcome::Rejected, + CliCommandOutcome::ParseFailed, + ]; + + for family in families { + assert!(family.as_str().len() <= 24); + assert!( + family + .as_str() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } + for outcome in outcomes { + assert!(outcome.code().len() <= 48); + assert!( + outcome + .code() + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_') + ); + } +} + +#[test] +fn command_summary_rejects_control_text_and_unbounded_tokens() { + assert!(CliCommandSummary::new(" ").is_none()); + assert!(CliCommandSummary::new("runtime\nload").is_none()); + assert!(CliCommandSummary::new(&"token ".repeat(33)).is_none()); + assert!(CliCommandSummary::new(&"x".repeat(257)).is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_arbitrary_values_and_preserves_safe_shape() { + assert!(CliCommandSummary::sanitize("mesh-llm load model-private").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load --port 41731 name [REDACTED]").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm load name [REDACTED]").is_some()); + assert!(CliCommandSummary::sanitize(&format!("mesh-llm {}", "load ".repeat(32))).is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_unknown_options_and_global_numeric_values() { + assert!(CliCommandSummary::sanitize("mesh-llm --api-key 123456").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load 123456 name [REDACTED]").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm load --port 41731 name [REDACTED]").is_some()); + assert!( + CliCommandSummary::sanitize("mesh-llm runtime install runtime_ref [REDACTED]").is_some() + ); +} + +#[test] +fn command_summary_sanitizer_rejects_non_canonical_whole_command_shapes() { + let summaries = [ + " ", + "mesh-llm models list --json --json", + "mesh-llm models --json list", + "mesh-llm load name [REDACTED] name [REDACTED]", + "mesh-llm gpus run-benchmark --backend cuda --json --json", + "mesh-llm load --port 41731 --port 41732", + "mesh-llm load --port 41731 name [REDACTED] --json", + "mesh-llm load name [REDACTED] status", + "mesh-llm models nonsense", + "mesh-llm load --json name [REDACTED]", + "mesh-llm runtime status name [REDACTED]", + "mesh-llm models list name [REDACTED] --json", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "non-canonical summary was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_non_canonical_ascii_whitespace() { + for summary in [ + " mesh-llm models list", + "mesh-llm models list ", + "mesh-llm models list", + "mesh-llm\tmodels list", + "mesh-llm models\nlist", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "non-canonical whitespace was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_conflicting_boolean_pairs() { + for summary in [ + "mesh-llm setup --service --no-service", + "mesh-llm setup --no-service --service", + "mesh-llm uninstall --purge-config --keep-config", + "mesh-llm uninstall --keep-config --purge-config", + "mesh-llm auth init --no-passphrase --keychain", + "mesh-llm auth init --keychain --no-passphrase", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "conflicting flags were accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_accepts_global_relay_redaction_only() { + assert!( + CliCommandSummary::sanitize("mesh-llm load name [REDACTED] --root-relay [REDACTED]") + .is_some() + ); + for summary in [ + "mesh-llm load name [REDACTED] --relay private-relay", + "mesh-llm load name [REDACTED] --root-relay [REDACTED] value", + "mesh-llm load name [REDACTED] --relay-auth private-token", + "mesh-llm load --root-relay [REDACTED] name [REDACTED]", + "mesh-llm load name [REDACTED] --relay-auth [REDACTED] --root-relay [REDACTED]", + ] { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "malformed global relay marker was accepted: {summary:?}" + ); + } +} + +#[test] +fn command_summary_sanitizer_contextualizes_enum_values() { + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --backend cuda").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime guardrails --mode metrics").is_some()); + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --backend rocm").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime guardrails --mode strict").is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_raw_options_in_wrong_command_contexts() { + assert!(CliCommandSummary::sanitize("mesh-llm load --mode enforce").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm runtime status --backend cuda").is_none()); + assert!(CliCommandSummary::sanitize("mesh-llm gpus run-benchmark --port 1234").is_none()); +} + +#[test] +fn command_summary_sanitizer_rejects_deep_static_prefix_without_panicking() { + let result = std::panic::catch_unwind(|| { + CliCommandSummary::sanitize( + "mesh-llm load unload status discover rotate-key setup --port 1234", + ) + }); + + assert!(result.is_ok()); + assert!(result.is_ok_and(|summary| summary.is_none())); +} + +#[test] +fn command_summary_sanitizer_rejects_inserted_prefix_tokens_before_raw_options() { + let summaries = [ + "mesh-llm gpus --draft run-benchmark --backend cuda", + "mesh-llm gpus run-benchmark model [REDACTED] --backend cuda", + "mesh-llm gpus run-benchmark detect --backend cuda", + "mesh-llm runtime guardrails status --mode metrics", + "mesh-llm gpus --json run-benchmark --backend cuda", + "mesh-llm doctor --json split --port 1234", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "malformed producer prefix was accepted: {summary}" + ); + } +} + +#[test] +fn command_summary_sanitizer_rejects_impossible_boolean_and_raw_option_orders() { + let summaries = [ + "mesh-llm runtime guardrails --mode metrics --port 41731 --json", + "mesh-llm runtime bootstrap --port 41731 --json", + "mesh-llm runtime remote --port 41731 --json --endpoint [REDACTED]", + "mesh-llm runtime remote --endpoint [REDACTED] --json --port 41731", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_none(), + "impossible producer ordering was accepted: {summary}" + ); + } +} + +#[test] +fn command_summary_sanitizer_accepts_every_producer_raw_option_context() { + let summaries = [ + "mesh-llm status --port 41731", + "mesh-llm load --port 41731 name [REDACTED]", + "mesh-llm unload --port 41731 name [REDACTED]", + "mesh-llm goose --port 41731 --model [REDACTED]", + "mesh-llm claude --port 41731 --model [REDACTED]", + "mesh-llm doctor split --port 41731 --model-ref [REDACTED]", + "mesh-llm doctor split --json --port 41731 --model-ref [REDACTED]", + "mesh-llm doctor split --json --port 41731 --model-ref [REDACTED]", + "mesh-llm gpus run-benchmark --backend cuda", + "mesh-llm gpus run-benchmark --backend cuda --json", + "mesh-llm runtime status --port 41731", + "mesh-llm runtime load --port 41731 name [REDACTED]", + "mesh-llm runtime unload --port 41731 name [REDACTED]", + "mesh-llm runtime guardrails --mode metrics --port 41731", + "mesh-llm runtime guardrails --mode metrics --json --port 41731", + "mesh-llm runtime bootstrap --port 41731", + "mesh-llm runtime bootstrap --json --port 41731", + "mesh-llm runtime remote --port 41731 --endpoint [REDACTED]", + "mesh-llm runtime remote --json --port 41731 --endpoint [REDACTED]", + "mesh-llm runtime remote-model --port 41731 --endpoint [REDACTED] --model [REDACTED]", + "mesh-llm runtime remote-model --json --port 41731 --endpoint [REDACTED] --model [REDACTED]", + "mesh-llm runtime apply-config --port 41731 --endpoint [REDACTED] --expected-revision [REDACTED] --config [REDACTED]", + "mesh-llm runtime apply-config --json --port 41731 --endpoint [REDACTED] --expected-revision [REDACTED] --config [REDACTED]", + ]; + + for summary in summaries { + assert!( + CliCommandSummary::sanitize(summary).is_some(), + "producer-reachable summary was rejected: {summary}" + ); + } +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar.rs b/crates/mesh-llm-events/src/command_summary_grammar.rs new file mode 100644 index 0000000000..c45b9c0f74 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar.rs @@ -0,0 +1,28 @@ +#[path = "command_summary_grammar/descriptors.rs"] +mod descriptors; +#[path = "command_summary_grammar/raw_options.rs"] +mod raw_options; +#[path = "command_summary_grammar/validation.rs"] +mod validation; +#[path = "command_summary_grammar/vocabulary.rs"] +mod vocabulary; + +use descriptors::DESCRIPTOR_GROUPS; +use validation::validate_descriptor; + +pub(super) fn is_safe_summary(value: &str) -> bool { + let tokens = value.split(' ').collect::>(); + !tokens.is_empty() + && tokens + .iter() + .all(|token| !token.is_empty() && !token.chars().any(char::is_whitespace)) + && DESCRIPTOR_GROUPS + .iter() + .flat_map(|group| group.iter()) + .copied() + .any(|descriptor| validate_descriptor(&tokens, descriptor)) +} + +#[cfg(test)] +#[path = "command_summary_grammar/tests.rs"] +mod tests; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs new file mode 100644 index 0000000000..fbf079db91 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors.rs @@ -0,0 +1,72 @@ +#[path = "descriptors/auth.rs"] +mod auth; +#[path = "descriptors/models.rs"] +mod models; +#[path = "descriptors/plugins_benchmark.rs"] +mod plugins_benchmark; +#[path = "descriptors/runtime.rs"] +mod runtime; +#[path = "descriptors/top_level.rs"] +mod top_level; + +#[derive(Clone, Copy)] +pub(super) enum RawKind { + Backend, + Mode, + None, +} + +#[derive(Clone, Copy)] +pub(super) struct Descriptor { + pub(super) path: &'static [&'static str], + pub(super) booleans: &'static [&'static str], + pub(super) redacted: &'static [&'static str], + pub(super) conflicts: &'static [&'static [&'static str]], + pub(super) has_port: bool, + pub(super) raw: RawKind, +} + +const NONE: &[&str] = &[]; +const JSON: &[&str] = &["--json"]; +const YES_JSON: &[&str] = &["--yes", "--json"]; +const REDACTED_NAME: &[&str] = &["name"]; +const NO_CONFLICTS: &[&[&str]] = &[]; + +pub(super) const GLOBAL_REDACTED: &[&str] = &["--join", "--root-relay", "--relay-auth"]; + +const fn descriptor( + path: &'static [&'static str], + booleans: &'static [&'static str], + redacted: &'static [&'static str], + has_port: bool, + raw: RawKind, +) -> Descriptor { + descriptor_with_conflicts(path, booleans, redacted, NO_CONFLICTS, has_port, raw) +} + +const fn descriptor_with_conflicts( + path: &'static [&'static str], + booleans: &'static [&'static str], + redacted: &'static [&'static str], + conflicts: &'static [&'static [&'static str]], + has_port: bool, + raw: RawKind, +) -> Descriptor { + Descriptor { + path, + booleans, + redacted, + conflicts, + has_port, + raw, + } +} + +pub(super) const DESCRIPTOR_GROUPS: &[&[Descriptor]] = &[ + top_level::DESCRIPTORS, + plugins_benchmark::PLUGIN_DESCRIPTORS, + models::DESCRIPTORS, + plugins_benchmark::BENCHMARK_DESCRIPTORS, + runtime::DESCRIPTORS, + auth::DESCRIPTORS, +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs new file mode 100644 index 0000000000..8758c3f408 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/auth.rs @@ -0,0 +1,125 @@ +use super::{Descriptor, NONE, RawKind, descriptor, descriptor_with_conflicts}; + +const AUTH_INIT_FLAGS: &[&str] = &["--force", "--no-passphrase", "--keychain"]; +const AUTH_INIT_CONFLICTS: &[&[&str]] = &[&["--no-passphrase", "--keychain"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "auth", "init"], + AUTH_INIT_FLAGS, + &["--owner-key"], + AUTH_INIT_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "status"], + NONE, + &[ + "--owner-key", + "--node-key", + "--node-ownership", + "--trust-store", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "sign-node"], + NONE, + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "renew-node"], + NONE, + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "verify-node"], + NONE, + &[ + "--file", + "--node-id", + "--trust-store", + "--verify-trust-policy", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "rotate-node"], + &["--revoke-current"], + &[ + "--owner-key", + "--node-key", + "--out", + "--hostname-hint", + "--node-label", + "--expires-in-hours", + "--reason", + "--trust-store", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "revoke-owner"], + NONE, + &["owner_id", "--reason", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "revoke-node"], + NONE, + &["--cert-id", "--node-id", "--reason", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "rotate-owner"], + &["--no-passphrase", "--force"], + &["--owner-key"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "add"], + NONE, + &["owner_id", "--label", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "remove"], + NONE, + &["owner_id", "--trust-store"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "auth", "trust", "list"], + NONE, + &["--trust-store"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs new file mode 100644 index 0000000000..04b5e3f288 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/models.rs @@ -0,0 +1,119 @@ +use super::{Descriptor, JSON, NONE, RawKind, YES_JSON, descriptor, descriptor_with_conflicts}; + +const MODEL_PACKAGE_FLAGS: &[&str] = &[ + "--experimental", + "--dry-run", + "--confirm", + "--follow", + "--list", + "--update-script", + "--json", +]; +const SEARCH_FLAGS: &[&str] = &["--gguf", "--mlx", "--catalog", "--json"]; +const MODEL_CERTIFY_FLAGS: &[&str] = &["--json", "--package-only"]; +const MODEL_SEARCH_CONFLICTS: &[&[&str]] = &[&["--gguf", "--mlx"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor( + &["mesh-llm", "models", "package"], + MODEL_PACKAGE_FLAGS, + &[ + "source_repo", + "--quant", + "--target", + "--model-id", + "--flavor", + "--timeout", + "--mesh-llm-ref", + "--status", + "--logs", + "--cancel", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "recommended"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "installed"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "cleanup"], + YES_JSON, + &["--unused-since"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "prune"], + YES_JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "certify"], + MODEL_CERTIFY_FLAGS, + &[ + "model", + "--report-out", + "--api-base", + "--prompt", + "--max-tokens", + ], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "list"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "models", "search"], + SEARCH_FLAGS, + &["query", "--limit", "--sort"], + MODEL_SEARCH_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "show"], + JSON, + &["model"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "download"], + &["--draft", "--direct", "--json"], + &["model"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "updates"], + &["--all", "--check", "--json"], + &["repo"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "models", "delete"], + YES_JSON, + &["model"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs new file mode 100644 index 0000000000..70502d6f0b --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/plugins_benchmark.rs @@ -0,0 +1,122 @@ +use super::{Descriptor, NONE, REDACTED_NAME, RawKind, descriptor, descriptor_with_conflicts}; + +const PLUGIN_INSTALL_CONFLICTS: &[&[&str]] = &[&["reference", "--archive"]]; +const TUNE_FLAGS: &[&str] = &[ + "--json", + "--no-speculative-tune", + "--apply", + "--replace-existing", + "--launch-args", + "--debug-telemetry", +]; +const TUNE_CONFLICTS: &[&[&str]] = &[ + &["--model", "--models"], + &["--no-speculative-tune", "--speculative-types"], + &["--no-speculative-tune", "--spec-draft-models"], + &["--no-speculative-tune", "--spec-draft-max-tokens"], + &["--no-speculative-tune", "--spec-draft-min-tokens"], + &["--no-speculative-tune", "--spec-draft-acceptance-threshold"], + &["--no-speculative-tune", "--spec-draft-split-probability"], + &["--no-speculative-tune", "--spec-ngram-min"], + &["--no-speculative-tune", "--spec-ngram-max"], +]; + +pub(super) const PLUGIN_DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "plugins", "install"], + NONE, + &["reference", "--archive", "--name", "--version"], + PLUGIN_INSTALL_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "update"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "enable"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "disable"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "delete"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "info"], + NONE, + REDACTED_NAME, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "search"], + NONE, + &["query"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "plugins", "list"], + NONE, + NONE, + false, + RawKind::None, + ), +]; + +pub(super) const BENCHMARK_DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "benchmark", "tune"], + TUNE_FLAGS, + &[ + "--model", + "--models", + "--ctx-sizes", + "--batch-sizes", + "--ubatch-sizes", + "--mmap-values", + "--mlock-values", + "--flash-attention", + "--speculative-types", + "--spec-draft-models", + "--spec-draft-max-tokens", + "--spec-draft-min-tokens", + "--spec-ngram-min", + "--spec-ngram-max", + "--spec-draft-acceptance-threshold", + "--spec-draft-split-probability", + "--throughput-tolerance-pct", + "--max-tokens", + "--startup-timeout-secs", + "--request-timeout-secs", + "--prompt", + ], + TUNE_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "benchmark", "import-prompts"], + NONE, + &["--source", "--limit", "--max-tokens", "--output"], + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs new file mode 100644 index 0000000000..c28a5c8ce5 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/runtime.rs @@ -0,0 +1,96 @@ +use super::{Descriptor, JSON, NONE, RawKind, descriptor, descriptor_with_conflicts}; + +const RUNTIME_LIST_FLAGS: &[&str] = &["--available", "--installed", "--json"]; +const RUNTIME_PRUNE_FLAGS: &[&str] = &["--active-only", "--json"]; +const RUNTIME_LIST_CONFLICTS: &[&[&str]] = &[&["--available", "--installed"]]; +const REMOTE_MODEL_CONFLICTS: &[&[&str]] = &[&["--model", "--instance-id"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor(&["mesh-llm", "runtime"], NONE, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "runtime", "status"], + NONE, + NONE, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "load"], + NONE, + &["name"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "unload"], + NONE, + &["name"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "guardrails"], + JSON, + NONE, + true, + RawKind::Mode, + ), + descriptor( + &["mesh-llm", "runtime", "bootstrap"], + JSON, + NONE, + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "runtime", "list"], + RUNTIME_LIST_FLAGS, + &["--manifest", "--bundle-dir", "--cache-dir"], + RUNTIME_LIST_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "install"], + JSON, + &["runtime_ref", "--manifest", "--bundle-dir", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "remove"], + JSON, + &["native_runtime_id", "--mesh-version", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "prune"], + RUNTIME_PRUNE_FLAGS, + &["--mesh-version", "--cache-dir"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "remote"], + JSON, + &["--endpoint"], + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "runtime", "remote-model"], + JSON, + &["--endpoint", "--model", "--profile", "--instance-id"], + REMOTE_MODEL_CONFLICTS, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "runtime", "apply-config"], + JSON, + &["--endpoint", "--expected-revision", "--config"], + true, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs new file mode 100644 index 0000000000..991b64d1b5 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/descriptors/top_level.rs @@ -0,0 +1,189 @@ +use super::{ + Descriptor, JSON, NONE, REDACTED_NAME, RawKind, descriptor, descriptor_with_conflicts, +}; + +const SETUP_FLAGS: &[&str] = &[ + "--yes", + "--no-interactive", + "--service", + "--no-service", + "--skip-runtime", + "--verbose", +]; +const UNINSTALL_FLAGS: &[&str] = &[ + "--dry-run", + "--yes", + "--keep-cache", + "--keep-service-files", + "--purge-config", + "--keep-config", + "--json", + "--verbose", +]; +const MODEL_PREPARE_FLAGS: &[&str] = &[ + "--dry-run", + "--confirm", + "--follow", + "--json", + "--list", + "--update-script", +]; +const SETUP_CONFLICTS: &[&[&str]] = &[&["--service", "--no-service"]]; +const UNINSTALL_CONFLICTS: &[&[&str]] = &[&["--purge-config", "--keep-config"]]; +const UPDATE_CONFLICTS: &[&[&str]] = &[&["--flavor", "--detect-flavor"]]; +const SKILLS_INSTALL_CONFLICTS: &[&[&str]] = &[&["--agent", "--all"]]; + +pub(super) const DESCRIPTORS: &[Descriptor] = &[ + descriptor_with_conflicts( + &["mesh-llm", "setup"], + SETUP_FLAGS, + NONE, + SETUP_CONFLICTS, + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "uninstall"], + UNINSTALL_FLAGS, + &["--binary-path"], + UNINSTALL_CONFLICTS, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "download"], + &["--draft"], + &["name"], + false, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "update"], + &["--detect-flavor"], + &["--version", "--flavor"], + UPDATE_CONFLICTS, + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "status"], NONE, NONE, true, RawKind::None), + descriptor( + &["mesh-llm", "load"], + NONE, + REDACTED_NAME, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "unload"], + NONE, + REDACTED_NAME, + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "discover"], + &["--auto"], + &["--name", "--model", "--min-vram", "--region", "--relay"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "rotate-key"], + NONE, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "goose"], + NONE, + &["--model"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "claude"], + NONE, + &["--model"], + true, + RawKind::None, + ), + descriptor( + &["mesh-llm", "pi"], + &["--write"], + &["--model", "--host"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "opencode"], + &["--write"], + &["--model", "--host"], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "stop"], NONE, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "external-plugin"], + NONE, + &["argv"], + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "model-prepare"], + MODEL_PREPARE_FLAGS, + &[ + "source_repo", + "--quant", + "--target", + "--model-id", + "--flavor", + "--timeout", + "--mesh-llm-ref", + "--status", + "--logs", + "--cancel", + ], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "gpus"], JSON, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "gpus", "detect"], + JSON, + NONE, + false, + RawKind::None, + ), + descriptor( + &["mesh-llm", "gpus", "run-benchmark"], + JSON, + NONE, + false, + RawKind::Backend, + ), + descriptor( + &["mesh-llm", "config", "validate"], + JSON, + &["--config-path"], + false, + RawKind::None, + ), + descriptor(&["mesh-llm", "doctor"], JSON, NONE, false, RawKind::None), + descriptor( + &["mesh-llm", "doctor", "split"], + JSON, + &["--model-ref", "--output-dir"], + true, + RawKind::None, + ), + descriptor_with_conflicts( + &["mesh-llm", "skills", "install"], + &["--all", "--dry-run", "--force"], + &["--agent"], + SKILLS_INSTALL_CONFLICTS, + false, + RawKind::None, + ), +]; diff --git a/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs b/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs new file mode 100644 index 0000000000..090946f488 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/raw_options.rs @@ -0,0 +1,111 @@ +use super::vocabulary::is_static_summary_token; + +#[derive(Clone, Copy)] +struct CommandPath<'a> { + tokens: [Option<&'a str>; 4], + len: usize, + valid: bool, +} + +impl CommandPath<'_> { + fn matches(self, expected: &[&str]) -> bool { + self.valid + && self.len == expected.len() + && expected + .iter() + .enumerate() + .all(|(index, token)| self.tokens[index] == Some(*token)) + } +} + +pub(super) fn raw_option_is_allowed(tokens: &[&str], index: usize, option: &str) -> bool { + let prefix = &tokens[..index]; + match option { + "--backend" => matches_backend_prefix(prefix), + "--mode" => matches_mode_prefix(prefix), + "--port" => matches_port_prefix(prefix), + _ => false, + } +} + +fn command_path<'a>(tokens: &'a [&'a str], end: usize) -> CommandPath<'a> { + let mut path = CommandPath { + tokens: [None; 4], + len: 0, + valid: true, + }; + for token in &tokens[..end] { + if !is_static_summary_token(token) { + continue; + } + if path.len >= path.tokens.len() { + path.valid = false; + return path; + } + path.tokens[path.len] = Some(*token); + path.len += 1; + } + path +} + +fn matches_backend_prefix(prefix: &[&str]) -> bool { + let path = command_path(prefix, prefix.len()); + path.matches(&["mesh-llm", "gpus", "run-benchmark"]) + && prefix == ["mesh-llm", "gpus", "run-benchmark"] +} + +fn matches_mode_prefix(prefix: &[&str]) -> bool { + prefix == ["mesh-llm", "runtime", "guardrails"] +} + +fn matches_port_prefix(prefix: &[&str]) -> bool { + matches!( + prefix, + ["mesh-llm", "status"] + | ["mesh-llm", "load"] + | ["mesh-llm", "unload"] + | ["mesh-llm", "goose"] + | ["mesh-llm", "claude"] + | ["mesh-llm", "doctor", "split"] + | ["mesh-llm", "doctor", "split", "--json"] + | ["mesh-llm", "doctor", "split", "--json", "--json"] + | ["mesh-llm", "runtime", "status"] + | ["mesh-llm", "runtime", "load"] + | ["mesh-llm", "runtime", "unload"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "disabled"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "metrics"] + | ["mesh-llm", "runtime", "guardrails", "--mode", "enforce"] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "disabled", + "--json", + ] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "metrics", + "--json", + ] + | [ + "mesh-llm", + "runtime", + "guardrails", + "--mode", + "enforce", + "--json", + ] + | ["mesh-llm", "runtime", "bootstrap"] + | ["mesh-llm", "runtime", "bootstrap", "--json"] + | ["mesh-llm", "runtime", "remote"] + | ["mesh-llm", "runtime", "remote", "--json"] + | ["mesh-llm", "runtime", "remote-model"] + | ["mesh-llm", "runtime", "remote-model", "--json"] + | ["mesh-llm", "runtime", "apply-config"] + | ["mesh-llm", "runtime", "apply-config", "--json"] + ) +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/tests.rs b/crates/mesh-llm-events/src/command_summary_grammar/tests.rs new file mode 100644 index 0000000000..4be1eb992b --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/tests.rs @@ -0,0 +1,107 @@ +use super::descriptors::{DESCRIPTOR_GROUPS, RawKind}; +use super::is_safe_summary; +use super::raw_options::raw_option_is_allowed; +use super::vocabulary::{is_boolean_option, is_redacted_marker, is_static_summary_token}; + +#[test] +fn vocabulary_classifies_static_boolean_and_redacted_tokens() { + assert!(is_static_summary_token("mesh-llm")); + assert!(is_static_summary_token("run-benchmark")); + assert!(!is_static_summary_token("private-model")); + + assert!(is_boolean_option("--json")); + assert!(is_boolean_option("--no-speculative-tune")); + assert!(!is_boolean_option("--backend")); + + assert!(is_redacted_marker("model")); + assert!(is_redacted_marker("--relay-auth")); + assert!(!is_redacted_marker("[REDACTED]")); +} + +#[test] +fn raw_options_are_limited_to_candidate_producer_contexts() { + let backend = ["mesh-llm", "gpus", "run-benchmark", "--backend", "cuda"]; + let mode = ["mesh-llm", "runtime", "guardrails", "--mode", "metrics"]; + let port = ["mesh-llm", "doctor", "split", "--json", "--port", "41731"]; + + assert!(raw_option_is_allowed(&backend, 3, "--backend")); + assert!(raw_option_is_allowed(&mode, 3, "--mode")); + assert!(raw_option_is_allowed(&port, 4, "--port")); + assert!(!raw_option_is_allowed(&backend, 3, "--mode")); +} + +#[test] +fn benchmark_tune_rejects_each_speculative_option_when_tuning_is_disabled() { + let speculative_options = [ + "--speculative-types", + "--spec-draft-models", + "--spec-draft-max-tokens", + "--spec-draft-min-tokens", + "--spec-draft-acceptance-threshold", + "--spec-draft-split-probability", + "--spec-ngram-min", + "--spec-ngram-max", + ]; + + for option in speculative_options { + let summary = format!("mesh-llm benchmark tune --no-speculative-tune {option} [REDACTED]"); + assert!(!is_safe_summary(&summary), "accepted conflict: {option}"); + } +} + +#[test] +fn ports_accept_only_ascii_decimal_u16_values() { + for port in ["0", "1", "65535"] { + let summary = format!("mesh-llm status --port {port}"); + assert!(is_safe_summary(&summary), "rejected valid port: {port}"); + } + + for port in ["+1", "-1", "65536", "1.0", "١"] { + let summary = format!("mesh-llm status --port {port}"); + assert!(!is_safe_summary(&summary), "accepted invalid port: {port}"); + } +} + +#[test] +fn every_descriptor_accepts_its_complete_canonical_shape() { + for descriptor in DESCRIPTOR_GROUPS.iter().flat_map(|group| group.iter()) { + let mut prefix = descriptor.path.to_vec(); + match descriptor.raw { + RawKind::Backend => prefix.extend_from_slice(&["--backend", "cuda"]), + RawKind::Mode => prefix.extend_from_slice(&["--mode", "metrics"]), + RawKind::None => {} + } + let mut tokens = prefix.clone(); + tokens.extend_from_slice(descriptor.booleans); + if descriptor.has_port { + tokens.extend_from_slice(&["--port", "41731"]); + } + for marker in descriptor.redacted { + tokens.extend_from_slice(&[*marker, "[REDACTED]"]); + } + if descriptor + .conflicts + .iter() + .any(|pair| pair.iter().all(|flag| tokens.contains(flag))) + { + assert!(!is_safe_summary(&tokens.join(" "))); + continue; + } + let summary = tokens.join(" "); + if tokens.len() <= 32 && summary.chars().count() <= 256 { + assert!(is_safe_summary(&summary), "descriptor rejected: {summary}"); + } else { + for marker in descriptor.booleans.iter().chain(descriptor.redacted) { + let mut single = prefix.clone(); + single.push(*marker); + if descriptor.redacted.contains(marker) { + single.push("[REDACTED]"); + } + assert!( + is_safe_summary(&single.join(" ")), + "descriptor marker rejected: {marker}" + ); + } + } + } +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/validation.rs b/crates/mesh-llm-events/src/command_summary_grammar/validation.rs new file mode 100644 index 0000000000..08aa7578e2 --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/validation.rs @@ -0,0 +1,123 @@ +use super::descriptors::{Descriptor, GLOBAL_REDACTED, RawKind}; +use super::raw_options::raw_option_is_allowed; +use super::vocabulary::{is_boolean_option, is_redacted_marker, is_static_summary_token}; + +#[derive(Clone, Copy)] +enum Phase { + Booleans, + Port, + Redacted, +} + +fn descriptor_matches(tokens: &[&str], descriptor: Descriptor) -> bool { + descriptor.path.len() <= tokens.len() + && descriptor + .path + .iter() + .enumerate() + .all(|(index, token)| is_static_summary_token(token) && tokens[index] == *token) +} + +fn raw_value_is_valid(raw: RawKind, tokens: &[&str], index: usize) -> Option { + let (option, valid_values): (&str, &[&str]) = match raw { + RawKind::Backend => ("--backend", &["metal", "cuda", "hip", "intel"]), + RawKind::Mode => ("--mode", &["disabled", "metrics", "enforce"]), + RawKind::None => return Some(index), + }; + (tokens.get(index) == Some(&option) + && tokens + .get(index + 1) + .is_some_and(|value| valid_values.contains(value))) + .then_some(index + 2) +} + +pub(super) fn validate_descriptor(tokens: &[&str], descriptor: Descriptor) -> bool { + if !descriptor_matches(tokens, descriptor) { + return false; + } + let Some(mut index) = raw_value_is_valid(descriptor.raw, tokens, descriptor.path.len()) else { + return false; + }; + let mut phase = Phase::Booleans; + let mut seen_booleans = Vec::new(); + let mut seen_redacted = Vec::new(); + let mut seen_tokens = Vec::new(); + let mut port_seen = false; + let mut global_phase = false; + let mut last_global_rank = 0; + + while index < tokens.len() { + let Some(token) = tokens.get(index) else { + return false; + }; + if is_boolean_option(token) { + if !matches!(phase, Phase::Booleans) + || !descriptor.booleans.contains(token) + || seen_booleans.contains(token) + { + return false; + } + seen_booleans.push(*token); + seen_tokens.push(*token); + index += 1; + continue; + } + if *token == "--port" { + if matches!(phase, Phase::Redacted) + || port_seen + || !descriptor.has_port + || !raw_option_is_allowed(tokens, index, token) + { + return false; + } + let Some(value) = tokens.get(index + 1) else { + return false; + }; + if value.is_empty() + || !value.bytes().all(|byte| byte.is_ascii_digit()) + || value.parse::().is_err() + { + return false; + } + port_seen = true; + phase = Phase::Port; + index += 2; + continue; + } + if is_redacted_marker(token) { + let is_global = GLOBAL_REDACTED.contains(token); + if global_phase && !is_global { + return false; + } + if is_global { + let global_rank = match *token { + "--join" => 1, + "--root-relay" => 2, + "--relay-auth" => 3, + _ => 0, + }; + if global_rank <= last_global_rank { + return false; + } + global_phase = true; + last_global_rank = global_rank; + } + if (!descriptor.redacted.contains(token) && !GLOBAL_REDACTED.contains(token)) + || seen_redacted.contains(token) + || tokens.get(index + 1) != Some(&"[REDACTED]") + { + return false; + } + seen_redacted.push(*token); + seen_tokens.push(*token); + phase = Phase::Redacted; + index += 2; + continue; + } + return false; + } + !descriptor + .conflicts + .iter() + .any(|pair| pair.iter().all(|flag| seen_tokens.contains(flag))) +} diff --git a/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs b/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs new file mode 100644 index 0000000000..c3e048c7ce --- /dev/null +++ b/crates/mesh-llm-events/src/command_summary_grammar/vocabulary.rs @@ -0,0 +1,205 @@ +pub(super) fn is_static_summary_token(token: &str) -> bool { + matches!( + token, + "mesh-llm" + | "load" + | "unload" + | "status" + | "discover" + | "rotate-key" + | "goose" + | "claude" + | "pi" + | "opencode" + | "stop" + | "external-plugin" + | "setup" + | "uninstall" + | "gpus" + | "detect" + | "run-benchmark" + | "config" + | "validate" + | "doctor" + | "split" + | "skills" + | "install" + | "plugins" + | "update" + | "enable" + | "disable" + | "delete" + | "info" + | "search" + | "list" + | "models" + | "package" + | "recommended" + | "installed" + | "cleanup" + | "prune" + | "certify" + | "show" + | "download" + | "updates" + | "benchmark" + | "tune" + | "import-prompts" + | "model-prepare" + | "runtime" + | "guardrails" + | "bootstrap" + | "remove" + | "remote" + | "remote-model" + | "apply-config" + | "auth" + | "init" + | "sign-node" + | "renew-node" + | "verify-node" + | "rotate-node" + | "revoke-owner" + | "revoke-node" + | "rotate-owner" + | "trust" + | "add" + ) +} + +pub(super) fn is_boolean_option(token: &str) -> bool { + matches!( + token, + "--draft" + | "--detect-flavor" + | "--yes" + | "--no-interactive" + | "--service" + | "--no-service" + | "--skip-runtime" + | "--verbose" + | "--dry-run" + | "--keep-cache" + | "--keep-config" + | "--keep-service-files" + | "--json" + | "--auto" + | "--write" + | "--all" + | "--force" + | "--experimental" + | "--confirm" + | "--follow" + | "--list" + | "--update-script" + | "--gguf" + | "--mlx" + | "--catalog" + | "--direct" + | "--check" + | "--no-speculative-tune" + | "--apply" + | "--replace-existing" + | "--launch-args" + | "--debug-telemetry" + | "--available" + | "--installed" + | "--active-only" + | "--keychain" + | "--no-passphrase" + | "--revoke-current" + | "--package-only" + | "--purge-config" + ) +} + +pub(super) fn is_redacted_marker(token: &str) -> bool { + matches!( + token, + "argv" + | "model" + | "name" + | "native_runtime_id" + | "owner_id" + | "query" + | "reference" + | "repo" + | "runtime_ref" + | "source_repo" + | "--version" + | "--flavor" + | "--binary-path" + | "--name" + | "--model" + | "--min-vram" + | "--region" + | "--relay" + | "--root-relay" + | "--join" + | "--relay-auth" + | "--sort" + | "--host" + | "--config-path" + | "--model-ref" + | "--output-dir" + | "--agent" + | "--archive" + | "--quant" + | "--target" + | "--model-id" + | "--timeout" + | "--mesh-llm-ref" + | "--status" + | "--logs" + | "--cancel" + | "--unused-since" + | "--report-out" + | "--api-base" + | "--prompt" + | "--max-tokens" + | "--limit" + | "--models" + | "--ctx-sizes" + | "--batch-sizes" + | "--ubatch-sizes" + | "--mmap-values" + | "--mlock-values" + | "--flash-attention" + | "--speculative-types" + | "--spec-draft-models" + | "--spec-draft-max-tokens" + | "--spec-draft-min-tokens" + | "--spec-ngram-min" + | "--spec-ngram-max" + | "--spec-draft-acceptance-threshold" + | "--spec-draft-split-probability" + | "--throughput-tolerance-pct" + | "--startup-timeout-secs" + | "--request-timeout-secs" + | "--source" + | "--output" + | "--manifest" + | "--bundle-dir" + | "--cache-dir" + | "--mesh-version" + | "--endpoint" + | "--profile" + | "--instance-id" + | "--expected-revision" + | "--config" + | "--owner-key" + | "--node-key" + | "--node-ownership" + | "--trust-store" + | "--out" + | "--hostname-hint" + | "--node-label" + | "--expires-in-hours" + | "--file" + | "--node-id" + | "--verify-trust-policy" + | "--reason" + | "--cert-id" + | "--label" + ) +} diff --git a/crates/mesh-llm-events/src/lib.rs b/crates/mesh-llm-events/src/lib.rs index 0d05646684..ad373ac962 100644 --- a/crates/mesh-llm-events/src/lib.rs +++ b/crates/mesh-llm-events/src/lib.rs @@ -14,7 +14,8 @@ pub mod terminal_progress; mod command_lifecycle; pub use audit::*; pub use command_lifecycle::{ - CliCommandFamily, CliCommandOutcome, emit_cli_command_event, set_cli_command_event_verbose, + CliCommandFamily, CliCommandOutcome, CliCommandSummary, emit_cli_command_event, + set_cli_command_event_verbose, }; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] diff --git a/crates/mesh-llm-events/src/logging/envelope.rs b/crates/mesh-llm-events/src/logging/envelope.rs index 84debfdb04..b399e0fa1f 100644 --- a/crates/mesh-llm-events/src/logging/envelope.rs +++ b/crates/mesh-llm-events/src/logging/envelope.rs @@ -80,6 +80,7 @@ impl CanonicalPresentationContext { | "management_get_other" | "management_post" | "management_put" + | "management_patch" | "management_delete" | "management_other", ) => "management", @@ -127,6 +128,7 @@ fn closed_route(value: &str) -> Option { | "management_get_other" | "management_post" | "management_put" + | "management_patch" | "management_delete" | "management_other" ); @@ -142,11 +144,15 @@ pub fn closed_source(value: &str) -> Option { } /// Trim, uppercase, and accept only the closed HTTP method vocabulary -/// (`GET` | `POST` | `PUT` | `DELETE` | `OTHER`). Unknown or empty methods are +/// (`GET` | `POST` | `PUT` | `PATCH` | `DELETE` | `OTHER`). Unknown or empty methods are /// discarded rather than echoed to a console. pub fn closed_method(value: &str) -> Option { let value = value.trim().to_ascii_uppercase(); - matches!(value.as_str(), "GET" | "POST" | "PUT" | "DELETE" | "OTHER").then_some(value) + matches!( + value.as_str(), + "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "OTHER" + ) + .then_some(value) } /// Current canonical logging schema version. Bump on additive changes to the envelope shape. diff --git a/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs b/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs index f4a9f35612..efb9521584 100644 --- a/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/api/management_lifecycle.rs @@ -41,36 +41,43 @@ impl ManagementLifecycleContext { } pub(super) fn request_id_from_raw(raw: &[u8]) -> RequestId { - let value = crate::api::access::request_header(raw, "x-request-id") + let mut headers = [httparse::EMPTY_HEADER; 64]; + let mut request = httparse::Request::new(&mut headers); + let Ok(httparse::Status::Complete(_)) = request.parse(raw) else { + return RequestId::default(); + }; + let mut request_id_headers = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")); + let Some(header) = request_id_headers.next() else { + return RequestId::default(); + }; + if request_id_headers.next().is_some() { + return RequestId::default(); + } + std::str::from_utf8(header.value) .ok() - .flatten() - .and_then(openai_frontend::parse_request_id); - value.unwrap_or_default() + .and_then(openai_frontend::parse_request_id) + .unwrap_or_default() } pub(super) fn eligible_management_route(method: &str, path: &str) -> bool { - !(is_read_only_observation_method(method) - || path == "/models" - || path.starts_with("/v1/") - || path == "/api/chat" - || path == "/api/responses" - || path == "/api/objects" - || path.starts_with("/api/objects/") - || path == "/mesh/hook" - || path == "/api/logs" - || path.starts_with("/api/logs/")) + is_mutation_method(method) + && path != "/models" + && !path.starts_with("/v1/") + && path != "/api/chat" + && path != "/api/responses" + && path != "/api/objects" + && !path.starts_with("/api/objects/") + && path != "/mesh/hook" + && path != "/api/logs" + && !path.starts_with("/api/logs/") && (path.starts_with("/api/") || path == "/mcp") } -/// Read-only console observation must never become workload ledger traffic. -/// -/// These endpoints are frequently polled or held open by the local console. -/// Recording them as ordinary management work makes request counts describe -/// the observer rather than the workload it is observing. This applies to all -/// safe observation methods, including future console endpoints. Mutations -/// continue through the management lifecycle and remain auditable. -fn is_read_only_observation_method(method: &str) -> bool { - matches!(method, "GET" | "HEAD" | "OPTIONS") +fn is_mutation_method(method: &str) -> bool { + matches!(method, "POST" | "PUT" | "PATCH" | "DELETE") } pub(super) fn method_route_label(method: &str, path: &str) -> &'static str { @@ -82,6 +89,7 @@ pub(super) fn method_route_label(method: &str, path: &str) -> &'static str { ("GET", _) => "management_get_other", ("POST", _) => "management_post", ("PUT", _) => "management_put", + ("PATCH", _) => "management_patch", ("DELETE", _) => "management_delete", _ => "management_other", } @@ -190,6 +198,20 @@ mod tests { } } + #[test] + fn duplicate_request_ids_are_replaced_instead_of_selecting_the_first() { + let first = "00000000-0000-4000-8000-000000000051"; + let second = "00000000-0000-4000-8000-000000000052"; + let raw = format!( + "POST /api/runtime/config/validate HTTP/1.1\r\nHost: localhost\r\nx-request-id: {first}\r\nX-Request-Id: {second}\r\nContent-Length: 0\r\n\r\n" + ); + + let request_id = request_id_from_raw(raw.as_bytes()); + + assert_ne!(request_id.as_uuid().to_string(), first); + assert_ne!(request_id.as_uuid().to_string(), second); + } + #[tokio::test] async fn trusted_local_and_server_failures_use_bounded_terminal_states() { let (_, rejected) = lifecycle(RequestId::new()); @@ -234,32 +256,53 @@ mod tests { } #[test] - fn logs_openai_and_all_read_only_management_routes_are_excluded() { - for path in [ - "/api/logs/requests", - "/api/logs/events", - "/api/status", - "/api/events", - "/api/runtime/events", + fn management_lifecycle_routes_are_mutation_only() { + let mutation_methods = ["POST", "PUT", "PATCH", "DELETE"]; + let observation_methods = ["GET", "HEAD", "OPTIONS", "TRACE", "CONNECT"]; + let management_paths = ["/api/runtime/control", "/api/plugins/install", "/mcp"]; + let excluded_paths = [ "/v1/chat/completions", "/models", "/api/chat", "/api/responses", "/api/objects", + "/api/objects/item", + "/api/logs", + "/api/logs/requests", "/mesh/hook", - ] { - assert!( - !eligible_management_route("GET", path), - "{path} must stay excluded" - ); + ]; + + for method in mutation_methods { + for path in management_paths { + assert!( + eligible_management_route(method, path), + "{method} {path} must be eligible" + ); + } } - assert!(!eligible_management_route("GET", "/api/models")); - assert!(!eligible_management_route("HEAD", "/api/plugins")); - assert!(!eligible_management_route( - "OPTIONS", - "/api/runtime/control" - )); - assert!(eligible_management_route("POST", "/api/runtime/control")); - assert!(eligible_management_route("POST", "/api/status")); + for method in observation_methods { + for path in management_paths { + assert!( + !eligible_management_route(method, path), + "{method} {path} must be excluded" + ); + } + } + for method in mutation_methods.into_iter().chain(observation_methods) { + for path in excluded_paths { + assert!( + !eligible_management_route(method, path), + "{method} {path} must stay excluded" + ); + } + } + } + + #[test] + fn patch_management_routes_keep_their_method_label() { + assert_eq!( + method_route_label("PATCH", "/api/plugins/demo/web-ui/config"), + "management_patch" + ); } } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs index 529ad8a0a0..bec384ca9d 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/delete.rs @@ -236,7 +236,9 @@ fn prepare_terminal_delete( }); } match facade.request(&request_id)? { - Some(record) if record.outcome == "active" => return Err(LogsError::ActiveRequest), + Some(record) if record.request.outcome == "active" => { + return Err(LogsError::ActiveRequest); + } Some(_) => {} None => return Err(LogsError::NotFound), } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs index 1b3b5db1d9..4d5321d774 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/dto.rs @@ -2,12 +2,14 @@ use base64::Engine; use mesh_llm_events::logging::envelope::CanonicalEnvelope; use mesh_llm_events::logging::events::{LifecycleEvent, TokenUsage}; use mesh_llm_log_store::{ - ArtifactContent, ArtifactRecord, AuditEntryRow, AuditEntrySeverity, EventRecord, ProxyRecord, - RequestRecord, + ArtifactContent, ArtifactRecord, AuditEntryDetail, AuditEntrySeverity, EventRecord, + ProxyRecord, RequestRecordWithCaller, }; use serde::Serialize; use std::collections::BTreeMap; +use mesh_llm_events::CliCommandSummary; + use super::{LogsError, event_kind}; use crate::logging::RequestSummaryEntry; @@ -30,37 +32,75 @@ pub(crate) struct RequestDto { provider: Option, engine: Option, status_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_path_type: Option, source: &'static str, } impl RequestDto { - pub(super) fn durable(record: RequestRecord) -> Self { + pub(super) fn durable(record: RequestRecordWithCaller) -> Self { + let request = record.request; Self { - request_id: record.request_id, - outcome: record.outcome, - created_at: record.created_at, - terminal_at: record.terminal_at, - route: record.route.as_deref().map(safe_metadata), - model: record.model.as_deref().map(safe_metadata), - provider: record.provider.as_deref().map(safe_metadata), - engine: record.engine.as_deref().map(safe_metadata), - status_code: record.status_code, + request_id: request.request_id, + outcome: request.outcome, + created_at: request.created_at, + terminal_at: request.terminal_at, + route: request.route.as_deref().map(safe_metadata), + model: request.model.as_deref().map(safe_metadata), + provider: request.provider.as_deref().map(safe_metadata), + engine: request.engine.as_deref().map(safe_metadata), + status_code: request.status_code, + caller_endpoint_id: record.caller_endpoint_id.as_deref().map(safe_metadata), + caller_addr: record.caller_addr.as_deref().map(safe_metadata), + caller_path_type: record.caller_path_type.as_deref().map(safe_metadata), source: "durable", } } - pub(super) fn active(entry: RequestSummaryEntry, metadata: Option) -> Self { + pub(super) fn active( + entry: RequestSummaryEntry, + metadata: Option, + ) -> Self { let summary_metadata = entry.metadata.clone(); let metadata = metadata.map(|record| { ( - record.route.as_deref().map(safe_metadata), - record.model.as_deref().map(safe_metadata), - record.provider.as_deref().map(safe_metadata), - record.engine.as_deref().map(safe_metadata), - record.status_code, + record.request.route.as_deref().map(safe_metadata), + record.request.model.as_deref().map(safe_metadata), + record.request.provider.as_deref().map(safe_metadata), + record.request.engine.as_deref().map(safe_metadata), + record.request.status_code, + record.caller_endpoint_id.as_deref().map(safe_metadata), + record.caller_addr.as_deref().map(safe_metadata), + record.caller_path_type.as_deref().map(safe_metadata), ) }); - let (route, model, provider, engine, status_code) = metadata.unwrap_or_default(); + let ( + route, + model, + provider, + engine, + status_code, + caller_endpoint_id, + caller_addr, + caller_path_type, + ) = metadata.unwrap_or_default(); + let active_caller = ( + summary_metadata.caller_endpoint_id().map(safe_metadata), + summary_metadata.caller_addr().map(safe_metadata), + summary_metadata.caller_path_type().map(str::to_owned), + ); + let caller = if active_caller.0.is_some() + || active_caller.1.is_some() + || active_caller.2.is_some() + { + active_caller + } else { + (caller_endpoint_id, caller_addr, caller_path_type) + }; Self { request_id: entry.request_id, outcome: entry.state, @@ -71,6 +111,9 @@ impl RequestDto { provider: summary_metadata.provider().map(safe_metadata).or(provider), engine: summary_metadata.engine().map(safe_metadata).or(engine), status_code, + caller_endpoint_id: caller.0, + caller_addr: caller.1, + caller_path_type: caller.2, source: "active", } } @@ -314,6 +357,10 @@ pub(crate) struct AuditDto { #[serde(skip_serializing_if = "Option::is_none")] subject_id: Option, #[serde(skip_serializing_if = "Option::is_none")] + remote_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] operation_id: Option, #[serde(skip_serializing_if = "Option::is_none")] request_id: Option, @@ -322,13 +369,16 @@ pub(crate) struct AuditDto { #[serde(skip_serializing_if = "Option::is_none")] outcome: Option, #[serde(skip_serializing_if = "Option::is_none")] + command_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] duration_ms: Option, #[serde(skip_serializing_if = "BTreeMap::is_empty")] numeric_summaries: BTreeMap, } -impl From for AuditDto { - fn from(row: AuditEntryRow) -> Self { +impl From for AuditDto { + fn from(detail: AuditEntryDetail) -> Self { + let row = detail.entry; Self { sequence: row.sequence, entry_id: row.entry_id, @@ -343,10 +393,16 @@ impl From for AuditDto { context_version: row.context_version, subject_kind: row.subject_kind, subject_id: row.subject_id.as_deref().map(safe_metadata), + remote_addr: detail.remote_addr, + path_type: detail.path_type, operation_id: row.operation_id.as_deref().map(safe_metadata), request_id: row.correlation_request_id.as_deref().map(safe_metadata), reason_code: row.reason_code, outcome: row.outcome, + command_summary: detail + .command_summary + .and_then(|summary| CliCommandSummary::sanitize(&summary)) + .map(|summary| summary.as_str().to_owned()), duration_ms: row.duration_ms, numeric_summaries: row.numeric_summaries, } @@ -426,7 +482,7 @@ mod usage_tests { } } -fn safe_metadata(value: &str) -> String { +pub(super) fn safe_metadata(value: &str) -> String { let trimmed = value.trim(); let path_shaped = trimmed.starts_with('/') || trimmed.starts_with("~/") @@ -445,6 +501,43 @@ mod tests { envelope::CanonicalEnvelope, identifiers::{EventId, RequestId}, }; + use mesh_llm_log_store::{LogStore, RealClock, RequestRecord}; + use std::sync::Arc; + + fn detailed_request( + request: RequestRecord, + caller: (Option<&str>, Option<&str>, Option<&str>), + ) -> RequestRecordWithCaller { + let root = tempfile::tempdir().expect("temporary detailed request store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open request store"); + let insert_result = match caller { + (None, None, None) => store.upsert_summary_metadata( + &request.request_id, + request.model.as_deref(), + request.route.as_deref(), + request.provider.as_deref(), + request.engine.as_deref(), + &request.created_at, + ), + (caller_endpoint_id, caller_addr, caller_path_type) => store + .upsert_summary_metadata_with_caller( + &request.request_id, + request.model.as_deref(), + request.route.as_deref(), + request.provider.as_deref(), + request.engine.as_deref(), + caller_endpoint_id, + caller_addr, + caller_path_type, + &request.created_at, + ), + }; + insert_result.expect("insert detailed request"); + store + .query_request_with_caller(&request.request_id) + .expect("query detailed request") + .expect("detailed request") + } #[test] fn event_dto_projects_authoritative_usage_and_terminal_status() { @@ -531,20 +624,27 @@ mod tests { #[test] fn metadata_and_proxy_dtos_do_not_leak_paths_or_credentials() { - let dto = RequestDto::durable(RequestRecord { - request_id: uuid::Uuid::new_v4().to_string(), - outcome: "completed".into(), - created_at: "2026-01-01T00:00:00Z".into(), - terminal_at: None, - route: Some("/private/secret".into()), - model: Some("Bearer secret-token".into()), - provider: None, - engine: None, - status_code: None, - }); + let dto = RequestDto::durable(detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "completed".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: Some("/private/secret".into()), + model: Some("Bearer secret-token".into()), + provider: None, + engine: None, + status_code: None, + }, + (None, None, None), + )); let json = serde_json::to_string(&dto).expect("serialize dto"); assert!(!json.contains("secret-token")); assert!(!json.contains("/private/secret")); + let json: serde_json::Value = serde_json::from_str(&json).expect("request DTO JSON"); + assert!(json.get("callerEndpointId").is_none()); + assert!(json.get("callerAddr").is_none()); + assert!(json.get("callerPathType").is_none()); let proxy = ProxyDto::from(ProxyRecord { attempt_id: uuid::Uuid::new_v4().to_string(), @@ -563,6 +663,130 @@ mod tests { assert!(!json.contains("token=secret")); } + #[test] + fn active_and_durable_request_dtos_expose_local_http_socket_addresses() { + for caller_addr in [ + "127.0.0.1:40123", + "192.0.2.10:40123", + "[2001:db8::10]:40123", + ] { + let metadata = crate::logging::RequestSummaryMetadata::default().with_caller_identity( + None, + Some(caller_addr), + Some(crate::logging::CallerPathType::LocalHttp), + ); + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata, + }, + None, + ); + let durable = RequestDto::durable(detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "completed".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: Some("2026-01-01T00:00:01Z".into()), + route: None, + model: None, + provider: None, + engine: None, + status_code: Some(200), + }, + (None, Some(caller_addr), Some("local_http")), + )); + + for json in [ + serde_json::to_value(active).expect("active DTO"), + serde_json::to_value(durable).expect("durable DTO"), + ] { + assert_eq!(json["callerAddr"], caller_addr); + assert_eq!(json["callerPathType"], "local_http"); + assert!(json.get("callerEndpointId").is_none()); + } + } + } + + #[test] + fn active_caller_tuple_is_selected_atomically_before_durable_fallback() { + let endpoint_id = "abababababababababababababababababababababababababababababababab"; + let active_metadata = crate::logging::RequestSummaryMetadata::default() + .with_caller_identity(Some(endpoint_id), None, None); + let durable_local = detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: None, + model: None, + provider: None, + engine: None, + status_code: None, + }, + (None, Some("127.0.0.1:40123"), Some("local_http")), + ); + + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata: active_metadata, + }, + Some(durable_local), + ); + let json = serde_json::to_value(active).expect("active DTO"); + + assert_eq!(json["callerEndpointId"], endpoint_id); + assert!(json.get("callerAddr").is_none()); + assert!(json.get("callerPathType").is_none()); + } + + #[test] + fn active_request_without_caller_uses_complete_durable_caller_tuple() { + let endpoint_id = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac"; + let durable = detailed_request( + RequestRecord { + request_id: uuid::Uuid::new_v4().to_string(), + outcome: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + route: None, + model: None, + provider: None, + engine: None, + status_code: None, + }, + ( + Some(endpoint_id), + Some("192.0.2.44:11204"), + Some("remote_quic_http"), + ), + ); + + let active = RequestDto::active( + crate::logging::RequestSummaryEntry { + request_id: uuid::Uuid::new_v4().to_string(), + state: "active".into(), + created_at: "2026-01-01T00:00:00Z".into(), + terminal_at: None, + metadata: crate::logging::RequestSummaryMetadata::default(), + }, + Some(durable), + ); + let json = serde_json::to_value(active).expect("active DTO"); + + assert_eq!(json["callerEndpointId"], endpoint_id); + assert_eq!(json["callerAddr"], "192.0.2.44:11204"); + assert_eq!(json["callerPathType"], "remote_quic_http"); + } + #[test] fn intentionally_omitted_artifact_is_unavailable_not_missing() { let record = ArtifactRecord { diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs index 3d7afa2441..c351018c37 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/error.rs @@ -123,7 +123,7 @@ impl LogsError { Self::SchemaIncompatible { .. } => ( 503, "logging_schema_incompatible", - "the local log database schema is incompatible with this MeshLLM version", + "the local log database schema is unknown or incompatible", ), }; let details = match self { @@ -223,12 +223,12 @@ mod tests { fn schema_incompatibility_preserves_only_the_version_pair() { assert_eq!( LogsError::from(mesh_llm_log_store::LogStoreError::SchemaIncompatible { - found: 14, - supported: 11, + found: 2, + supported: 1, }), LogsError::SchemaIncompatible { - found: 14, - supported: 11, + found: 2, + supported: 1, } ); } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs index c1dc724846..0fcadbbfbe 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol.rs @@ -1,15 +1,16 @@ use mesh_llm_events::logging::envelope::CanonicalEnvelope; use mesh_llm_events::logging::events::LifecycleEvent; use mesh_llm_events::logging::replay::ReplayChannel; -use mesh_llm_log_store::{AuditEntryRow, AuditEntrySeverity}; use serde::Serialize; -use std::collections::BTreeMap; +use super::super::dto::safe_metadata; use super::super::event_kind; use super::query::{AuditCursor, Cursor}; -use crate::logging::OperationalAuditContext; +use crate::logging::ReplayRecord; use crate::logging::RequestSummaryEventSnapshots; -use crate::logging::{AuditReplayRecord, ReplayRecord}; + +mod audit_entry; +pub(super) use audit_entry::{audit_entry_frame, durable_audit_entry_frame}; pub(in crate::api::routes::logs) const MAX_FRAME_BYTES: usize = 16 * 1024; @@ -56,6 +57,12 @@ struct PublicRequest { model: Option, provider: Option, engine: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + caller_path_type: Option, status_code: Option, source: &'static str, } @@ -146,10 +153,13 @@ fn request_projection( outcome: snapshot.state().to_owned(), created_at: snapshot.created_at().to_owned(), terminal_at: snapshot.terminal_at().map(str::to_owned), - route: metadata.route().map(str::to_owned), - model: metadata.model().map(str::to_owned), - provider: metadata.provider().map(str::to_owned), - engine: metadata.engine().map(str::to_owned), + route: metadata.route().map(safe_metadata), + model: metadata.model().map(safe_metadata), + provider: metadata.provider().map(safe_metadata), + engine: metadata.engine().map(safe_metadata), + caller_endpoint_id: metadata.caller_endpoint_id().map(safe_metadata), + caller_addr: metadata.caller_addr().map(safe_metadata), + caller_path_type: metadata.caller_path_type().map(safe_metadata), status_code: lifecycle_status_code(&envelope.event), source: "active", }) @@ -222,164 +232,6 @@ fn frame(event: &str, id: &str, data: &T) -> Result { (frame.len() <= MAX_FRAME_BYTES).then_some(frame).ok_or(()) } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct AuditEntryData { - entry_id: String, - occurred_at: String, - source: String, - code: String, - #[serde(skip_serializing_if = "Option::is_none")] - severity: Option, - #[serde(skip_serializing_if = "Option::is_none")] - context_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - subject_kind: Option, - #[serde(skip_serializing_if = "Option::is_none")] - subject_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - operation_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - request_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - reason_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - outcome: Option, - #[serde(skip_serializing_if = "Option::is_none")] - duration_ms: Option, - #[serde(skip_serializing_if = "BTreeMap::is_empty")] - numeric_summaries: BTreeMap, - sequence: u64, -} - -/// Audit entry frame: privacy-safe projection of an audit replay record. -/// Never contains `canonical_envelope` or arbitrary `detail_json`. -pub(super) fn audit_entry_frame(record: &AuditReplayRecord) -> Result { - let payload: serde_json::Value = serde_json::from_str(&record.entry.payload).map_err(|_| ())?; - let entry_id = payload - .get("entry_id") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let occurred_at = payload - .get("occurred_at") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let source = payload - .get("source") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let code = payload - .get("code") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_owned(); - let severity = payload - .get("severity") - .and_then(|v| v.as_str()) - .map(str::to_owned); - let context_version = payload - .get("context_version") - .and_then(serde_json::Value::as_u64) - .and_then(|value| u8::try_from(value).ok()) - .filter(|value| *value == 1); - let context_string = |key: &str| { - context_version.and_then(|_| { - payload - .get(key) - .and_then(serde_json::Value::as_str) - .filter(|value| !value.is_empty() && value.chars().count() <= 256) - .map(str::to_owned) - }) - }; - let context_code = |key: &str| { - context_version.and_then(|_| { - payload - .get(key) - .and_then(serde_json::Value::as_str) - .filter(|value| OperationalAuditContext::valid_static_code(value)) - .map(str::to_owned) - }) - }; - let subject_kind = context_version.and_then(|_| { - payload - .get("subject_kind") - .and_then(serde_json::Value::as_str) - .and_then(crate::logging::OperationalAuditSubjectKind::parse) - .map(|kind| kind.as_str().to_owned()) - }); - let numeric_summaries = context_version.map_or_else(BTreeMap::new, |_| { - payload - .get("numeric_summaries") - .and_then(serde_json::Value::as_object) - .into_iter() - .flatten() - .filter(|(key, _)| OperationalAuditContext::valid_static_code(key)) - .filter_map(|(key, value)| value.as_u64().map(|value| (key.clone(), value))) - .take(8) - .collect() - }); - - let data = AuditEntryData { - entry_id, - occurred_at, - source, - code, - severity, - context_version, - subject_kind, - subject_id: context_string("subject_id"), - operation_id: context_string("operation_id"), - request_id: context_string("request_id"), - reason_code: context_code("reason_code"), - outcome: context_code("outcome"), - duration_ms: context_version.and_then(|_| { - payload - .get("duration_ms") - .and_then(serde_json::Value::as_u64) - }), - numeric_summaries, - sequence: record.sequence, - }; - frame( - "audit_entry", - &AuditCursor(record.sequence).event_id(), - &data, - ) -} - -/// Render a row read back from the durable store. This is the production audit -/// reconciliation path and deliberately uses the database sequence as the SSE -/// cursor so entries written by another process share the same ordering. -pub(super) fn durable_audit_entry_frame(record: AuditEntryRow) -> Result { - let sequence = u64::try_from(record.sequence).map_err(|_| ())?; - let severity = record.severity.map(|severity| match severity { - AuditEntrySeverity::Info => "info".to_string(), - AuditEntrySeverity::Warning => "warning".to_string(), - AuditEntrySeverity::Error => "error".to_string(), - }); - let data = AuditEntryData { - entry_id: record.entry_id, - occurred_at: record.occurred_at, - source: record.source, - code: record.code, - severity, - context_version: record.context_version, - subject_kind: record.subject_kind, - subject_id: record.subject_id, - operation_id: record.operation_id, - request_id: record.correlation_request_id, - reason_code: record.reason_code, - outcome: record.outcome, - duration_ms: record.duration_ms, - numeric_summaries: record.numeric_summaries, - sequence, - }; - frame("audit_entry", &AuditCursor(sequence).event_id(), &data) -} - /// Audit gap frame: points to `/api/logs/audit` for recovery. pub(super) fn audit_gap_frame( from_sequence: u64, @@ -393,109 +245,6 @@ pub(super) fn audit_gap_frame( #[cfg(test)] mod tests { use super::*; - use crate::logging::BusEntry; - - fn audit_record(sequence: u64) -> AuditReplayRecord { - AuditReplayRecord { - entry: BusEntry { - payload: serde_json::json!({ - "kind": "audit", - "entry_id": "test-entry-id", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "runtime", - "code": "startup_complete", - "severity": "info", - }) - .to_string(), - channel_hint: 2, - }, - sequence, - cursor: sequence, - } - } - - #[test] - fn audit_entry_frame_shape_and_fields() { - let record = audit_record(7); - let frame = audit_entry_frame(&record).expect("audit entry frame"); - - assert!(frame.contains("event: audit_entry")); - assert!(frame.contains("id: a1:7")); - assert!(frame.contains("\"entryId\":\"test-entry-id\"")); - assert!(frame.contains("\"occurredAt\":\"2026-01-01T00:00:00Z\"")); - assert!(frame.contains("\"source\":\"runtime\"")); - assert!(frame.contains("\"code\":\"startup_complete\"")); - assert!(frame.contains("\"severity\":\"info\"")); - assert!(frame.contains("\"sequence\":7")); - assert!(!frame.contains("canonical_envelope")); - assert!(!frame.contains("detail_json")); - assert!(frame.len() <= MAX_FRAME_BYTES); - } - - #[test] - fn audit_entry_frame_omits_severity_when_none() { - let mut record = audit_record(3); - record.entry.payload = serde_json::json!({ - "kind": "audit", - "entry_id": "id-3", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "cli", - "code": "command_executed", - }) - .to_string(); - let frame = audit_entry_frame(&record).expect("audit entry without severity"); - assert!(!frame.contains("severity")); - } - - #[test] - fn audit_entry_frame_filters_invalid_typed_context_fields() { - let mut record = audit_record(4); - record.entry.payload = serde_json::json!({ - "kind": "audit", - "entry_id": "id-4", - "occurred_at": "2026-01-01T00:00:00Z", - "source": "runtime", - "code": "startup_complete", - "context_version": 1, - "subject_kind": "not_a_subject", - "reason_code": "NOT VALID", - "outcome": "completed", - "numeric_summaries": { - "bad-key": 0, - "metric_0": 0, - "metric_1": 1, - "metric_2": 2, - "metric_3": 3, - "metric_4": 4, - "metric_5": 5, - "metric_6": 6, - "metric_7": 7, - "metric_8": 8 - } - }) - .to_string(); - let frame = audit_entry_frame(&record).expect("audit context frame"); - let data = frame - .lines() - .find_map(|line| line.strip_prefix("data: ")) - .and_then(|line| serde_json::from_str::(line).ok()) - .expect("audit frame data"); - - assert!(data.get("subjectKind").is_none()); - assert!(data.get("reasonCode").is_none()); - assert_eq!(data["outcome"], "completed"); - assert_eq!(data["numericSummaries"].as_object().unwrap().len(), 8); - assert!(data["numericSummaries"].get("metric_7").is_some()); - assert!(data["numericSummaries"].get("metric_8").is_none()); - assert!(data["numericSummaries"].get("bad-key").is_none()); - } - - #[test] - fn audit_entry_frame_rejects_malformed_payload() { - let mut record = audit_record(1); - record.entry.payload = "not-json".to_string(); - assert!(audit_entry_frame(&record).is_err()); - } #[test] fn audit_gap_frame_carries_audit_endpoint() { diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs new file mode 100644 index 0000000000..8e88d14a84 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry.rs @@ -0,0 +1,233 @@ +use mesh_llm_events::CliCommandSummary; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntrySeverity}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::net::SocketAddr; + +use super::super::query::AuditCursor; +use crate::api::routes::logs::dto::safe_metadata; +use crate::logging::{AuditReplayRecord, OperationalAuditContext}; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct AuditEntryData { + entry_id: String, + occurred_at: String, + source: String, + code: String, + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + context_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + subject_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + subject_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + remote_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path_type: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + operation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reason_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + outcome: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + numeric_summaries: BTreeMap, + sequence: u64, +} + +#[derive(Default)] +struct AuditPathProjection { + remote_addr: Option, + path_type: Option<&'static str>, +} + +fn audit_path_projection( + context_version: Option, + path_type: Option<&str>, + remote_addr: Option<&str>, +) -> AuditPathProjection { + if context_version != Some(1) { + return AuditPathProjection::default(); + } + match path_type { + Some("direct") => AuditPathProjection { + remote_addr: remote_addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + path_type: Some("direct"), + }, + Some("relay") => AuditPathProjection { + remote_addr: None, + path_type: Some("relay"), + }, + Some(_) | None => AuditPathProjection::default(), + } +} + +/// Audit entry frame: privacy-safe projection of an audit replay record. +/// Never contains `canonical_envelope` or arbitrary `detail_json`. +pub(in super::super) fn audit_entry_frame(record: &AuditReplayRecord) -> Result { + let payload: serde_json::Value = serde_json::from_str(&record.entry.payload).map_err(|_| ())?; + let entry_id = payload + .get("entry_id") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let occurred_at = payload + .get("occurred_at") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let source = payload + .get("source") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let code = payload + .get("code") + .and_then(|v| v.as_str()) + .ok_or(())? + .to_owned(); + let severity = payload + .get("severity") + .and_then(|v| v.as_str()) + .map(str::to_owned); + let context_version = payload + .get("context_version") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .filter(|value| *value == 1); + let context_string = |key: &str| { + context_version.and_then(|_| { + payload + .get(key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty() && value.chars().count() <= 256) + .map(str::to_owned) + }) + }; + let context_code = |key: &str| { + context_version.and_then(|_| { + payload + .get(key) + .and_then(serde_json::Value::as_str) + .filter(|value| OperationalAuditContext::valid_static_code(value)) + .map(str::to_owned) + }) + }; + let subject_kind = context_version.and_then(|_| { + payload + .get("subject_kind") + .and_then(serde_json::Value::as_str) + .and_then(crate::logging::OperationalAuditSubjectKind::parse) + .map(|kind| kind.as_str().to_owned()) + }); + let numeric_summaries = context_version.map_or_else(BTreeMap::new, |_| { + payload + .get("numeric_summaries") + .and_then(serde_json::Value::as_object) + .into_iter() + .flatten() + .filter(|(key, _)| OperationalAuditContext::valid_static_code(key)) + .filter_map(|(key, value)| value.as_u64().map(|value| (key.clone(), value))) + .take(8) + .collect() + }); + let path = audit_path_projection( + context_version, + payload.get("path_type").and_then(serde_json::Value::as_str), + payload + .get("remote_addr") + .and_then(serde_json::Value::as_str), + ); + + let data = AuditEntryData { + entry_id, + occurred_at, + source, + code, + severity, + context_version, + subject_kind, + subject_id: context_string("subject_id"), + remote_addr: path.remote_addr, + path_type: path.path_type, + operation_id: context_string("operation_id"), + request_id: context_string("request_id"), + reason_code: context_code("reason_code"), + outcome: context_code("outcome"), + command_summary: context_version.and_then(|_| { + payload + .get("command_summary") + .and_then(serde_json::Value::as_str) + .and_then(CliCommandSummary::sanitize) + .map(|summary| summary.as_str().to_owned()) + }), + duration_ms: context_version.and_then(|_| { + payload + .get("duration_ms") + .and_then(serde_json::Value::as_u64) + }), + numeric_summaries, + sequence: record.sequence, + }; + super::frame( + "audit_entry", + &AuditCursor(record.sequence).event_id(), + &data, + ) +} + +/// Render a row read back from the durable store. This is the production audit +/// reconciliation path and deliberately uses the database sequence as the SSE +/// cursor so entries written by another process share the same ordering. +pub(in super::super) fn durable_audit_entry_frame(detail: AuditEntryDetail) -> Result { + let record = detail.entry; + let sequence = u64::try_from(record.sequence).map_err(|_| ())?; + let path = audit_path_projection( + record.context_version, + detail.path_type.as_deref(), + detail.remote_addr.as_deref(), + ); + let severity = record.severity.map(|severity| match severity { + AuditEntrySeverity::Info => "info".to_string(), + AuditEntrySeverity::Warning => "warning".to_string(), + AuditEntrySeverity::Error => "error".to_string(), + }); + let data = AuditEntryData { + entry_id: record.entry_id, + occurred_at: record.occurred_at, + source: record.source, + code: record.code, + severity, + context_version: record.context_version, + subject_kind: record.subject_kind, + subject_id: record.subject_id.as_deref().map(safe_metadata), + remote_addr: path.remote_addr, + path_type: path.path_type, + operation_id: record.operation_id.as_deref().map(safe_metadata), + request_id: record.correlation_request_id.as_deref().map(safe_metadata), + reason_code: record.reason_code, + outcome: record.outcome, + command_summary: detail + .command_summary + .and_then(|summary| CliCommandSummary::sanitize(&summary)) + .map(|summary| summary.as_str().to_owned()), + duration_ms: record.duration_ms, + numeric_summaries: record.numeric_summaries, + sequence, + }; + super::frame("audit_entry", &AuditCursor(sequence).event_id(), &data) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs new file mode 100644 index 0000000000..3928f39feb --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/protocol/audit_entry/tests.rs @@ -0,0 +1,342 @@ +use super::*; +use crate::logging::{AuditReplayRecord, BusEntry}; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntryFilters, LogStore, RealClock}; +use std::sync::Arc; + +use super::super::MAX_FRAME_BYTES; + +fn audit_record(sequence: u64) -> AuditReplayRecord { + AuditReplayRecord { + entry: BusEntry { + payload: serde_json::json!({ + "kind": "audit", + "entry_id": "test-entry-id", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "runtime", + "code": "startup_complete", + "severity": "info", + }) + .to_string(), + channel_hint: 2, + }, + sequence, + cursor: sequence, + } +} + +fn durable_audit_detail(entry_id: &str, detail: serde_json::Value) -> AuditEntryDetail { + let root = tempfile::tempdir().expect("temporary durable audit store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit store"); + store + .insert_audit_entry( + entry_id, + None, + "2026-01-01T00:00:00Z", + "cli", + "command_completed", + Some(&detail.to_string()), + ) + .expect("insert durable audit entry"); + store + .list_audit_entry_details_after_sequence(0, 1, AuditEntryFilters::default()) + .expect("query durable audit entry") + .into_iter() + .next() + .expect("durable audit entry") +} + +fn frame_data(frame: &str) -> serde_json::Value { + frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .and_then(|line| serde_json::from_str::(line).ok()) + .expect("audit frame data") +} + +#[test] +fn audit_entry_frame_shape_and_fields() { + let record = audit_record(7); + let frame = audit_entry_frame(&record).expect("audit entry frame"); + + assert!(frame.contains("event: audit_entry")); + assert!(frame.contains("id: a1:7")); + assert!(frame.contains("\"entryId\":\"test-entry-id\"")); + assert!(frame.contains("\"occurredAt\":\"2026-01-01T00:00:00Z\"")); + assert!(frame.contains("\"source\":\"runtime\"")); + assert!(frame.contains("\"code\":\"startup_complete\"")); + assert!(frame.contains("\"severity\":\"info\"")); + assert!(frame.contains("\"sequence\":7")); + assert!(!frame.contains("canonical_envelope")); + assert!(!frame.contains("detail_json")); + assert!(frame.len() <= MAX_FRAME_BYTES); +} + +#[test] +fn audit_entry_frame_omits_severity_when_none() { + let mut record = audit_record(3); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-3", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_executed", + }) + .to_string(); + let frame = audit_entry_frame(&record).expect("audit entry without severity"); + assert!(!frame.contains("severity")); +} + +#[test] +fn audit_entry_frame_filters_invalid_typed_context_fields() { + let mut record = audit_record(4); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-4", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "runtime", + "code": "startup_complete", + "context_version": 1, + "subject_kind": "not_a_subject", + "reason_code": "NOT VALID", + "outcome": "completed", + "numeric_summaries": { + "bad-key": 0, + "metric_0": 0, + "metric_1": 1, + "metric_2": 2, + "metric_3": 3, + "metric_4": 4, + "metric_5": 5, + "metric_6": 6, + "metric_7": 7, + "metric_8": 8 + } + }) + .to_string(); + let frame = audit_entry_frame(&record).expect("audit context frame"); + let data = frame_data(&frame); + + assert!(data.get("subjectKind").is_none()); + assert!(data.get("reasonCode").is_none()); + assert_eq!(data["outcome"], "completed"); + assert_eq!(data["numericSummaries"].as_object().unwrap().len(), 8); + assert!(data["numericSummaries"].get("metric_7").is_some()); + assert!(data["numericSummaries"].get("metric_8").is_none()); + assert!(data["numericSummaries"].get("bad-key").is_none()); +} + +#[test] +fn live_audit_path_projection_is_privacy_filtered() { + let cases = [ + ( + 1, + "direct", + Some("[2001:0db8:0:0:0:0:0:1]:443"), + Some("direct"), + Some("[2001:db8::1]:443"), + ), + ( + 1, + "relay", + Some("private-relay.example:443"), + Some("relay"), + None, + ), + ( + 1, + "direct", + Some("private-hostname:443"), + Some("direct"), + None, + ), + (1, "unknown", Some("192.0.2.1:443"), None, None), + (2, "direct", Some("192.0.2.1:443"), None, None), + ]; + + for (context_version, path_type, remote_addr, expected_path, expected_addr) in cases { + let mut record = audit_record(20); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "mesh-path", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "mesh", + "code": "peer_path_observed", + "context_version": context_version, + "path_type": path_type, + "remote_addr": remote_addr, + "canonical_envelope": "must-not-leak", + "detail_json": "must-not-leak" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit path frame"); + let data = frame_data(&frame); + + assert_eq!( + data.get("pathType").and_then(|value| value.as_str()), + expected_path + ); + assert_eq!( + data.get("remoteAddr").and_then(|value| value.as_str()), + expected_addr + ); + assert!(data.get("canonical_envelope").is_none()); + assert!(data.get("detail_json").is_none()); + } +} + +#[test] +fn audit_entry_frame_rejects_malformed_payload() { + let mut record = audit_record(1); + record.entry.payload = "not-json".to_string(); + assert!(audit_entry_frame(&record).is_err()); +} + +#[test] +fn live_audit_frame_drops_malformed_command_summary() { + let mut record = audit_record(8); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-8", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm gpus --draft run-benchmark --backend cuda" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_drops_duplicate_command_summary_flags() { + let mut record = audit_record(13); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-13", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm models list --json --json" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_drops_deep_malformed_command_summary() { + let mut record = audit_record(11); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-11", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm load unload status discover rotate-key setup --port 1234" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn live_audit_frame_preserves_valid_command_summary() { + let mut record = audit_record(10); + record.entry.payload = serde_json::json!({ + "kind": "audit", + "entry_id": "id-10", + "occurred_at": "2026-01-01T00:00:00Z", + "source": "cli", + "code": "command_completed", + "context_version": 1, + "command_summary": "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + }) + .to_string(); + + let frame = audit_entry_frame(&record).expect("live audit frame"); + assert!(frame.contains( + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + )); +} + +#[test] +fn durable_audit_frame_drops_malformed_command_summary() { + let record = durable_audit_detail( + "id-9", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm gpus --draft run-benchmark --backend cuda", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn durable_audit_frame_drops_deep_malformed_command_summary() { + let record = durable_audit_detail( + "id-12", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm load unload status discover rotate-key setup --port 1234", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(!frame.contains("commandSummary")); +} + +#[test] +fn durable_audit_frame_preserves_valid_command_summary() { + let record = durable_audit_detail( + "id-11", + serde_json::json!({ + "context_version": 1, + "command_summary": "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + assert!(frame.contains( + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + )); +} + +#[test] +fn durable_audit_frame_redacts_unsafe_rest_parity_metadata() { + let record = durable_audit_detail( + "id-14", + serde_json::json!({ + "context_version": 1, + "subject_id": "https://alice:subject-secret@example.test/model?api_key=subject-query", + "operation_id": "/Users/alice/private-operation", + "request_id": "request-7?token=request-secret", + }), + ); + + let frame = durable_audit_entry_frame(record).expect("durable audit frame"); + let data = frame_data(&frame); + + assert_eq!(data["subjectId"], "[REDACTED]"); + assert_eq!(data["operationId"], "[REDACTED]"); + assert_eq!(data["requestId"], "[REDACTED]"); + + let serialized = data.to_string(); + for unsafe_value in [ + "alice", + "subject-secret", + "subject-query", + "/Users/alice/private-operation", + "request-secret", + ] { + assert!(!serialized.contains(unsafe_value)); + } +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs index 5345c22771..69c68559cf 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session.rs @@ -118,11 +118,11 @@ impl ReplaySession { pub(super) fn durable_audit_frames( &mut self, - records: Vec, + records: Vec, ) -> Vec { let mut frames = Vec::with_capacity(records.len()); for record in records { - let Ok(sequence) = u64::try_from(record.sequence) else { + let Ok(sequence) = u64::try_from(record.entry.sequence) else { continue; }; if sequence <= self.audit_cursor.sequence() { @@ -512,6 +512,10 @@ impl ConnectionQueue { } impl ConnectionReceiver { + pub(super) fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + pub(super) async fn recv(&mut self) -> Option { if self.cancelled.load(Ordering::Acquire) { return None; diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs index 69b768bdea..2e824a0e3f 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests.rs @@ -4,12 +4,16 @@ use mesh_llm_events::logging::events::LifecycleEvent; use mesh_llm_events::logging::identifiers::{EventId, RequestId}; use mesh_llm_events::logging::replay::ReplaySequence; +use super::super::protocol::MAX_FRAME_BYTES; use super::*; use crate::logging::{ - LoggingService, ReplayBus, RequestSummaryEntry, RequestSummaryEventSnapshots, + CallerPathType, LoggingService, ReplayBus, RequestSummaryEntry, RequestSummaryEventSnapshots, RequestSummaryMetadata, TerminalOutcome, }; +mod audit; +mod queue; + fn summary(created_at: &str, state: &str, metadata: RequestSummaryMetadata) -> RequestSummaryEntry { RequestSummaryEntry { request_id: "test-request-summary".into(), @@ -601,295 +605,203 @@ fn terminal_replay_uses_enriched_summary_metadata_for_filters() { } #[test] -fn heartbeat_is_an_sse_comment() { - assert_eq!(super::super::heartbeat_frame(), ": keepalive\n\n"); -} - -#[tokio::test] -async fn queue_bounds_slow_consumers_and_cancellation() { - let (queue, mut receiver) = ConnectionQueue::new(1); - queue.try_send("first".into()).expect("first fits"); - assert_eq!( - queue.try_send("second".into()), - Err(QueueError::SlowConsumer) - ); - assert_eq!(receiver.recv().await.as_deref(), Some("first")); - queue.cancel(); - assert_eq!( - queue.try_send("after-cancel".into()), - Err(QueueError::Cancelled) - ); - assert!(receiver.recv().await.is_none()); -} - -#[tokio::test] -async fn queue_times_out_a_slow_socket_writer_without_unbounded_growth() { - let (queue, _receiver) = ConnectionQueue::new(1); - queue.try_send("first".into()).unwrap(); - assert_eq!( - queue - .send_with_timeout("second".into(), std::time::Duration::from_millis(5)) - .await, - Err(QueueError::SlowConsumer) - ); -} - -fn audit_subscription(cursor: AuditCursor) -> Subscription { - Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(AuditSelection { - cursor, - source: None, - severity: None, - }), - } -} - -fn push_audit(bus: &ReplayBus, payload: &str) { - bus.push_audit_replay(payload.to_string(), 2); -} - -fn audit_payload( - entry_id: &str, - occurred_at: &str, - source: &str, - code: &str, - severity: Option<&str>, -) -> String { - let mut obj = serde_json::json!({ - "kind": "audit", - "entry_id": entry_id, - "occurred_at": occurred_at, - "source": source, - "code": code, - }); - if let Some(s) = severity { - obj.as_object_mut() - .unwrap() - .insert("severity".into(), serde_json::json!(s)); - } - obj.to_string() -} - -#[test] -fn audit_replay_emits_ordered_frames_with_a1_ids() { +fn lifecycle_replay_projects_canonical_caller_attribution() { let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload( - "id-1", - "2026-01-01T00:00:01Z", - "runtime", - "startup", - Some("info"), + let cases = [ + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.71:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.71:11204"), + Some("remote_quic_http"), ), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "mesh", "peer_joined", None), - ); - push_audit( - &bus, - &audit_payload( - "id-3", - "2026-01-01T00:00:03Z", - "cli", - "command", - Some("warning"), + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"), + Some("192.0.2.72:11204"), + Some(CallerPathType::Relay), + ), + Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"), + None, + Some("relay"), ), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); - assert_eq!(frames.len(), 3); - assert!(frames[0].contains("event: audit_entry")); - assert!(frames[0].contains("id: a1:1")); - assert!(frames[1].contains("id: a1:2")); - assert!(frames[2].contains("id: a1:3")); - assert!(frames[0].contains("\"entryId\":\"id-1\"")); - assert!(frames[1].contains("\"entryId\":\"id-2\"")); - assert!(frames[2].contains("\"entryId\":\"id-3\"")); -} - -#[test] -fn audit_live_updates_are_incremental_and_advance_filtered_records() { - let bus = ReplayBus::new(4); - let mut selected = audit_subscription(AuditCursor(0)); - selected.audit.as_mut().unwrap().source = Some("runtime".into()); - let mut session = ReplaySession::new(selected); - let mut updates = bus.subscribe_updates(); - - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "ignored", None), - ); - let filtered = updates.try_recv().expect("filtered audit update"); - assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); - assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); - - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "selected", None), - ); - let update = updates.try_recv().expect("selected audit update"); - let frames = session.next_update_frames(&bus, &update, None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id: a1:2")); - assert!(frames[0].contains("\"entryId\":\"id-2\"")); -} - -#[test] -fn audit_replay_dedupes_records_at_or_below_cursor() { - let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), - ); - push_audit( - &bus, - &audit_payload("id-3", "2026-01-01T00:00:03Z", "runtime", "c", None), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(2)), None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id: a1:3")); - assert!(frames[0].contains("id-3")); -} - -#[test] -fn audit_replay_filters_by_source() { - let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "a", None), - ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), - ); - push_audit( - &bus, - &audit_payload("id-3", "2026-01-01T00:00:03Z", "mesh", "c", None), - ); - - let sel = AuditSelection { - cursor: AuditCursor(0), - source: Some("mesh".to_owned()), - severity: None, - }; - let sub = Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(sel.clone()), - }; - let frames = replay_frames(&bus, &sub, None); - assert_eq!(frames.len(), 2); - assert!(frames[0].contains("id-1")); - assert!(frames[1].contains("id-3")); - assert!(!frames[0].contains("id-2")); + ( + RequestSummaryMetadata::from_parts(None, None, None, None).with_caller_identity( + Some("fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"), + None, + None, + ), + Some("fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"), + None, + None, + ), + (RequestSummaryMetadata::default(), None, None, None), + ]; + + for (sequence, (metadata, endpoint_id, caller_addr, path_type)) in cases.into_iter().enumerate() + { + entry_with_metadata( + &bus, + ReplayChannel::Requests, + sequence as u64 + 1, + RequestId::new(), + metadata, + ); + let frame = replay_frames( + &bus, + &subscription(vec![ReplayChannel::Requests], Cursor::default()), + None, + ) + .pop() + .expect("caller attribution replay frame"); + let data = frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .and_then(|line| serde_json::from_str::(line).ok()) + .expect("lifecycle replay data"); + let request = &data["request"]; + + assert_eq!( + request + .get("callerEndpointId") + .and_then(|value| value.as_str()), + endpoint_id + ); + assert!(endpoint_id.is_some() || request.get("callerEndpointId").is_none()); + assert_eq!( + request.get("callerAddr").and_then(|value| value.as_str()), + caller_addr + ); + assert!(caller_addr.is_some() || request.get("callerAddr").is_none()); + assert_eq!( + request + .get("callerPathType") + .and_then(|value| value.as_str()), + path_type + ); + assert!(path_type.is_some() || request.get("callerPathType").is_none()); + } } #[test] -fn audit_replay_filters_by_severity() { +fn lifecycle_replay_sanitizes_deserialized_metadata_at_the_sse_boundary() { let bus = ReplayBus::new(4); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", Some("info")), - ); - push_audit( - &bus, - &audit_payload( - "id-2", - "2026-01-01T00:00:02Z", - "runtime", - "b", - Some("warning"), - ), + let request_id = RequestId::new(); + let sequence = 1; + let occurred_at = "2026-08-03T00:00:01Z"; + let envelope = CanonicalEnvelope::new( + EventId::new(), + request_id, + ReplayChannel::Requests, + sequence, + occurred_at.into(), + LifecycleEvent::Admitted { + model: None, + method: None, + }, ); - push_audit( - &bus, - &audit_payload( - "id-3", - "2026-01-01T00:00:03Z", - "runtime", - "c", - Some("error"), - ), + let unsafe_route = "/private/route?token=route-secret"; + let unsafe_model = "https://model:secret@example.test/private"; + let unsafe_provider = "Bearer provider-secret"; + let unsafe_engine = r"C:\Users\operator\private-engine"; + let unsafe_endpoint = "https://operator:credential@example.test/private?token=secret"; + let unsafe_addr = "/Users/operator/.ssh/id_ed25519"; + let unsafe_path_type = "/private/caller-path"; + bus.push_replay( + serde_json::json!({ + "canonical_envelope": envelope, + "request_summary_snapshots": { + "after": { + "created_at": occurred_at, + "state": "active", + "terminal_at": null, + "metadata": { + "route": unsafe_route, + "model": unsafe_model, + "provider": unsafe_provider, + "engine": unsafe_engine, + "caller_endpoint_id": unsafe_endpoint, + "caller_addr": unsafe_addr, + "caller_path_type": "remote_quic_http" + } + } + } + }) + .to_string(), + 0, + ReplaySequence::next(ReplayChannel::Requests, sequence), ); - - let sel = AuditSelection { - cursor: AuditCursor(0), - source: None, - severity: Some("warning".to_owned()), - }; - let sub = Subscription { - channels: Vec::new(), - filters: Default::default(), - cursor: Cursor::default(), - audit: Some(sel), - }; - let frames = replay_frames(&bus, &sub, None); - assert_eq!(frames.len(), 1); - assert!(frames[0].contains("id-2")); -} - -#[test] -fn audit_gap_emits_recovery_endpoint() { - let bus = ReplayBus::new(1); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + let invalid_path_envelope = CanonicalEnvelope::new( + EventId::new(), + RequestId::new(), + ReplayChannel::Requests, + 2, + "2026-08-03T00:00:02Z".into(), + LifecycleEvent::Admitted { + model: None, + method: None, + }, ); - push_audit( - &bus, - &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + bus.push_replay( + serde_json::json!({ + "canonical_envelope": invalid_path_envelope, + "request_summary_snapshots": { + "after": { + "created_at": "2026-08-03T00:00:02Z", + "state": "active", + "terminal_at": null, + "metadata": { + "caller_path_type": unsafe_path_type + } + } + } + }) + .to_string(), + 0, + ReplaySequence::next(ReplayChannel::Requests, 2), ); let frames = replay_frames( &bus, - &audit_subscription(AuditCursor(0)), - Some("a1:2".to_owned()), - ); - assert!(frames[0].contains("event: replay_gap")); - assert!(frames[0].contains("/api/logs/audit")); - assert!(frames[0].contains("id: a1:1")); - assert!(frames[1].contains("event: audit_entry")); - assert!(frames[1].contains("id: a1:2")); -} - -#[test] -fn audit_frames_never_contain_lifecycle_fields() { - let bus = ReplayBus::new(3); - push_audit( - &bus, - &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), - ); - - let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); - assert_eq!(frames.len(), 1); - assert!(!frames[0].contains("canonical_envelope")); - assert!(!frames[0].contains("detail_json")); - assert!(!frames[0].contains("requestId")); + &subscription(vec![ReplayChannel::Requests], Cursor::default()), + None, + ); + let frame = frames + .iter() + .find(|frame| frame.contains("remote_quic_http")) + .expect("sanitized metadata replay frame"); + let data = frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .and_then(|line| serde_json::from_str::(line).ok()) + .expect("lifecycle replay data"); + let request = &data["request"]; + + assert_eq!(request["route"], "[REDACTED]"); + assert_eq!(request["model"], "[REDACTED]"); + assert_eq!(request["provider"], "[REDACTED]"); + assert_eq!(request["engine"], "[REDACTED]"); + assert_eq!(request["callerEndpointId"], "[REDACTED]"); + assert_eq!(request["callerAddr"], "[REDACTED]"); + assert_eq!(request["callerPathType"], "remote_quic_http"); + let emitted = frames.join("\n"); + for unsafe_value in [ + unsafe_route, + unsafe_model, + unsafe_provider, + unsafe_engine, + unsafe_endpoint, + unsafe_addr, + unsafe_path_type, + ] { + assert!(!emitted.contains(unsafe_value)); + } } #[test] -fn lifecycle_gap_regression_still_uses_requests_endpoint() { - let bus = ReplayBus::new(1); - entry(&bus, ReplayChannel::Requests, 1, RequestId::new()); - entry(&bus, ReplayChannel::Requests, 2, RequestId::new()); - - let frames = replay_frames( - &bus, - &subscription(vec![ReplayChannel::Requests], Cursor::default()), - Some("opaque-rest-cursor".into()), - ); - assert!(frames[0].contains("event: replay_gap")); - assert!(frames[0].contains("/api/logs/requests")); - assert!(!frames[0].contains("/api/logs/audit")); +fn heartbeat_is_an_sse_comment() { + assert_eq!(super::super::heartbeat_frame(), ": keepalive\n\n"); } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs new file mode 100644 index 0000000000..15488ee7c7 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/audit.rs @@ -0,0 +1,378 @@ +use super::*; +use mesh_llm_log_store::{AuditEntryDetail, AuditEntryFilters, LogStore, RealClock}; + +fn durable_path_details() -> Vec { + let root = tempfile::tempdir().expect("temporary durable audit store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit store"); + for (entry_id, subject_id, path_type, remote_addr) in [ + ( + "durable-direct", + "peer-direct", + "direct", + "[2001:0db8:0:0:0:0:0:1]:443", + ), + ( + "durable-relay", + "peer-relay", + "relay", + "private-relay.example:443", + ), + ] { + let detail = serde_json::json!({ + "context_version": 1, + "subject_kind": "mesh_peer", + "subject_id": subject_id, + "path_type": path_type, + "remote_addr": remote_addr, + }); + store + .insert_audit_entry( + entry_id, + None, + "2026-01-01T00:00:01Z", + "mesh", + "peer_path_observed", + Some(&detail.to_string()), + ) + .expect("insert durable audit entry"); + } + store + .list_audit_entry_details_after_sequence(0, 2, AuditEntryFilters::default()) + .expect("query durable audit entries") +} + +fn audit_subscription(cursor: AuditCursor) -> Subscription { + Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(AuditSelection { + cursor, + source: None, + severity: None, + }), + } +} + +fn push_audit(bus: &ReplayBus, payload: &str) { + bus.push_audit_replay(payload.to_string(), 2); +} + +fn audit_payload( + entry_id: &str, + occurred_at: &str, + source: &str, + code: &str, + severity: Option<&str>, +) -> String { + let mut obj = serde_json::json!({ + "kind": "audit", + "entry_id": entry_id, + "occurred_at": occurred_at, + "source": source, + "code": code, + }); + if let Some(s) = severity { + obj.as_object_mut() + .unwrap() + .insert("severity".into(), serde_json::json!(s)); + } + obj.to_string() +} + +#[test] +fn audit_replay_emits_ordered_frames_with_a1_ids() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload( + "id-1", + "2026-01-01T00:00:01Z", + "runtime", + "startup", + Some("info"), + ), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "mesh", "peer_joined", None), + ); + push_audit( + &bus, + &audit_payload( + "id-3", + "2026-01-01T00:00:03Z", + "cli", + "command", + Some("warning"), + ), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); + assert_eq!(frames.len(), 3); + assert!(frames[0].contains("event: audit_entry")); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[2].contains("id: a1:3")); + assert!(frames[0].contains("\"entryId\":\"id-1\"")); + assert!(frames[1].contains("\"entryId\":\"id-2\"")); + assert!(frames[2].contains("\"entryId\":\"id-3\"")); +} + +#[test] +fn audit_replay_session_projects_direct_and_relay_paths_without_raw_fields() { + let bus = ReplayBus::new(4); + for (entry_id, path_type, remote_addr) in [ + ("direct-entry", "direct", "192.0.2.44:11204"), + ("relay-entry", "relay", "private-relay.example:443"), + ] { + push_audit( + &bus, + &serde_json::json!({ + "kind": "audit", + "entry_id": entry_id, + "occurred_at": "2026-01-01T00:00:01Z", + "source": "mesh", + "code": "peer_path_observed", + "context_version": 1, + "path_type": path_type, + "remote_addr": remote_addr, + "canonical_envelope": "must-not-leak", + "detail_json": "must-not-leak" + }) + .to_string(), + ); + } + let mut session = ReplaySession::new(audit_subscription(AuditCursor(0))); + + let frames = session.next_frames(&bus, None); + + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[0].contains("\"pathType\":\"direct\"")); + assert!(frames[0].contains("\"remoteAddr\":\"192.0.2.44:11204\"")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[1].contains("\"pathType\":\"relay\"")); + assert!(!frames[1].contains("remoteAddr")); + assert!(!frames[1].contains("private-relay.example")); + assert!( + frames + .iter() + .all(|frame| !frame.contains("canonical_envelope")) + ); + assert!(frames.iter().all(|frame| !frame.contains("detail_json"))); + assert!(frames.iter().all(|frame| !frame.contains("must-not-leak"))); + assert!(frames.iter().all(|frame| frame.len() <= MAX_FRAME_BYTES)); +} + +#[test] +fn durable_audit_reconciliation_projects_paths_and_advances_cursor() { + let records = durable_path_details(); + let direct = records[0].clone(); + let relay = records[1].clone(); + let mut session = ReplaySession::new(audit_subscription(AuditCursor(0))); + + let frames = session.durable_audit_frames(vec![direct.clone(), relay.clone()]); + + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[0].contains("\"pathType\":\"direct\"")); + assert!(frames[0].contains("\"remoteAddr\":\"[2001:db8::1]:443\"")); + assert!(frames[1].contains("id: a1:2")); + assert!(frames[1].contains("\"pathType\":\"relay\"")); + assert!(!frames[1].contains("remoteAddr")); + assert!(!frames[1].contains("private-relay.example")); + assert!( + frames + .iter() + .all(|frame| !frame.contains("canonical_envelope")) + ); + assert!(frames.iter().all(|frame| !frame.contains("detail_json"))); + assert!(session.durable_audit_frames(vec![direct, relay]).is_empty()); + assert_eq!( + session.durable_audit_query().map(|(cursor, _)| cursor), + Some(2) + ); +} + +#[test] +fn audit_live_updates_are_incremental_and_advance_filtered_records() { + let bus = ReplayBus::new(4); + let mut selected = audit_subscription(AuditCursor(0)); + selected.audit.as_mut().unwrap().source = Some("runtime".into()); + let mut session = ReplaySession::new(selected); + let mut updates = bus.subscribe_updates(); + + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "ignored", None), + ); + let filtered = updates.try_recv().expect("filtered audit update"); + assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); + assert!(session.next_update_frames(&bus, &filtered, None).is_empty()); + + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "selected", None), + ); + let update = updates.try_recv().expect("selected audit update"); + let frames = session.next_update_frames(&bus, &update, None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id: a1:2")); + assert!(frames[0].contains("\"entryId\":\"id-2\"")); +} + +#[test] +fn audit_replay_dedupes_records_at_or_below_cursor() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + push_audit( + &bus, + &audit_payload("id-3", "2026-01-01T00:00:03Z", "runtime", "c", None), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(2)), None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id: a1:3")); + assert!(frames[0].contains("id-3")); +} + +#[test] +fn audit_replay_filters_by_source() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "mesh", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + push_audit( + &bus, + &audit_payload("id-3", "2026-01-01T00:00:03Z", "mesh", "c", None), + ); + + let sel = AuditSelection { + cursor: AuditCursor(0), + source: Some("mesh".to_owned()), + severity: None, + }; + let sub = Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(sel.clone()), + }; + let frames = replay_frames(&bus, &sub, None); + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("id-1")); + assert!(frames[1].contains("id-3")); + assert!(!frames[0].contains("id-2")); +} + +#[test] +fn audit_replay_filters_by_severity() { + let bus = ReplayBus::new(4); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", Some("info")), + ); + push_audit( + &bus, + &audit_payload( + "id-2", + "2026-01-01T00:00:02Z", + "runtime", + "b", + Some("warning"), + ), + ); + push_audit( + &bus, + &audit_payload( + "id-3", + "2026-01-01T00:00:03Z", + "runtime", + "c", + Some("error"), + ), + ); + + let sel = AuditSelection { + cursor: AuditCursor(0), + source: None, + severity: Some("warning".to_owned()), + }; + let sub = Subscription { + channels: Vec::new(), + filters: Default::default(), + cursor: Cursor::default(), + audit: Some(sel), + }; + let frames = replay_frames(&bus, &sub, None); + assert_eq!(frames.len(), 1); + assert!(frames[0].contains("id-2")); +} + +#[test] +fn audit_gap_emits_recovery_endpoint() { + let bus = ReplayBus::new(1); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + push_audit( + &bus, + &audit_payload("id-2", "2026-01-01T00:00:02Z", "runtime", "b", None), + ); + + let frames = replay_frames( + &bus, + &audit_subscription(AuditCursor(0)), + Some("a1:2".to_owned()), + ); + assert!(frames[0].contains("event: replay_gap")); + assert!(frames[0].contains("/api/logs/audit")); + assert!(frames[0].contains("id: a1:1")); + assert!(frames[1].contains("event: audit_entry")); + assert!(frames[1].contains("id: a1:2")); +} + +#[test] +fn audit_frames_never_contain_lifecycle_fields() { + let bus = ReplayBus::new(3); + push_audit( + &bus, + &audit_payload("id-1", "2026-01-01T00:00:01Z", "runtime", "a", None), + ); + + let frames = replay_frames(&bus, &audit_subscription(AuditCursor(0)), None); + assert_eq!(frames.len(), 1); + assert!(!frames[0].contains("canonical_envelope")); + assert!(!frames[0].contains("detail_json")); + assert!(!frames[0].contains("requestId")); +} + +#[test] +fn lifecycle_gap_regression_still_uses_requests_endpoint() { + let bus = ReplayBus::new(1); + entry(&bus, ReplayChannel::Requests, 1, RequestId::new()); + entry(&bus, ReplayChannel::Requests, 2, RequestId::new()); + + let frames = replay_frames( + &bus, + &subscription(vec![ReplayChannel::Requests], Cursor::default()), + Some("opaque-rest-cursor".into()), + ); + assert!(frames[0].contains("event: replay_gap")); + assert!(frames[0].contains("/api/logs/requests")); + assert!(!frames[0].contains("/api/logs/audit")); +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs new file mode 100644 index 0000000000..2ba10e6ef0 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/session/tests/queue.rs @@ -0,0 +1,30 @@ +use super::*; + +#[tokio::test] +async fn queue_bounds_slow_consumers_and_cancellation() { + let (queue, mut receiver) = ConnectionQueue::new(1); + queue.try_send("first".into()).expect("first fits"); + assert_eq!( + queue.try_send("second".into()), + Err(QueueError::SlowConsumer) + ); + assert_eq!(receiver.recv().await.as_deref(), Some("first")); + queue.cancel(); + assert_eq!( + queue.try_send("after-cancel".into()), + Err(QueueError::Cancelled) + ); + assert!(receiver.recv().await.is_none()); +} + +#[tokio::test] +async fn queue_times_out_a_slow_socket_writer_without_unbounded_growth() { + let (queue, _receiver) = ConnectionQueue::new(1); + queue.try_send("first".into()).unwrap(); + assert_eq!( + queue + .send_with_timeout("second".into(), std::time::Duration::from_millis(5)) + .await, + Err(QueueError::SlowConsumer) + ); +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs index fb4140504f..9b14e4a5fd 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/stream.rs @@ -59,14 +59,14 @@ async fn run( query_facade, subscription, recovery_cursor, - queue.clone(), + queue, updates, )); while let Some(frame) = receiver.recv().await { let write = tokio::time::timeout(WRITE_TIMEOUT, stream.write_all(frame.as_bytes())).await; if !matches!(write, Ok(Ok(()))) { - queue.cancel(); + receiver.cancel(); break; } } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs index 4c7c86cb8f..5e53d6e91c 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/export.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; -use mesh_llm_log_store::{ArtifactRecord, EventRecord, LogStoreError, RequestRecord}; +use mesh_llm_log_store::{ArtifactRecord, EventRecord, LogStoreError, RequestRecordWithCaller}; use serde::Serialize; use tokio::net::TcpStream; @@ -90,7 +90,7 @@ fn build_export( let request_ids = page .items .iter() - .map(|record| record.request_id.clone()) + .map(|record| record.request.request_id.clone()) .collect::>(); // Each item may consume at most the complete export row budget. Fetch one // extra child per owner so a partial child window remains detectable while @@ -122,7 +122,7 @@ fn build_export( set_resume_cursor(&mut export); break; } - let request_id = record.request_id.clone(); + let request_id = record.request.request_id.clone(); let built = export_item( ExportItemInput { record, @@ -219,7 +219,7 @@ fn set_resume_cursor(export: &mut ExportDto) { } struct ExportItemInput { - record: RequestRecord, + record: RequestRecordWithCaller, event_records: Vec, artifact_records: Vec, child_row_limit: usize, diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs index 2c683ef1fc..32e0e46237 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/mod.rs @@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet}; pub(crate) use dto::{ArtifactDto, AuditDto, EventDto, PageDto, ProxyDto, RequestDto}; pub(crate) use error::LogsError; use mesh_llm_events::logging::events::LifecycleEvent; -use mesh_llm_log_store::{ArtifactRecord, LogStoreError, QuerySort, RequestRecord}; +use mesh_llm_log_store::{ArtifactRecord, LogStoreError, QuerySort, RequestRecordWithCaller}; use self::dto::artifact_state; use self::parse::SourceFilter; @@ -601,7 +601,7 @@ fn load_active_metadata( active: &[RequestSummaryEntry], parsed: &parse::RequestListQuery, cursor_is_active: bool, -) -> Result, LogsError> { +) -> Result, LogsError> { let request_ids = if parsed.source != Some(SourceFilter::Durable) { active .iter() @@ -622,14 +622,14 @@ fn load_active_metadata( Ok(facade .requests_by_ids(&request_ids)? .into_iter() - .map(|record| (record.request_id.clone(), record)) + .map(|record| (record.request.request_id.clone(), record)) .collect()) } fn validate_cursor_scope( facade: &LoggingQueryFacade, active: &[RequestSummaryEntry], - active_metadata: &HashMap, + active_metadata: &HashMap, parsed: &parse::RequestListQuery, cursor_is_active: bool, ) -> Result<(), LogsError> { @@ -673,7 +673,7 @@ fn collect_durable( active_ids: &HashSet, parsed: &parse::RequestListQuery, cursor_is_active: bool, -) -> Result, LogsError> { +) -> Result, LogsError> { let mut query = parsed.store.clone(); if cursor_is_active { // An active-only cursor is not a durable row, so passing it to SQLite @@ -688,7 +688,9 @@ fn collect_durable( for _ in 0..max_pages { let page = facade.requests(&query)?; for record in page.items { - if !active_ids.contains(&record.request_id) && durable_after_cursor(&record, parsed) { + if !active_ids.contains(&record.request.request_id) + && durable_after_cursor(&record, parsed) + { result.push(record); } } @@ -700,11 +702,17 @@ fn collect_durable( Ok(result) } -fn durable_after_cursor(record: &RequestRecord, parsed: &parse::RequestListQuery) -> bool { +fn durable_after_cursor( + record: &RequestRecordWithCaller, + parsed: &parse::RequestListQuery, +) -> bool { let Some((timestamp, request_id)) = &parsed.cursor_boundary else { return true; }; - let row = (record.created_at.as_str(), record.request_id.as_str()); + let row = ( + record.request.created_at.as_str(), + record.request.request_id.as_str(), + ); let cursor = (timestamp.as_str(), request_id.as_str()); match parsed.store.sort { QuerySort::Ascending => row > cursor, @@ -714,7 +722,7 @@ fn durable_after_cursor(record: &RequestRecord, parsed: &parse::RequestListQuery fn active_matches( entry: &RequestSummaryEntry, - metadata: Option<&RequestRecord>, + metadata: Option<&RequestRecordWithCaller>, parsed: &parse::RequestListQuery, ) -> bool { if !active_matches_filters(entry, metadata, parsed) { @@ -733,16 +741,27 @@ fn active_matches( fn active_matches_filters( entry: &RequestSummaryEntry, - metadata: Option<&RequestRecord>, + metadata: Option<&RequestRecordWithCaller>, parsed: &parse::RequestListQuery, ) -> bool { let query = &parsed.store; - if let Some(route) = &query.route - && entry - .metadata - .route() - .or_else(|| metadata.and_then(|record| record.route.as_deref())) - != Some(route.as_str()) + let route = entry + .metadata + .route() + .or_else(|| metadata.and_then(|record| record.request.route.as_deref())); + if let Some(selected_route) = &query.route + && route != Some(selected_route.as_str()) + { + return false; + } + if query + .exclude_route + .as_deref() + .is_some_and(|excluded| route == Some(excluded)) + || query + .exclude_route_prefix + .as_deref() + .is_some_and(|prefix| route.is_some_and(|route| route.starts_with(prefix))) { return false; } @@ -750,7 +769,7 @@ fn active_matches_filters( && entry .metadata .model() - .or_else(|| metadata.and_then(|record| record.model.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.model.as_deref())) != Some(model.as_str()) { return false; @@ -759,7 +778,7 @@ fn active_matches_filters( && entry .metadata .provider() - .or_else(|| metadata.and_then(|record| record.provider.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.provider.as_deref())) != Some(provider.as_str()) { return false; @@ -768,13 +787,13 @@ fn active_matches_filters( && entry .metadata .engine() - .or_else(|| metadata.and_then(|record| record.engine.as_deref())) + .or_else(|| metadata.and_then(|record| record.request.engine.as_deref())) != Some(engine.as_str()) { return false; } if let Some(status_code) = query.status_code - && metadata.and_then(|record| record.status_code) != Some(i64::from(status_code)) + && metadata.and_then(|record| record.request.status_code) != Some(i64::from(status_code)) { return false; } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs index 8db6f0640e..f7611e3ad8 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/parse.rs @@ -34,6 +34,8 @@ pub(super) fn request_query(path: &str) -> Result { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, @@ -49,6 +51,8 @@ pub(super) fn request_query(path: &str) -> Result { "from" => query.from = Some(timestamp(&value)?), "to" => query.to = Some(timestamp(&value)?), "route" => query.route = Some(filter(value)?), + "exclude_route" => query.exclude_route = Some(filter(value)?), + "exclude_route_prefix" => query.exclude_route_prefix = Some(filter(value)?), "model" => query.model = Some(filter(value)?), "provider" => query.provider = Some(filter(value)?), "engine" => query.engine = Some(filter(value)?), @@ -593,6 +597,31 @@ mod tests { ); } + #[test] + fn parses_and_validates_request_route_exclusions() { + // Given / When + let query = request_query( + "/api/logs/requests?exclude_route=models&exclude_route_prefix=management_", + ) + .expect("parse route exclusions"); + + // Then + assert_eq!(query.store.exclude_route.as_deref(), Some("models")); + assert_eq!( + query.store.exclude_route_prefix.as_deref(), + Some("management_") + ); + for path in [ + "/api/logs/requests?exclude_route=", + "/api/logs/requests?exclude_route_prefix=%00", + ] { + assert!(matches!( + request_query(path), + Err(LogsError::InvalidQuery(_)) + )); + } + } + #[test] fn export_requires_a_bounded_reason_and_durable_scope() { assert!(matches!( diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs index 877e3a980f..8d9b86765d 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests.rs @@ -6,6 +6,9 @@ use openai_frontend::{ use super::*; +mod audit; +mod audit_sanitization; + #[test] fn export_route_classifies_before_method_validation() { let get_route = classify_mutating_route(Some("/api/logs/requests/export"), "GET") @@ -138,6 +141,91 @@ async fn list_merges_active_and_durable_without_duplicate_ids() { drop(active_guard); } +#[test] +fn route_exclusions_apply_to_every_merged_active_and_durable_page() { + // Given + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + for (request_id, route, created_at) in [ + ("durable-visible", Some("responses"), "2026-08-03T00:00:01Z"), + ( + "durable-hidden-management", + Some("management_get_status"), + "2026-08-03T00:00:02Z", + ), + ( + "durable-hidden-models", + Some("models"), + "2026-08-03T00:00:03Z", + ), + ] { + store + .insert_summary( + request_id, None, route, None, None, created_at, None, None, None, + ) + .expect("seed durable summary"); + } + let active = [ + ( + "active-hidden-management", + Some("management_post"), + "2026-08-03T00:00:04Z", + ), + ("active-visible-null", None, "2026-08-03T00:00:05Z"), + ( + "active-visible-chat", + Some("chat_completions"), + "2026-08-03T00:00:06Z", + ), + ( + "active-hidden-models", + Some("models"), + "2026-08-03T00:00:07Z", + ), + ] + .map(|(request_id, route, created_at)| RequestSummaryEntry { + request_id: request_id.to_string(), + state: "active".to_string(), + created_at: created_at.to_string(), + terminal_at: None, + metadata: crate::logging::RequestSummaryMetadata::from_parts(route, None, None, None), + }) + .to_vec(); + let facade = state.query_facade().expect("query facade"); + let base_path = + "/api/logs/requests?limit=1&exclude_route=models&exclude_route_prefix=management_"; + + // When + let first = list_requests_blocking( + facade.clone(), + active.clone(), + parse::request_query(base_path).expect("parse first page"), + ) + .expect("list first page"); + let first_cursor = first.next_cursor.expect("first page cursor"); + let second = list_requests_blocking( + facade.clone(), + active.clone(), + parse::request_query(&format!("{base_path}&cursor={first_cursor}")) + .expect("parse second page"), + ) + .expect("list second page"); + let second_cursor = second.next_cursor.expect("second page cursor"); + let third = list_requests_blocking( + facade, + active, + parse::request_query(&format!("{base_path}&cursor={second_cursor}")) + .expect("parse third page"), + ) + .expect("list third page"); + + // Then + assert_eq!(first.items[0].request_id(), "active-visible-chat"); + assert_eq!(second.items[0].request_id(), "active-visible-null"); + assert_eq!(third.items[0].request_id(), "durable-visible"); + assert!(third.next_cursor.is_none()); +} + #[test] fn active_request_time_bounds_compare_instants_within_the_boundary_second() { let (_temp, state) = runtime(); @@ -241,6 +329,11 @@ async fn active_request_uses_registered_metadata_before_durable_persistence() { None, None, None, + ) + .with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + Some("192.0.2.42:11204"), + Some(crate::logging::CallerPathType::RemoteQuicHttp), ), ); service.merge_request_metadata( @@ -277,10 +370,79 @@ async fn active_request_uses_registered_metadata_before_durable_persistence() { assert_eq!(json["items"][0]["model"], "acme/model"); assert_eq!(json["items"][0]["provider"], "mesh"); assert_eq!(json["items"][0]["engine"], "raw_ingress"); + assert_eq!( + json["items"][0]["callerEndpointId"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert_eq!(json["items"][0]["callerAddr"], "192.0.2.42:11204"); + assert_eq!(json["items"][0]["callerPathType"], "remote_quic_http"); assert_eq!(json["items"][0]["source"], "active"); drop(guard); } +#[tokio::test] +async fn active_and_durable_api_json_expose_endpoint_only_caller_without_path_fields() { + let (_temp, state) = runtime(); + let service = state.service_for_test().expect("logging service"); + let active_request_id = RequestId::new(); + let durable_request_id = RequestId::new(); + let active_endpoint_id = "ad".repeat(32); + let durable_endpoint_id = "ae".repeat(32); + let (active_guard, _) = service.register_request_with_metadata( + active_request_id, + crate::logging::RequestSummaryMetadata::from_parts( + Some("responses"), + Some("active-model"), + None, + None, + ) + .with_caller_identity(Some(&active_endpoint_id), None, None), + ); + let (durable_guard, _) = service.register_request_with_metadata( + durable_request_id, + crate::logging::RequestSummaryMetadata::from_parts( + Some("responses"), + Some("durable-model"), + None, + None, + ) + .with_caller_identity(Some(&durable_endpoint_id), None, None), + ); + service + .transition_terminal( + durable_request_id, + &durable_guard, + crate::logging::TerminalOutcome::Completed, + ) + .expect("durable request terminalizes"); + assert!(service.pump_sync().await > 0); + + let active_page = list_requests( + &state, + "/api/logs/requests?source=active&model=active-model", + ) + .await + .expect("active endpoint-only request"); + let durable_page = list_requests( + &state, + "/api/logs/requests?source=durable&model=durable-model&outcome=completed", + ) + .await + .expect("durable endpoint-only request"); + + for (page, endpoint_id) in [ + (active_page, active_endpoint_id.as_str()), + (durable_page, durable_endpoint_id.as_str()), + ] { + let json = serde_json::to_value(page).expect("request page JSON"); + assert_eq!(json["items"].as_array().expect("items").len(), 1); + assert_eq!(json["items"][0]["callerEndpointId"], endpoint_id); + assert!(json["items"][0].get("callerAddr").is_none()); + assert!(json["items"][0].get("callerPathType").is_none()); + } + drop(active_guard); +} + #[tokio::test] async fn active_request_listing_batches_durable_metadata_fallback() { let (_temp, state) = runtime(); @@ -345,6 +507,11 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica None, None, None, + ) + .with_caller_identity( + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + None, + Some(crate::logging::CallerPathType::Relay), ), ); service.merge_request_metadata( @@ -388,6 +555,12 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica assert_eq!(json["items"][0]["model"], "acme/model"); assert_eq!(json["items"][0]["provider"], "mesh"); assert_eq!(json["items"][0]["engine"], "raw_ingress"); + assert_eq!( + json["items"][0]["callerEndpointId"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert!(json["items"][0].get("callerAddr").is_none()); + assert_eq!(json["items"][0]["callerPathType"], "relay"); assert_eq!(json["items"][0]["source"], "durable"); let absent_detail = request_detail(&state, &absent.as_uuid().to_string()) @@ -398,6 +571,9 @@ async fn registered_metadata_is_persisted_and_durably_filterable_without_fabrica assert_eq!(absent_json["model"], serde_json::Value::Null); assert_eq!(absent_json["provider"], serde_json::Value::Null); assert_eq!(absent_json["engine"], serde_json::Value::Null); + assert!(absent_json.get("callerEndpointId").is_none()); + assert!(absent_json.get("callerAddr").is_none()); + assert!(absent_json.get("callerPathType").is_none()); } #[tokio::test] @@ -456,249 +632,3 @@ async fn detail_and_related_routes_are_typed_and_keep_envelope_private() { assert!(!json.contains("never-expose")); assert!(!json.contains("payload_json")); } - -#[tokio::test] -async fn audit_list_returns_sparse_dto_and_never_exposes_detail_json() { - let (_temp, state) = runtime(); - let store = state.store().expect("store"); - for (entry_id, occurred_at) in [ - ( - "00000000-0000-4000-8000-000000000001", - "2026-01-01T00:00:00Z", - ), - ( - "00000000-0000-4000-8000-000000000002", - "2026-01-01T00:00:01Z", - ), - ] { - store - .insert_audit_entry( - entry_id, - None, - occurred_at, - "runtime", - "startup_complete", - Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), - ) - .expect("seed audit row"); - } - - let page = list_audits(&state, "/api/logs/audit?limit=10") - .await - .expect("list audits"); - let json = serde_json::to_value(page).expect("serialize page"); - let items = json["items"].as_array().expect("items"); - assert_eq!(items.len(), 2); - let item = &items[0]; - assert_eq!(item["entryId"], "00000000-0000-4000-8000-000000000002"); - assert_eq!(item["occurredAt"], "2026-01-01T00:00:01.000000000Z"); - assert_eq!(item["source"], "runtime"); - assert_eq!(item["code"], "startup_complete"); - assert_eq!(item["severity"], "info"); - let first_sequence = items[0]["sequence"].as_u64().expect("positive sequence"); - let second_sequence = items[1]["sequence"].as_u64().expect("positive sequence"); - assert!(first_sequence > 0); - assert!(second_sequence > 0); - assert_ne!(first_sequence, second_sequence); - assert!(!json.to_string().contains("SENTINEL-AUDIT-SECRET")); - assert!(items.iter().all(|item| item.get("detailJson").is_none())); - assert!(!json.to_string().contains("requestId")); -} - -#[tokio::test] -async fn audit_pagination_resumes_correctly_with_next_cursor() { - let (_temp, state) = runtime(); - let store = state.store().expect("store"); - for i in 1..=3 { - store - .insert_audit_entry( - &format!("00000000-0000-4000-8000-00000000000{i}"), - None, - &format!("2026-01-01T00:00:0{i}Z"), - "cli", - &format!("action-{i}"), - None, - ) - .expect("seed audit row"); - } - - let first = list_audits(&state, "/api/logs/audit?limit=1") - .await - .expect("first page"); - let first_json = serde_json::to_value(&first).expect("serialize first page"); - let cursor = first.next_cursor.expect("has next cursor"); - assert_eq!(first_json["items"].as_array().expect("items").len(), 1); - - let second = list_audits(&state, &format!("/api/logs/audit?limit=1&cursor={cursor}")) - .await - .expect("second page"); - let second_json = serde_json::to_value(&second).expect("serialize second page"); - assert_eq!(second_json["items"].as_array().expect("items").len(), 1); - assert_ne!( - second_json["items"][0]["entryId"], first_json["items"][0]["entryId"], - "cursor should advance to a different row" - ); -} - -#[tokio::test] -async fn audit_filters_by_source() { - let (_temp, state) = runtime(); - let store = state.store().expect("store"); - store - .insert_audit_entry( - "00000000-0000-4000-8000-000000000010", - None, - "2026-01-01T00:00:00Z", - "mesh", - "peer_joined", - None, - ) - .expect("seed mesh row"); - store - .insert_audit_entry( - "00000000-0000-4000-8000-000000000011", - None, - "2026-01-01T00:00:01Z", - "runtime", - "startup_complete", - None, - ) - .expect("seed runtime row"); - - let page = list_audits(&state, "/api/logs/audit?source=mesh&limit=10") - .await - .expect("filter by source"); - let json = serde_json::to_value(page).expect("serialize page"); - assert_eq!(json["items"].as_array().expect("items").len(), 1); - assert_eq!(json["items"][0]["source"], "mesh"); -} - -#[tokio::test] -async fn audit_filters_by_severity() { - let (_temp, state) = runtime(); - let store = state.store().expect("store"); - store - .insert_audit_entry( - "00000000-0000-4000-8000-000000000020", - None, - "2026-01-01T00:00:00Z", - "logging_service", - "health_check", - Some(r#"{"severity":"info"}"#), - ) - .expect("seed info row"); - store - .insert_audit_entry( - "00000000-0000-4000-8000-000000000021", - None, - "2026-01-01T00:00:01Z", - "logging_service", - "disk_pressure", - Some(r#"{"severity":"warning"}"#), - ) - .expect("seed warning row"); - - let page = list_audits(&state, "/api/logs/audit?severity=warning&limit=10") - .await - .expect("filter by severity"); - let json = serde_json::to_value(page).expect("serialize page"); - assert_eq!(json["items"].as_array().expect("items").len(), 1); - assert_eq!(json["items"][0]["severity"], "warning"); -} - -#[tokio::test] -async fn audit_list_exposes_typed_context_without_arbitrary_detail() { - let (_temp, state) = runtime(); - state - .store() - .expect("store") - .insert_audit_entry( - "00000000-0000-4000-8000-000000000022", - None, - "2026-08-12T12:00:00Z", - "runtime", - "runtime_model_ready", - Some( - r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"secret":"SENTINEL-AUDIT-SECRET"}"#, - ), - ) - .expect("seed typed audit row"); - - let page = list_audits(&state, "/api/logs/audit?limit=10") - .await - .expect("list typed audit row"); - let json = serde_json::to_value(page).expect("serialize page"); - let row = &json["items"][0]; - assert_eq!(row["contextVersion"], 1); - assert_eq!(row["subjectKind"], "model"); - assert_eq!(row["subjectId"], "local-gguf/sha256-safe"); - assert_eq!(row["operationId"], "runtime-7"); - assert_eq!(row["outcome"], "ready"); - assert_eq!(row["durationMs"], 42); - assert!(!json.to_string().contains("SENTINEL-AUDIT-SECRET")); -} - -#[tokio::test] -async fn audit_filters_by_inclusive_canonical_time_bounds_before_pagination() { - let (_temp, state) = runtime(); - let store = state.store().expect("store"); - for (suffix, occurred_at) in [ - (30, "2026-01-01T00:00:00Z"), - (31, "2026-01-02T00:00:00Z"), - (32, "2026-01-03T00:00:00Z"), - ] { - store - .insert_audit_entry( - &format!("00000000-0000-4000-8000-{suffix:012}"), - None, - occurred_at, - "runtime", - "bounded_action", - None, - ) - .expect("seed bounded audit row"); - } - - let page = list_audits( - &state, - "/api/logs/audit?limit=1&from=2026-01-02T00%3A00%3A00Z&to=2026-01-02T00%3A00%3A00Z", - ) - .await - .expect("filter by inclusive bounds"); - let json = serde_json::to_value(page).expect("serialize page"); - assert_eq!(json["items"].as_array().expect("items").len(), 1); - assert_eq!( - json["items"][0]["occurredAt"], - "2026-01-02T00:00:00.000000000Z" - ); - assert!(json["nextCursor"].is_null()); -} - -#[tokio::test] -async fn audit_rejects_invalid_query_parameters() { - let (_temp, state) = runtime(); - - for path in [ - "/api/logs/audit?limit=0", - "/api/logs/audit?limit=101", - "/api/logs/audit?limit=abc", - "/api/logs/audit?source=bogus", - "/api/logs/audit?severity=bogus", - "/api/logs/audit?cursor=garbage", - "/api/logs/audit?from=not-a-time", - "/api/logs/audit?to=2026-01-01T00%3A00%3A00Z&from=2026-01-02T00%3A00%3A00Z", - "/api/logs/audit?unknown=1", - ] { - let result = list_audits(&state, path).await; - assert!(result.is_err(), "expected error for invalid query: {path}"); - } -} - -#[tokio::test] -async fn audit_unmatched_path_returns_not_found() { - assert!(matches!(classify("/api/logs/audit/extra"), Route::Unknown)); - assert!(matches!( - classify("/api/logs/audit/sub/path"), - Route::Unknown - )); -} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit.rs new file mode 100644 index 0000000000..117f8e09fa --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit.rs @@ -0,0 +1,390 @@ +use super::*; + +#[tokio::test] +async fn audit_list_returns_sparse_dto_and_never_exposes_detail_json() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + for (entry_id, occurred_at) in [ + ( + "00000000-0000-4000-8000-000000000001", + "2026-01-01T00:00:00Z", + ), + ( + "00000000-0000-4000-8000-000000000002", + "2026-01-01T00:00:01Z", + ), + ] { + store + .insert_audit_entry( + entry_id, + None, + occurred_at, + "runtime", + "startup_complete", + Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), + ) + .expect("seed audit row"); + } + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list audits"); + let json = serde_json::to_value(page).expect("serialize page"); + let items = json["items"].as_array().expect("items"); + assert_eq!(items.len(), 2); + let item = &items[0]; + assert_eq!(item["entryId"], "00000000-0000-4000-8000-000000000002"); + assert_eq!(item["occurredAt"], "2026-01-01T00:00:01.000000000Z"); + assert_eq!(item["source"], "runtime"); + assert_eq!(item["code"], "startup_complete"); + assert_eq!(item["severity"], "info"); + let first_sequence = items[0]["sequence"].as_u64().expect("positive sequence"); + let second_sequence = items[1]["sequence"].as_u64().expect("positive sequence"); + assert!(first_sequence > 0); + assert!(second_sequence > 0); + assert_ne!(first_sequence, second_sequence); + assert!(!json.to_string().contains("SENTINEL-AUDIT-SECRET")); + assert!(items.iter().all(|item| item.get("detailJson").is_none())); + assert!(!json.to_string().contains("requestId")); +} + +#[tokio::test] +async fn audit_pagination_resumes_correctly_with_next_cursor() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + for i in 1..=3 { + store + .insert_audit_entry( + &format!("00000000-0000-4000-8000-00000000000{i}"), + None, + &format!("2026-01-01T00:00:0{i}Z"), + "cli", + &format!("action-{i}"), + None, + ) + .expect("seed audit row"); + } + + let first = list_audits(&state, "/api/logs/audit?limit=1") + .await + .expect("first page"); + let first_json = serde_json::to_value(&first).expect("serialize first page"); + let cursor = first.next_cursor.expect("has next cursor"); + assert_eq!(first_json["items"].as_array().expect("items").len(), 1); + + let second = list_audits(&state, &format!("/api/logs/audit?limit=1&cursor={cursor}")) + .await + .expect("second page"); + let second_json = serde_json::to_value(&second).expect("serialize second page"); + assert_eq!(second_json["items"].as_array().expect("items").len(), 1); + assert_ne!( + second_json["items"][0]["entryId"], first_json["items"][0]["entryId"], + "cursor should advance to a different row" + ); +} + +#[tokio::test] +async fn audit_filters_by_source() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000010", + None, + "2026-01-01T00:00:00Z", + "mesh", + "peer_joined", + None, + ) + .expect("seed mesh row"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000011", + None, + "2026-01-01T00:00:01Z", + "runtime", + "startup_complete", + None, + ) + .expect("seed runtime row"); + + let page = list_audits(&state, "/api/logs/audit?source=mesh&limit=10") + .await + .expect("filter by source"); + let json = serde_json::to_value(page).expect("serialize page"); + assert_eq!(json["items"].as_array().expect("items").len(), 1); + assert_eq!(json["items"][0]["source"], "mesh"); +} + +#[tokio::test] +async fn audit_filters_by_severity() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000020", + None, + "2026-01-01T00:00:00Z", + "logging_service", + "health_check", + Some(r#"{"severity":"info"}"#), + ) + .expect("seed info row"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000021", + None, + "2026-01-01T00:00:01Z", + "logging_service", + "disk_pressure", + Some(r#"{"severity":"warning"}"#), + ) + .expect("seed warning row"); + + let page = list_audits(&state, "/api/logs/audit?severity=warning&limit=10") + .await + .expect("filter by severity"); + let json = serde_json::to_value(page).expect("serialize page"); + assert_eq!(json["items"].as_array().expect("items").len(), 1); + assert_eq!(json["items"][0]["severity"], "warning"); +} + +#[tokio::test] +async fn audit_list_exposes_typed_context_without_arbitrary_detail() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000022", + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_model_ready", + Some( + r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"secret":"SENTINEL-AUDIT-SECRET"}"#, + ), + ) + .expect("seed typed audit row"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list typed audit row"); + let json = serde_json::to_value(page).expect("serialize page"); + let row = &json["items"][0]; + assert_eq!(row["contextVersion"], 1); + assert_eq!(row["subjectKind"], "model"); + assert_eq!(row["subjectId"], "local-gguf/sha256-safe"); + assert_eq!(row["operationId"], "runtime-7"); + assert_eq!(row["outcome"], "ready"); + assert_eq!(row["durationMs"], 42); + assert!(!json.to_string().contains("SENTINEL-AUDIT-SECRET")); +} + +#[tokio::test] +async fn audit_list_omits_malformed_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000024", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm gpus --draft run-benchmark --backend cuda"}"#, + ), + ) + .expect("seed malformed command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list malformed command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); +} + +#[tokio::test] +async fn audit_list_omits_duplicate_command_summary_flags() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000027", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm models list --json --json"}"#), + ) + .expect("seed duplicate command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list duplicate command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); +} + +#[tokio::test] +async fn audit_list_omits_deep_malformed_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000026", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm load unload status discover rotate-key setup --port 1234"}"#, + ), + ) + .expect("seed deep malformed command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list deep malformed command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert!(json["items"][0].get("commandSummary").is_none()); +} + +#[tokio::test] +async fn audit_list_preserves_valid_command_summary() { + let (_temp, state) = runtime(); + state + .store() + .expect("store") + .insert_audit_entry( + "00000000-0000-4000-8000-000000000025", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]"}"#), + ) + .expect("seed valid command summary"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list valid command summary"); + let json = serde_json::to_value(page).expect("serialize audit page"); + assert_eq!( + json["items"][0]["commandSummary"], + "mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]" + ); +} + +#[tokio::test] +async fn audit_list_exposes_mesh_peer_direct_path_and_omits_relay_address() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000023", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-direct","remote_addr":"192.168.1.44:11204","path_type":"direct"}"#, + ), + ) + .expect("seed direct peer audit"); + store + .insert_audit_entry( + "00000000-0000-4000-8000-000000000024", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-relay","remote_addr":"203.0.113.10:443","path_type":"relay"}"#, + ), + ) + .expect("seed relay peer audit"); + + let page = list_audits(&state, "/api/logs/audit?limit=10") + .await + .expect("list peer audits"); + let json = serde_json::to_value(page).expect("serialize page"); + let direct = &json["items"][0]; + let relay = &json["items"][1]; + + assert_eq!(direct["subjectKind"], "mesh_peer"); + assert_eq!(direct["subjectId"], "peer-direct"); + assert_eq!(direct["remoteAddr"], "192.168.1.44:11204"); + assert_eq!(direct["pathType"], "direct"); + assert_eq!(relay["pathType"], "relay"); + assert!(relay.get("remoteAddr").is_none()); +} + +#[tokio::test] +async fn audit_filters_by_inclusive_canonical_time_bounds_before_pagination() { + let (_temp, state) = runtime(); + let store = state.store().expect("store"); + for (suffix, occurred_at) in [ + (30, "2026-01-01T00:00:00Z"), + (31, "2026-01-02T00:00:00Z"), + (32, "2026-01-03T00:00:00Z"), + ] { + store + .insert_audit_entry( + &format!("00000000-0000-4000-8000-{suffix:012}"), + None, + occurred_at, + "runtime", + "bounded_action", + None, + ) + .expect("seed bounded audit row"); + } + + let page = list_audits( + &state, + "/api/logs/audit?limit=1&from=2026-01-02T00%3A00%3A00Z&to=2026-01-02T00%3A00%3A00Z", + ) + .await + .expect("filter by inclusive bounds"); + let json = serde_json::to_value(page).expect("serialize page"); + assert_eq!(json["items"].as_array().expect("items").len(), 1); + assert_eq!( + json["items"][0]["occurredAt"], + "2026-01-02T00:00:00.000000000Z" + ); + assert!(json["nextCursor"].is_null()); +} + +#[tokio::test] +async fn audit_rejects_invalid_query_parameters() { + let (_temp, state) = runtime(); + + for path in [ + "/api/logs/audit?limit=0", + "/api/logs/audit?limit=101", + "/api/logs/audit?limit=abc", + "/api/logs/audit?source=bogus", + "/api/logs/audit?severity=bogus", + "/api/logs/audit?cursor=garbage", + "/api/logs/audit?from=not-a-time", + "/api/logs/audit?to=2026-01-01T00%3A00%3A00Z&from=2026-01-02T00%3A00%3A00Z", + "/api/logs/audit?unknown=1", + ] { + let result = list_audits(&state, path).await; + assert!(result.is_err(), "expected error for invalid query: {path}"); + } +} + +#[tokio::test] +async fn audit_unmatched_path_returns_not_found() { + assert!(matches!(classify("/api/logs/audit/extra"), Route::Unknown)); + assert!(matches!( + classify("/api/logs/audit/sub/path"), + Route::Unknown + )); +} diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs new file mode 100644 index 0000000000..8dc145dead --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/tests/audit_sanitization.rs @@ -0,0 +1,43 @@ +use super::*; + +#[tokio::test] +async fn audit_list_sanitizes_raw_actor_and_action_before_rest_serialization() { + // Given: a raw SQLite row that bypasses the typed write sanitizer. + let (_temp, state) = runtime(); + let raw_actor = "Bearer rest-raw-actor-secret"; + let raw_action = format!("REST-RAW-ACTION-SENTINEL-{}", "x".repeat(1_100)); + state + .store() + .expect("store") + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + ( + "00000000-0000-4000-8000-000000000099", + "2026-08-22T12:00:00.000000000Z", + raw_actor, + raw_action.as_str(), + r#"{"severity":"error"}"#, + ), + ) + .expect("insert raw audit row"); + + // When: the REST route reads and serializes the durable row. + let page = list_audits(&state, "/api/logs/audit?limit=1") + .await + .expect("list raw audit row"); + let json = serde_json::to_value(page).expect("serialize raw audit page"); + let row = &json["items"][0]; + + // Then: the API exposes only canonicalized scalar values. + assert_eq!(row["source"], "[REDACTED]"); + assert_eq!(row["severity"], "error"); + let code = row["code"].as_str().expect("audit code string"); + assert_eq!(code.chars().count(), 1_024); + assert!(code.ends_with("... [TRUNCATED]")); + let wire = json.to_string(); + assert!(!wire.contains(raw_actor)); + assert!(!wire.contains(&raw_action)); +} diff --git a/crates/mesh-llm-host-runtime/src/api/server.rs b/crates/mesh-llm-host-runtime/src/api/server.rs index a69a5c0ec8..4380de271a 100644 --- a/crates/mesh-llm-host-runtime/src/api/server.rs +++ b/crates/mesh-llm-host-runtime/src/api/server.rs @@ -240,6 +240,7 @@ pub(crate) async fn handle_request(mut stream: TcpStream, state: &MeshApi) -> an state.register_management_request( request_id, super::management_lifecycle::method_route_label(method, path_only), + source_addr, ) }) { return super::management_lifecycle::scope(lifecycle, dispatch).await; diff --git a/crates/mesh-llm-host-runtime/src/api/status.rs b/crates/mesh-llm-host-runtime/src/api/status.rs index 55b5b9eea8..19b382d734 100644 --- a/crates/mesh-llm-host-runtime/src/api/status.rs +++ b/crates/mesh-llm-host-runtime/src/api/status.rs @@ -1638,8 +1638,8 @@ mod tests { LoggingStatusPayload { metadata_available: false, metadata_state: "schema_incompatible", - schema_version: Some(14), - supported_schema_version: Some(11), + schema_version: Some(2), + supported_schema_version: Some(1), capture_mode: "unavailable", artifact_capture_available: false, artifact_capture_ready: false, diff --git a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs index 9d5bef4ccc..aa2ccfc9cd 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream.rs @@ -1,5 +1,7 @@ use super::*; +mod audit_sanitization; + #[tokio::test] async fn log_routes_reject_methods_and_invalid_paths_with_bounded_json_errors() { let requests = [ @@ -219,6 +221,45 @@ async fn logs_events_rejects_invalid_raw_requests_before_sse_headers() { } } +#[tokio::test] +#[serial] +async fn logs_events_audit_reconcile_failure_writes_one_terminal_frame_then_eof() { + let _temporary_directory = install_sse_logging().await; + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + b"GET /api/logs/events?audit=true&cursor=a1:18446744073709551615 HTTP/1.1\r\nHost: localhost\r\nAccept: text/event-stream\r\n\r\n", + ) + .await + .unwrap(); + + let mut response = read_until_contains( + &mut stream, + b"\"code\":\"audit_reconcile_failed\"", + Duration::from_secs(2), + ) + .await; + let mut remainder = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), stream.read_to_end(&mut remainder)) + .await + .expect("audit reconcile failure stream must reach EOF") + .unwrap(); + response.extend_from_slice(&remainder); + let response = String::from_utf8(response).unwrap(); + assert_eq!(response.matches("event: stream_error").count(), 1); + assert_eq!( + response + .matches("\"code\":\"audit_reconcile_failed\"") + .count(), + 1 + ); + + server.await.unwrap().unwrap(); + disable_sse_logging().await; +} + #[tokio::test] #[serial] async fn logs_events_stops_after_a_disconnected_tcp_client() { diff --git a/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs new file mode 100644 index 0000000000..8255d09ff8 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/tests/logs_api_routes/event_stream/audit_sanitization.rs @@ -0,0 +1,66 @@ +use super::*; + +#[tokio::test] +#[serial] +async fn logs_events_audit_sanitizes_raw_sql_actor_and_action() { + // Given: an audit row inserted outside the typed repository with unsafe scalars. + let _temporary_directory = install_sse_logging().await; + let logging = crate::logging_runtime_state().unwrap(); + let raw_actor = "Bearer sse-raw-actor-secret"; + let oversized_marker = "SSE-RAW-OVERSIZED-ACTION"; + let raw_action = format!("{}{oversized_marker}", "x".repeat(1_100)); + logging + .store() + .expect("installed log store") + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + ( + "00000000-0000-4000-8000-0000000000aa", + "2026-08-22T12:00:00.000000000Z", + raw_actor, + raw_action.as_str(), + r#"{"severity":"warning"}"#, + ), + ) + .expect("insert raw audit row"); + let expected_code = mesh_llm_events::audit::SanitizedAuditScalar::sanitize(&raw_action); + let expected_code_json = format!( + "\"code\":{}", + serde_json::to_string(expected_code.as_str()).expect("serialize expected audit code") + ); + + // When: an audit SSE client reconciles durable rows through the management route. + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + b"GET /api/logs/events?audit=true&cursor=a1:0 HTTP/1.1\r\nHost: localhost\r\nAccept: text/event-stream\r\n\r\n", + ) + .await + .unwrap(); + let response = read_until_contains( + &mut stream, + expected_code_json.as_bytes(), + Duration::from_secs(2), + ) + .await; + let response = String::from_utf8(response).unwrap(); + + // Then: only the canonical scalar values cross the SSE boundary. + assert!(response.contains("HTTP/1.1 200 OK")); + assert!(response.contains("event: audit_entry")); + assert!(response.contains("\"source\":\"[REDACTED]\"")); + assert!(response.contains(&expected_code_json)); + assert!(!response.contains(raw_actor)); + assert!(!response.contains("sse-raw-actor-secret")); + assert!(!response.contains(&raw_action)); + assert!(!response.contains(oversized_marker)); + + drop(stream); + server.abort(); + disable_sse_logging().await; +} diff --git a/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs b/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs index 954ae34f71..d8f5414c13 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/management_http.rs @@ -62,13 +62,142 @@ async fn management_health_is_a_json_liveness_response() { .await; server.await.unwrap().unwrap(); - assert!(response.starts_with("HTTP/1.1 200 OK"), "response: {response}"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); assert!(response.contains("Content-Type: application/json")); assert_eq!(json_body(&response)["status"], "ok"); assert_eq!(json_body(&response)["mesh"]["status"], "standalone"); assert_eq!(json_body(&response)["serving"]["status"], "idle"); } +#[tokio::test] +#[serial] +async fn trusted_local_management_mutation_persists_the_tcp_caller_address() { + let root = tempfile::tempdir().expect("temporary logging root"); + let config = mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(root.path().to_path_buf()), + ..Default::default() + }; + crate::initialize_logging_foundation(&config).await; + + let state = build_test_mesh_api().await; + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind management listener"); + let address = listener.local_addr().expect("management listener address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept management caller"); + handle_request(stream, &state).await + }); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let body = r#""active""#; + let request = format!( + "PUT /api/runtime/activity/override HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + request_id.as_uuid(), + body.len(), + body, + ); + let mut client = TcpStream::connect(address) + .await + .expect("connect management caller"); + let caller_addr = client + .local_addr() + .expect("management caller address") + .to_string(); + client + .write_all(request.as_bytes()) + .await + .expect("write management mutation"); + client.shutdown().await.expect("finish management request"); + let mut response = Vec::new(); + client + .read_to_end(&mut response) + .await + .expect("read management response"); + server + .await + .expect("management task joins") + .expect("request"); + assert!( + String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK"), + "trusted-local mutation succeeds" + ); + + let logging = crate::logging_runtime_state().expect("installed logging runtime"); + let request_key = request_id.as_uuid().to_string(); + let active = logging + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_key) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); + logging.pump_persistence_for_test().await; + let durable = logging + .store() + .expect("metadata store") + .query_request_with_caller(&request_key) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); +} + +#[tokio::test] +#[serial] +async fn logs_and_read_only_management_routes_never_self_record() { + let root = tempfile::tempdir().expect("temporary logging root"); + let config = mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(root.path().to_path_buf()), + ..Default::default() + }; + crate::initialize_logging_foundation(&config).await; + let state = build_test_mesh_api().await; + + let mut request_ids = Vec::new(); + for path in ["/api/logs/requests", "/api/status"] { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let (address, server) = spawn_management_test_server(state.clone()).await; + let response = send_management_request( + address, + format!( + "GET {path} HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ), + ) + .await; + server + .await + .expect("management task joins") + .expect("request"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); + request_ids.push(request_id); + } + + let service = crate::logging_runtime_state() + .expect("installed logging runtime") + .service_for_test() + .expect("logging service"); + for request_id in request_ids { + assert!( + service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .is_none(), + "excluded route must not register a lifecycle" + ); + } +} + #[tokio::test] async fn management_health_remains_available_in_headless_mode() { let state = build_test_mesh_api().await; @@ -81,7 +210,10 @@ async fn management_health_remains_available_in_headless_mode() { .await; server.await.unwrap().unwrap(); - assert!(response.starts_with("HTTP/1.1 200 OK"), "response: {response}"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "response: {response}" + ); assert_eq!(json_body(&response)["status"], "ok"); } @@ -113,14 +245,13 @@ async fn management_health_reports_client_worker_and_serving_modes() { .await; server.await.unwrap().unwrap(); assert_eq!(json_body(&worker_response)["mode"], "worker"); - assert_eq!( - json_body(&worker_response)["serving"]["status"], - "idle" - ); + assert_eq!(json_body(&worker_response)["serving"]["status"], "idle"); state.update(true, true).await; - node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }).await; - node.set_hosted_models(vec!["served-model".to_string()]).await; + node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }) + .await; + node.set_hosted_models(vec!["served-model".to_string()]) + .await; let (address, server) = spawn_management_test_server(state).await; let serving_response = send_management_request( address, @@ -323,7 +454,8 @@ async fn management_health_reports_cached_plugin_models_for_serving_host() { let plugin_manager = build_inference_endpoint_plugin_manager(&["plugin-model"]).await; let state = build_test_mesh_api_with_plugin_manager(3131, plugin_manager).await; let node = state.node().await; - node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }).await; + node.set_role(crate::mesh::NodeRole::Host { http_port: 9337 }) + .await; let (address, server) = spawn_management_test_server(state).await; let response = send_management_request( diff --git a/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs b/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs new file mode 100644 index 0000000000..8efc9ddf33 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/api/tests/management_request_id.rs @@ -0,0 +1,113 @@ +use super::*; + +const CLIENT_REQUEST_ID: &str = "00000000-0000-4000-8000-000000000041"; +const SECOND_CLIENT_REQUEST_ID: &str = "00000000-0000-4000-8000-000000000042"; + +struct RequestIdObservation { + response_id: String, + lifecycle_id: String, + lifecycle_state: String, +} + +async fn observe_management_request_id(request_id_headers: &str) -> RequestIdObservation { + let temporary_directory = tempfile::tempdir().expect("temporary logging root"); + crate::initialize_logging_foundation(&mesh_llm_config::LoggingConfig { + enabled: true, + application_state_root: Some(temporary_directory.path().join("logging")), + replay_capacity: 16, + ..Default::default() + }) + .await; + let state = build_test_mesh_api().await; + let (address, server) = spawn_management_test_server(state).await; + let body = r#"{"toml":"version = 1\n","path":"x"}"#; + let response = send_management_request( + address, + format!( + "POST /api/runtime/config/validate HTTP/1.1\r\nHost: localhost\r\n{request_id_headers}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ), + ) + .await; + server + .await + .expect("management task joins") + .expect("management request succeeds"); + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + + let response_ids = response + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("x-request-id")) + .map(|(_, value)| value.trim().to_string()) + .collect::>(); + assert_eq!( + response_ids.len(), + 1, + "response must contain one request ID" + ); + let response_id = response_ids[0].clone(); + let summary = crate::logging_runtime_state() + .expect("installed logging runtime") + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&response_id) + .expect("response request ID identifies the lifecycle"); + let observation = RequestIdObservation { + response_id, + lifecycle_id: summary.request_id, + lifecycle_state: summary.state, + }; + + crate::initialize_logging_foundation(&mesh_llm_config::LoggingConfig { + enabled: false, + ..Default::default() + }) + .await; + observation +} + +#[tokio::test] +#[serial] +async fn single_valid_client_request_id_is_the_canonical_management_id() { + let observation = + observe_management_request_id(&format!("x-request-id: {CLIENT_REQUEST_ID}\r\n")).await; + + assert_eq!(observation.response_id, CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_id, CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_state, "completed"); +} + +#[tokio::test] +#[serial] +async fn duplicate_client_request_ids_generate_a_new_canonical_management_id() { + let observation = observe_management_request_id(&format!( + "x-request-id: {CLIENT_REQUEST_ID}\r\nX-Request-Id: {SECOND_CLIENT_REQUEST_ID}\r\n" + )) + .await; + + assert_ne!(observation.response_id, CLIENT_REQUEST_ID); + assert_ne!(observation.response_id, SECOND_CLIENT_REQUEST_ID); + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} + +#[tokio::test] +#[serial] +async fn malformed_client_request_id_generates_a_canonical_management_id() { + let observation = observe_management_request_id("x-request-id: not-a-uuid\r\n").await; + + assert_ne!(observation.response_id, "not-a-uuid"); + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} + +#[tokio::test] +#[serial] +async fn missing_client_request_id_generates_a_canonical_management_id() { + let observation = observe_management_request_id("").await; + + assert_eq!(observation.lifecycle_id, observation.response_id); + assert!(uuid::Uuid::parse_str(&observation.response_id).is_ok()); +} diff --git a/crates/mesh-llm-host-runtime/src/api/tests/mod.rs b/crates/mesh-llm-host-runtime/src/api/tests/mod.rs index 2ace24238a..0932f42a88 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/mod.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/mod.rs @@ -50,6 +50,7 @@ impl Drop for HomeEnvGuard { mod apply_config_diagnostics; mod apply_config_validation_authority; mod logs_api_routes; +mod management_request_id; mod runtime_config; mod runtime_config_validation_authority; mod runtime_control_state; diff --git a/crates/mesh-llm-host-runtime/src/lib.rs b/crates/mesh-llm-host-runtime/src/lib.rs index 72236b8f91..558321d204 100644 --- a/crates/mesh-llm-host-runtime/src/lib.rs +++ b/crates/mesh-llm-host-runtime/src/lib.rs @@ -33,8 +33,8 @@ pub use crypto::{ verify_release_attestation, }; pub use logging::{ - LoggingRuntimeState, OperationalAuditContext, OperationalAuditRecord, OperationalAuditSeverity, - OperationalAuditSubjectKind, + LoggingRuntimeState, OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, OperationalAuditSubjectKind, }; pub use mesh::requirements::{ BootstrapStatus, DIRECT_NODE_ADMISSION_PROOF_MAX_CLOCK_SKEW_MS, DirectNodeAdmissionProof, diff --git a/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs index e6f8d4896f..03849e016e 100644 --- a/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/management_lifecycle.rs @@ -1,15 +1,20 @@ //! Metadata-only lifecycle ownership for one management API request. -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, +use std::{ + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, }; use mesh_llm_events::logging::{ events::LifecycleEvent, identifiers::RequestId, replay::ReplayChannel, }; -use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome}; +use super::{ + CallerPathType, LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome, +}; pub(crate) struct ManagementRequestLifecycle { request_id: RequestId, @@ -24,6 +29,25 @@ impl ManagementRequestLifecycle { request_id: RequestId, method_route: &'static str, ) -> Self { + Self::register_with_caller_addr(service, request_id, method_route, None) + } + + pub(crate) fn register_with_source_addr( + service: Arc, + request_id: RequestId, + method_route: &'static str, + source_addr: SocketAddr, + ) -> Self { + Self::register_with_caller_addr(service, request_id, method_route, Some(source_addr)) + } + + fn register_with_caller_addr( + service: Arc, + request_id: RequestId, + method_route: &'static str, + source_addr: Option, + ) -> Self { + let caller_addr = source_addr.map(|addr| addr.to_string()); let metadata = RequestSummaryMetadata::from_parts( Some(method_route), None, @@ -31,7 +55,12 @@ impl ManagementRequestLifecycle { Some(method_route), ) .with_source(Some("direct_http")) - .with_method(Some(management_method_label(method_route))); + .with_method(Some(management_method_label(method_route))) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let (guard, _) = service.register_request_with_metadata(request_id, metadata.clone()); if let Ok(payload) = serde_json::to_string(&LifecycleEvent::RouteSelected { model: None, @@ -90,6 +119,8 @@ fn management_method_label(method_route: &str) -> &'static str { "POST" } else if method_route == "management_put" { "PUT" + } else if method_route == "management_patch" { + "PATCH" } else if method_route == "management_delete" { "DELETE" } else { @@ -135,6 +166,52 @@ mod tests { assert!(summary.metadata.model().is_none()); assert_eq!(summary.metadata.provider(), Some("management_api")); assert_eq!(summary.metadata.engine(), Some("management_get_status")); + assert!(summary.metadata.caller_endpoint_id().is_none()); + assert!(summary.metadata.caller_addr().is_none()); + assert!(summary.metadata.caller_path_type().is_none()); + } + + #[test] + fn management_registration_captures_truthful_local_http_caller() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let request_id = RequestId::new(); + let source_addr = "[::1]:40123".parse().expect("IPv6 source address"); + let lifecycle = ManagementRequestLifecycle::register_with_source_addr( + Arc::clone(&service), + request_id, + "management_post", + source_addr, + ); + + lifecycle.finish_status(200); + + let summary = service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("terminal request summary"); + assert_eq!(summary.metadata.caller_addr(), Some("[::1]:40123")); + assert_eq!(summary.metadata.caller_path_type(), Some("local_http")); + assert!(summary.metadata.caller_endpoint_id().is_none()); + } + + #[test] + fn patch_management_registration_preserves_the_http_method() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let request_id = RequestId::new(); + let lifecycle = ManagementRequestLifecycle::register( + Arc::clone(&service), + request_id, + "management_patch", + ); + + lifecycle.finish_status(200); + + let summary = service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("terminal PATCH request summary"); + assert_eq!(summary.metadata.route(), Some("management_patch")); + assert_eq!(summary.metadata.method(), Some("PATCH")); } #[test] diff --git a/crates/mesh-llm-host-runtime/src/logging/mod.rs b/crates/mesh-llm-host-runtime/src/logging/mod.rs index 9f91933548..4ae3ee5101 100644 --- a/crates/mesh-llm-host-runtime/src/logging/mod.rs +++ b/crates/mesh-llm-host-runtime/src/logging/mod.rs @@ -51,12 +51,12 @@ pub(crate) use openai_lifecycle::{ }; pub use persistence::LogStoreSink; pub(crate) use raw_mesh_lifecycle::{ - ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRemoteSuppressionLease, - RawMeshRequestLifecycle, + ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRemoteAttributionLease, + RawMeshRemoteSuppressionLease, RawMeshRequestLifecycle, }; pub use registry::{ActiveRequestSnapshot, RegistryConfig, RequestRegistry, RequestSummaryEntry}; pub(crate) use registry::{RequestSummaryEventSnapshots, RequestSummarySnapshot}; -pub(crate) use request_metadata::RequestSummaryMetadata; +pub(crate) use request_metadata::{CallerPathType, RequestSummaryMetadata}; #[cfg(test)] pub use runtime_state::ArtifactCaptureRequest; pub(crate) use runtime_state::LoggingQueryFacade; @@ -66,8 +66,8 @@ pub use sequences::SequenceGenerators; pub(crate) use service::ArtifactUnavailableReason; pub use service::{ ArtifactCaptureEntry, ArtifactPersistenceStatus, Clock, LoggingService, - OperationalAuditContext, OperationalAuditRecord, OperationalAuditSeverity, - OperationalAuditSubjectKind, PersistSink, ServiceConfig, SystemClock, + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, OperationalAuditSubjectKind, PersistSink, ServiceConfig, SystemClock, }; pub(crate) use webhook_delivery::{ RandomWebhookJitter, ReqwestWebhookTransport, SystemWebhookWorkerClock, WebhookDeliveryWorker, diff --git a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs index 72fc1ec87d..e06f75d4b8 100644 --- a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs @@ -19,6 +19,7 @@ use openai_frontend::{ OpenAiUsage, }; +use super::raw_mesh_lifecycle::FrontendAdmissionDecision; use super::{ ArtifactUnavailableReason, LifecycleGuard, LoggingService, ProxyAttemptFinish, RawMeshLifecycleOwners, RawMeshProxyAttempt, RawMeshRequestLifecycle, RequestSummaryMetadata, @@ -434,28 +435,28 @@ impl OpenAiLifecycleLoggingAdapter { fn admit(&self, context: &OpenAiLifecycleContext) { let request_id = context.request_id; - // A raw mesh ingress request owns its lifecycle before it reaches an - // embedded frontend. Direct frontend loopback traffic never claims this - // registry and remains owned by this adapter. - if self.raw_mesh_owners.is_claimed(request_id) { - return; - } - let mut tracked = lock_recover(&self.tracked); - if tracked.requests.contains_key(&request_id) || !tracked.make_room() { - return; - } + self.raw_mesh_owners.admit_frontend(request_id, || { + let mut tracked = lock_recover(&self.tracked); + if tracked.requests.contains_key(&request_id) { + return FrontendAdmissionDecision::Rejected; + } + let Ok(evicted) = tracked.make_room() else { + return FrontendAdmissionDecision::Rejected; + }; - let (guard, _) = self.service.register_request_with_metadata( - request_id, - RequestSummaryMetadata::from_openai_frontend_route(context.route) - .with_source(Some("direct_http")) - .with_method(Some(openai_method_label(context.method))), - ); - tracked.requests.insert( - request_id, - TrackedRequest::Active(ActiveRequest::new(guard)), - ); - tracked.insertion_order.push_back(request_id); + let (guard, _) = self.service.register_request_with_metadata( + request_id, + RequestSummaryMetadata::from_openai_frontend_route(context.route) + .with_source(Some("direct_http")) + .with_method(Some(openai_method_label(context.method))), + ); + tracked.requests.insert( + request_id, + TrackedRequest::Active(ActiveRequest::new(guard)), + ); + tracked.insertion_order.push_back(request_id); + FrontendAdmissionDecision::Registered { evicted } + }); } fn backend_dispatched(&self, request_id: RequestId, operation: OpenAiBackendOperation) { @@ -653,6 +654,7 @@ impl OpenAiLifecycleLoggingAdapter { let _ = self .service .transition_terminal(request_id, &guard, outcome); + self.raw_mesh_owners.release_frontend(request_id); } #[cfg(test)] @@ -670,19 +672,20 @@ const fn openai_method_label(method: OpenAiRequestMethod) -> &'static str { } impl TrackedRequests { - fn make_room(&mut self) -> bool { - while self.requests.len() >= MAX_TRACKED_REQUESTS { - let Some(oldest) = self.insertion_order.pop_front() else { - return false; - }; - if matches!(self.requests.get(&oldest), Some(TrackedRequest::Terminal)) { - self.requests.remove(&oldest); - continue; - } + fn make_room(&mut self) -> Result, ()> { + if self.requests.len() < MAX_TRACKED_REQUESTS { + return Ok(None); + } + let Some(oldest) = self.insertion_order.pop_front() else { + return Err(()); + }; + if matches!(self.requests.get(&oldest), Some(TrackedRequest::Terminal)) { + self.requests.remove(&oldest); + Ok(Some(oldest)) + } else { self.insertion_order.push_front(oldest); - return false; + Err(()) } - true } fn is_active(&self, request_id: RequestId) -> bool { @@ -952,6 +955,41 @@ mod tests { assert_eq!(adapter.tracked_len(), 1); } + #[test] + fn terminalization_releases_frontend_owner_without_disturbing_raw_owner() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let adapter = OpenAiLifecycleLoggingAdapter::new(Arc::clone(&service), Arc::clone(&owners)); + let frontend_request_id = RequestId::new(); + let frontend_context = context(frontend_request_id); + let raw_request_id = RequestId::new(); + let raw = RawMeshRequestLifecycle::register( + Arc::clone(&service), + Arc::clone(&owners), + raw_request_id, + ) + .expect("raw request should own its lifecycle"); + adapter.observe(&OpenAiLifecycleEvent::Admitted { + context: frontend_context.clone(), + }); + assert!(owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + + adapter.observe(&OpenAiLifecycleEvent::NonStreamTerminal { + context: frontend_context.clone(), + result: OpenAiTerminalResult::Completed { status_code: 200 }, + }); + adapter.observe(&OpenAiLifecycleEvent::NonStreamTerminal { + context: frontend_context, + result: OpenAiTerminalResult::Completed { status_code: 200 }, + }); + + assert!(!owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + assert_eq!(adapter.tracked_len(), 1); + raw.terminal(TerminalOutcome::Completed); + } + #[test] fn backend_stream_and_usage_events_map_to_canonical_children_once() { let (service, adapter) = adapter(); @@ -1132,6 +1170,104 @@ mod tests { ); } + #[test] + fn markerless_authenticated_attribution_and_frontend_admission_choose_one_parent_in_both_orders() + { + for attribution_first in [true, false] { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let adapter = + OpenAiLifecycleLoggingAdapter::new(Arc::clone(&service), Arc::clone(&owners)); + let request_id = RequestId::new(); + let endpoint_id = "83".repeat(32); + let authenticated = RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.83:11204"), + Some(super::super::CallerPathType::RemoteQuicHttp), + ); + let request_metadata = RequestSummaryMetadata::from_parts( + Some("chat_completions"), + Some("model-a"), + Some("openai_frontend"), + Some("chat_completion"), + ) + .with_source(Some("direct_http")) + .with_method(Some("POST")); + + let attribution = if attribution_first { + super::super::RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + authenticated.clone(), + ) + .expect("authenticated attribution") + } else { + adapter.admit(&context(request_id)); + service.merge_request_metadata(request_id, request_metadata.clone()); + super::super::RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + authenticated, + ) + .expect("authenticated attribution") + }; + if attribution_first { + adapter.admit(&context(request_id)); + } + + let raw = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + request_metadata, + ); + match raw { + Some(raw) => raw.terminal(TerminalOutcome::Completed), + None => adapter.terminal(request_id, TerminalOutcome::Completed), + } + drop(attribution); + + let events = canonical_events(&service); + assert_eq!( + count_events(&events, |event| matches!( + event, + LifecycleEvent::Admitted { .. } + )), + 1, + "admission order: attribution_first={attribution_first}" + ); + assert_eq!( + count_events(&events, |event| matches!( + event, + LifecycleEvent::Completed { .. } + )), + 1, + "terminal order: attribution_first={attribution_first}" + ); + let summary = service + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("terminal summary"); + assert_eq!(summary.metadata.route(), Some("chat_completions")); + assert_eq!(summary.metadata.model(), Some("model-a")); + assert_eq!(summary.metadata.provider(), Some("openai_frontend")); + assert_eq!(summary.metadata.engine(), Some("chat_completion")); + assert_eq!(summary.metadata.source(), Some("direct_http")); + assert_eq!(summary.metadata.method(), Some("POST")); + assert_eq!( + summary.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(summary.metadata.caller_addr(), Some("192.0.2.83:11204")); + assert_eq!( + summary.metadata.caller_path_type(), + Some("remote_quic_http") + ); + } + } + #[test] fn ingress_attachment_keeps_direct_frontend_on_one_parent() { let service = Arc::new(LoggingService::new_disabled(Default::default())); diff --git a/crates/mesh-llm-host-runtime/src/logging/persistence.rs b/crates/mesh-llm-host-runtime/src/logging/persistence.rs index ca79ed5f22..25da1c86cf 100644 --- a/crates/mesh-llm-host-runtime/src/logging/persistence.rs +++ b/crates/mesh-llm-host-runtime/src/logging/persistence.rs @@ -143,12 +143,15 @@ impl LogStoreSink { impl PersistSink for LogStoreSink { async fn persist_summary(&self, entry: RequestSummaryEntry) -> Result<(), String> { self.run_blocking(move |store| { - store.upsert_summary_metadata( + store.upsert_summary_metadata_with_caller( &entry.request_id, entry.metadata.model(), entry.metadata.route(), entry.metadata.provider(), entry.metadata.engine(), + entry.metadata.caller_endpoint_id(), + entry.metadata.caller_addr(), + entry.metadata.caller_path_type(), &entry.created_at, ) }) diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs index 32574d094b..bf200876ec 100644 --- a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle.rs @@ -5,33 +5,52 @@ //! Direct loopback requests still belong to the frontend observer because they //! never claim this raw-ingress ownership. -use std::{ - collections::HashMap, - sync::{ - Arc, Mutex, - atomic::{AtomicU64, Ordering}, - }, -}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::sync::{Arc, Mutex}; -use mesh_llm_events::logging::{ - events::LifecycleEvent, - identifiers::{AttemptId, RequestId}, - proxy::ProxyRecord, - replay::ReplayChannel, -}; +use mesh_llm_events::logging::identifiers::RequestId; -use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata, TerminalOutcome}; +#[cfg(test)] +use mesh_llm_events::logging::events::LifecycleEvent; + +use super::{LifecycleGuard, LoggingService, RequestSummaryMetadata}; + +#[cfg(test)] +use super::TerminalOutcome; + +mod event_emission; +mod ownership; +mod proxy_attempts; +mod remote_attribution; -const MAX_RAW_MESH_LIFECYCLE_OWNERS: usize = 1_024; -const MAX_LOGGED_COMPLETION_TOKENS: u64 = u32::MAX as u64; +pub(crate) use proxy_attempts::{ProxyAttemptFinish, RawMeshProxyAttempt}; +pub(crate) use remote_attribution::{RawMeshRemoteAttributionLease, RawMeshRemoteSuppressionLease}; + +const MAX_TRACKED_REQUESTS: usize = 1_024; + +#[cfg(test)] +const MAX_RAW_MESH_LIFECYCLE_OWNERS: usize = MAX_TRACKED_REQUESTS; #[derive(Default)] pub(crate) struct RawMeshLifecycleOwners { - active: Mutex>, - remote_suppressions: Mutex>, + coordination: Mutex, next_token: AtomicU64, } +#[derive(Default)] +struct LifecycleCoordination { + owners: HashMap, + raw_owner_count: usize, + remote_attributions: HashMap, + remote_suppressions: HashMap, +} + +enum LifecycleOwnerEntry { + Raw(RawMeshLifecycleEntry), + Frontend, +} + struct RawMeshLifecycleEntry { guard: LifecycleGuard, token: u64, @@ -47,488 +66,28 @@ struct RemoteSuppressionEntry { leases: u32, } -pub(crate) struct RawMeshRequestLifecycle { - service: Arc, - owners: Arc, - request_id: RequestId, +struct RemoteAttributionEntry { token: u64, - guard: LifecycleGuard, + leases: u32, + metadata: RequestSummaryMetadata, } -/// One transport attempt that is already owned by a raw mesh request parent. -/// -/// The attempt identifier is created by the canonical lifecycle recorder and -/// is reused for durable proxy metadata; this type cannot create a second -/// terminal owner. -pub(crate) struct RawMeshProxyAttempt { - attempt_id: AttemptId, - started_at: String, +enum RemoteAttributionPlacement { + Applied, + Pending(u64), } -/// Sanitized terminal metadata for one durable proxy attempt record. -pub(crate) struct ProxyAttemptFinish { - pub(crate) target: &'static str, - pub(crate) provider: Option<&'static str>, - pub(crate) engine: Option<&'static str>, - pub(crate) status_code: Option, - pub(crate) lifecycle_error: Option<&'static str>, - pub(crate) error: Option<&'static str>, +pub(crate) enum FrontendAdmissionDecision { + Rejected, + Registered { evicted: Option }, } -/// A fail-open, process-local marker for a trusted remote tunnel request. -/// -/// Unlike a raw ingress lifecycle, this never registers a logging parent. It -/// only prevents the target's embedded frontend observer from registering a -/// duplicate parent while the authenticated tunnel relay is active. -pub(crate) struct RawMeshRemoteSuppressionLease { +pub(crate) struct RawMeshRequestLifecycle { + service: Arc, owners: Arc, request_id: RequestId, token: u64, -} - -impl RawMeshLifecycleOwners { - pub(crate) fn is_claimed(&self, request_id: RequestId) -> bool { - lock_recover(&self.active).contains_key(&request_id) - || lock_recover(&self.remote_suppressions).contains_key(&request_id) - } - - fn acquire_remote_suppression(&self, request_id: RequestId) -> Option { - let mut suppressions = lock_recover(&self.remote_suppressions); - if let Some(existing) = suppressions.get_mut(&request_id) { - existing.leases = existing.leases.saturating_add(1); - return Some(existing.token); - } - if suppressions.len() >= MAX_RAW_MESH_LIFECYCLE_OWNERS { - return None; - } - - let token = self.next_token.fetch_add(1, Ordering::Relaxed); - suppressions.insert(request_id, RemoteSuppressionEntry { token, leases: 1 }); - Some(token) - } - - fn release_remote_suppression(&self, request_id: RequestId, token: u64) { - let mut suppressions = lock_recover(&self.remote_suppressions); - let Some(entry) = suppressions.get_mut(&request_id) else { - return; - }; - if entry.token != token { - return; - } - if entry.leases > 1 { - entry.leases -= 1; - } else { - suppressions.remove(&request_id); - } - } - - fn claim( - &self, - service: &LoggingService, - request_id: RequestId, - metadata: RequestSummaryMetadata, - ) -> Option<(LifecycleGuard, u64)> { - let mut active = lock_recover(&self.active); - if let Some(existing) = active.get(&request_id) { - let claim = (existing.guard.clone(), existing.token); - drop(active); - service.merge_request_metadata(request_id, metadata); - return Some(claim); - } - if active.len() >= MAX_RAW_MESH_LIFECYCLE_OWNERS { - return None; - } - - let (guard, _) = service.register_request_with_metadata(request_id, metadata); - let token = self.next_token.fetch_add(1, Ordering::Relaxed); - active.insert( - request_id, - RawMeshLifecycleEntry { - guard: guard.clone(), - token, - route_selected: false, - stream_started: false, - first_token_recorded: false, - stream_completed: false, - stream_error: false, - }, - ); - Some((guard, token)) - } - - fn emit_route_selected( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - model: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.route_selected { - false - } else { - entry.route_selected = true; - true - } - }; - if !should_emit { - return; - } - - let metadata = RequestSummaryMetadata::from_parts(None, model, provider, engine); - service.merge_request_metadata(request_id, metadata.clone()); - let event = LifecycleEvent::RouteSelected { - model: metadata.model().map(str::to_owned), - provider: metadata.provider().map(str::to_owned), - engine: metadata.engine().map(str::to_owned), - }; - if let Ok(payload) = serde_json::to_string(&event) { - let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); - } - } - - fn emit_stream_started( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - model: Option<&str>, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.stream_started || entry.stream_completed { - false - } else { - entry.stream_started = true; - entry.first_token_recorded = false; - entry.stream_completed = false; - entry.stream_error = false; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamStarted { - model: bounded_route_metadata(model), - }, - ); - } - } - - fn emit_stream_chunk(&self, service: &LoggingService, request_id: RequestId, token: u64) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token - || !entry.stream_started - || entry.stream_completed - || entry.stream_error - || entry.first_token_recorded - { - false - } else { - entry.first_token_recorded = true; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamChunk { tokens: None }, - ); - } - } - - fn emit_stream_completed( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - usage: Option, - ) { - let usage = usage.and_then(|usage| { - mesh_llm_events::logging::events::TokenUsage::from_counts( - usage.prompt_tokens, - usage.completion_tokens, - usage.total_tokens, - ) - }); - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token - || !entry.stream_started - || entry.stream_completed - || entry.stream_error - { - false - } else { - entry.stream_completed = true; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamCompleted { - tokens: bounded_completion_tokens( - usage.and_then(|usage| usage.completion_tokens), - ), - usage, - }, - ); - } - } - - fn emit_stream_error( - &self, - service: &LoggingService, - request_id: RequestId, - token: u64, - label: &'static str, - ) { - let should_emit = { - let mut active = lock_recover(&self.active); - let Some(entry) = active.get_mut(&request_id) else { - return; - }; - if entry.token != token || entry.stream_completed || entry.stream_error { - false - } else { - entry.stream_error = true; - entry.stream_started = false; - true - } - }; - if should_emit { - enqueue_stream_event( - service, - request_id, - LifecycleEvent::StreamError { - error: Some(label.to_owned()), - }, - ); - } - } - - fn release(&self, request_id: RequestId, token: u64) { - let mut active = lock_recover(&self.active); - if active - .get(&request_id) - .is_some_and(|entry| entry.token == token) - { - active.remove(&request_id); - } - } -} - -fn enqueue_stream_event(service: &LoggingService, request_id: RequestId, event: LifecycleEvent) { - if let Ok(payload) = serde_json::to_string(&event) { - let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); - } -} - -impl RawMeshRequestLifecycle { - pub(crate) fn request_id(&self) -> RequestId { - self.request_id - } - - pub(crate) fn register( - service: Arc, - owners: Arc, - request_id: RequestId, - ) -> Option { - Self::register_with_metadata( - service, - owners, - request_id, - RequestSummaryMetadata::default(), - ) - } - - /// Register a raw ingress parent with the metadata known before routing. - pub(crate) fn register_with_metadata( - service: Arc, - owners: Arc, - request_id: RequestId, - metadata: RequestSummaryMetadata, - ) -> Option { - let (guard, token) = owners.claim(&service, request_id, metadata)?; - Some(Self { - service, - owners, - request_id, - token, - guard, - }) - } - - pub(crate) fn route_selected(&self, model: Option<&str>) { - self.route_selected_with_metadata(model, Some("mesh"), Some("raw_ingress")); - } - - pub(crate) fn route_selected_with_metadata( - &self, - model: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - ) { - self.owners.emit_route_selected( - &self.service, - self.request_id, - self.token, - model, - provider, - engine, - ); - } - - pub(crate) fn stream_started(&self, model: Option<&str>) { - self.owners - .emit_stream_started(&self.service, self.request_id, self.token, model); - } - - /// Record the first produced stream chunk. The canonical event contract - /// deliberately keeps this metadata-only and represents the first-token - /// boundary as the first `stream_chunk` event. - pub(crate) fn stream_first_token(&self) { - self.owners - .emit_stream_chunk(&self.service, self.request_id, self.token); - } - - pub(crate) fn stream_chunk(&self) { - self.owners - .emit_stream_chunk(&self.service, self.request_id, self.token); - } - - pub(crate) fn stream_completed( - &self, - usage: Option, - ) { - self.owners - .emit_stream_completed(&self.service, self.request_id, self.token, usage); - } - - pub(crate) fn stream_error(&self, label: &'static str) { - self.owners - .emit_stream_error(&self.service, self.request_id, self.token, label); - } - - pub(crate) fn stream_cancelled(&self) { - self.stream_error("client_disconnected"); - } - - /// Start one bounded transport attempt beneath this raw mesh parent. - pub(crate) fn start_attempt(&self) -> AttemptId { - self.service.start_attempt(self.request_id, &self.guard) - } - - /// Start one lifecycle attempt and retain only the bounded metadata needed - /// to persist its transport result later. - pub(crate) fn start_proxy_attempt(&self) -> RawMeshProxyAttempt { - RawMeshProxyAttempt { - attempt_id: self.start_attempt(), - started_at: self.service.proxy_record_timestamp(), - } - } - - /// Complete one previously started raw mesh transport attempt. - pub(crate) fn complete_attempt(&self, attempt_id: AttemptId, status_code: u16) { - self.service - .complete_attempt(self.request_id, attempt_id, Some(status_code)); - } - - /// Fail one previously started raw mesh transport attempt with a bounded - /// static outcome label. The parent remains active for later targets. - pub(crate) fn fail_attempt(&self, attempt_id: AttemptId, label: &'static str) { - self.service - .fail_attempt(self.request_id, attempt_id, label.to_owned()); - } - - /// Finish the existing lifecycle attempt and enqueue one metadata-only - /// durable proxy record. Persistence is deliberately fail-open and this - /// method never terminalizes the parent request. - pub(crate) fn finish_proxy_attempt( - &self, - attempt: RawMeshProxyAttempt, - finish: ProxyAttemptFinish, - ) { - let ProxyAttemptFinish { - target, - provider, - engine, - status_code, - lifecycle_error, - error, - } = finish; - let completed_at = self.service.proxy_record_timestamp(); - let mut record = ProxyRecord::new( - attempt.attempt_id, - self.request_id, - target.to_owned(), - attempt.started_at, - ); - record.provider = provider.map(str::to_owned); - record.engine = engine.map(str::to_owned); - - if let Some(status_code) = status_code { - self.complete_attempt(attempt.attempt_id, status_code); - record.complete(status_code, completed_at); - record.error = error.map(str::to_owned); - } else { - let lifecycle_error = lifecycle_error.unwrap_or("retryable_unavailable"); - let error = error.unwrap_or("unavailable"); - self.fail_attempt(attempt.attempt_id, lifecycle_error); - record.fail(error.to_owned(), completed_at); - } - - let _ = self.service.enqueue_proxy_record(record); - } - - pub(crate) fn terminal(&self, outcome: TerminalOutcome) { - let _ = self - .service - .transition_terminal(self.request_id, &self.guard, outcome); - self.owners.release(self.request_id, self.token); - } -} - -impl RawMeshRemoteSuppressionLease { - pub(crate) fn acquire( - owners: Arc, - request_id: RequestId, - ) -> Option { - let token = owners.acquire_remote_suppression(request_id)?; - Some(Self { - owners, - request_id, - token, - }) - } -} - -impl Drop for RawMeshRemoteSuppressionLease { - fn drop(&mut self) { - self.owners - .release_remote_suppression(self.request_id, self.token); - } + guard: LifecycleGuard, } fn lock_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { @@ -537,28 +96,5 @@ fn lock_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -const MAX_ROUTE_METADATA_CHARS: usize = 64; - -fn bounded_route_metadata(value: Option<&str>) -> Option { - let value = value?; - let (value, _) = super::policy::apply_redaction(value); - let bounded: String = value - .chars() - .take(MAX_ROUTE_METADATA_CHARS) - .map(|character| { - if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { - character - } else { - '_' - } - }) - .collect(); - (!bounded.is_empty()).then_some(bounded) -} - -fn bounded_completion_tokens(tokens: Option) -> Option { - tokens.filter(|value| *value <= MAX_LOGGED_COMPLETION_TOKENS) -} - #[cfg(test)] mod tests; diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs new file mode 100644 index 0000000000..b4268cc4c2 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/event_emission.rs @@ -0,0 +1,235 @@ +use mesh_llm_events::logging::{ + events::{LifecycleEvent, TokenUsage}, + identifiers::RequestId, + replay::ReplayChannel, +}; + +use super::{ + LifecycleOwnerEntry, LoggingService, RawMeshLifecycleOwners, RawMeshRequestLifecycle, + lock_recover, +}; + +const MAX_LOGGED_COMPLETION_TOKENS: u64 = u32::MAX as u64; +const MAX_ROUTE_METADATA_CHARS: usize = 64; + +impl RawMeshLifecycleOwners { + fn emit_stream_started( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + model: Option<&str>, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.stream_started || entry.stream_completed { + false + } else { + entry.stream_started = true; + entry.first_token_recorded = false; + entry.stream_completed = false; + entry.stream_error = false; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamStarted { + model: bounded_route_metadata(model), + }, + ); + } + } + + fn emit_stream_chunk(&self, service: &LoggingService, request_id: RequestId, token: u64) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token + || !entry.stream_started + || entry.stream_completed + || entry.stream_error + || entry.first_token_recorded + { + false + } else { + entry.first_token_recorded = true; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamChunk { tokens: None }, + ); + } + } + + fn emit_stream_completed( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + usage: Option, + ) { + let usage = usage.and_then(|usage| { + TokenUsage::from_counts( + usage.prompt_tokens, + usage.completion_tokens, + usage.total_tokens, + ) + }); + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token + || !entry.stream_started + || entry.stream_completed + || entry.stream_error + { + false + } else { + entry.stream_completed = true; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamCompleted { + tokens: bounded_completion_tokens( + usage.and_then(|usage| usage.completion_tokens), + ), + usage, + }, + ); + } + } + + fn emit_stream_error( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + label: &'static str, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.stream_completed || entry.stream_error { + false + } else { + entry.stream_error = true; + entry.stream_started = false; + true + } + }; + if should_emit { + enqueue_stream_event( + service, + request_id, + LifecycleEvent::StreamError { + error: Some(label.to_owned()), + }, + ); + } + } +} + +impl RawMeshRequestLifecycle { + pub(crate) fn route_selected(&self, model: Option<&str>) { + self.route_selected_with_metadata(model, Some("mesh"), Some("raw_ingress")); + } + + pub(crate) fn route_selected_with_metadata( + &self, + model: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + ) { + self.owners.emit_route_selected( + &self.service, + self.request_id, + self.token, + model, + provider, + engine, + ); + } + + pub(crate) fn stream_started(&self, model: Option<&str>) { + self.owners + .emit_stream_started(&self.service, self.request_id, self.token, model); + } + + /// Record the first produced stream chunk. The canonical event contract + /// deliberately keeps this metadata-only and represents the first-token + /// boundary as the first `stream_chunk` event. + pub(crate) fn stream_first_token(&self) { + self.owners + .emit_stream_chunk(&self.service, self.request_id, self.token); + } + + pub(crate) fn stream_chunk(&self) { + self.owners + .emit_stream_chunk(&self.service, self.request_id, self.token); + } + + pub(crate) fn stream_completed(&self, usage: Option) { + self.owners + .emit_stream_completed(&self.service, self.request_id, self.token, usage); + } + + pub(crate) fn stream_error(&self, label: &'static str) { + self.owners + .emit_stream_error(&self.service, self.request_id, self.token, label); + } + + pub(crate) fn stream_cancelled(&self) { + self.stream_error("client_disconnected"); + } +} + +fn enqueue_stream_event(service: &LoggingService, request_id: RequestId, event: LifecycleEvent) { + if let Ok(payload) = serde_json::to_string(&event) { + let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); + } +} + +fn bounded_route_metadata(value: Option<&str>) -> Option { + let value = value?; + let (value, _) = super::super::policy::apply_redaction(value); + let bounded: String = value + .chars() + .take(MAX_ROUTE_METADATA_CHARS) + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { + character + } else { + '_' + } + }) + .collect(); + (!bounded.is_empty()).then_some(bounded) +} + +fn bounded_completion_tokens(tokens: Option) -> Option { + tokens.filter(|value| *value <= MAX_LOGGED_COMPLETION_TOKENS) +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs new file mode 100644 index 0000000000..07573346e2 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/ownership.rs @@ -0,0 +1,208 @@ +use std::sync::{Arc, atomic::Ordering}; + +use mesh_llm_events::logging::{ + events::LifecycleEvent, identifiers::RequestId, replay::ReplayChannel, +}; + +use super::super::TerminalOutcome; + +use super::{ + FrontendAdmissionDecision, LifecycleGuard, LifecycleOwnerEntry, LoggingService, + MAX_TRACKED_REQUESTS, RawMeshLifecycleEntry, RawMeshLifecycleOwners, RawMeshRequestLifecycle, + RequestSummaryMetadata, lock_recover, +}; + +impl RawMeshLifecycleOwners { + pub(crate) fn is_claimed(&self, request_id: RequestId) -> bool { + let coordination = lock_recover(&self.coordination); + coordination.owners.contains_key(&request_id) + || coordination.remote_attributions.contains_key(&request_id) + || coordination.remote_suppressions.contains_key(&request_id) + } + + /// Coordinate frontend eligibility and registration with raw claims, + /// authenticated attribution, and suppression placement. The callback may + /// briefly lock the frontend tracker and service registry; callers must not + /// enter this method while holding either inner lock. + pub(crate) fn admit_frontend( + &self, + request_id: RequestId, + register: impl FnOnce() -> FrontendAdmissionDecision, + ) { + let mut coordination = lock_recover(&self.coordination); + if coordination.owners.contains_key(&request_id) + || coordination.remote_attributions.contains_key(&request_id) + || coordination.remote_suppressions.contains_key(&request_id) + { + return; + } + let FrontendAdmissionDecision::Registered { evicted } = register() else { + return; + }; + if let Some(evicted) = evicted + && matches!( + coordination.owners.get(&evicted), + Some(LifecycleOwnerEntry::Frontend) + ) + { + coordination.owners.remove(&evicted); + } + coordination + .owners + .insert(request_id, LifecycleOwnerEntry::Frontend); + } + + pub(crate) fn release_frontend(&self, request_id: RequestId) { + let mut coordination = lock_recover(&self.coordination); + if matches!( + coordination.owners.get(&request_id), + Some(LifecycleOwnerEntry::Frontend) + ) { + coordination.owners.remove(&request_id); + } + } + + fn claim( + &self, + service: &LoggingService, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option<(LifecycleGuard, u64)> { + let mut coordination = lock_recover(&self.coordination); + if let Some(existing) = coordination.owners.get(&request_id) { + return match existing { + LifecycleOwnerEntry::Raw(existing) => { + let claim = (existing.guard.clone(), existing.token); + service.merge_request_metadata(request_id, metadata); + Some(claim) + } + LifecycleOwnerEntry::Frontend => { + service.merge_request_metadata(request_id, metadata); + None + } + }; + } + if coordination.raw_owner_count >= MAX_TRACKED_REQUESTS { + return None; + } + + let mut metadata = metadata; + if let Some(attribution) = coordination.remote_attributions.remove(&request_id) { + metadata.merge_authenticated_remote_caller(attribution.metadata); + } + let (guard, _) = service.register_request_with_metadata(request_id, metadata); + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination.owners.insert( + request_id, + LifecycleOwnerEntry::Raw(RawMeshLifecycleEntry { + guard: guard.clone(), + token, + route_selected: false, + stream_started: false, + first_token_recorded: false, + stream_completed: false, + stream_error: false, + }), + ); + coordination.raw_owner_count += 1; + Some((guard, token)) + } + + pub(super) fn emit_route_selected( + &self, + service: &LoggingService, + request_id: RequestId, + token: u64, + model: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + ) { + let should_emit = { + let mut coordination = lock_recover(&self.coordination); + let Some(LifecycleOwnerEntry::Raw(entry)) = coordination.owners.get_mut(&request_id) + else { + return; + }; + if entry.token != token || entry.route_selected { + false + } else { + entry.route_selected = true; + true + } + }; + if !should_emit { + return; + } + + let metadata = RequestSummaryMetadata::from_parts(None, model, provider, engine); + service.merge_request_metadata(request_id, metadata.clone()); + let event = LifecycleEvent::RouteSelected { + model: metadata.model().map(str::to_owned), + provider: metadata.provider().map(str::to_owned), + engine: metadata.engine().map(str::to_owned), + }; + if let Ok(payload) = serde_json::to_string(&event) { + let _ = service.enqueue_event(request_id, ReplayChannel::Operations, payload); + } + } + + fn release(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + if matches!( + coordination.owners.get(&request_id), + Some(LifecycleOwnerEntry::Raw(entry)) if entry.token == token + ) { + coordination.owners.remove(&request_id); + coordination.raw_owner_count -= 1; + } + } +} + +impl RawMeshRequestLifecycle { + pub(crate) fn request_id(&self) -> RequestId { + self.request_id + } + + pub(crate) fn register( + service: Arc, + owners: Arc, + request_id: RequestId, + ) -> Option { + Self::register_with_metadata( + service, + owners, + request_id, + RequestSummaryMetadata::default(), + ) + } + + /// Register a raw ingress parent with the metadata known before routing. + pub(crate) fn register_with_metadata( + service: Arc, + owners: Arc, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + let (guard, token) = owners.claim(&service, request_id, metadata)?; + Some(Self { + service, + owners, + request_id, + token, + guard, + }) + } + + pub(crate) fn terminal(&self, outcome: TerminalOutcome) { + let _ = self + .service + .transition_terminal(self.request_id, &self.guard, outcome); + self.owners.release(self.request_id, self.token); + } +} + +impl Drop for RawMeshRequestLifecycle { + fn drop(&mut self) { + self.owners.release(self.request_id, self.token); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs new file mode 100644 index 0000000000..0a45fbfd44 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/proxy_attempts.rs @@ -0,0 +1,93 @@ +use mesh_llm_events::logging::identifiers::AttemptId; +use mesh_llm_events::logging::proxy::ProxyRecord; + +use super::RawMeshRequestLifecycle; + +/// One transport attempt that is already owned by a raw mesh request parent. +/// +/// The attempt identifier is created by the canonical lifecycle recorder and +/// is reused for durable proxy metadata; this type cannot create a second +/// terminal owner. +pub(crate) struct RawMeshProxyAttempt { + attempt_id: AttemptId, + started_at: String, +} + +/// Sanitized terminal metadata for one durable proxy attempt record. +pub(crate) struct ProxyAttemptFinish { + pub(crate) target: &'static str, + pub(crate) provider: Option<&'static str>, + pub(crate) engine: Option<&'static str>, + pub(crate) status_code: Option, + pub(crate) lifecycle_error: Option<&'static str>, + pub(crate) error: Option<&'static str>, +} + +impl RawMeshRequestLifecycle { + /// Start one bounded transport attempt beneath this raw mesh parent. + pub(crate) fn start_attempt(&self) -> AttemptId { + self.service.start_attempt(self.request_id, &self.guard) + } + + /// Start one lifecycle attempt and retain only the bounded metadata needed + /// to persist its transport result later. + pub(crate) fn start_proxy_attempt(&self) -> RawMeshProxyAttempt { + RawMeshProxyAttempt { + attempt_id: self.start_attempt(), + started_at: self.service.proxy_record_timestamp(), + } + } + + /// Complete one previously started raw mesh transport attempt. + pub(crate) fn complete_attempt(&self, attempt_id: AttemptId, status_code: u16) { + self.service + .complete_attempt(self.request_id, attempt_id, Some(status_code)); + } + + /// Fail one previously started raw mesh transport attempt with a bounded + /// static outcome label. The parent remains active for later targets. + pub(crate) fn fail_attempt(&self, attempt_id: AttemptId, label: &'static str) { + self.service + .fail_attempt(self.request_id, attempt_id, label.to_owned()); + } + + /// Finish the existing lifecycle attempt and enqueue one metadata-only + /// durable proxy record. Persistence is deliberately fail-open and this + /// method never terminalizes the parent request. + pub(crate) fn finish_proxy_attempt( + &self, + attempt: RawMeshProxyAttempt, + finish: ProxyAttemptFinish, + ) { + let ProxyAttemptFinish { + target, + provider, + engine, + status_code, + lifecycle_error, + error, + } = finish; + let completed_at = self.service.proxy_record_timestamp(); + let mut record = ProxyRecord::new( + attempt.attempt_id, + self.request_id, + target.to_owned(), + attempt.started_at, + ); + record.provider = provider.map(str::to_owned); + record.engine = engine.map(str::to_owned); + + if let Some(status_code) = status_code { + self.complete_attempt(attempt.attempt_id, status_code); + record.complete(status_code, completed_at); + record.error = error.map(str::to_owned); + } else { + let lifecycle_error = lifecycle_error.unwrap_or("retryable_unavailable"); + let error = error.unwrap_or("unavailable"); + self.fail_attempt(attempt.attempt_id, lifecycle_error); + record.fail(error.to_owned(), completed_at); + } + + let _ = self.service.enqueue_proxy_record(record); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs new file mode 100644 index 0000000000..02e307cb64 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/remote_attribution.rs @@ -0,0 +1,157 @@ +use std::sync::{Arc, atomic::Ordering}; + +use mesh_llm_events::logging::identifiers::RequestId; + +use super::{ + LoggingService, MAX_TRACKED_REQUESTS, RawMeshLifecycleOwners, RemoteAttributionEntry, + RemoteAttributionPlacement, RemoteSuppressionEntry, RequestSummaryMetadata, lock_recover, +}; + +/// A fail-open, process-local marker for a trusted remote tunnel request. +/// +/// Unlike a raw ingress lifecycle, this never registers a logging parent. It +/// only prevents the target's embedded frontend observer from registering a +/// duplicate parent while the authenticated tunnel relay is active. +pub(crate) struct RawMeshRemoteSuppressionLease { + owners: Arc, + request_id: RequestId, + token: u64, +} + +pub(crate) struct RawMeshRemoteAttributionLease { + owners: Arc, + request_id: RequestId, + token: Option, +} + +impl RawMeshLifecycleOwners { + fn acquire_remote_suppression(&self, request_id: RequestId) -> Option { + let mut coordination = lock_recover(&self.coordination); + if let Some(existing) = coordination.remote_suppressions.get_mut(&request_id) { + existing.leases = existing.leases.saturating_add(1); + return Some(existing.token); + } + if coordination.remote_suppressions.len() >= MAX_TRACKED_REQUESTS { + return None; + } + + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination + .remote_suppressions + .insert(request_id, RemoteSuppressionEntry { token, leases: 1 }); + Some(token) + } + + fn attribute_remote_caller( + &self, + service: &LoggingService, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + if !metadata.has_authenticated_remote_caller() { + return None; + } + let mut coordination = lock_recover(&self.coordination); + if coordination.owners.contains_key(&request_id) { + service.merge_authenticated_remote_caller(request_id, metadata); + return Some(RemoteAttributionPlacement::Applied); + } + + if let Some(existing) = coordination.remote_attributions.get_mut(&request_id) { + existing.leases = existing.leases.saturating_add(1); + return Some(RemoteAttributionPlacement::Pending(existing.token)); + } + if coordination.remote_attributions.len() >= MAX_TRACKED_REQUESTS { + return None; + } + let token = self.next_token.fetch_add(1, Ordering::Relaxed); + coordination.remote_attributions.insert( + request_id, + RemoteAttributionEntry { + token, + leases: 1, + metadata, + }, + ); + Some(RemoteAttributionPlacement::Pending(token)) + } + + fn release_remote_attribution(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + let Some(entry) = coordination.remote_attributions.get_mut(&request_id) else { + return; + }; + if entry.token != token { + return; + } + if entry.leases > 1 { + entry.leases -= 1; + } else { + coordination.remote_attributions.remove(&request_id); + } + } + + fn release_remote_suppression(&self, request_id: RequestId, token: u64) { + let mut coordination = lock_recover(&self.coordination); + let Some(entry) = coordination.remote_suppressions.get_mut(&request_id) else { + return; + }; + if entry.token != token { + return; + } + if entry.leases > 1 { + entry.leases -= 1; + } else { + coordination.remote_suppressions.remove(&request_id); + } + } +} + +impl RawMeshRemoteSuppressionLease { + pub(crate) fn acquire( + owners: Arc, + request_id: RequestId, + ) -> Option { + let token = owners.acquire_remote_suppression(request_id)?; + Some(Self { + owners, + request_id, + token, + }) + } +} + +impl RawMeshRemoteAttributionLease { + pub(crate) fn acquire( + service: &LoggingService, + owners: Arc, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + let placement = owners.attribute_remote_caller(service, request_id, metadata)?; + Some(Self { + owners, + request_id, + token: match placement { + RemoteAttributionPlacement::Applied => None, + RemoteAttributionPlacement::Pending(token) => Some(token), + }, + }) + } +} + +impl Drop for RawMeshRemoteAttributionLease { + fn drop(&mut self) { + if let Some(token) = self.token { + self.owners + .release_remote_attribution(self.request_id, token); + } + } +} + +impl Drop for RawMeshRemoteSuppressionLease { + fn drop(&mut self) { + self.owners + .release_remote_suppression(self.request_id, self.token); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs index 36112c7685..916082e529 100644 --- a/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs +++ b/crates/mesh-llm-host-runtime/src/logging/raw_mesh_lifecycle/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::logging::{OpenAiLifecycleAttachment, ServiceConfig}; +use crate::logging::{CallerPathType, OpenAiLifecycleAttachment, ServiceConfig}; fn recorded_events(service: &LoggingService) -> Vec { service @@ -15,6 +15,31 @@ fn recorded_events(service: &LoggingService) -> Vec { .collect() } +#[test] +fn frontend_release_is_idempotent_and_preserves_raw_ownership() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let raw_request_id = RequestId::new(); + let raw = RawMeshRequestLifecycle::register( + Arc::clone(&service), + Arc::clone(&owners), + raw_request_id, + ) + .expect("raw request should own its lifecycle"); + let frontend_request_id = RequestId::new(); + owners.admit_frontend(frontend_request_id, || { + FrontendAdmissionDecision::Registered { evicted: None } + }); + + owners.release_frontend(frontend_request_id); + owners.release_frontend(frontend_request_id); + owners.release_frontend(raw_request_id); + + assert!(!owners.is_claimed(frontend_request_id)); + assert!(owners.is_claimed(raw_request_id)); + raw.terminal(TerminalOutcome::Completed); +} + #[test] fn raw_mesh_lifecycle_orders_metadata_events_without_payloads() { let service = Arc::new(LoggingService::new_disabled(Default::default())); @@ -365,6 +390,45 @@ fn raw_mesh_owner_reuses_one_guard_and_terminalizes_once() { assert!(!owners.is_claimed(request_id)); } +#[test] +fn dropping_unterminalized_raw_mesh_lifecycle_releases_owner_slot() { + // Given + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let lifecycle = + RawMeshRequestLifecycle::register(service, Arc::clone(&owners), request_id).unwrap(); + + // When + drop(lifecycle); + + // Then + assert!(!owners.is_claimed(request_id)); +} + +#[test] +fn dropping_stale_duplicate_does_not_release_replacement_owner() { + // Given + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let first = + RawMeshRequestLifecycle::register(Arc::clone(&service), Arc::clone(&owners), request_id) + .unwrap(); + let stale_duplicate = + RawMeshRequestLifecycle::register(Arc::clone(&service), Arc::clone(&owners), request_id) + .unwrap(); + first.terminal(TerminalOutcome::Completed); + let _replacement = + RawMeshRequestLifecycle::register(service, Arc::clone(&owners), request_id).unwrap(); + + // When + drop(stale_duplicate); + + // Then + assert!(owners.is_claimed(request_id)); +} + #[test] fn claimed_plan_failure_terminalizes_without_route_selected_event() { let service = Arc::new(LoggingService::new_disabled(Default::default())); @@ -483,3 +547,140 @@ fn remote_suppression_cap_fails_open_without_registering_parents() { drop(leases); } + +#[test] +fn dropping_first_pending_remote_attribution_releases_token_zero() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let endpoint_id = "81".repeat(32); + let attribution = RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.81:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ) + .expect("first pending attribution"); + + drop(attribution); + let lifecycle = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ) + .expect("local lifecycle"); + + let summary = service + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active summary"); + assert_eq!(summary.metadata.caller_endpoint_id(), None); + assert_eq!(summary.metadata.caller_addr(), Some("127.0.0.1:40123")); + assert_eq!(summary.metadata.caller_path_type(), Some("local_http")); + lifecycle.terminal(TerminalOutcome::Completed); +} + +#[test] +fn remote_attribution_rejects_incomplete_authenticated_tuple() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + + assert!( + RawMeshRemoteAttributionLease::acquire( + &service, + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("192.0.2.82:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ) + .is_none() + ); +} + +#[test] +fn pending_endpoint_only_attribution_survives_provisional_local_registration() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let owners = Arc::new(RawMeshLifecycleOwners::default()); + let request_id = RequestId::new(); + let endpoint_id = "83".repeat(32); + let attribution = RawMeshRemoteAttributionLease::acquire( + &service, + Arc::clone(&owners), + request_id, + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None), + ) + .expect("pending endpoint-only attribution"); + + let lifecycle = RawMeshRequestLifecycle::register_with_metadata( + Arc::clone(&service), + owners, + request_id, + RequestSummaryMetadata::default().with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ) + .expect("local lifecycle"); + + let summary = service + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active summary"); + assert_eq!( + summary.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(summary.metadata.caller_addr(), None); + assert_eq!(summary.metadata.caller_path_type(), None); + lifecycle.terminal(TerminalOutcome::Completed); + drop(attribution); +} + +#[test] +fn remote_attribution_rejects_invalid_endpoint_only_variants() { + let service = Arc::new(LoggingService::new_disabled(Default::default())); + let endpoint_id = "84".repeat(32); + + for metadata in [ + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.84:11204"), + None, + ), + RequestSummaryMetadata::default().with_caller_identity( + Some("not-an-authenticated-endpoint"), + None, + None, + ), + RequestSummaryMetadata::default().with_caller_identity(None, None, None), + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ] { + assert!( + RawMeshRemoteAttributionLease::acquire( + &service, + Arc::new(RawMeshLifecycleOwners::default()), + RequestId::new(), + metadata, + ) + .is_none() + ); + } +} diff --git a/crates/mesh-llm-host-runtime/src/logging/registry.rs b/crates/mesh-llm-host-runtime/src/logging/registry.rs index 617de073ee..d63c088374 100644 --- a/crates/mesh-llm-host-runtime/src/logging/registry.rs +++ b/crates/mesh-llm-host-runtime/src/logging/registry.rs @@ -321,6 +321,29 @@ impl RequestRegistry { }) } + pub(crate) fn merge_authenticated_remote_caller( + &self, + request_id: &str, + metadata: RequestSummaryMetadata, + ) -> Option { + let mut active = self.active.lock().expect("registry mutex poisoned"); + if let Some(entry) = active.get_mut(request_id) { + return entry + .metadata + .merge_authenticated_remote_caller(metadata) + .then(|| entry.clone()); + } + drop(active); + + let mut recent = self.recent.lock().expect("registry mutex poisoned"); + recent.get_mut(request_id).and_then(|entry| { + entry + .metadata + .merge_authenticated_remote_caller(metadata) + .then(|| entry.clone()) + }) + } + /// Atomically move an active request to recent while retaining metadata /// merged before its terminal transition. pub(crate) fn terminalize( diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs index 0a40581616..0cdb32542b 100644 --- a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs @@ -9,6 +9,10 @@ use openai_frontend::OpenAiFrontendRoute; use super::policy::{RedactMode, apply_redaction}; +mod caller_identity; + +pub(crate) use caller_identity::{CallerPathType, authenticated_endpoint_id}; + const MAX_REQUEST_METADATA_CHARS: usize = 64; /// The small, privacy-safe metadata projection shared by active summaries, @@ -27,6 +31,12 @@ pub(crate) struct RequestSummaryMetadata { source: Option, #[serde(default, skip_serializing_if = "Option::is_none")] method: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_endpoint_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + caller_path_type: Option, } impl RequestSummaryMetadata { @@ -44,6 +54,9 @@ impl RequestSummaryMetadata { engine: bounded_metadata(engine), source: None, method: None, + caller_endpoint_id: None, + caller_addr: None, + caller_path_type: None, } } @@ -114,18 +127,33 @@ impl RequestSummaryMetadata { && self.engine.is_none() && self.source.is_none() && self.method.is_none() + && self.caller_endpoint_id.is_none() + && self.caller_addr.is_none() + && self.caller_path_type.is_none() } /// Preserve the first truthful value for each field. A later source can /// fill a missing classification but cannot overwrite an earlier one. pub(crate) fn merge_missing(&mut self, update: Self) -> bool { + let Self { + route, + model, + provider, + engine, + source, + method, + caller_endpoint_id, + caller_addr, + caller_path_type, + } = update; let mut changed = false; - changed |= merge_field(&mut self.route, update.route); - changed |= merge_field(&mut self.model, update.model); - changed |= merge_field(&mut self.provider, update.provider); - changed |= merge_field(&mut self.engine, update.engine); - changed |= merge_field(&mut self.source, update.source); - changed |= merge_field(&mut self.method, update.method); + changed |= merge_field(&mut self.route, route); + changed |= merge_field(&mut self.model, model); + changed |= merge_field(&mut self.provider, provider); + changed |= merge_field(&mut self.engine, engine); + changed |= merge_field(&mut self.source, source); + changed |= merge_field(&mut self.method, method); + changed |= self.merge_missing_caller(caller_endpoint_id, caller_addr, caller_path_type); changed } } @@ -235,14 +263,14 @@ mod tests { .with_source(Some("10.0.0.1")) .is_empty() ); - // Unrecognized or empty methods classify as OTHER rather than being - // silently dropped, so the method field is never absent. assert_eq!( RequestSummaryMetadata::from_parts(None, None, None, None) .with_method(Some("PATCH")) .method(), - Some("OTHER") + Some("PATCH") ); + // Unrecognized or empty methods classify as OTHER rather than being + // silently dropped, so the method field is never absent. assert_eq!( RequestSummaryMetadata::from_parts(None, None, None, None) .with_method(Some("")) diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs new file mode 100644 index 0000000000..810495bb03 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity.rs @@ -0,0 +1,126 @@ +use std::net::SocketAddr; + +use super::RequestSummaryMetadata; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CallerPathType { + LocalHttp, + RemoteQuicHttp, + Relay, +} + +impl CallerPathType { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::LocalHttp => "local_http", + Self::RemoteQuicHttp => "remote_quic_http", + Self::Relay => "relay", + } + } +} + +impl RequestSummaryMetadata { + pub(crate) fn with_caller_identity( + mut self, + endpoint_id: Option<&str>, + addr: Option<&str>, + path_type: Option, + ) -> Self { + let endpoint_id = endpoint_id + .filter(|value| authenticated_endpoint_id(value)) + .map(str::to_owned); + let (caller_endpoint_id, caller_addr, caller_path_type) = match path_type { + Some(CallerPathType::LocalHttp) if endpoint_id.is_none() => { + match bounded_caller_addr(addr) { + Some(addr) => (None, Some(addr), path_type), + None => (None, None, None), + } + } + Some(CallerPathType::RemoteQuicHttp) if endpoint_id.is_some() => { + (endpoint_id, bounded_caller_addr(addr), path_type) + } + Some(CallerPathType::Relay) if endpoint_id.is_some() => (endpoint_id, None, path_type), + None if endpoint_id.is_some() && addr.is_none() => (endpoint_id, None, None), + Some( + CallerPathType::LocalHttp | CallerPathType::RemoteQuicHttp | CallerPathType::Relay, + ) + | None => (None, None, None), + }; + self.caller_endpoint_id = caller_endpoint_id; + self.caller_addr = caller_addr; + self.caller_path_type = caller_path_type; + self + } + + pub(crate) fn caller_endpoint_id(&self) -> Option<&str> { + self.caller_endpoint_id.as_deref() + } + + pub(crate) fn caller_addr(&self) -> Option<&str> { + self.caller_addr.as_deref() + } + + pub(crate) fn caller_path_type(&self) -> Option<&'static str> { + self.caller_path_type.map(CallerPathType::as_str) + } + + pub(crate) fn merge_authenticated_remote_caller(&mut self, update: Self) -> bool { + if !update.has_authenticated_remote_caller() || self.has_authenticated_remote_caller() { + return false; + } + + let changed = self.caller_endpoint_id != update.caller_endpoint_id + || self.caller_addr != update.caller_addr + || self.caller_path_type != update.caller_path_type; + self.caller_endpoint_id = update.caller_endpoint_id; + self.caller_addr = update.caller_addr; + self.caller_path_type = update.caller_path_type; + changed + } + + pub(super) fn merge_missing_caller( + &mut self, + caller_endpoint_id: Option, + caller_addr: Option, + caller_path_type: Option, + ) -> bool { + let current_is_empty = self.caller_endpoint_id.is_none() + && self.caller_addr.is_none() + && self.caller_path_type.is_none(); + let update_is_empty = + caller_endpoint_id.is_none() && caller_addr.is_none() && caller_path_type.is_none(); + if !current_is_empty || update_is_empty { + return false; + } + self.caller_endpoint_id = caller_endpoint_id; + self.caller_addr = caller_addr; + self.caller_path_type = caller_path_type; + true + } + + pub(crate) fn has_authenticated_remote_caller(&self) -> bool { + self.caller_endpoint_id + .as_deref() + .is_some_and(authenticated_endpoint_id) + && match self.caller_path_type { + Some(CallerPathType::LocalHttp) => false, + Some(CallerPathType::RemoteQuicHttp) => true, + Some(CallerPathType::Relay) | None => self.caller_addr.is_none(), + } + } +} + +fn bounded_caller_addr(value: Option<&str>) -> Option { + value? + .parse::() + .ok() + .map(|addr| addr.to_string()) +} + +pub(crate) fn authenticated_endpoint_id(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs new file mode 100644 index 0000000000..3834a539bd --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata/caller_identity/tests.rs @@ -0,0 +1,92 @@ +use super::*; + +#[test] +fn local_http_caller_identity_keeps_ipv4_and_ipv6_socket_addresses() { + for caller_addr in [ + "127.0.0.1:40123", + "192.0.2.10:40123", + "[2001:db8::10]:40123", + ] { + let metadata = RequestSummaryMetadata::default().with_caller_identity( + None, + Some(caller_addr), + Some(CallerPathType::LocalHttp), + ); + + assert_eq!(metadata.caller_addr(), Some(caller_addr)); + assert_eq!(metadata.caller_path_type(), Some("local_http")); + } +} + +#[test] +fn caller_path_vocabulary_rejects_stage_transport_as_a_request_caller() { + let parsed = serde_json::from_str::("\"remote_quic_stage\""); + + assert!(parsed.is_err()); +} + +#[test] +fn authenticated_endpoint_only_caller_is_preserved_without_address_or_path() { + let endpoint_id = "ab".repeat(32); + + let metadata = + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None); + + assert_eq!(metadata.caller_endpoint_id(), Some(endpoint_id.as_str())); + assert_eq!(metadata.caller_addr(), None); + assert_eq!(metadata.caller_path_type(), None); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[test] +fn endpoint_only_caller_rejects_partial_or_unauthenticated_tuples() { + let endpoint_id = "cd".repeat(32); + for metadata in [ + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.80:11204"), + None, + ), + RequestSummaryMetadata::default().with_caller_identity( + Some("not-an-authenticated-endpoint"), + None, + None, + ), + RequestSummaryMetadata::default().with_caller_identity(None, None, None), + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), + ] { + assert!(!metadata.has_authenticated_remote_caller()); + assert_eq!(metadata.caller_endpoint_id(), None); + } +} + +#[test] +fn first_authenticated_endpoint_only_caller_cannot_be_overwritten() { + let first_endpoint_id = "de".repeat(32); + let later_endpoint_id = "ef".repeat(32); + let mut metadata = RequestSummaryMetadata::default().with_caller_identity( + Some(&first_endpoint_id), + None, + None, + ); + + let changed = metadata.merge_authenticated_remote_caller( + RequestSummaryMetadata::default().with_caller_identity( + Some(&later_endpoint_id), + Some("192.0.2.81:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), + ); + + assert!(!changed); + assert_eq!( + metadata.caller_endpoint_id(), + Some(first_endpoint_id.as_str()) + ); + assert_eq!(metadata.caller_addr(), None); + assert_eq!(metadata.caller_path_type(), None); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs index 1ef4e51f5a..2d9dcf73f1 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state.rs @@ -14,9 +14,9 @@ use std::time::Duration; use mesh_llm_events::logging::identifiers::EventId; use mesh_llm_log_store::{ ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE, ArtifactContent, ArtifactRecord, - ArtifactRedactor, AuditEntryFilters, AuditEntryRow, Clock as StoreClock, EventRecord, - FailOpenArtifactCapture, LogStore, LogStoreError, Page, PageQuery, ProxyQuery, ProxyRecord, - QueryPage, RealClock, RequestQuery, RequestRecord, + ArtifactRedactor, AuditEntryDetail, AuditEntryFilters, AuditEntryRow, Clock as StoreClock, + EventRecord, FailOpenArtifactCapture, LogStore, LogStoreError, Page, PageQuery, ProxyQuery, + ProxyRecord, QueryPage, RealClock, RequestQuery, RequestRecord, RequestRecordWithCaller, }; #[cfg(test)] use mesh_llm_log_store::{ArtifactCaptureDisabledReason, ArtifactCaptureOutcome}; @@ -82,8 +82,8 @@ pub(crate) struct LoggingRuntimeStatus { } /// Stable, path-free explanation for metadata availability. Schema versions -/// are safe to expose to trusted-local clients and make upgrade/downgrade -/// failures actionable without leaking the database location or SQLite text. +/// are safe to expose to trusted-local clients without leaking the database +/// location or SQLite text. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum LoggingMetadataState { Ready, @@ -759,12 +759,33 @@ impl LoggingRuntimeState { RawMeshRemoteSuppressionLease::acquire(Arc::clone(&self.raw_mesh_owners), request_id) } + pub(crate) fn attribute_remote_tunneled_request( + &self, + request_id: mesh_llm_events::logging::identifiers::RequestId, + metadata: RequestSummaryMetadata, + ) -> Option { + if self.retired.load(Ordering::Acquire) { + return None; + } + let service = self.service.as_ref()?; + if !service.is_startable() { + return None; + } + super::RawMeshRemoteAttributionLease::acquire( + service, + Arc::clone(&self.raw_mesh_owners), + request_id, + metadata, + ) + } + /// Register one already-parsed management API request with bounded /// metadata. Disabled, unavailable, and retired logging stay fail-open. pub(crate) fn register_management_request( &self, request_id: mesh_llm_events::logging::identifiers::RequestId, method_route: &'static str, + source_addr: Option, ) -> Option { if self.retired.load(Ordering::Acquire) { return None; @@ -773,11 +794,15 @@ impl LoggingRuntimeState { if !service.is_startable() { return None; } - Some(ManagementRequestLifecycle::register( - service, - request_id, - method_route, - )) + Some(match source_addr { + Some(source_addr) => ManagementRequestLifecycle::register_with_source_addr( + service, + request_id, + method_route, + source_addr, + ), + None => ManagementRequestLifecycle::register(service, request_id, method_route), + }) } /// Record one static operational audit without exposing the service to diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs index 9916db3f65..fbb0e1e06f 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/query_facade.rs @@ -45,30 +45,33 @@ impl LoggingQueryFacade { self.service.registry_ref().snapshot_active() } - pub(crate) fn request(&self, request_id: &str) -> Result, LogStoreError> { + pub(crate) fn request( + &self, + request_id: &str, + ) -> Result, LogStoreError> { #[cfg(test)] self.query_counter .point_requests .fetch_add(1, Ordering::Relaxed); - self.store.query_request(request_id) + self.store.query_request_with_caller(request_id) } pub(crate) fn requests_by_ids( &self, request_ids: &[String], - ) -> Result, LogStoreError> { + ) -> Result, LogStoreError> { #[cfg(test)] self.query_counter .batch_requests .fetch_add(1, Ordering::Relaxed); - self.store.query_requests_by_ids(request_ids) + self.store.query_requests_by_ids_with_caller(request_ids) } pub(crate) fn requests( &self, query: &RequestQuery, - ) -> Result, LogStoreError> { - self.store.query_requests(query) + ) -> Result, LogStoreError> { + self.store.query_requests_with_caller(query) } pub(crate) fn events( @@ -124,8 +127,9 @@ impl LoggingQueryFacade { limit: Option, after_cursor: Option<&str>, filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - self.store.list_audit_entries(limit, after_cursor, filters) + ) -> Result, LogStoreError> { + self.store + .list_audit_entry_details(limit, after_cursor, filters) } pub(crate) fn audit_entries_after_sequence( @@ -133,9 +137,9 @@ impl LoggingQueryFacade { after_sequence: u64, limit: usize, filters: AuditEntryFilters, - ) -> Result, LogStoreError> { + ) -> Result, LogStoreError> { self.store - .list_audit_entries_after_sequence(after_sequence, limit, filters) + .list_audit_entry_details_after_sequence(after_sequence, limit, filters) } /// Read content only through the fail-open capture owner. The route core diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs index 1e7d34e096..c1a83f8613 100644 --- a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests.rs @@ -2,7 +2,11 @@ use mesh_llm_log_store::{ArtifactPrivacy, MaintenanceExecutionControl}; use std::path::Path; use super::*; -use crate::logging::TerminalOutcome; +use crate::logging::{CallerPathType, TerminalOutcome}; + +mod artifact_capture; +mod remote_caller_attribution; +mod service_lifecycle; #[derive(Default)] struct RejectPrivacy; @@ -58,589 +62,60 @@ fn artifact_config() -> mesh_llm_config::LoggingConfig { } #[test] -fn production_ingress_wiring_advertises_capture_only_when_storage_is_usable() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); - - let status = state.status(); - assert!(status.artifact_capture_available); - assert!(status.artifact_capture_ready); -} - -#[tokio::test] -async fn ingress_attachment_captures_redacted_request_and_response_bodies() { +fn remote_tunnel_suppression_merges_authenticated_caller_into_existing_parent() { let root = tempfile::tempdir().expect("temporary logging root"); let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); let state = Arc::new(LoggingRuntimeState::initialize( &foundation, - &artifact_config(), + &mesh_llm_config::LoggingConfig::default(), )); let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); let mut attachment = state.openai_ingress_attachment( request_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), - ); - - attachment.capture_request_body( - br#"{"authorization":"Bearer test-secret","model":"safe"}"#, - Some("application/json"), - ); - attachment.route_observer().capture_response_body( - br#"{"id":"chatcmpl-safe","api_key":"test-secret"}"#, - Some("application/json; charset=utf-8"), - ); - attachment.terminal(TerminalOutcome::CompletedWithStatus(201)); - - // Summary, admission, both artifact commands, and terminal state share one - // serial persistence owner. Ingress itself performs no SQLite/file work. - assert_eq!( - state - .service - .as_ref() - .expect("logging service") - .pump_sync() - .await, - 5 + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions") + .with_source(Some("direct_http")) + .with_method(Some("POST")) + .with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ), ); - - let store = state.store().expect("metadata store"); - let request_key = request_id.as_uuid().to_string(); - let artifact_page = store - .query_artifacts( - &request_key, - &mesh_llm_log_store::PageQuery { - limit: 10, - cursor: None, - sort: mesh_llm_log_store::QuerySort::Ascending, - }, + let endpoint_id = "61".repeat(32); + let attribution = state + .attribute_remote_tunneled_request( + request_id, + RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + Some("192.0.2.61:11204"), + Some(CallerPathType::RemoteQuicHttp), + ), ) - .expect("query artifacts"); - assert_eq!(artifact_page.items.len(), 2); - let request_artifact = artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "request") - .expect("request artifact") - .artifact_id - .clone(); - let response_artifact = artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "response") - .expect("response artifact") - .artifact_id - .clone(); - let artifacts = state.query_facade().expect("query facade"); - let request_content = artifacts - .read_artifact(&request_artifact) - .expect("request body"); - let response_content = artifacts - .read_artifact(&response_artifact) - .expect("response body"); - let request_text = String::from_utf8(request_content.bytes).expect("utf8 request artifact"); - let response_text = String::from_utf8(response_content.bytes).expect("utf8 response artifact"); - assert!(!request_text.contains("test-secret")); - assert!(!response_text.contains("test-secret")); - assert!(response_text.contains("chatcmpl-safe")); - let admitted_at = store - .get_summary(&request_key) - .unwrap() - .expect("summary") - .created_at; - assert!( - artifact_page - .items - .iter() - .all(|artifact| artifact.occurred_at >= admitted_at), - "artifact metadata must use the lifecycle clock at capture time" - ); - assert_eq!( - artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "request") - .and_then(|artifact| artifact.media_kind.as_deref()), - Some("application/json"), - "the parsed OpenAI JSON boundary supplies this closed semantic media kind" - ); - assert_eq!( - artifact_page - .items - .iter() - .find(|artifact| artifact.kind == "response") - .and_then(|artifact| artifact.media_kind.as_deref()), - Some("application/json") - ); - let events = store.list_events_for_summary(&request_key).unwrap(); - assert_eq!(events.len(), 2, "one admitted and one terminal event"); - assert_ne!(events[0].event_id, events[1].event_id); -} + .expect("remote attribution lease"); + let lease = state + .suppress_remote_tunneled_request(request_id) + .expect("remote suppression lease"); -#[tokio::test] -async fn streaming_and_oversized_responses_persist_explicit_unavailable_metadata() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = Arc::new(LoggingRuntimeState::initialize( - &foundation, - &artifact_config(), - )); - - let streaming_id = mesh_llm_events::logging::identifiers::RequestId::new(); - let streaming = state.openai_ingress_attachment( - streaming_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), - ); - streaming.route_observer().capture_response_unavailable( - super::super::ArtifactUnavailableReason::StreamingResponseNotAssembled, - ); - - let oversized_id = mesh_llm_events::logging::identifiers::RequestId::new(); - let oversized = state.openai_ingress_attachment( - oversized_id, - RequestSummaryMetadata::from_openai_ingress_path("/v1/responses"), - ); - oversized - .route_observer() - .capture_response_body(&vec![b'x'; 4_097], Some("not a valid media type")); - - state + let active = state .service .as_ref() .expect("logging service") - .pump_sync() - .await; - let store = state.store().expect("metadata store"); - for (request_id, reason) in [ - (streaming_id, "streaming_response_not_assembled"), - (oversized_id, "capture_content_limit_exceeded"), - ] { - let page = store - .query_artifacts( - &request_id.as_uuid().to_string(), - &mesh_llm_log_store::PageQuery { - limit: 10, - cursor: None, - sort: mesh_llm_log_store::QuerySort::Ascending, - }, - ) - .expect("query unavailable artifact"); - assert_eq!(page.items.len(), 1); - let artifact = &page.items[0]; - assert_eq!(artifact.unavailable_reason.as_deref(), Some(reason)); - assert_eq!(artifact.bytes, 0); - assert!(artifact.checksum.is_none()); - assert!(!artifact.missing); - assert!(!artifact.corrupt); - } -} - -#[tokio::test] -async fn disabled_webhook_config_starts_no_delivery_scheduler() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - - let service = state - .start_persistence_worker() - .await - .expect("logging service starts"); - - assert!(!state.has_webhook_delivery_worker_for_test()); - state.shutdown_cleanup_worker().await; - assert!(service.shutdown().await); -} - -#[tokio::test] -async fn enabled_webhook_config_starts_and_retires_one_delivery_scheduler() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - webhook: mesh_llm_config::LoggingWebhookConfig { - enabled: true, - url: Some("http://127.0.0.1:9444/webhook".to_string()), - ..Default::default() - }, - ..Default::default() - }; - let state = LoggingRuntimeState::initialize(&foundation, &config); - - state - .start_persistence_worker() - .await - .expect("logging service starts"); - assert!(state.has_webhook_delivery_worker_for_test()); - - assert!(state.retire_and_shutdown().await); - assert!(!state.has_webhook_delivery_worker_for_test()); -} - -#[test] -fn metadata_only_never_opens_or_writes_artifact_capture() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let opened = Arc::new(AtomicBool::new(false)); - let opened_by_callback = Arc::clone(&opened); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &mesh_llm_config::LoggingConfig::default(), - move |_, _, _| { - opened_by_callback.store(true, Ordering::SeqCst); - Err(LogStoreError::PrivacyNotGuaranteed) - }, - ); - - assert!(!opened.load(Ordering::SeqCst)); - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "metadata-only", - request_id: "request", - kind: "body", - occurred_at: "2025-01-01T00:00:00Z", - content: b"must not be stored", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("metadata-only capture is disabled"); - assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); - assert_eq!( - std::fs::read_dir(foundation.artifact_dir()) - .expect("artifact directory") - .count(), - 0 - ); -} - -#[test] -fn metadata_only_query_facade_deletes_terminal_metadata_without_artifact_capture() { - struct NeverCancelled; - - impl MaintenanceExecutionControl for NeverCancelled { - fn is_cancelled(&self) -> bool { - false - } - } - - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - let store = state.store().expect("metadata store"); - let request_id = "00000000-0000-4000-8000-000000000143"; - store - .insert_summary( - request_id, - Some("model"), - Some("route"), - None, - None, - "2025-01-01T00:00:00Z", - None, - None, - None, - ) - .expect("summary"); - store - .conn() - .execute( - "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", - [request_id], - ) - .expect("terminal summary"); - let request = mesh_llm_log_store::DeleteOneRequest::new( - mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(143)), - request_id, - mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), - ) - .expect("delete request"); - let facade = state.query_facade().expect("query facade"); - - facade - .prepare_delete_request(&request, &NeverCancelled) - .expect("metadata-only preparation"); - let receipt = facade - .execute_prepared_delete_request(&request, &NeverCancelled) - .expect("metadata-only execution"); - + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active raw mesh parent"); assert_eq!( - receipt.state, - mesh_llm_log_store::MaintenanceReceiptState::Completed + active.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) ); - assert!(store.query_request(request_id).unwrap().is_none()); -} - -#[test] -fn metadata_only_query_facade_rejects_artifact_pointers_without_preparing_delete() { - struct NeverCancelled; - - impl MaintenanceExecutionControl for NeverCancelled { - fn is_cancelled(&self) -> bool { - false - } - } + assert_eq!(active.metadata.caller_addr(), Some("192.0.2.61:11204")); + assert_eq!(active.metadata.caller_path_type(), Some("remote_quic_http")); + assert_eq!(active.metadata.route(), Some("chat_completions")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - let store = state.store().expect("metadata store"); - let request_id = "00000000-0000-4000-8000-000000000144"; - store - .insert_summary( - request_id, - Some("model"), - Some("route"), - None, - None, - "2025-01-01T00:00:00Z", - None, - None, - None, - ) - .expect("summary"); - store - .conn() - .execute( - "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", - [request_id], - ) - .expect("terminal summary"); - store - .insert_unavailable_artifact_pointer(mesh_llm_log_store::UnavailableArtifactPointer { - artifact_id: "00000000-0000-4000-8000-000000000244", - request_id, - occurred_at: "2025-01-01T00:00:01Z", - kind: "response", - media_kind: Some("text/plain"), - version: 1, - reason: "artifact_capture_disabled", - }) - .expect("artifact pointer"); - let request = mesh_llm_log_store::DeleteOneRequest::new( - mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(144)), - request_id, - mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), - ) - .expect("delete request"); - - assert!(matches!( - state - .query_facade() - .expect("query facade") - .prepare_delete_request(&request, &NeverCancelled), - Err(LogStoreError::ArtifactDeletionUnavailable) - )); - assert!(store.query_request(request_id).unwrap().is_some()); - assert!(store.delete_one_receipt(&request).unwrap().is_none()); -} - -#[test] -fn privacy_failure_disables_only_artifacts_and_records_one_sanitized_marker() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &artifact_config(), - |artifact_root, clock, store| { - FailOpenArtifactCapture::open_with_privacy( - artifact_root, - clock, - store, - canonical_artifact_redactor(), - Arc::new(RejectPrivacy), - ) - }, - ); - - assert_eq!( - state.health(), - LoggingRuntimeHealth { - metadata_available: true, - artifact_capture_available: false, - artifact_capture_ready: false, - artifact_capture_degradation: Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE,), - } - ); - let store = state.store().expect("metadata store remains available"); - assert_eq!(marker_audit_count(&store), 1); - - store - .insert_summary( - "metadata-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - store - .insert_audit_entry( - "metadata-audit", - Some("metadata-request"), - &store.now(), - "test", - "metadata_still_available", - None, - ) - .expect("metadata audit insert"); - - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "artifact-after-disable", - request_id: "metadata-request", - kind: "request_body", - occurred_at: &store.now(), - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("disabled capture is fail-open"); - assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); - let repeated_outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "artifact-after-disable-again", - request_id: "metadata-request", - kind: "request_body", - occurred_at: &store.now(), - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("repeated disabled capture is fail-open"); - assert!(matches!( - repeated_outcome, - ArtifactCaptureOutcome::Disabled(_) - )); - assert_eq!(marker_audit_count(&store), 1); - assert!(store.get_summary("metadata-request").unwrap().is_some()); - let total_audits: i64 = store - .conn() - .query_row("SELECT COUNT(*) FROM audit_entries", [], |row| row.get(0)) - .expect("count all audits"); - assert_eq!(total_audits, 2); -} - -#[test] -fn write_time_privacy_failure_publishes_one_marker_and_keeps_metadata_available() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( - &foundation, - &artifact_config(), - |artifact_root, clock, store| { - FailOpenArtifactCapture::open_with_privacy( - artifact_root, - clock, - store, - canonical_artifact_redactor(), - Arc::new(RejectArtifactFiles), - ) - }, - ); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "write-time-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - - let occurred_at = store.now(); - let write = |artifact_id| ArtifactCaptureRequest { - artifact_id, - request_id: "write-time-request", - kind: "request_body", - occurred_at: &occurred_at, - content: b"redacted", - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }; - assert!(matches!( - state.write_artifact(write("write-time-artifact")).unwrap(), - ArtifactCaptureOutcome::Disabled(_) - )); - assert!(matches!( - state - .write_artifact(write("write-time-artifact-again")) - .unwrap(), - ArtifactCaptureOutcome::Disabled(_) - )); - - assert_eq!( - state.health().artifact_capture_degradation, - Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE) - ); - assert!(state.health().metadata_available); - assert_eq!(marker_audit_count(&store), 1); -} - -#[test] -fn raw_artifact_bytes_are_redacted_before_the_capture_boundary() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "raw-artifact-request", - None, - None, - None, - None, - &store.now(), - None, - None, - None, - ) - .expect("metadata summary insert"); - - let secret = b"Bearer super-secret-token-0123456789"; - let outcome = state - .write_artifact(ArtifactCaptureRequest { - artifact_id: "raw-artifact", - request_id: "raw-artifact-request", - kind: "request_body", - occurred_at: &store.now(), - content: secret, - media_kind: Some("text/plain"), - version: 1, - truncated: false, - }) - .expect("artifact write"); - assert!(matches!( - outcome, - ArtifactCaptureOutcome::Written(receipt) if receipt.redacted - )); - - let stored = std::fs::read( - foundation - .artifact_dir() - .join("raw-artifact-request") - .join("raw-artifact"), - ) - .expect("read stored artifact"); - assert!(!stored.windows(secret.len()).any(|window| window == secret)); - assert_eq!(stored, b"[REDACTED]"); - assert!( - store - .get_artifact_pointer("raw-artifact") - .expect("artifact pointer") - .expect("pointer present") - .redacted - ); + drop(lease); + drop(attribution); + attachment.terminal(TerminalOutcome::Completed); } #[test] @@ -688,15 +163,16 @@ fn unavailable_foundation_is_sanitized_and_fail_open() { #[test] fn incompatible_schema_is_reported_without_exposing_storage_details() { + let incompatible_version = mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION + 1; let state = LoggingRuntimeState::unavailable(LoggingMetadataState::SchemaIncompatible { - found: 99, + found: incompatible_version, supported: mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION, }); let status = state.status(); assert!(!status.metadata_available); assert_eq!(status.metadata_state, "schema_incompatible"); - assert_eq!(status.schema_version, Some(99)); + assert_eq!(status.schema_version, Some(incompatible_version)); assert_eq!( status.supported_schema_version, Some(mesh_llm_log_store::LOG_STORE_SCHEMA_VERSION) @@ -704,198 +180,6 @@ fn incompatible_schema_is_reported_without_exposing_storage_details() { assert!(state.store().is_none()); } -#[tokio::test] -async fn awaited_runtime_startup_cleanup_uses_injected_store_time_before_ready() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - retention_ttl_secs: 3_600, - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = LoggingRuntimeState::initialize_with_store_clock_for_test( - &foundation, - &config, - Arc::new(FixedStoreClock("2026-08-03T12:00:00Z")), - ); - let store = state.store().expect("metadata store available"); - store - .insert_summary( - "expired-before-startup", - None, - None, - None, - None, - "2026-08-03T10:00:00Z", - None, - None, - None, - ) - .expect("insert stale summary"); - store - .insert_summary( - "retained-after-startup", - None, - None, - None, - None, - "2026-08-03T11:30:01Z", - None, - None, - None, - ) - .expect("insert retained summary"); - for (request_id, terminal_at) in [ - ("expired-before-startup", "2026-08-03T10:00:00Z"), - ("retained-after-startup", "2026-08-03T11:30:01Z"), - ] { - store - .write_terminal_event( - request_id, - &format!("terminal-{request_id}"), - r#"{"type":"completed"}"#, - "completed", - None, - terminal_at, - ) - .expect("write deterministic terminal record"); - } - - let service = state - .start_persistence_worker() - .await - .expect("service becomes ready after cleanup outcome"); - - assert!( - store - .get_summary("expired-before-startup") - .expect("load stale summary") - .is_none(), - "startup cleanup completed before the ready service was returned" - ); - assert!( - store - .get_summary("retained-after-startup") - .expect("load retained summary") - .is_some() - ); - assert_eq!(state.status().cleanup_last_outcome, Some("completed")); - assert!(matches!( - state.status().cleanup_last_deleted_count, - Some(count) if count >= 1 - )); - - state.shutdown_cleanup_worker().await; - assert!(service.shutdown().await); -} - -#[tokio::test] -async fn concurrent_starts_publish_one_cleanup_scheduler_and_truthful_status() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); - let barrier = Arc::new(tokio::sync::Barrier::new(3)); - - let first_state = Arc::clone(&state); - let first_barrier = Arc::clone(&barrier); - let first = tokio::spawn(async move { - first_barrier.wait().await; - first_state.start_persistence_worker().await - }); - let second_state = Arc::clone(&state); - let second_barrier = Arc::clone(&barrier); - let second = tokio::spawn(async move { - second_barrier.wait().await; - second_state.start_persistence_worker().await - }); - - // Release both calls from the same scheduling boundary. The - // state-local candidate count is incremented at construction under - // the activation gate, so it proves no losing task was ever spawned. - barrier.wait().await; - let first_service = first - .await - .expect("first start task joins") - .expect("first start returns the ready service"); - let second_service = second - .await - .expect("second start task joins") - .expect("second start returns the ready service"); - - assert!(Arc::ptr_eq(&first_service, &second_service)); - assert_eq!(state.cleanup_candidate_count_for_test(), 1); - assert!(state.has_cleanup_worker_for_test()); - assert_eq!(state.status().cleanup_worker_state, "running"); - assert_eq!(state.status().persistence_worker_state, "running"); - - // Shutdown drains the one operational audit produced by the one - // startup cleanup. A losing scheduler would produce a second audit or - // leave a second task able to race a later cleanup pass. - state.shutdown_cleanup_worker().await; - assert!(first_service.shutdown().await); - assert!(!state.has_cleanup_worker_for_test()); - assert_eq!(state.status().cleanup_worker_state, "stopped"); - assert_eq!(state.status().persistence_worker_state, "stopped"); - let cleanup_audits: i64 = state - .store() - .expect("metadata store available") - .conn() - .query_row( - "SELECT COUNT(*) FROM audit_entries WHERE action = 'logging_cleanup_completed'", - [], - |row| row.get(0), - ) - .expect("count startup cleanup audits"); - assert_eq!(cleanup_audits, 1); -} - -#[tokio::test] -async fn retirement_after_cleanup_candidate_publication_leaves_no_worker_on_displaced_state() { - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let config = mesh_llm_config::LoggingConfig { - cleanup_cadence_secs: 86_400, - ..Default::default() - }; - let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); - let hook = state.install_cleanup_publish_hook_for_test(); - let displaced_service = Arc::clone( - state - .service - .as_ref() - .expect("healthy state owns a persistence service"), - ); - - let starting_state = Arc::clone(&state); - let start = tokio::spawn(async move { starting_state.start_persistence_worker().await }); - - // The candidate is already atomically published, but the starter has - // not yet observed readiness. Retire the displaced state in this - // exact window and prove retirement cancels and joins that task. - hook.candidate_created.wait().await; - state.retire_and_shutdown().await; - hook.resume_install.wait().await; - - assert!(start.await.expect("start task joins").is_none()); - assert!(state.is_retired()); - assert!( - !state.has_cleanup_worker_for_test(), - "a retired state must not retain a cleanup task handle" - ); - assert_eq!(state.status().cleanup_worker_state, "stopped"); - assert!(!displaced_service.is_startable()); - assert!(!displaced_service.is_spawned()); - assert_eq!( - state.status().persistence_worker_state, - "stopped", - "replacement must have joined the displaced persistence worker" - ); -} - #[test] fn store_open_failure_is_fail_open_without_exposing_the_failed_path() { let root = tempfile::tempdir().expect("temporary logging root"); @@ -934,20 +218,3 @@ fn applies_retention_and_replay_limits_together_to_the_installed_service() { state.apply_dynamic_limits(next).expect("healthy runtime"); assert_eq!(state.dynamic_limits(), Some(next)); } - -#[test] -fn openai_lifecycle_observer_snapshot_is_absent_when_disabled_or_retired() { - assert!( - LoggingRuntimeState::unavailable(LoggingMetadataState::StorageUnavailable) - .openai_lifecycle_observer() - .is_none() - ); - - let root = tempfile::tempdir().expect("temporary logging root"); - let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); - let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); - assert!(state.openai_lifecycle_observer().is_some()); - - state.retired.store(true, Ordering::Release); - assert!(state.openai_lifecycle_observer().is_none()); -} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs new file mode 100644 index 0000000000..b5d7979355 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/artifact_capture.rs @@ -0,0 +1,547 @@ +use super::*; + +#[test] +fn production_ingress_wiring_advertises_capture_only_when_storage_is_usable() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); + + let status = state.status(); + assert!(status.artifact_capture_available); + assert!(status.artifact_capture_ready); +} + +#[tokio::test] +async fn ingress_attachment_captures_redacted_request_and_response_bodies() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &artifact_config(), + )); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let mut attachment = state.openai_ingress_attachment( + request_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), + ); + + attachment.capture_request_body( + br#"{"authorization":"Bearer test-secret","model":"safe"}"#, + Some("application/json"), + ); + attachment.route_observer().capture_response_body( + br#"{"id":"chatcmpl-safe","api_key":"test-secret"}"#, + Some("application/json; charset=utf-8"), + ); + attachment.terminal(TerminalOutcome::CompletedWithStatus(201)); + + // Summary, admission, both artifact commands, and terminal state share one + // serial persistence owner. Ingress itself performs no SQLite/file work. + assert_eq!( + state + .service + .as_ref() + .expect("logging service") + .pump_sync() + .await, + 5 + ); + + let store = state.store().expect("metadata store"); + let request_key = request_id.as_uuid().to_string(); + let artifact_page = store + .query_artifacts( + &request_key, + &mesh_llm_log_store::PageQuery { + limit: 10, + cursor: None, + sort: mesh_llm_log_store::QuerySort::Ascending, + }, + ) + .expect("query artifacts"); + assert_eq!(artifact_page.items.len(), 2); + let request_artifact = artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "request") + .expect("request artifact") + .artifact_id + .clone(); + let response_artifact = artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "response") + .expect("response artifact") + .artifact_id + .clone(); + let artifacts = state.query_facade().expect("query facade"); + let request_content = artifacts + .read_artifact(&request_artifact) + .expect("request body"); + let response_content = artifacts + .read_artifact(&response_artifact) + .expect("response body"); + let request_text = String::from_utf8(request_content.bytes).expect("utf8 request artifact"); + let response_text = String::from_utf8(response_content.bytes).expect("utf8 response artifact"); + assert!(!request_text.contains("test-secret")); + assert!(!response_text.contains("test-secret")); + assert!(response_text.contains("chatcmpl-safe")); + let admitted_at = store + .get_summary(&request_key) + .unwrap() + .expect("summary") + .created_at; + assert!( + artifact_page + .items + .iter() + .all(|artifact| artifact.occurred_at >= admitted_at), + "artifact metadata must use the lifecycle clock at capture time" + ); + assert_eq!( + artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "request") + .and_then(|artifact| artifact.media_kind.as_deref()), + Some("application/json"), + "the parsed OpenAI JSON boundary supplies this closed semantic media kind" + ); + assert_eq!( + artifact_page + .items + .iter() + .find(|artifact| artifact.kind == "response") + .and_then(|artifact| artifact.media_kind.as_deref()), + Some("application/json") + ); + let events = store.list_events_for_summary(&request_key).unwrap(); + assert_eq!(events.len(), 2, "one admitted and one terminal event"); + assert_ne!(events[0].event_id, events[1].event_id); +} + +#[tokio::test] +async fn streaming_and_oversized_responses_persist_explicit_unavailable_metadata() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &artifact_config(), + )); + + let streaming_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let streaming = state.openai_ingress_attachment( + streaming_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/chat/completions"), + ); + streaming.route_observer().capture_response_unavailable( + crate::logging::ArtifactUnavailableReason::StreamingResponseNotAssembled, + ); + + let oversized_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let oversized = state.openai_ingress_attachment( + oversized_id, + RequestSummaryMetadata::from_openai_ingress_path("/v1/responses"), + ); + oversized + .route_observer() + .capture_response_body(&vec![b'x'; 4_097], Some("not a valid media type")); + + state + .service + .as_ref() + .expect("logging service") + .pump_sync() + .await; + let store = state.store().expect("metadata store"); + for (request_id, reason) in [ + (streaming_id, "streaming_response_not_assembled"), + (oversized_id, "capture_content_limit_exceeded"), + ] { + let page = store + .query_artifacts( + &request_id.as_uuid().to_string(), + &mesh_llm_log_store::PageQuery { + limit: 10, + cursor: None, + sort: mesh_llm_log_store::QuerySort::Ascending, + }, + ) + .expect("query unavailable artifact"); + assert_eq!(page.items.len(), 1); + let artifact = &page.items[0]; + assert_eq!(artifact.unavailable_reason.as_deref(), Some(reason)); + assert_eq!(artifact.bytes, 0); + assert!(artifact.checksum.is_none()); + assert!(!artifact.missing); + assert!(!artifact.corrupt); + } +} + +#[test] +fn metadata_only_never_opens_or_writes_artifact_capture() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let opened = Arc::new(AtomicBool::new(false)); + let opened_by_callback = Arc::clone(&opened); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &mesh_llm_config::LoggingConfig::default(), + move |_, _, _| { + opened_by_callback.store(true, Ordering::SeqCst); + Err(LogStoreError::PrivacyNotGuaranteed) + }, + ); + + assert!(!opened.load(Ordering::SeqCst)); + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "metadata-only", + request_id: "request", + kind: "body", + occurred_at: "2025-01-01T00:00:00Z", + content: b"must not be stored", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("metadata-only capture is disabled"); + assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); + assert_eq!( + std::fs::read_dir(foundation.artifact_dir()) + .expect("artifact directory") + .count(), + 0 + ); +} + +#[test] +fn metadata_only_query_facade_deletes_terminal_metadata_without_artifact_capture() { + struct NeverCancelled; + + impl MaintenanceExecutionControl for NeverCancelled { + fn is_cancelled(&self) -> bool { + false + } + } + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + let store = state.store().expect("metadata store"); + let request_id = "00000000-0000-4000-8000-000000000143"; + store + .insert_summary( + request_id, + Some("model"), + Some("route"), + None, + None, + "2025-01-01T00:00:00Z", + None, + None, + None, + ) + .expect("summary"); + store + .conn() + .execute( + "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", + [request_id], + ) + .expect("terminal summary"); + let request = mesh_llm_log_store::DeleteOneRequest::new( + mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(143)), + request_id, + mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), + ) + .expect("delete request"); + let facade = state.query_facade().expect("query facade"); + + facade + .prepare_delete_request(&request, &NeverCancelled) + .expect("metadata-only preparation"); + let receipt = facade + .execute_prepared_delete_request(&request, &NeverCancelled) + .expect("metadata-only execution"); + + assert_eq!( + receipt.state, + mesh_llm_log_store::MaintenanceReceiptState::Completed + ); + assert!(store.query_request(request_id).unwrap().is_none()); +} + +#[test] +fn metadata_only_query_facade_rejects_artifact_pointers_without_preparing_delete() { + struct NeverCancelled; + + impl MaintenanceExecutionControl for NeverCancelled { + fn is_cancelled(&self) -> bool { + false + } + } + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + let store = state.store().expect("metadata store"); + let request_id = "00000000-0000-4000-8000-000000000144"; + store + .insert_summary( + request_id, + Some("model"), + Some("route"), + None, + None, + "2025-01-01T00:00:00Z", + None, + None, + None, + ) + .expect("summary"); + store + .conn() + .execute( + "UPDATE summaries SET state = 'completed' WHERE request_id = ?1", + [request_id], + ) + .expect("terminal summary"); + store + .insert_unavailable_artifact_pointer(mesh_llm_log_store::UnavailableArtifactPointer { + artifact_id: "00000000-0000-4000-8000-000000000244", + request_id, + occurred_at: "2025-01-01T00:00:01Z", + kind: "response", + media_kind: Some("text/plain"), + version: 1, + reason: "artifact_capture_disabled", + }) + .expect("artifact pointer"); + let request = mesh_llm_log_store::DeleteOneRequest::new( + mesh_llm_log_store::MaintenanceOperationId::new(uuid::Uuid::from_u128(144)), + request_id, + mesh_llm_log_store::MaintenanceReason::try_from("operator delete").expect("reason"), + ) + .expect("delete request"); + + assert!(matches!( + state + .query_facade() + .expect("query facade") + .prepare_delete_request(&request, &NeverCancelled), + Err(LogStoreError::ArtifactDeletionUnavailable) + )); + assert!(store.query_request(request_id).unwrap().is_some()); + assert!(store.delete_one_receipt(&request).unwrap().is_none()); +} + +#[test] +fn privacy_failure_disables_only_artifacts_and_records_one_sanitized_marker() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &artifact_config(), + |artifact_root, clock, store| { + FailOpenArtifactCapture::open_with_privacy( + artifact_root, + clock, + store, + canonical_artifact_redactor(), + Arc::new(RejectPrivacy), + ) + }, + ); + + assert_eq!( + state.health(), + LoggingRuntimeHealth { + metadata_available: true, + artifact_capture_available: false, + artifact_capture_ready: false, + artifact_capture_degradation: Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE,), + } + ); + let store = state.store().expect("metadata store remains available"); + assert_eq!(marker_audit_count(&store), 1); + + store + .insert_summary( + "metadata-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + store + .insert_audit_entry( + "metadata-audit", + Some("metadata-request"), + &store.now(), + "test", + "metadata_still_available", + None, + ) + .expect("metadata audit insert"); + + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "artifact-after-disable", + request_id: "metadata-request", + kind: "request_body", + occurred_at: &store.now(), + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("disabled capture is fail-open"); + assert!(matches!(outcome, ArtifactCaptureOutcome::Disabled(_))); + let repeated_outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "artifact-after-disable-again", + request_id: "metadata-request", + kind: "request_body", + occurred_at: &store.now(), + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("repeated disabled capture is fail-open"); + assert!(matches!( + repeated_outcome, + ArtifactCaptureOutcome::Disabled(_) + )); + assert_eq!(marker_audit_count(&store), 1); + assert!(store.get_summary("metadata-request").unwrap().is_some()); + let total_audits: i64 = store + .conn() + .query_row("SELECT COUNT(*) FROM audit_entries", [], |row| row.get(0)) + .expect("count all audits"); + assert_eq!(total_audits, 2); +} + +#[test] +fn write_time_privacy_failure_publishes_one_marker_and_keeps_metadata_available() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize_with_capture_opener_for_test( + &foundation, + &artifact_config(), + |artifact_root, clock, store| { + FailOpenArtifactCapture::open_with_privacy( + artifact_root, + clock, + store, + canonical_artifact_redactor(), + Arc::new(RejectArtifactFiles), + ) + }, + ); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "write-time-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + + let occurred_at = store.now(); + let write = |artifact_id| ArtifactCaptureRequest { + artifact_id, + request_id: "write-time-request", + kind: "request_body", + occurred_at: &occurred_at, + content: b"redacted", + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }; + assert!(matches!( + state.write_artifact(write("write-time-artifact")).unwrap(), + ArtifactCaptureOutcome::Disabled(_) + )); + assert!(matches!( + state + .write_artifact(write("write-time-artifact-again")) + .unwrap(), + ArtifactCaptureOutcome::Disabled(_) + )); + + assert_eq!( + state.health().artifact_capture_degradation, + Some(ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE) + ); + assert!(state.health().metadata_available); + assert_eq!(marker_audit_count(&store), 1); +} + +#[test] +fn raw_artifact_bytes_are_redacted_before_the_capture_boundary() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &artifact_config()); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "raw-artifact-request", + None, + None, + None, + None, + &store.now(), + None, + None, + None, + ) + .expect("metadata summary insert"); + + let secret = b"Bearer super-secret-token-0123456789"; + let outcome = state + .write_artifact(ArtifactCaptureRequest { + artifact_id: "raw-artifact", + request_id: "raw-artifact-request", + kind: "request_body", + occurred_at: &store.now(), + content: secret, + media_kind: Some("text/plain"), + version: 1, + truncated: false, + }) + .expect("artifact write"); + assert!(matches!( + outcome, + ArtifactCaptureOutcome::Written(receipt) if receipt.redacted + )); + + let stored = std::fs::read( + foundation + .artifact_dir() + .join("raw-artifact-request") + .join("raw-artifact"), + ) + .expect("read stored artifact"); + assert!(!stored.windows(secret.len()).any(|window| window == secret)); + assert_eq!(stored, b"[REDACTED]"); + assert!( + store + .get_artifact_pointer("raw-artifact") + .expect("artifact pointer") + .expect("pointer present") + .redacted + ); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs new file mode 100644 index 0000000000..13a405aae7 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/remote_caller_attribution.rs @@ -0,0 +1,314 @@ +use super::*; + +fn initialized_state() -> (Arc, tempfile::TempDir) { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = Arc::new(LoggingRuntimeState::initialize( + &foundation, + &mesh_llm_config::LoggingConfig::default(), + )); + (state, root) +} + +fn provisional_local_metadata() -> RequestSummaryMetadata { + RequestSummaryMetadata::from_parts( + Some("responses"), + Some("model-a"), + Some("provider-a"), + Some("engine-a"), + ) + .with_source(Some("direct_http")) + .with_method(Some("POST")) + .with_caller_identity( + None, + Some("127.0.0.1:40123"), + Some(CallerPathType::LocalHttp), + ) +} + +fn authenticated_direct_metadata(endpoint_id: &str) -> RequestSummaryMetadata { + RequestSummaryMetadata::default().with_caller_identity( + Some(endpoint_id), + Some("192.0.2.71:11204"), + Some(CallerPathType::RemoteQuicHttp), + ) +} + +fn authenticated_relay_metadata(endpoint_id: &str) -> RequestSummaryMetadata { + RequestSummaryMetadata::default().with_caller_identity( + Some(endpoint_id), + None, + Some(CallerPathType::Relay), + ) +} + +fn assert_active_caller( + state: &LoggingRuntimeState, + request_id: mesh_llm_events::logging::identifiers::RequestId, + endpoint_id: &str, + address: Option<&str>, + path_type: &str, +) { + let active = state + .service + .as_ref() + .expect("logging service") + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active request"); + assert_eq!(active.metadata.caller_endpoint_id(), Some(endpoint_id)); + assert_eq!(active.metadata.caller_addr(), address); + assert_eq!(active.metadata.caller_path_type(), Some(path_type)); + assert_eq!(active.metadata.route(), Some("responses")); + assert_eq!(active.metadata.model(), Some("model-a")); + assert_eq!(active.metadata.provider(), Some("provider-a")); + assert_eq!(active.metadata.engine(), Some("engine-a")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); +} + +#[tokio::test] +async fn production_order_authenticated_direct_survives_local_registration_and_persistence() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "71".repeat(32); + + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_direct_metadata(&endpoint_id)) + .expect("remote attribution lease"); + let suppression = state + .suppress_remote_tunneled_request(request_id) + .expect("remote suppression lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller( + &state, + request_id, + &endpoint_id, + Some("192.0.2.71:11204"), + "remote_quic_http", + ); + attachment.terminal(TerminalOutcome::Completed); + drop(suppression); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr.as_deref(), Some("192.0.2.71:11204")); + assert_eq!( + durable.caller_path_type.as_deref(), + Some("remote_quic_http") + ); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn reverse_order_authenticated_relay_atomically_clears_local_address() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "72".repeat(32); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + let first = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("first remote attribution lease"); + let second = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("idempotent remote attribution lease"); + + assert_active_caller(&state, request_id, &endpoint_id, None, "relay"); + attachment.terminal(TerminalOutcome::Completed); + drop(first); + drop(second); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type.as_deref(), Some("relay")); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn markerless_authenticated_relay_survives_production_order_without_suppression() { + let (state, _root) = initialized_state(); + let (forwarded, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), + ); + let request_id = request_id.expect("tunnel-generated request ID"); + let endpoint_id = "73".repeat(32); + + assert_eq!( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(&forwarded), + Some(request_id) + ); + + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_relay_metadata(&endpoint_id)) + .expect("markerless remote attribution lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller(&state, request_id, &endpoint_id, None, "relay"); + attachment.terminal(TerminalOutcome::Completed); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type.as_deref(), Some("relay")); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} + +#[tokio::test] +async fn no_header_authenticated_direct_survives_active_and_durable_lifecycle() { + let (state, _root) = initialized_state(); + let (forwarded, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), + ); + let request_id = request_id.expect("tunnel-generated request ID"); + let endpoint_id = "76".repeat(32); + + assert_eq!( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(&forwarded), + Some(request_id) + ); + let attribution = state + .attribute_remote_tunneled_request(request_id, authenticated_direct_metadata(&endpoint_id)) + .expect("markerless remote attribution lease"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + + assert_active_caller( + &state, + request_id, + &endpoint_id, + Some("192.0.2.71:11204"), + "remote_quic_http", + ); + attachment.terminal(TerminalOutcome::Completed); + drop(attribution); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr.as_deref(), Some("192.0.2.71:11204")); + assert_eq!( + durable.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + +#[tokio::test] +async fn endpoint_only_caller_survives_pending_active_terminal_and_durable_lifecycle() { + let (state, _root) = initialized_state(); + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let endpoint_id = "79".repeat(32); + let later_endpoint_id = "7a".repeat(32); + + let endpoint_only = state + .attribute_remote_tunneled_request( + request_id, + RequestSummaryMetadata::default().with_caller_identity(Some(&endpoint_id), None, None), + ) + .expect("pending endpoint-only attribution"); + let mut attachment = state.openai_ingress_attachment(request_id, provisional_local_metadata()); + let direct = state + .attribute_remote_tunneled_request( + request_id, + authenticated_direct_metadata(&later_endpoint_id), + ) + .expect("later direct attribution lease"); + let relay = state + .attribute_remote_tunneled_request( + request_id, + authenticated_relay_metadata(&later_endpoint_id), + ) + .expect("later relay attribution lease"); + + let active = state + .service + .as_ref() + .expect("logging service") + .registry_ref() + .get_active(&request_id.as_uuid().to_string()) + .expect("active request"); + assert_eq!( + active.metadata.caller_endpoint_id(), + Some(endpoint_id.as_str()) + ); + assert_eq!(active.metadata.caller_addr(), None); + assert_eq!(active.metadata.caller_path_type(), None); + assert_eq!(active.metadata.route(), Some("responses")); + assert_eq!(active.metadata.model(), Some("model-a")); + assert_eq!(active.metadata.provider(), Some("provider-a")); + assert_eq!(active.metadata.engine(), Some("engine-a")); + assert_eq!(active.metadata.source(), Some("direct_http")); + assert_eq!(active.metadata.method(), Some("POST")); + + attachment.terminal(TerminalOutcome::Completed); + drop(relay); + drop(direct); + drop(endpoint_only); + state.pump_persistence_for_test().await; + + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("query request") + .expect("durable request"); + assert_eq!( + durable.caller_endpoint_id.as_deref(), + Some(endpoint_id.as_str()) + ); + assert_eq!(durable.caller_addr, None); + assert_eq!(durable.caller_path_type, None); + assert_eq!(durable.request.model.as_deref(), Some("model-a")); + assert_eq!(durable.request.route.as_deref(), Some("responses")); + assert_eq!(durable.request.provider.as_deref(), Some("provider-a")); + assert_eq!(durable.request.engine.as_deref(), Some("engine-a")); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs new file mode 100644 index 0000000000..c66c48944f --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/runtime_state/tests/service_lifecycle.rs @@ -0,0 +1,250 @@ +use super::*; + +#[tokio::test] +async fn disabled_webhook_config_starts_no_delivery_scheduler() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + + let service = state + .start_persistence_worker() + .await + .expect("logging service starts"); + + assert!(!state.has_webhook_delivery_worker_for_test()); + state.shutdown_cleanup_worker().await; + assert!(service.shutdown().await); +} + +#[tokio::test] +async fn enabled_webhook_config_starts_and_retires_one_delivery_scheduler() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + webhook: mesh_llm_config::LoggingWebhookConfig { + enabled: true, + url: Some("http://127.0.0.1:9444/webhook".to_string()), + ..Default::default() + }, + ..Default::default() + }; + let state = LoggingRuntimeState::initialize(&foundation, &config); + + state + .start_persistence_worker() + .await + .expect("logging service starts"); + assert!(state.has_webhook_delivery_worker_for_test()); + + assert!(state.retire_and_shutdown().await); + assert!(!state.has_webhook_delivery_worker_for_test()); +} + +#[tokio::test] +async fn awaited_runtime_startup_cleanup_uses_injected_store_time_before_ready() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + retention_ttl_secs: 3_600, + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = LoggingRuntimeState::initialize_with_store_clock_for_test( + &foundation, + &config, + Arc::new(FixedStoreClock("2026-08-03T12:00:00Z")), + ); + let store = state.store().expect("metadata store available"); + store + .insert_summary( + "expired-before-startup", + None, + None, + None, + None, + "2026-08-03T10:00:00Z", + None, + None, + None, + ) + .expect("insert stale summary"); + store + .insert_summary( + "retained-after-startup", + None, + None, + None, + None, + "2026-08-03T11:30:01Z", + None, + None, + None, + ) + .expect("insert retained summary"); + for (request_id, terminal_at) in [ + ("expired-before-startup", "2026-08-03T10:00:00Z"), + ("retained-after-startup", "2026-08-03T11:30:01Z"), + ] { + store + .write_terminal_event( + request_id, + &format!("terminal-{request_id}"), + r#"{"type":"completed"}"#, + "completed", + None, + terminal_at, + ) + .expect("write deterministic terminal record"); + } + + let service = state + .start_persistence_worker() + .await + .expect("service becomes ready after cleanup outcome"); + + assert!( + store + .get_summary("expired-before-startup") + .expect("load stale summary") + .is_none(), + "startup cleanup completed before the ready service was returned" + ); + assert!( + store + .get_summary("retained-after-startup") + .expect("load retained summary") + .is_some() + ); + assert_eq!(state.status().cleanup_last_outcome, Some("completed")); + assert!(matches!( + state.status().cleanup_last_deleted_count, + Some(count) if count >= 1 + )); + + state.shutdown_cleanup_worker().await; + assert!(service.shutdown().await); +} + +#[tokio::test] +async fn concurrent_starts_publish_one_cleanup_scheduler_and_truthful_status() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + + let first_state = Arc::clone(&state); + let first_barrier = Arc::clone(&barrier); + let first = tokio::spawn(async move { + first_barrier.wait().await; + first_state.start_persistence_worker().await + }); + let second_state = Arc::clone(&state); + let second_barrier = Arc::clone(&barrier); + let second = tokio::spawn(async move { + second_barrier.wait().await; + second_state.start_persistence_worker().await + }); + + // Release both calls from the same scheduling boundary. The + // state-local candidate count is incremented at construction under + // the activation gate, so it proves no losing task was ever spawned. + barrier.wait().await; + let first_service = first + .await + .expect("first start task joins") + .expect("first start returns the ready service"); + let second_service = second + .await + .expect("second start task joins") + .expect("second start returns the ready service"); + + assert!(Arc::ptr_eq(&first_service, &second_service)); + assert_eq!(state.cleanup_candidate_count_for_test(), 1); + assert!(state.has_cleanup_worker_for_test()); + assert_eq!(state.status().cleanup_worker_state, "running"); + assert_eq!(state.status().persistence_worker_state, "running"); + + // Shutdown drains the one operational audit produced by the one + // startup cleanup. A losing scheduler would produce a second audit or + // leave a second task able to race a later cleanup pass. + state.shutdown_cleanup_worker().await; + assert!(first_service.shutdown().await); + assert!(!state.has_cleanup_worker_for_test()); + assert_eq!(state.status().cleanup_worker_state, "stopped"); + assert_eq!(state.status().persistence_worker_state, "stopped"); + let cleanup_audits: i64 = state + .store() + .expect("metadata store available") + .conn() + .query_row( + "SELECT COUNT(*) FROM audit_entries WHERE action = 'logging_cleanup_completed'", + [], + |row| row.get(0), + ) + .expect("count startup cleanup audits"); + assert_eq!(cleanup_audits, 1); +} + +#[tokio::test] +async fn retirement_after_cleanup_candidate_publication_leaves_no_worker_on_displaced_state() { + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let config = mesh_llm_config::LoggingConfig { + cleanup_cadence_secs: 86_400, + ..Default::default() + }; + let state = Arc::new(LoggingRuntimeState::initialize(&foundation, &config)); + let hook = state.install_cleanup_publish_hook_for_test(); + let displaced_service = Arc::clone( + state + .service + .as_ref() + .expect("healthy state owns a persistence service"), + ); + + let starting_state = Arc::clone(&state); + let start = tokio::spawn(async move { starting_state.start_persistence_worker().await }); + + // The candidate is already atomically published, but the starter has + // not yet observed readiness. Retire the displaced state in this + // exact window and prove retirement cancels and joins that task. + hook.candidate_created.wait().await; + state.retire_and_shutdown().await; + hook.resume_install.wait().await; + + assert!(start.await.expect("start task joins").is_none()); + assert!(state.is_retired()); + assert!( + !state.has_cleanup_worker_for_test(), + "a retired state must not retain a cleanup task handle" + ); + assert_eq!(state.status().cleanup_worker_state, "stopped"); + assert!(!displaced_service.is_startable()); + assert!(!displaced_service.is_spawned()); + assert_eq!( + state.status().persistence_worker_state, + "stopped", + "replacement must have joined the displaced persistence worker" + ); +} + +#[test] +fn openai_lifecycle_observer_snapshot_is_absent_when_disabled_or_retired() { + assert!( + LoggingRuntimeState::unavailable(LoggingMetadataState::StorageUnavailable) + .openai_lifecycle_observer() + .is_none() + ); + + let root = tempfile::tempdir().expect("temporary logging root"); + let foundation = LoggingFoundation::init(true, Some(&root.path().to_path_buf())); + let state = LoggingRuntimeState::initialize(&foundation, &Default::default()); + assert!(state.openai_lifecycle_observer().is_some()); + + state.retired.store(true, Ordering::Release); + assert!(state.openai_lifecycle_observer().is_none()); +} diff --git a/crates/mesh-llm-host-runtime/src/logging/service.rs b/crates/mesh-llm-host-runtime/src/logging/service.rs index 06eddf6330..caf51dec23 100644 --- a/crates/mesh-llm-host-runtime/src/logging/service.rs +++ b/crates/mesh-llm-host-runtime/src/logging/service.rs @@ -29,8 +29,8 @@ mod operational_audit; mod persistence_delivery; use event_delivery::{EventDelivery, ServiceLifecycleRecorder, enqueue_event_with_delivery}; pub use operational_audit::{ - OperationalAuditContext, OperationalAuditRecord, OperationalAuditRecordBuilder, - OperationalAuditSeverity, OperationalAuditSubjectKind, + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditRecordBuilder, OperationalAuditSeverity, OperationalAuditSubjectKind, }; pub(crate) use persistence_delivery::PersistenceWorkerState; pub use persistence_delivery::WorkerHandle; @@ -740,6 +740,29 @@ impl LoggingService { } } + pub(crate) fn merge_authenticated_remote_caller( + &self, + request_id: RequestId, + metadata: RequestSummaryMetadata, + ) -> bool { + let Some(summary) = self + .registry + .merge_authenticated_remote_caller(&request_id.as_uuid().to_string(), metadata) + else { + return false; + }; + if self.sink.is_some() { + offer_summary_persistence( + &self.delivery, + &self.persistence_queue_drops, + &self.persistence_outstanding, + &self.metrics, + summary, + ); + } + true + } + /// Record the beginning of a transport attempt under an existing request. /// The returned branded identifier is used by its completion or failure and /// never changes the parent request lifecycle. @@ -895,29 +918,24 @@ impl LoggingService { pub(crate) fn retire(&self) { self.startable.store(false, Ordering::Release); } - pub(crate) fn is_startable(&self) -> bool { self.startable.load(Ordering::Acquire) } - #[cfg(test)] pub(crate) fn with_shutdown_drain_timeout(mut self, timeout: Duration) -> Self { self.shutdown_drain_timeout = timeout; self } - /// Check if the service is currently spawned and running. For observability / tests. #[allow(dead_code)] pub fn is_spawned(&self) -> bool { self.spawned.load(Ordering::Acquire) } - /// Clone writer for external observation of drop counters. #[allow(dead_code)] pub fn writer_ref(&self) -> Arc { Arc::clone(&self.writer) } - #[cfg(test)] pub(crate) fn worker_handle_lock_for_test( &self, diff --git a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs index 9046245de9..a02fe566fe 100644 --- a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs +++ b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs @@ -1,171 +1,8 @@ //! Bounded operational-audit records admitted by the logging service. -use std::collections::BTreeMap; +mod context; -const OPERATIONAL_AUDIT_CONTEXT_VERSION: u8 = 1; -const MAX_CONTEXT_VALUE_CHARS: usize = 256; -const MAX_NUMERIC_SUMMARIES: usize = 8; - -/// Closed subject vocabulary for diagnostic operational-audit context. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum OperationalAuditSubjectKind { - Runtime, - Model, - RuntimeInstance, - CliCommand, -} - -impl OperationalAuditSubjectKind { - pub const fn as_str(self) -> &'static str { - match self { - Self::Runtime => "runtime", - Self::Model => "model", - Self::RuntimeInstance => "runtime_instance", - Self::CliCommand => "cli_command", - } - } - - pub(crate) fn parse(value: &str) -> Option { - match value { - "runtime" => Some(Self::Runtime), - "model" => Some(Self::Model), - "runtime_instance" => Some(Self::RuntimeInstance), - "cli_command" => Some(Self::CliCommand), - _ => None, - } - } -} - -/// Versioned, bounded context shared by audit replay and durable storage. -/// -/// Values are sanitized before this type accepts them. Arbitrary detail JSON, -/// command arguments, error text, and payload bodies cannot enter this shape. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct OperationalAuditContext { - subject_kind: Option, - subject_id: Option, - operation_id: Option, - request_id: Option, - reason_code: Option<&'static str>, - outcome: Option<&'static str>, - duration_ms: Option, - numeric_summaries: BTreeMap<&'static str, u64>, -} - -impl OperationalAuditContext { - pub fn new() -> Self { - Self::default() - } - - pub fn subject(mut self, kind: OperationalAuditSubjectKind, id: &str) -> Self { - self.subject_kind = Some(kind); - self.subject_id = bounded_context_value(id); - self - } - - pub fn operation_id(mut self, operation_id: &str) -> Self { - self.operation_id = bounded_context_value(operation_id); - self - } - - pub fn request_id(mut self, request_id: &str) -> Self { - self.request_id = bounded_context_value(request_id); - self - } - - pub const fn reason_code(mut self, reason_code: &'static str) -> Self { - if Self::valid_static_code(reason_code) { - self.reason_code = Some(reason_code); - } - self - } - - pub const fn outcome(mut self, outcome: &'static str) -> Self { - if Self::valid_static_code(outcome) { - self.outcome = Some(outcome); - } - self - } - - pub const fn duration_ms(mut self, duration_ms: u64) -> Self { - self.duration_ms = Some(duration_ms); - self - } - - pub fn numeric_summary(mut self, key: &'static str, value: u64) -> Self { - if self.numeric_summaries.len() < MAX_NUMERIC_SUMMARIES && Self::valid_static_code(key) { - self.numeric_summaries.insert(key, value); - } - self - } - - pub(crate) const fn valid_static_code(value: &str) -> bool { - valid_static_code(value) - } - - pub(crate) fn fields(&self) -> serde_json::Map { - let mut fields = serde_json::Map::new(); - fields.insert( - "context_version".into(), - serde_json::json!(OPERATIONAL_AUDIT_CONTEXT_VERSION), - ); - if let Some(kind) = self.subject_kind { - fields.insert("subject_kind".into(), serde_json::json!(kind.as_str())); - } - insert_optional_string(&mut fields, "subject_id", self.subject_id.as_deref()); - insert_optional_string(&mut fields, "operation_id", self.operation_id.as_deref()); - insert_optional_string(&mut fields, "request_id", self.request_id.as_deref()); - insert_optional_string(&mut fields, "reason_code", self.reason_code); - insert_optional_string(&mut fields, "outcome", self.outcome); - if let Some(duration_ms) = self.duration_ms { - fields.insert("duration_ms".into(), serde_json::json!(duration_ms)); - } - if !self.numeric_summaries.is_empty() { - fields.insert( - "numeric_summaries".into(), - serde_json::json!(self.numeric_summaries), - ); - } - fields - } -} - -fn insert_optional_string( - fields: &mut serde_json::Map, - key: &'static str, - value: Option<&str>, -) { - if let Some(value) = value { - fields.insert(key.into(), serde_json::json!(value)); - } -} - -const fn valid_static_code(value: &str) -> bool { - if value.is_empty() || value.len() > 64 { - return false; - } - let bytes = value.as_bytes(); - let mut index = 0; - while index < bytes.len() { - let byte = bytes[index]; - if !(byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') { - return false; - } - index += 1; - } - true -} - -fn bounded_context_value(value: &str) -> Option { - let sanitized = crate::logging::policy::sanitize_paths_in_text(value); - let sanitized = crate::logging::policy::redact_urls_in_text(&sanitized); - let sanitized = crate::logging::policy::apply_redaction(&sanitized).0; - let trimmed = sanitized.trim(); - if trimmed.is_empty() { - return None; - } - Some(trimmed.chars().take(MAX_CONTEXT_VALUE_CHARS).collect()) -} +pub use context::{OperationalAuditContext, OperationalAuditPathType, OperationalAuditSubjectKind}; /// Static, bounded operational audit data admitted by the logging service. #[derive(Clone, Debug, Eq, PartialEq)] @@ -286,77 +123,3 @@ impl OperationalAuditSeverity { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn typed_context_is_sanitized_bounded_and_cardinality_limited() { - let long_subject = format!("{}?api_key=private", "model".repeat(80)); - let mut context = OperationalAuditContext::new() - .subject(OperationalAuditSubjectKind::Model, &long_subject) - .operation_id("runtime-7") - .reason_code("load_failed") - .outcome("failed") - .duration_ms(42); - for index in 0..12 { - let key = match index { - 0 => "metric_0", - 1 => "metric_1", - 2 => "metric_2", - 3 => "metric_3", - 4 => "metric_4", - 5 => "metric_5", - 6 => "metric_6", - 7 => "metric_7", - 8 => "metric_8", - 9 => "metric_9", - 10 => "metric_10", - _ => "metric_11", - }; - context = context.numeric_summary(key, index); - } - - let fields = context.fields(); - assert_eq!(fields["context_version"], 1); - assert_eq!(fields["subject_kind"], "model"); - assert_eq!(fields["operation_id"], "runtime-7"); - assert_eq!(fields["reason_code"], "load_failed"); - assert_eq!(fields["outcome"], "failed"); - assert_eq!(fields["duration_ms"], 42); - assert!(fields["subject_id"].as_str().unwrap().chars().count() <= 256); - assert!(!fields["subject_id"].as_str().unwrap().contains("private")); - assert_eq!(fields["numeric_summaries"].as_object().unwrap().len(), 8); - } - - #[test] - fn invalid_static_codes_are_not_admitted() { - let fields = OperationalAuditContext::new() - .reason_code("NOT VALID") - .outcome("also-not-valid") - .numeric_summary("bad-key", 1) - .fields(); - assert!(fields.get("reason_code").is_none()); - assert!(fields.get("outcome").is_none()); - assert!(fields.get("numeric_summaries").is_none()); - } - - #[test] - fn context_values_redact_url_credentials_and_query_secrets() { - let fields = OperationalAuditContext::new() - .subject( - OperationalAuditSubjectKind::Model, - "https://alice:top-secret@example.test/model?api_key=query-secret&safe=1", - ) - .fields(); - let subject_id = fields["subject_id"].as_str().expect("subject id"); - - assert!(!subject_id.contains("alice")); - assert!(!subject_id.contains("top-secret")); - assert!(!subject_id.contains("query-secret")); - assert!(subject_id.contains("[REDACTED]@example.test")); - assert!(subject_id.contains("api_key=[REDACTED]")); - assert!(subject_id.contains("safe=1")); - } -} diff --git a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs new file mode 100644 index 0000000000..d117c96806 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs @@ -0,0 +1,247 @@ +use std::{collections::BTreeMap, net::SocketAddr}; + +use mesh_llm_events::CliCommandSummary; + +const OPERATIONAL_AUDIT_CONTEXT_VERSION: u8 = 1; +const MAX_CONTEXT_VALUE_CHARS: usize = 256; +const MAX_NUMERIC_SUMMARIES: usize = 8; + +/// Closed subject vocabulary for diagnostic operational-audit context. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationalAuditSubjectKind { + Runtime, + Model, + RuntimeInstance, + CliCommand, +} + +impl OperationalAuditSubjectKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Runtime => "runtime", + Self::Model => "model", + Self::RuntimeInstance => "runtime_instance", + Self::CliCommand => "cli_command", + } + } + + pub(crate) fn parse(value: &str) -> Option { + match value { + "runtime" => Some(OperationalAuditContextSubjectKind::Public(Self::Runtime)), + "model" => Some(OperationalAuditContextSubjectKind::Public(Self::Model)), + "runtime_instance" => Some(OperationalAuditContextSubjectKind::Public( + Self::RuntimeInstance, + )), + "cli_command" => Some(OperationalAuditContextSubjectKind::Public(Self::CliCommand)), + "mesh_peer" => Some(OperationalAuditContextSubjectKind::MeshPeer), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum OperationalAuditContextSubjectKind { + Public(OperationalAuditSubjectKind), + MeshPeer, +} + +impl OperationalAuditContextSubjectKind { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Public(kind) => kind.as_str(), + Self::MeshPeer => "mesh_peer", + } + } +} + +/// Closed connection-path vocabulary for mesh peer audit context. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationalAuditPathType { + Direct, + Relay, +} + +impl OperationalAuditPathType { + pub const fn as_str(self) -> &'static str { + match self { + Self::Direct => "direct", + Self::Relay => "relay", + } + } +} + +/// Versioned, bounded context shared by audit replay and durable storage. +/// +/// Values are sanitized before this type accepts them. Arbitrary detail JSON, +/// command arguments, error text, and payload bodies cannot enter this shape. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct OperationalAuditContext { + subject_kind: Option, + subject_id: Option, + remote_addr: Option, + path_type: Option, + operation_id: Option, + request_id: Option, + reason_code: Option<&'static str>, + outcome: Option<&'static str>, + duration_ms: Option, + numeric_summaries: BTreeMap<&'static str, u64>, + command_summary: Option, +} + +impl OperationalAuditContext { + pub fn new() -> Self { + Self::default() + } + + pub fn subject(mut self, kind: OperationalAuditSubjectKind, id: &str) -> Self { + self.subject_kind = Some(OperationalAuditContextSubjectKind::Public(kind)); + self.subject_id = bounded_context_value(id); + self + } + + pub(crate) fn mesh_peer_subject(mut self, id: &str) -> Self { + self.subject_kind = Some(OperationalAuditContextSubjectKind::MeshPeer); + self.subject_id = bounded_context_value(id); + self + } + + pub fn operation_id(mut self, operation_id: &str) -> Self { + self.operation_id = bounded_context_value(operation_id); + self + } + + pub const fn network_path( + mut self, + path_type: OperationalAuditPathType, + remote_addr: Option, + ) -> Self { + self.path_type = Some(path_type); + self.remote_addr = match path_type { + OperationalAuditPathType::Direct => remote_addr, + OperationalAuditPathType::Relay => None, + }; + self + } + + pub fn request_id(mut self, request_id: &str) -> Self { + self.request_id = bounded_context_value(request_id); + self + } + + pub const fn reason_code(mut self, reason_code: &'static str) -> Self { + if Self::valid_static_code(reason_code) { + self.reason_code = Some(reason_code); + } + self + } + + pub const fn outcome(mut self, outcome: &'static str) -> Self { + if Self::valid_static_code(outcome) { + self.outcome = Some(outcome); + } + self + } + + pub const fn duration_ms(mut self, duration_ms: u64) -> Self { + self.duration_ms = Some(duration_ms); + self + } + + pub fn numeric_summary(mut self, key: &'static str, value: u64) -> Self { + if self.numeric_summaries.len() < MAX_NUMERIC_SUMMARIES && Self::valid_static_code(key) { + self.numeric_summaries.insert(key, value); + } + self + } + + pub fn command_summary(mut self, summary: &str) -> Self { + self.command_summary = + CliCommandSummary::sanitize(summary).map(|summary| summary.as_str().to_owned()); + self + } + + pub(crate) const fn valid_static_code(value: &str) -> bool { + valid_static_code(value) + } + + pub(crate) fn fields(&self) -> serde_json::Map { + let mut fields = serde_json::Map::new(); + fields.insert( + "context_version".into(), + serde_json::json!(OPERATIONAL_AUDIT_CONTEXT_VERSION), + ); + if let Some(kind) = self.subject_kind { + fields.insert("subject_kind".into(), serde_json::json!(kind.as_str())); + } + insert_optional_string(&mut fields, "subject_id", self.subject_id.as_deref()); + if let Some(remote_addr) = self.remote_addr { + fields.insert( + "remote_addr".into(), + serde_json::json!(remote_addr.to_string()), + ); + } + if let Some(path_type) = self.path_type { + fields.insert("path_type".into(), serde_json::json!(path_type.as_str())); + } + insert_optional_string(&mut fields, "operation_id", self.operation_id.as_deref()); + insert_optional_string(&mut fields, "request_id", self.request_id.as_deref()); + insert_optional_string(&mut fields, "reason_code", self.reason_code); + insert_optional_string(&mut fields, "outcome", self.outcome); + if let Some(duration_ms) = self.duration_ms { + fields.insert("duration_ms".into(), serde_json::json!(duration_ms)); + } + if !self.numeric_summaries.is_empty() { + fields.insert( + "numeric_summaries".into(), + serde_json::json!(self.numeric_summaries), + ); + } + insert_optional_string( + &mut fields, + "command_summary", + self.command_summary.as_deref(), + ); + fields + } +} + +fn insert_optional_string( + fields: &mut serde_json::Map, + key: &'static str, + value: Option<&str>, +) { + if let Some(value) = value { + fields.insert(key.into(), serde_json::json!(value)); + } +} + +const fn valid_static_code(value: &str) -> bool { + if value.is_empty() || value.len() > 64 { + return false; + } + let bytes = value.as_bytes(); + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if !(byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') { + return false; + } + index += 1; + } + true +} + +fn bounded_context_value(value: &str) -> Option { + let sanitized = crate::logging::policy::sanitize_paths_in_text(value); + let sanitized = crate::logging::policy::redact_urls_in_text(&sanitized); + let sanitized = crate::logging::policy::apply_redaction(&sanitized).0; + let trimmed = sanitized.trim(); + if trimmed.is_empty() { + return None; + } + Some(trimmed.chars().take(MAX_CONTEXT_VALUE_CHARS).collect()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context/tests.rs b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context/tests.rs new file mode 100644 index 0000000000..26c4d76216 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context/tests.rs @@ -0,0 +1,125 @@ +use super::*; + +#[test] +fn typed_context_is_sanitized_bounded_and_cardinality_limited() { + let long_subject = format!("{}?api_key=private", "model".repeat(80)); + let mut context = OperationalAuditContext::new() + .subject(OperationalAuditSubjectKind::Model, &long_subject) + .operation_id("runtime-7") + .reason_code("load_failed") + .outcome("failed") + .duration_ms(42); + for index in 0..12 { + let key = match index { + 0 => "metric_0", + 1 => "metric_1", + 2 => "metric_2", + 3 => "metric_3", + 4 => "metric_4", + 5 => "metric_5", + 6 => "metric_6", + 7 => "metric_7", + 8 => "metric_8", + 9 => "metric_9", + 10 => "metric_10", + _ => "metric_11", + }; + context = context.numeric_summary(key, index); + } + + let fields = context.fields(); + assert_eq!(fields["context_version"], 1); + assert_eq!(fields["subject_kind"], "model"); + assert_eq!(fields["operation_id"], "runtime-7"); + assert_eq!(fields["reason_code"], "load_failed"); + assert_eq!(fields["outcome"], "failed"); + assert_eq!(fields["duration_ms"], 42); + assert!(fields["subject_id"].as_str().unwrap().chars().count() <= 256); + assert!(!fields["subject_id"].as_str().unwrap().contains("private")); + assert_eq!(fields["numeric_summaries"].as_object().unwrap().len(), 8); +} + +#[test] +fn invalid_static_codes_are_not_admitted() { + let fields = OperationalAuditContext::new() + .reason_code("NOT VALID") + .outcome("also-not-valid") + .numeric_summary("bad-key", 1) + .fields(); + assert!(fields.get("reason_code").is_none()); + assert!(fields.get("outcome").is_none()); + assert!(fields.get("numeric_summaries").is_none()); +} + +#[test] +fn context_values_redact_url_credentials_and_query_secrets() { + let fields = OperationalAuditContext::new() + .subject( + OperationalAuditSubjectKind::Model, + "https://alice:top-secret@example.test/model?api_key=query-secret&safe=1", + ) + .fields(); + let subject_id = fields["subject_id"].as_str().expect("subject id"); + + assert!(!subject_id.contains("alice")); + assert!(!subject_id.contains("top-secret")); + assert!(!subject_id.contains("query-secret")); + assert!(subject_id.contains("[REDACTED]@example.test")); + assert!(subject_id.contains("api_key=[REDACTED]")); + assert!(subject_id.contains("safe=1")); +} + +#[test] +fn mesh_peer_direct_path_preserves_identity_address_and_path_type() { + let remote_addr = "192.168.1.44:11204".parse().expect("socket address"); + + let fields = OperationalAuditContext::new() + .mesh_peer_subject("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + .network_path(OperationalAuditPathType::Direct, Some(remote_addr)) + .fields(); + + assert_eq!(fields["context_version"], 1); + assert_eq!(fields["subject_kind"], "mesh_peer"); + assert_eq!( + fields["subject_id"], + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ); + assert_eq!(fields["remote_addr"], "192.168.1.44:11204"); + assert_eq!(fields["path_type"], "direct"); +} + +#[test] +fn mesh_peer_relay_path_omits_observed_address() { + let relay_addr = "203.0.113.10:443".parse().expect("socket address"); + + let fields = OperationalAuditContext::new() + .mesh_peer_subject("peer-hex") + .network_path(OperationalAuditPathType::Relay, Some(relay_addr)) + .fields(); + + assert_eq!(fields["path_type"], "relay"); + assert!(fields.get("remote_addr").is_none()); +} + +#[test] +fn command_summary_is_serialized_with_context_bounds_and_redaction() { + let fields = OperationalAuditContext::new() + .command_summary("mesh-llm load name [REDACTED]") + .fields(); + let summary = fields["command_summary"].as_str().expect("command summary"); + assert!(summary.chars().count() <= 256); + assert_eq!(summary, "mesh-llm load name [REDACTED]"); +} + +#[test] +fn command_summary_context_drops_malformed_values_and_overlong_token_lists() { + let fields = OperationalAuditContext::new() + .command_summary("mesh-llm load private-model-name") + .fields(); + assert!(fields.get("command_summary").is_none()); + + let fields = OperationalAuditContext::new() + .command_summary(&format!("mesh-llm {}", "x ".repeat(32))) + .fields(); + assert!(fields.get("command_summary").is_none()); +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections.rs b/crates/mesh-llm-host-runtime/src/mesh/connections.rs index 93036d8aba..34dd91ba6c 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/connections.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/connections.rs @@ -1,6 +1,9 @@ use super::*; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +mod inbound; +mod tunnel; + pub(crate) struct NodeHardwareSnapshot { /// Accelerator-resident capacity advertised for mesh stage placement. pub(crate) vram_bytes: u64, @@ -26,6 +29,7 @@ pub(crate) struct DetectedVramLog { } pub(crate) struct AcceptedMeshStream { + pub(crate) remote: EndpointId, pub(crate) send: iroh::endpoint::SendStream, pub(crate) recv: iroh::endpoint::RecvStream, pub(crate) stream_type: u8, @@ -631,7 +635,8 @@ impl Node { "Tunnel to {} failed, broadcasting death", peer_id.fmt_short() ); - self.handle_peer_death(peer_id).await; + self.handle_peer_death_with_reason(peer_id, MeshPeerRemovalReason::TunnelOpenFailed) + .await; } result @@ -655,7 +660,6 @@ impl Node { let node = self.clone(); tokio::spawn(async move { if let Err(e) = node.handle_incoming(incoming).await { - record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed); tracing::warn!("Incoming connection error: {e}"); } }); @@ -681,7 +685,6 @@ impl Node { let node = self.clone(); tokio::spawn(Box::pin(async move { if let Err(error) = node.handle_control_incoming(incoming).await { - record_mesh_operational_event(MeshOperationalEvent::ControlHandlerFailed); tracing::debug!("Control-plane incoming connection error: {error}"); } })); @@ -744,187 +747,8 @@ impl Node { } }); } - - pub(crate) async fn handle_incoming(&self, incoming: iroh::endpoint::Incoming) -> Result<()> { - let mut accepting = incoming.accept()?; - let alpn = accepting.alpn().await?; - let conn = accepting.await?; - let remote = conn.remote_id(); - if self.handle_stage_alpn(&alpn, conn.clone(), remote).await { - return Ok(()); - } - record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted); - tracing::info!("Inbound connection from {}", remote.fmt_short()); - - // Store connection for stream dispatch (tunneling, route requests, etc.) - // Don't add to peer list yet — only gossip exchange promotes to peer. - let (was_dead, admitted) = self.remember_incoming_connection(remote, &conn).await; - self.capture_connection_event(ConnectionCaptureEvent { - event: "peer_connection_accepted", - remote, - direction: "inbound", - phase: "accept", - protocol: Some(connection_protocol(&conn)), - path_type: None, - rtt_ms: None, - admitted_peer: Some(admitted), - reason: was_dead.then_some("previously_dead"), - }); - self.capture_selected_connection_path(remote, &conn, "inbound_connection_accept_path"); - - // If this peer was previously dead, immediately gossip to restore their - // assigned/routable state in our peer list. Without this, models served by the - // reconnecting peer stay invisible until the next heartbeat (up to 60s). - if was_dead { - self.spawn_reconnect_gossip(conn.clone(), remote); - } - - self.dispatch_streams(conn, remote).await; - Ok(()) - } - - pub(crate) async fn handle_stage_alpn( - &self, - alpn: &[u8], - conn: Connection, - remote: EndpointId, - ) -> bool { - if alpn != skippy_protocol::STAGE_ALPN_V2 { - return false; - } - tracing::info!( - "Inbound skippy stage connection from {}", - remote.fmt_short() - ); - self.dispatch_stage_streams(conn, remote).await; - true - } - - pub(crate) async fn handle_control_incoming( - &self, - incoming: iroh::endpoint::Incoming, - ) -> Result<()> { - let mut accepting = incoming.accept()?; - let alpn = accepting.alpn().await?; - if alpn.as_slice() != ALPN_CONTROL_V1 { - record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); - anyhow::bail!( - "unexpected control-plane ALPN {:?}", - String::from_utf8_lossy(&alpn) - ); - } - let conn = accepting.await?; - let remote = conn.remote_id(); - record_mesh_operational_event(MeshOperationalEvent::ControlConnectionAccepted); - let permits = control_stream_semaphore(); - loop { - let permit = match permits.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => break, - }; - let (mut send, mut recv) = match conn.accept_bi().await { - Ok(streams) => streams, - Err(error) => { - tracing::debug!( - "Control-plane connection from {} closed: {error}", - remote.fmt_short() - ); - break; - } - }; - let node = self.clone(); - tokio::spawn(Box::pin(async move { - let _permit = permit; - if let Err(error) = node - .handle_control_stream(remote, &mut send, &mut recv) - .await - { - tracing::debug!( - "Control-plane stream from {} failed: {error}", - remote.fmt_short() - ); - } - })); - } - Ok(()) - } } impl Node { - pub(crate) async fn accept_mesh_stream( - &self, - conn: &Connection, - remote: EndpointId, - protocol: ControlProtocol, - ) -> Result { - let (send, mut recv) = conn.accept_bi().await.map_err(|error| { - tracing::info!("Connection to {} closed: {error}", remote.fmt_short()); - self.capture_connection_event(ConnectionCaptureEvent { - event: "peer_connection_closed", - remote, - direction: "unknown", - phase: "accept_bi", - protocol: Some(protocol), - path_type: None, - rtt_ms: None, - admitted_peer: None, - reason: Some("accept_bi_error"), - }); - })?; - let mut type_buf = [0u8; 1]; - if !matches!( - tokio::time::timeout( - MESH_STREAM_TYPE_READ_TIMEOUT, - recv.read_exact(&mut type_buf), - ) - .await, - Ok(Ok(_)) - ) { - let _ = recv.stop(0u32.into()); - return Err(()); - } - Ok(AcceptedMeshStream { - send, - recv, - stream_type: type_buf[0], - }) - } - - pub(crate) async fn admitted_mesh_stream( - &self, - remote: EndpointId, - protocol: ControlProtocol, - stream_type: u8, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> Option { - let capture_streams = self.swarm_capture_enabled(); - if stream_allowed_before_admission(stream_type, self.trust_policy) { - if capture_streams { - self.capture_stream_observation(remote, stream_type, protocol, true); - } - return Some((send, recv)); - } - let admitted = { - let state = self.state.lock().await; - state.peers.get(&remote).is_some_and(PeerInfo::is_admitted) - }; - if capture_streams { - self.capture_stream_observation(remote, stream_type, protocol, admitted); - } - if admitted { - Some((send, recv)) - } else { - self.capture_stream_rejected(remote, stream_type, protocol, "unadmitted_peer"); - tracing::warn!( - "Quarantine: stream {:#04x} from unadmitted peer {} rejected — peer must complete gossip first", - stream_type, - remote.fmt_short() - ); - drop((send, recv)); - None - } - } - pub(crate) async fn recover_closed_connection( &self, remote: EndpointId, @@ -939,7 +763,8 @@ impl Node { .await; } ClosedConnectionRecovery::RemovePeer => { - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::ClosedConnectionNoAddress) + .await; } ClosedConnectionRecovery::AlreadyReplaced => {} } @@ -957,7 +782,8 @@ impl Node { } _ => { tracing::info!("Reconnect to {} failed — removing peer", remote.fmt_short()); - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::ReconnectFailed) + .await; } } } @@ -1059,7 +885,8 @@ impl Node { "Reconnect gossip to {} failed — peer is dead, removing", remote.fmt_short() ); - self.remove_peer(remote).await; + self.remove_peer(remote, MeshPeerRemovalReason::RecoveredGossipFailed) + .await; } pub(crate) async fn recovered_connection_gossip_ok( @@ -1076,15 +903,6 @@ impl Node { .unwrap_or(false) } - /// Dispatch bi-streams on a connection by type byte - pub(crate) fn dispatch_streams( - &self, - conn: Connection, - remote: EndpointId, - ) -> std::pin::Pin + Send + '_>> { - Box::pin(self._dispatch_streams(conn, remote)) - } - pub(crate) fn spawn_gossip_stream( &self, remote: EndpointId, @@ -1281,49 +1099,6 @@ impl Node { } } impl Node { - pub(crate) async fn dispatch_mesh_stream( - &self, - remote: EndpointId, - protocol: ControlProtocol, - stream_type: u8, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if stream_type == STREAM_TUNNEL { - return self.forward_tunnel_stream(send, recv).await; - } - if stream_type == STREAM_TUNNEL_HTTP { - return self.forward_tunnel_http_stream(send, recv).await; - } - - self.spawn_non_tunnel_mesh_stream(remote, protocol, stream_type, send, recv); - true - } - - pub(crate) async fn forward_tunnel_stream( - &self, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if self.tunnel_tx.send((send, recv)).await.is_err() { - tracing::warn!("Tunnel receiver dropped"); - return false; - } - true - } - - pub(crate) async fn forward_tunnel_http_stream( - &self, - send: iroh::endpoint::SendStream, - recv: iroh::endpoint::RecvStream, - ) -> bool { - if self.tunnel_http_tx.send((send, recv)).await.is_err() { - tracing::warn!("HTTP tunnel receiver dropped"); - return false; - } - true - } - pub(crate) fn spawn_non_tunnel_mesh_stream( &self, remote: EndpointId, @@ -1380,7 +1155,8 @@ impl Node { .insert(leaving_id, std::time::Instant::now()); state.connections.remove(&leaving_id); drop(state); - self.remove_peer(leaving_id).await; + self.remove_peer(leaving_id, MeshPeerRemovalReason::CleanShutdown) + .await; } pub(crate) async fn decode_peer_leaving( @@ -1437,59 +1213,8 @@ impl Node { } } } - - pub(crate) async fn _dispatch_streams(&self, conn: Connection, remote: EndpointId) { - let protocol = connection_protocol(&conn); - let dispatcher_stable_id = conn.stable_id(); - loop { - let accepted = match self.accept_mesh_stream(&conn, remote, protocol).await { - Ok(accepted) => accepted, - Err(()) => { - self.recover_closed_connection(remote, dispatcher_stable_id) - .await; - break; - } - }; - let Some((send, recv)) = self - .admitted_mesh_stream( - remote, - protocol, - accepted.stream_type, - accepted.send, - accepted.recv, - ) - .await - else { - continue; - }; - if !self - .dispatch_mesh_stream(remote, protocol, accepted.stream_type, send, recv) - .await - { - break; - } - } - } } impl Node { - pub(crate) async fn remove_connection_if_stable_id( - &self, - peer_id: EndpointId, - conn: &Connection, - ) -> Option { - let stable_id = conn.stable_id(); - let mut state = self.state.lock().await; - if state - .connections - .get(&peer_id) - .is_some_and(|current| current.stable_id() == stable_id) - { - state.connections.remove(&peer_id) - } else { - None - } - } - pub(crate) async fn reserve_pending_connection( &self, peer_id: EndpointId, diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs new file mode 100644 index 0000000000..b735041ece --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs @@ -0,0 +1,250 @@ +use super::*; +use crate::logging::OperationalAuditContext; + +mod stage; + +impl Node { + pub(crate) async fn handle_incoming(&self, incoming: iroh::endpoint::Incoming) -> Result<()> { + let started = std::time::Instant::now(); + let mut accepting = incoming.accept().inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::AcceptSetup.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let alpn = accepting.alpn().await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::AlpnRead.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let conn = accepting.await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureBoundary::Handshake.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let remote = conn.remote_id(); + if self.handle_stage_alpn(&alpn, conn.clone(), remote).await { + return Ok(()); + } + tracing::info!("Inbound connection from {}", remote.fmt_short()); + + // Store connection for stream dispatch (tunneling, route requests, etc.) + // Don't add to peer list yet — only gossip exchange promotes to peer. + let (was_dead, admitted) = self.remember_incoming_connection(remote, &conn).await; + let inbound_outcome = if was_dead { + MeshQuicInboundOutcome::Readmitted + } else { + MeshQuicInboundOutcome::Accepted + }; + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(inbound_outcome), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary("protocol_gen", u64::from(NODE_PROTOCOL_GENERATION)), + ); + self.capture_connection_event(ConnectionCaptureEvent { + event: "peer_connection_accepted", + remote, + direction: "inbound", + phase: "accept", + protocol: Some(connection_protocol(&conn)), + path_type: None, + rtt_ms: None, + admitted_peer: Some(admitted), + reason: was_dead.then_some("previously_dead"), + }); + self.capture_selected_connection_path(remote, &conn, "inbound_connection_accept_path"); + + // If this peer was previously dead, immediately gossip to restore their + // assigned/routable state in our peer list. Without this, models served by the + // reconnecting peer stay invisible until the next heartbeat (up to 60s). + if was_dead { + self.spawn_reconnect_gossip(conn.clone(), remote); + } + + self.dispatch_streams(conn, remote).await; + Ok(()) + } + + pub(crate) async fn handle_control_incoming( + &self, + incoming: iroh::endpoint::Incoming, + ) -> Result<()> { + let started = std::time::Instant::now(); + let mut accepting = incoming.accept().inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::AcceptSetup.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let alpn = accepting.alpn().await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::AlpnRead.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + if alpn.as_slice() != ALPN_CONTROL_V1 { + record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); + anyhow::bail!("unsupported control-plane ALPN"); + } + let conn = accepting.await.inspect_err(|_| { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::Handshake.failure_class(), + ), + OperationalAuditContext::new().duration_ms(elapsed_ms_u64(started.elapsed())), + ); + })?; + let remote = conn.remote_id(); + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlConnectionAccepted, + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary("protocol_gen", u64::from(NODE_PROTOCOL_GENERATION)), + ); + let permits = control_stream_semaphore(); + loop { + let permit_started = std::time::Instant::now(); + let permit = match permits.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::CapacityPermit.failure_class(), + ), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .duration_ms(elapsed_ms_u64(permit_started.elapsed())), + ); + break; + } + }; + let stream_accept_started = std::time::Instant::now(); + let (mut send, mut recv) = match conn.accept_bi().await { + Ok(streams) => streams, + Err(error) => { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::StreamAccept.failure_class(), + ), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .duration_ms(elapsed_ms_u64(stream_accept_started.elapsed())), + ); + tracing::debug!( + "Control-plane connection from {} closed: {error}", + remote.fmt_short() + ); + break; + } + }; + let node = self.clone(); + let peer_context = + mesh_peer_operational_context(remote, selected_path_observation(&conn)); + tokio::spawn(Box::pin(async move { + let _permit = permit; + let dispatch_started = std::time::Instant::now(); + if let Err(error) = node + .handle_control_stream(remote, &mut send, &mut recv) + .await + { + record_mesh_operational_event_with_context( + MeshOperationalEvent::ControlHandlerFailed( + MeshHandlerFailureBoundary::ControlDispatch.failure_class(), + ), + peer_context.duration_ms(elapsed_ms_u64(dispatch_started.elapsed())), + ); + tracing::debug!( + "Control-plane stream from {} failed: {error}", + remote.fmt_short() + ); + } + })); + } + Ok(()) + } + + pub(crate) async fn accept_mesh_stream( + &self, + conn: &Connection, + protocol: ControlProtocol, + ) -> Result { + let remote = conn.remote_id(); + let (send, mut recv) = conn.accept_bi().await.map_err(|error| { + tracing::info!("Connection to {} closed: {error}", remote.fmt_short()); + self.capture_connection_event(ConnectionCaptureEvent { + event: "peer_connection_closed", + remote, + direction: "unknown", + phase: "accept_bi", + protocol: Some(protocol), + path_type: None, + rtt_ms: None, + admitted_peer: None, + reason: Some("accept_bi_error"), + }); + })?; + let mut type_buf = [0u8; 1]; + if !matches!( + tokio::time::timeout( + MESH_STREAM_TYPE_READ_TIMEOUT, + recv.read_exact(&mut type_buf), + ) + .await, + Ok(Ok(_)) + ) { + let _ = recv.stop(0u32.into()); + return Err(()); + } + Ok(AcceptedMeshStream { + remote, + send, + recv, + stream_type: type_buf[0], + }) + } + + pub(crate) async fn admitted_mesh_stream( + &self, + remote: EndpointId, + protocol: ControlProtocol, + stream_type: u8, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> Option { + let capture_streams = self.swarm_capture_enabled(); + if stream_allowed_before_admission(stream_type, self.trust_policy) { + if capture_streams { + self.capture_stream_observation(remote, stream_type, protocol, true); + } + return Some((send, recv)); + } + let admitted = { + let state = self.state.lock().await; + state.peers.get(&remote).is_some_and(PeerInfo::is_admitted) + }; + if capture_streams { + self.capture_stream_observation(remote, stream_type, protocol, admitted); + } + if admitted { + Some((send, recv)) + } else { + self.capture_stream_rejected(remote, stream_type, protocol, "unadmitted_peer"); + tracing::warn!( + "Quarantine: stream {:#04x} from unadmitted peer {} rejected — peer must complete gossip first", + stream_type, + remote.fmt_short() + ); + drop((send, recv)); + None + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs new file mode 100644 index 0000000000..7357a8029a --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs @@ -0,0 +1,28 @@ +use super::super::*; + +impl Node { + pub(crate) async fn handle_stage_alpn( + &self, + alpn: &[u8], + conn: Connection, + remote: EndpointId, + ) -> bool { + if alpn != skippy_protocol::STAGE_ALPN_V2 { + return false; + } + tracing::info!( + "Inbound skippy stage connection from {}", + remote.fmt_short() + ); + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + mesh_peer_operational_context(remote, selected_path_observation(&conn)) + .numeric_summary( + "protocol_gen", + u64::from(skippy_protocol::STAGE_PROTOCOL_GENERATION), + ), + ); + self.dispatch_stage_streams(conn, remote).await; + true + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs b/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs new file mode 100644 index 0000000000..fc0386e335 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs @@ -0,0 +1,122 @@ +use super::*; + +impl Node { + /// Dispatch bi-streams on a connection by type byte + pub(crate) fn dispatch_streams( + &self, + conn: Connection, + remote: EndpointId, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(self._dispatch_streams(conn, remote)) + } + + pub(crate) async fn dispatch_mesh_stream( + &self, + remote: EndpointId, + protocol: ControlProtocol, + stream_type: u8, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if stream_type == STREAM_TUNNEL { + return self.forward_tunnel_stream(send, recv).await; + } + if stream_type == STREAM_TUNNEL_HTTP { + return self.forward_tunnel_http_stream(remote, send, recv).await; + } + + self.spawn_non_tunnel_mesh_stream(remote, protocol, stream_type, send, recv); + true + } + + pub(crate) async fn forward_tunnel_stream( + &self, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if self.tunnel_tx.send((send, recv)).await.is_err() { + tracing::warn!("Tunnel receiver dropped"); + return false; + } + true + } + + pub(crate) async fn forward_tunnel_http_stream( + &self, + remote: EndpointId, + send: iroh::endpoint::SendStream, + recv: iroh::endpoint::RecvStream, + ) -> bool { + if self + .tunnel_http_tx + .send((remote, send, recv)) + .await + .is_err() + { + tracing::warn!("HTTP tunnel receiver dropped"); + return false; + } + true + } + + pub(crate) async fn _dispatch_streams(&self, conn: Connection, remote: EndpointId) { + let protocol = connection_protocol(&conn); + let dispatcher_stable_id = conn.stable_id(); + loop { + let accepted = match self.accept_mesh_stream(&conn, protocol).await { + Ok(accepted) => accepted, + Err(()) => { + self.recover_closed_connection(remote, dispatcher_stable_id) + .await; + break; + } + }; + let Some((send, recv)) = self + .admitted_mesh_stream( + accepted.remote, + protocol, + accepted.stream_type, + accepted.send, + accepted.recv, + ) + .await + else { + continue; + }; + if !self + .dispatch_mesh_stream(accepted.remote, protocol, accepted.stream_type, send, recv) + .await + { + break; + } + } + } + + pub(crate) async fn authenticated_peer_path( + &self, + remote: EndpointId, + ) -> Option { + let conn = self.state.lock().await.connections.get(&remote).cloned()?; + (conn.remote_id() == remote) + .then(|| selected_path_observation(&conn)) + .flatten() + } + + pub(crate) async fn remove_connection_if_stable_id( + &self, + peer_id: EndpointId, + conn: &Connection, + ) -> Option { + let stable_id = conn.stable_id(); + let mut state = self.state.lock().await; + if state + .connections + .get(&peer_id) + .is_some_and(|current| current.stable_id() == stable_id) + { + state.connections.remove(&peer_id) + } else { + None + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs index d40f33ac5c..866c0056be 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/gossip.rs @@ -2,11 +2,12 @@ //! and peer list management (add/remove/update). use super::{ - DEAD_PEER_TTL, DisplayLatencySource, InviteTokenMaterial, MeshOperationalEvent, ModelDemand, - ModelRuntimeDescriptor, Node, NodeRole, PEER_CONNECT_AND_GOSSIP_TIMEOUT, PEER_STALE_SECS, - PeerAnnouncement, PeerInfo, ServedModelDescriptor, SignedNodeOwnership, connect_mesh, - elapsed_ms_u64, emit_mesh_info, infer_remote_served_descriptors, parse_invite_token, - record_mesh_operational_event, + DEAD_PEER_TTL, DisplayLatencySource, InviteTokenMaterial, MeshOperationalEvent, + MeshPeerRemovalReason, MeshPolicyRejectionReason, ModelDemand, ModelRuntimeDescriptor, Node, + NodeRole, PEER_CONNECT_AND_GOSSIP_TIMEOUT, PEER_STALE_SECS, PeerAnnouncement, PeerInfo, + ServedModelDescriptor, SignedNodeOwnership, connect_mesh, elapsed_ms_u64, emit_mesh_info, + infer_remote_served_descriptors, mesh_peer_operational_context, parse_invite_token, + record_mesh_operational_event, record_mesh_operational_event_with_context, }; use crate::crypto::{OwnershipSummary, verify_node_ownership}; use crate::mesh::peer_state::{PropagatedLatencyObservation, policy_accepts_peer}; @@ -519,7 +520,12 @@ impl Node { ); } if self - .add_peer_after_direct_requirements_validated(remote, addr.clone(), ann) + .add_peer_after_direct_requirements_validated( + remote, + addr.clone(), + ann, + context.negotiated_protocol_generation, + ) .await { if let Some(ref their_id) = ann.mesh_id { @@ -884,8 +890,14 @@ impl Node { let _ = self.peer_change_tx.send(admitted_count); } drop(state); - if newly_rejected { - record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected); + if newly_rejected + && let Some(reason) = + MeshPolicyRejectionReason::from_ownership_status(&owner_summary.status) + { + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipPolicyRejected(reason), + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await), + ); } true } @@ -971,7 +983,11 @@ impl Node { .count(); drop(state); self.capture_peer_observation("peer_direct_add", &peer, "direct", None); - record_mesh_operational_event(MeshOperationalEvent::GossipDirectPeerPromoted); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipDirectPeerPromoted, + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await) + .numeric_summary("direct_peers", count as u64), + ); let _ = self.peer_change_tx.send(count); self.emit_plugin_mesh_event( crate::plugin::proto::mesh_event::Kind::PeerUp, @@ -1571,7 +1587,7 @@ impl Node { Ok(()) } - pub(super) async fn remove_peer(&self, id: EndpointId) { + pub(super) async fn remove_peer(&self, id: EndpointId, reason: MeshPeerRemovalReason) { let mut state = self.state.lock().await; // Always clear any rejection-tracking entry so the map stays bounded. state.policy_rejected_peers.remove(&id); @@ -1598,14 +1614,18 @@ impl Node { self.capture_peer_lifecycle_event(PeerLifecycleCaptureEvent { event: "peer_removed", peer: id, - reason: "remove_peer", + reason: reason.reason_code(), reporter: None, last_seen_age_ms: Some(last_seen_age_ms), last_mentioned_age_ms: Some(last_mentioned_age_ms), had_connection: Some(had_connection), bridge_id, }); - record_mesh_operational_event(MeshOperationalEvent::GossipPeerRemoved); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipPeerRemoved(reason), + mesh_peer_operational_context(id, peer.selected_path) + .numeric_summary("direct_peers", count as u64), + ); let _ = self.peer_change_tx.send(count); self.emit_plugin_mesh_event( crate::plugin::proto::mesh_event::Kind::PeerDown, @@ -1651,8 +1671,13 @@ impl Node { } return; } - self.add_peer_after_direct_requirements_validated(id, addr, ann) - .await; + self.add_peer_after_direct_requirements_validated( + id, + addr, + ann, + negotiated_protocol_generation, + ) + .await; } pub(crate) async fn add_peer_after_direct_requirements_validated( @@ -1660,6 +1685,7 @@ impl Node { id: EndpointId, addr: EndpointAddr, ann: &PeerAnnouncement, + _negotiated_protocol_generation: Option, ) -> bool { // Reject ingest from peers below the supported version floor. They // are not added to local state, do not appear in /api/status, and @@ -1671,7 +1697,10 @@ impl Node { id.fmt_short(), ann.version ); - record_mesh_operational_event(MeshOperationalEvent::GossipIncompatibleVersionRejected); + record_mesh_operational_event_with_context( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + mesh_peer_operational_context(id, self.authenticated_peer_path(id).await), + ); self.remove_disallowed_peer(id).await; return false; } diff --git a/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs b/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs index 58450143f8..e8a94504e6 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/heartbeat.rs @@ -5,9 +5,10 @@ //! PeerDown messages are broadcast to the mesh when a peer is confirmed dead. use super::{ - ConnectionCaptureEvent, ControlProtocol, DEAD_PEER_TTL, ModelRuntimeDescriptor, Node, - PEER_DOWN_REPORTER_COOLDOWN_SECS, PEER_STALE_SECS, PeerInfo, PeerLifecycleCaptureEvent, - ServedModelDescriptor, connect_mesh, connection_protocol, endpoint_id_hex, + ConnectionCaptureEvent, ControlProtocol, DEAD_PEER_TTL, MeshPeerRemovalReason, + ModelRuntimeDescriptor, Node, PEER_DOWN_REPORTER_COOLDOWN_SECS, PEER_STALE_SECS, PeerInfo, + PeerLifecycleCaptureEvent, ServedModelDescriptor, connect_mesh, connection_protocol, + endpoint_id_hex, }; use crate::protocol::{ NODE_PROTOCOL_GENERATION, STREAM_PEER_DOWN, STREAM_PEER_LEAVING, write_len_prefixed, @@ -960,7 +961,8 @@ impl Node { None, ) .await; - self.remove_peer(stale_id).await; + self.remove_peer(stale_id, MeshPeerRemovalReason::StaleDirectAndTransitive) + .await; self.state.lock().await.connections.remove(&stale_id); } } @@ -1014,6 +1016,15 @@ impl Node { /// Handle a peer death: remove from state, broadcast to all other peers. pub async fn handle_peer_death(&self, dead_id: EndpointId) { + self.handle_peer_death_with_reason(dead_id, MeshPeerRemovalReason::HeartbeatUnreachable) + .await; + } + + pub(crate) async fn handle_peer_death_with_reason( + &self, + dead_id: EndpointId, + reason: MeshPeerRemovalReason, + ) { super::emit_mesh_warning(format!( "⚠️ Peer {} died — removing and broadcasting", dead_id.fmt_short() @@ -1029,11 +1040,11 @@ impl Node { self.capture_peer_lifecycle_snapshot( "peer_dead_marked", dead_id, - "handle_peer_death", + reason.reason_code(), None, ) .await; - self.remove_peer(dead_id).await; + self.remove_peer(dead_id, reason).await; self.broadcast_peer_down(dead_id).await; } diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index f42aa1618e..da9a96d511 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -110,7 +110,13 @@ use connections::*; pub(crate) use host_role_claims::{HostRoleClaim, HostRoleClaims}; use model_identity::*; use node_identity::*; -use operational_logging::{MeshOperationalEvent, record_mesh_operational_event}; +#[cfg(test)] +use operational_logging::capture_mesh_operational_audits; +use operational_logging::{ + MeshHandlerFailureBoundary, MeshOperationalEvent, MeshPeerRemovalReason, + MeshPolicyRejectionReason, MeshQuicInboundOutcome, mesh_peer_operational_context, + record_mesh_operational_event, record_mesh_operational_event_with_context, +}; use owner_control::*; use owner_lifecycle_cache::*; use peer_state::*; diff --git a/crates/mesh-llm-host-runtime/src/mesh/node.rs b/crates/mesh-llm-host-runtime/src/mesh/node.rs index 9b6cf897df..7d680ba913 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/node.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/node.rs @@ -115,8 +115,11 @@ pub struct Node { pub(crate) runtime_data_producer: crate::runtime_data::RuntimeDataProducer, pub(crate) tunnel_tx: tokio::sync::mpsc::Sender<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, - pub(crate) tunnel_http_tx: - tokio::sync::mpsc::Sender<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, + pub(crate) tunnel_http_tx: tokio::sync::mpsc::Sender<( + EndpointId, + iroh::endpoint::SendStream, + iroh::endpoint::RecvStream, + )>, pub(crate) stage_transport_tx: tokio::sync::mpsc::Sender<( EndpointId, iroh::endpoint::SendStream, diff --git a/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs b/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs index d2a33e8911..c42a2a3412 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/operational_logging.rs @@ -1,203 +1,305 @@ //! Bounded, metadata-only operational audit vocabulary for mesh boundaries. //! -//! These events deliberately accept no peer, endpoint, address, token, ALPN, -//! or error values. The service receives only the static level and code below. +//! Dynamic context is restricted to authenticated peer identity and selected +//! connection-path observations. Tokens, ALPN bytes, hostnames, and raw errors +//! never enter this adapter. +//! Pre-authentication failures remain identity-free; they carry only static +//! outcome/reason codes and an elapsed duration when that boundary owns a timer. #[cfg(test)] use crate::logging::LoggingService; -use crate::logging::{OperationalAuditRecord, OperationalAuditSeverity}; +use crate::logging::{ + OperationalAuditContext, OperationalAuditPathType, OperationalAuditRecord, + OperationalAuditSeverity, +}; +use crate::mesh::SelectedPathObservation; +use iroh::EndpointId; -const OPERATIONAL_AUDIT_INFO: &str = "info"; -const OPERATIONAL_AUDIT_WARNING: &str = "warning"; +mod vocabulary; + +#[cfg(test)] +use vocabulary::MeshHandlerFailureClass; +pub(crate) use vocabulary::{ + MeshHandlerFailureBoundary, MeshOperationalEvent, MeshPeerRemovalReason, + MeshPolicyRejectionReason, MeshQuicInboundOutcome, +}; +use vocabulary::{OPERATIONAL_AUDIT_INFO, OPERATIONAL_AUDIT_WARNING}; const OPERATIONAL_AUDIT_SOURCE: &str = "mesh"; -fn operational_audit_record(code: &'static str, level: &'static str) -> OperationalAuditRecord { +fn operational_audit_record( + event: MeshOperationalEvent, + context: Option, +) -> OperationalAuditRecord { + let level = event.level(); let severity = match level { OPERATIONAL_AUDIT_INFO => OperationalAuditSeverity::Info, OPERATIONAL_AUDIT_WARNING => OperationalAuditSeverity::Warning, _ => OperationalAuditSeverity::Error, }; - OperationalAuditRecord::builder(OPERATIONAL_AUDIT_SOURCE, code) + let record = OperationalAuditRecord::builder(OPERATIONAL_AUDIT_SOURCE, event.code()) .severity(severity) - .build() -} - -/// Static outcomes that are safe to publish through the local operational log. -/// -/// Variants and codes follow the reviewed mesh audit vocabulary. `QuicInboundAccepted` -/// describes a non-Skippy inbound QUIC connection accepted after the Skippy ALPN -/// exclusion; there is no `quic_alpn_accepted` code because no explicit mesh-ALPN -/// validation branch exists in the inbound path. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum MeshOperationalEvent { - QuicHandlerFailed, - QuicInboundAccepted, - ControlHandlerFailed, - ControlAlpnRejected, - ControlConnectionAccepted, - GossipPolicyRejected, - GossipDirectPeerPromoted, - GossipIncompatibleVersionRejected, - GossipPeerRemoved, - AutoJoinSucceeded, - AutoJoinFailed, -} - -impl MeshOperationalEvent { - const fn level(self) -> &'static str { - match self { - Self::QuicInboundAccepted - | Self::ControlConnectionAccepted - | Self::GossipDirectPeerPromoted - | Self::GossipPeerRemoved - | Self::AutoJoinSucceeded => OPERATIONAL_AUDIT_INFO, - Self::QuicHandlerFailed - | Self::ControlHandlerFailed - | Self::ControlAlpnRejected - | Self::GossipPolicyRejected - | Self::GossipIncompatibleVersionRejected - | Self::AutoJoinFailed => OPERATIONAL_AUDIT_WARNING, - } + .build(); + if matches!( + event, + MeshOperationalEvent::AutoJoinSucceeded | MeshOperationalEvent::AutoJoinFailed + ) { + return record; } + let Some(context) = context else { + return record; + }; + record.with_context(event.decorate_context(context)) +} - const fn code(self) -> &'static str { - match self { - Self::QuicHandlerFailed => "mesh_quic_handler_failed", - Self::QuicInboundAccepted => "mesh_quic_inbound_accepted", - Self::ControlHandlerFailed => "mesh_control_handler_failed", - Self::ControlAlpnRejected => "mesh_control_alpn_rejected", - Self::ControlConnectionAccepted => "mesh_control_connection_accepted", - Self::GossipPolicyRejected => "gossip_policy_rejected", - Self::GossipDirectPeerPromoted => "gossip_direct_peer_promoted", - Self::GossipIncompatibleVersionRejected => "gossip_incompatible_version_rejected", - Self::GossipPeerRemoved => "gossip_peer_removed", - Self::AutoJoinSucceeded => "mesh_auto_join_succeeded", - Self::AutoJoinFailed => "mesh_auto_join_failed", - } +pub(crate) fn mesh_peer_operational_context( + peer: EndpointId, + path: Option, +) -> OperationalAuditContext { + let context = OperationalAuditContext::new().mesh_peer_subject(&hex::encode(peer.as_bytes())); + let Some(path) = path else { + return context; + }; + match path.path_type { + "direct" => context.network_path( + OperationalAuditPathType::Direct, + path.observed_direct_remote_addr, + ), + "relay" => context.network_path(OperationalAuditPathType::Relay, None), + _ => context, } } /// Record one mesh boundary result through the process-local logging service. /// Logging is optional and this intentionally never affects mesh serving. pub(crate) fn record_mesh_operational_event(event: MeshOperationalEvent) { + let record = operational_audit_record(event, Some(OperationalAuditContext::new())); + #[cfg(test)] + capture_mesh_operational_audit_for_test(&record); let Some(state) = crate::logging_runtime_state() else { return; }; - let _ = state.write_operational_audit(operational_audit_record(event.code(), event.level())); + let _ = state.write_operational_audit(record); +} + +pub(crate) fn record_mesh_operational_event_with_context( + event: MeshOperationalEvent, + context: OperationalAuditContext, +) { + let record = operational_audit_record(event, Some(context)); + #[cfg(test)] + capture_mesh_operational_audit_for_test(&record); + let Some(state) = crate::logging_runtime_state() else { + return; + }; + let _ = state.write_operational_audit(record); +} + +#[cfg(test)] +static MESH_OPERATIONAL_AUDIT_CAPTURE: std::sync::Mutex< + Option>, +> = std::sync::Mutex::new(None); + +#[cfg(test)] +pub(crate) struct MeshOperationalAuditCaptureGuard { + sender: tokio::sync::mpsc::UnboundedSender, +} + +#[cfg(test)] +impl Drop for MeshOperationalAuditCaptureGuard { + fn drop(&mut self) { + let mut capture = MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock"); + if capture + .as_ref() + .is_some_and(|sender| sender.same_channel(&self.sender)) + { + *capture = None; + } + } +} + +#[cfg(test)] +pub(crate) fn capture_mesh_operational_audits() -> ( + tokio::sync::mpsc::UnboundedReceiver, + MeshOperationalAuditCaptureGuard, +) { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); + *MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock") = Some(sender.clone()); + (receiver, MeshOperationalAuditCaptureGuard { sender }) +} + +#[cfg(test)] +fn capture_mesh_operational_audit_for_test(record: &OperationalAuditRecord) { + if let Some(sender) = MESH_OPERATIONAL_AUDIT_CAPTURE + .lock() + .expect("mesh operational audit capture lock") + .as_ref() + { + let _ = sender.send(record.clone()); + } } #[cfg(test)] -fn record_mesh_operational_event_with_service( +fn record_mesh_operational_event_with_context_and_service( service: &LoggingService, event: MeshOperationalEvent, + context: OperationalAuditContext, ) { - let _ = service.write_operational_audit(operational_audit_record(event.code(), event.level())); + let _ = service.write_operational_audit(operational_audit_record(event, Some(context))); } #[cfg(test)] mod tests { use super::{ - MeshOperationalEvent, record_mesh_operational_event, - record_mesh_operational_event_with_service, + MeshHandlerFailureBoundary, MeshHandlerFailureClass, MeshOperationalEvent, + MeshPeerRemovalReason, MeshPolicyRejectionReason, MeshQuicInboundOutcome, + capture_mesh_operational_audits, mesh_peer_operational_context, + record_mesh_operational_event, record_mesh_operational_event_with_context_and_service, }; - use crate::logging::{LoggingService, ServiceConfig}; - - fn recorded_audits(service: &LoggingService) -> Vec { - service - .bus_ref() - .drain() - .into_iter() - .map(|entry| { - let audit: serde_json::Value = - serde_json::from_str(&entry.payload).expect("audit payload"); - serde_json::json!({ - "kind": "audit", - "level": audit["severity"], - "message": audit["code"], - }) - }) - .collect() + use crate::crypto::OwnershipStatus; + use crate::logging::{LoggingService, OperationalAuditContext, ServiceConfig}; + use crate::mesh::SelectedPathObservation; + use iroh::{EndpointId, SecretKey}; + use serial_test::serial; + use tokio::sync::mpsc::error::TryRecvError; + + #[test] + #[serial] + fn stale_capture_guard_does_not_clear_replacement() { + let (_receiver_a, guard_a) = capture_mesh_operational_audits(); + let (mut receiver_b, guard_b) = capture_mesh_operational_audits(); + + drop(guard_a); + record_mesh_operational_event(MeshOperationalEvent::ControlConnectionAccepted); + + let captured = loop { + match receiver_b.try_recv() { + Ok(record) if record.code() == "mesh_control_connection_accepted" => break record, + Ok(_) => {} + Err(error) => panic!("replacement capture must receive the audit: {error}"), + } + }; + assert_eq!(captured.code(), "mesh_control_connection_accepted"); + + drop(guard_b); + record_mesh_operational_event(MeshOperationalEvent::ControlAlpnRejected); + + loop { + match receiver_b.try_recv() { + Ok(record) => assert_ne!(record.code(), "mesh_control_alpn_rejected"), + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => panic!("active capture registration remains"), + } + } } #[test] - fn mesh_boundary_outcomes_emit_exact_static_audits_without_raw_metadata() { + fn mesh_boundary_outcomes_emit_authenticated_peer_context_without_raw_secrets() { let service = LoggingService::new_disabled(ServiceConfig::default()); - let events = [ - MeshOperationalEvent::QuicInboundAccepted, - MeshOperationalEvent::ControlAlpnRejected, - MeshOperationalEvent::ControlHandlerFailed, + let peer = EndpointId::from(SecretKey::from_bytes(&[0x42; 32]).public()); + let peer_hex = hex::encode(peer.as_bytes()); + let direct_addr = "192.0.2.42:11204".parse().expect("direct address"); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(17), + observed_direct_remote_addr: Some(direct_addr), + }), + ) + .numeric_summary("direct_peers", 3); + record_mesh_operational_event_with_context_and_service( + &service, MeshOperationalEvent::GossipDirectPeerPromoted, - MeshOperationalEvent::GossipPolicyRejected, - MeshOperationalEvent::AutoJoinFailed, - ]; - - for event in events { - record_mesh_operational_event_with_service(&service, event); - } + context, + ); - let audits = recorded_audits(&service); + let audit: serde_json::Value = serde_json::from_str( + &service + .bus_ref() + .drain() + .into_iter() + .next() + .expect("peer audit") + .payload, + ) + .expect("audit payload"); + assert_eq!(audit["kind"], "audit"); + assert_eq!(audit["severity"], "info"); + assert_eq!(audit["code"], "gossip_direct_peer_promoted"); + assert_eq!(audit["subject_kind"], "mesh_peer"); + assert_eq!(audit["subject_id"], peer_hex); + assert_eq!(audit["path_type"], "direct"); + assert_eq!(audit["remote_addr"], direct_addr.to_string()); + assert_eq!(audit["outcome"], "promoted"); + assert!(audit.get("duration_ms").is_none()); assert_eq!( - audits, - vec![ - serde_json::json!({ - "kind": "audit", - "level": "info", - "message": "mesh_quic_inbound_accepted", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_control_alpn_rejected", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_control_handler_failed", - }), - serde_json::json!({ - "kind": "audit", - "level": "info", - "message": "gossip_direct_peer_promoted", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "gossip_policy_rejected", - }), - serde_json::json!({ - "kind": "audit", - "level": "warning", - "message": "mesh_auto_join_failed", - }), - ] + audit["numeric_summaries"], + serde_json::json!({"direct_peers": 3}) ); - let serialized = serde_json::to_string(&audits).expect("serialized audit payloads"); + let serialized = serde_json::to_string(&audit).expect("serialized audit payload"); for raw_value in [ "node=untrusted-lab-host", - "peer=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "token=mesh-secret-bootstrap-token", + "mesh-llm/1-private-alpn", + "connection refused at secret.example.test", + "untrusted-lab-host.example.test", ] { assert!( !serialized.contains(raw_value), - "raw mesh metadata must not enter the audit payload" + "raw secret, ALPN, error, and hostname data must not enter the audit payload" ); } } #[test] - fn mesh_operational_vocabulary_is_bounded_and_identifier_free() { + fn relay_peer_context_never_records_an_address() { + let peer = EndpointId::from(SecretKey::from_bytes(&[0x24; 32]).public()); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "relay", + rtt_ms: Some(31), + observed_direct_remote_addr: Some( + "203.0.113.10:443".parse().expect("relay-shaped address"), + ), + }), + ); + let service = LoggingService::new_disabled(ServiceConfig::default()); + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + context, + ); + + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("audit payload"); + assert_eq!(audit["subject_id"], hex::encode(peer.as_bytes())); + assert_eq!(audit["path_type"], "relay"); + assert!(audit.get("remote_addr").is_none()); + assert!(audit.get("numeric_summaries").is_none()); + assert_eq!(audit["outcome"], "accepted"); + } + + #[test] + fn mesh_operational_vocabulary_is_bounded_and_static() { let events = [ - MeshOperationalEvent::QuicHandlerFailed, - MeshOperationalEvent::QuicInboundAccepted, - MeshOperationalEvent::ControlHandlerFailed, + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Capacity), MeshOperationalEvent::ControlAlpnRejected, MeshOperationalEvent::ControlConnectionAccepted, - MeshOperationalEvent::GossipPolicyRejected, + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), MeshOperationalEvent::GossipDirectPeerPromoted, MeshOperationalEvent::GossipIncompatibleVersionRejected, - MeshOperationalEvent::GossipPeerRemoved, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), MeshOperationalEvent::AutoJoinSucceeded, MeshOperationalEvent::AutoJoinFailed, ]; @@ -214,19 +316,373 @@ mod tests { } } + #[test] + fn peer_removal_reason_vocabulary_is_exactly_the_bounded_control_flow_causes() { + let cases = [ + ( + MeshPeerRemovalReason::StaleDirectAndTransitive, + "stale_direct_and_transitive", + ), + ( + MeshPeerRemovalReason::HeartbeatUnreachable, + "heartbeat_unreachable", + ), + ( + MeshPeerRemovalReason::PeerDownProbeFailed, + "peer_down_probe_failed", + ), + ( + MeshPeerRemovalReason::ClosedConnectionNoAddress, + "closed_connection_no_address", + ), + (MeshPeerRemovalReason::ReconnectFailed, "reconnect_failed"), + ( + MeshPeerRemovalReason::RecoveredGossipFailed, + "recovered_gossip_failed", + ), + (MeshPeerRemovalReason::CleanShutdown, "clean_shutdown"), + ( + MeshPeerRemovalReason::TunnelOpenFailed, + "tunnel_open_failed", + ), + ]; + + for (reason, expected_code) in cases { + let code = reason.reason_code(); + assert_eq!(code, expected_code); + assert!(code.len() <= 48, "reason code must stay bounded: {code}"); + assert!( + code.bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_'), + "reason code must be a static identifier: {code}" + ); + } + } + + #[test] + fn gossip_peer_removed_audit_carries_only_cause_count_and_peer_path_identity() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + let peer = EndpointId::from(SecretKey::from_bytes(&[0x51; 32]).public()); + let context = mesh_peer_operational_context( + peer, + Some(SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(11), + observed_direct_remote_addr: Some( + "192.0.2.51:1151".parse().expect("direct peer address"), + ), + }), + ) + .numeric_summary("direct_peers", 2); + + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::HeartbeatUnreachable), + context, + ); + + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("removal audit JSON"); + assert_eq!(audit["reason_code"], "heartbeat_unreachable"); + assert_eq!( + audit["numeric_summaries"], + serde_json::json!({"direct_peers": 2}) + ); + assert_eq!(audit["subject_id"], hex::encode(peer.as_bytes())); + assert_eq!(audit["path_type"], "direct"); + assert_eq!(audit["remote_addr"], "192.0.2.51:1151"); + for diagnostic_only_field in [ + "last_seen_age_ms", + "last_mentioned_age_ms", + "had_connection", + "bridge", + "reporter", + ] { + assert!(audit.get(diagnostic_only_field).is_none()); + } + } + + #[test] + fn reviewed_audit_outcomes_and_rejection_reasons_are_exact() { + let cases = [ + ( + MeshOperationalEvent::GossipDirectPeerPromoted, + "promoted", + None, + ), + ( + MeshOperationalEvent::ControlAlpnRejected, + "rejected", + Some("alpn_unsupported"), + ), + ( + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), + "rejected", + Some("owner_attestation_required"), + ), + ( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + "rejected", + Some("protocol_version_unsupported"), + ), + ( + MeshOperationalEvent::GossipPeerRemoved( + MeshPeerRemovalReason::HeartbeatUnreachable, + ), + "removed", + Some("heartbeat_unreachable"), + ), + ( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Readmitted), + "readmitted", + None, + ), + ]; + + for (event, expected_outcome, expected_reason) in cases { + assert_eq!(event.outcome(), expected_outcome); + assert_eq!(event.reason_code(), expected_reason); + } + } + + #[test] + fn pre_auth_alpn_rejection_stays_sparse() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + record_mesh_operational_event_with_context_and_service( + &service, + MeshOperationalEvent::ControlAlpnRejected, + OperationalAuditContext::new(), + ); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = serde_json::from_str(&payload).expect("pre-auth audit JSON"); + + assert_eq!(audit["outcome"], "rejected"); + assert_eq!(audit["reason_code"], "alpn_unsupported"); + for unavailable_field in [ + "subject_kind", + "subject_id", + "remote_addr", + "path_type", + "duration_ms", + "numeric_summaries", + ] { + assert!(audit.get(unavailable_field).is_none()); + } + } + + #[test] + fn reviewed_audit_numeric_summaries_and_handler_duration_are_exact() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + let cases = [ + ( + MeshOperationalEvent::GossipDirectPeerPromoted, + OperationalAuditContext::new().numeric_summary("direct_peers", 4), + serde_json::json!({"direct_peers": 4}), + None, + ), + ( + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), + OperationalAuditContext::new().numeric_summary("direct_peers", 3), + serde_json::json!({"direct_peers": 3}), + None, + ), + ( + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), + OperationalAuditContext::new(), + serde_json::json!({}), + None, + ), + ( + MeshOperationalEvent::GossipIncompatibleVersionRejected, + OperationalAuditContext::new() + .numeric_summary("peer_gen", 0) + .numeric_summary("local_gen", 1), + serde_json::json!({"local_gen": 1, "peer_gen": 0}), + None, + ), + ( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + OperationalAuditContext::new().numeric_summary("protocol_gen", 1), + serde_json::json!({"protocol_gen": 1}), + None, + ), + ( + MeshOperationalEvent::ControlConnectionAccepted, + OperationalAuditContext::new().numeric_summary("protocol_gen", 1), + serde_json::json!({"protocol_gen": 1}), + None, + ), + ( + MeshOperationalEvent::ControlAlpnRejected, + OperationalAuditContext::new(), + serde_json::json!({}), + None, + ), + ( + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), + OperationalAuditContext::new().duration_ms(29), + serde_json::json!({}), + Some(29), + ), + ( + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Internal), + OperationalAuditContext::new().duration_ms(37), + serde_json::json!({}), + Some(37), + ), + ]; + + for (event, context, expected_summaries, expected_duration) in cases { + record_mesh_operational_event_with_context_and_service(&service, event, context); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = + serde_json::from_str(&payload).expect("reviewed audit JSON"); + assert_eq!( + audit + .get("numeric_summaries") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + expected_summaries, + ); + assert_eq!( + audit.get("duration_ms").and_then(serde_json::Value::as_u64), + expected_duration + ); + } + } + + #[test] + fn auto_join_audits_remain_generic_and_context_free() { + let service = LoggingService::new_disabled(ServiceConfig::default()); + + for event in [ + MeshOperationalEvent::AutoJoinSucceeded, + MeshOperationalEvent::AutoJoinFailed, + ] { + record_mesh_operational_event_with_context_and_service( + &service, + event, + OperationalAuditContext::new() + .outcome("must_not_escape") + .reason_code("must_not_escape") + .numeric_summary("must_not_escape", 1), + ); + let payload = service.bus_ref().drain().remove(0).payload; + let audit: serde_json::Value = + serde_json::from_str(&payload).expect("generic auto-join audit JSON"); + + for context_field in [ + "context_version", + "outcome", + "reason_code", + "numeric_summaries", + ] { + assert!(audit.get(context_field).is_none()); + } + } + } + + #[test] + fn handler_failure_vocabulary_is_exactly_the_production_backed_classes() { + let classes = [ + MeshHandlerFailureClass::Capacity, + MeshHandlerFailureClass::Internal, + ]; + assert_eq!( + classes.map(MeshHandlerFailureClass::reason_code), + ["capacity", "internal"] + ); + } + + #[test] + fn ownership_status_maps_exhaustively_to_gate_specific_policy_reasons() { + let cases = [ + (OwnershipStatus::Verified, None), + ( + OwnershipStatus::Unsigned, + Some("owner_attestation_required"), + ), + (OwnershipStatus::Expired, Some("owner_attestation_expired")), + ( + OwnershipStatus::InvalidSignature, + Some("owner_attestation_invalid"), + ), + ( + OwnershipStatus::MismatchedNodeId, + Some("owner_attestation_node_mismatch"), + ), + (OwnershipStatus::RevokedOwner, Some("owner_revoked")), + ( + OwnershipStatus::RevokedCert, + Some("owner_attestation_revoked"), + ), + (OwnershipStatus::RevokedNodeId, Some("owner_node_revoked")), + ( + OwnershipStatus::UnsupportedProtocol, + Some("owner_attestation_protocol_unsupported"), + ), + (OwnershipStatus::UntrustedOwner, Some("owner_untrusted")), + ]; + + for (status, expected_reason) in cases { + assert_eq!( + MeshPolicyRejectionReason::from_ownership_status(&status) + .map(MeshPolicyRejectionReason::reason_code), + expected_reason, + ); + } + } + + #[test] + fn handler_failure_boundaries_map_only_when_truthfully_classifiable() { + let cases = [ + ( + MeshHandlerFailureBoundary::AcceptSetup, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::AlpnRead, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::Handshake, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::CapacityPermit, + MeshHandlerFailureClass::Capacity, + ), + ( + MeshHandlerFailureBoundary::StreamAccept, + MeshHandlerFailureClass::Internal, + ), + ( + MeshHandlerFailureBoundary::ControlDispatch, + MeshHandlerFailureClass::Internal, + ), + ]; + for (boundary, expected) in cases { + assert_eq!(boundary.failure_class(), expected); + } + } + #[test] fn mesh_operational_vocabulary_maps_each_variant_to_its_reviewed_code() { let cases = [ ( - MeshOperationalEvent::QuicHandlerFailed, + MeshOperationalEvent::QuicHandlerFailed(MeshHandlerFailureClass::Internal), "mesh_quic_handler_failed", ), ( - MeshOperationalEvent::QuicInboundAccepted, + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), "mesh_quic_inbound_accepted", ), ( - MeshOperationalEvent::ControlHandlerFailed, + MeshOperationalEvent::ControlHandlerFailed(MeshHandlerFailureClass::Internal), "mesh_control_handler_failed", ), ( @@ -238,7 +694,9 @@ mod tests { "mesh_control_connection_accepted", ), ( - MeshOperationalEvent::GossipPolicyRejected, + MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + ), "gossip_policy_rejected", ), ( @@ -250,7 +708,7 @@ mod tests { "gossip_incompatible_version_rejected", ), ( - MeshOperationalEvent::GossipPeerRemoved, + MeshOperationalEvent::GossipPeerRemoved(MeshPeerRemovalReason::CleanShutdown), "gossip_peer_removed", ), ( @@ -276,9 +734,15 @@ mod tests { // When the process-local logging runtime state is absent the adapter // must be a no-op; when a concurrent logging test installed state the // bounded write path must equally never panic (fail-open by contract). - record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed); + record_mesh_operational_event(MeshOperationalEvent::QuicHandlerFailed( + MeshHandlerFailureClass::Internal, + )); record_mesh_operational_event(MeshOperationalEvent::AutoJoinFailed); - record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected); - record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted); + record_mesh_operational_event(MeshOperationalEvent::GossipPolicyRejected( + MeshPolicyRejectionReason::AttestationRequired, + )); + record_mesh_operational_event(MeshOperationalEvent::QuicInboundAccepted( + MeshQuicInboundOutcome::Accepted, + )); } } diff --git a/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs b/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs new file mode 100644 index 0000000000..a90cd5f06c --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/operational_logging/vocabulary.rs @@ -0,0 +1,228 @@ +use crate::crypto::OwnershipStatus; +use crate::logging::OperationalAuditContext; + +pub(super) const OPERATIONAL_AUDIT_INFO: &str = "info"; +pub(super) const OPERATIONAL_AUDIT_WARNING: &str = "warning"; + +/// Static outcomes that are safe to publish through the local operational log. +/// +/// Variants and codes follow the reviewed mesh audit vocabulary. `QuicInboundAccepted` +/// covers authenticated mesh and Skippy-stage QUIC connections; its `protocol_gen` +/// summary is the generation of the negotiated protocol family. There is no +/// `quic_alpn_accepted` code because no explicit mesh-ALPN validation branch exists. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshOperationalEvent { + QuicHandlerFailed(MeshHandlerFailureClass), + QuicInboundAccepted(MeshQuicInboundOutcome), + ControlHandlerFailed(MeshHandlerFailureClass), + ControlAlpnRejected, + ControlConnectionAccepted, + GossipPolicyRejected(MeshPolicyRejectionReason), + GossipDirectPeerPromoted, + GossipIncompatibleVersionRejected, + GossipPeerRemoved(MeshPeerRemovalReason), + AutoJoinSucceeded, + AutoJoinFailed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshPeerRemovalReason { + StaleDirectAndTransitive, + HeartbeatUnreachable, + PeerDownProbeFailed, + ClosedConnectionNoAddress, + ReconnectFailed, + RecoveredGossipFailed, + CleanShutdown, + TunnelOpenFailed, +} + +impl MeshPeerRemovalReason { + pub(crate) const fn reason_code(self) -> &'static str { + match self { + Self::StaleDirectAndTransitive => "stale_direct_and_transitive", + Self::HeartbeatUnreachable => "heartbeat_unreachable", + Self::PeerDownProbeFailed => "peer_down_probe_failed", + Self::ClosedConnectionNoAddress => "closed_connection_no_address", + Self::ReconnectFailed => "reconnect_failed", + Self::RecoveredGossipFailed => "recovered_gossip_failed", + Self::CleanShutdown => "clean_shutdown", + Self::TunnelOpenFailed => "tunnel_open_failed", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshQuicInboundOutcome { + Accepted, + Readmitted, +} + +impl MeshQuicInboundOutcome { + const fn outcome(self) -> &'static str { + match self { + Self::Accepted => "accepted", + Self::Readmitted => "readmitted", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshHandlerFailureClass { + Capacity, + Internal, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshPolicyRejectionReason { + AttestationRequired, + AttestationExpired, + AttestationInvalid, + AttestationNodeMismatch, + Revoked, + AttestationRevoked, + NodeRevoked, + AttestationProtocolUnsupported, + Untrusted, +} + +impl MeshPolicyRejectionReason { + pub(crate) const fn from_ownership_status(status: &OwnershipStatus) -> Option { + match status { + OwnershipStatus::Verified => None, + OwnershipStatus::Unsigned => Some(Self::AttestationRequired), + OwnershipStatus::Expired => Some(Self::AttestationExpired), + OwnershipStatus::InvalidSignature => Some(Self::AttestationInvalid), + OwnershipStatus::MismatchedNodeId => Some(Self::AttestationNodeMismatch), + OwnershipStatus::RevokedOwner => Some(Self::Revoked), + OwnershipStatus::RevokedCert => Some(Self::AttestationRevoked), + OwnershipStatus::RevokedNodeId => Some(Self::NodeRevoked), + OwnershipStatus::UnsupportedProtocol => Some(Self::AttestationProtocolUnsupported), + OwnershipStatus::UntrustedOwner => Some(Self::Untrusted), + } + } + + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::AttestationRequired => "owner_attestation_required", + Self::AttestationExpired => "owner_attestation_expired", + Self::AttestationInvalid => "owner_attestation_invalid", + Self::AttestationNodeMismatch => "owner_attestation_node_mismatch", + Self::Revoked => "owner_revoked", + Self::AttestationRevoked => "owner_attestation_revoked", + Self::NodeRevoked => "owner_node_revoked", + Self::AttestationProtocolUnsupported => "owner_attestation_protocol_unsupported", + Self::Untrusted => "owner_untrusted", + } + } +} + +impl MeshHandlerFailureClass { + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::Capacity => "capacity", + Self::Internal => "internal", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MeshHandlerFailureBoundary { + AcceptSetup, + AlpnRead, + Handshake, + CapacityPermit, + StreamAccept, + ControlDispatch, +} + +impl MeshHandlerFailureBoundary { + pub(crate) const fn failure_class(self) -> MeshHandlerFailureClass { + match self { + Self::AcceptSetup + | Self::AlpnRead + | Self::Handshake + | Self::StreamAccept + | Self::ControlDispatch => MeshHandlerFailureClass::Internal, + Self::CapacityPermit => MeshHandlerFailureClass::Capacity, + } + } +} + +impl MeshOperationalEvent { + pub(super) const fn level(self) -> &'static str { + match self { + Self::QuicInboundAccepted(_) + | Self::ControlConnectionAccepted + | Self::GossipDirectPeerPromoted + | Self::GossipPeerRemoved(_) + | Self::AutoJoinSucceeded => OPERATIONAL_AUDIT_INFO, + Self::QuicHandlerFailed(_) + | Self::ControlHandlerFailed(_) + | Self::ControlAlpnRejected + | Self::GossipPolicyRejected(_) + | Self::GossipIncompatibleVersionRejected + | Self::AutoJoinFailed => OPERATIONAL_AUDIT_WARNING, + } + } + + pub(super) const fn code(self) -> &'static str { + match self { + Self::QuicHandlerFailed(_) => "mesh_quic_handler_failed", + Self::QuicInboundAccepted(_) => "mesh_quic_inbound_accepted", + Self::ControlHandlerFailed(_) => "mesh_control_handler_failed", + Self::ControlAlpnRejected => "mesh_control_alpn_rejected", + Self::ControlConnectionAccepted => "mesh_control_connection_accepted", + Self::GossipPolicyRejected(_) => "gossip_policy_rejected", + Self::GossipDirectPeerPromoted => "gossip_direct_peer_promoted", + Self::GossipIncompatibleVersionRejected => "gossip_incompatible_version_rejected", + Self::GossipPeerRemoved(_) => "gossip_peer_removed", + Self::AutoJoinSucceeded => "mesh_auto_join_succeeded", + Self::AutoJoinFailed => "mesh_auto_join_failed", + } + } + + pub(super) const fn outcome(self) -> &'static str { + match self { + Self::QuicInboundAccepted(outcome) => outcome.outcome(), + Self::ControlConnectionAccepted => "accepted", + Self::GossipDirectPeerPromoted => "promoted", + Self::GossipPeerRemoved(_) => "removed", + Self::AutoJoinSucceeded => "succeeded", + Self::QuicHandlerFailed(_) | Self::ControlHandlerFailed(_) | Self::AutoJoinFailed => { + "failed" + } + Self::ControlAlpnRejected + | Self::GossipPolicyRejected(_) + | Self::GossipIncompatibleVersionRejected => "rejected", + } + } + + pub(super) const fn reason_code(self) -> Option<&'static str> { + match self { + Self::QuicHandlerFailed(class) | Self::ControlHandlerFailed(class) => { + Some(class.reason_code()) + } + Self::ControlAlpnRejected => Some("alpn_unsupported"), + Self::GossipPolicyRejected(reason) => Some(reason.reason_code()), + Self::GossipIncompatibleVersionRejected => Some("protocol_version_unsupported"), + Self::GossipPeerRemoved(reason) => Some(reason.reason_code()), + Self::AutoJoinFailed => Some("candidate_failed"), + Self::QuicInboundAccepted(_) + | Self::ControlConnectionAccepted + | Self::GossipDirectPeerPromoted + | Self::AutoJoinSucceeded => None, + } + } + + pub(super) const fn decorate_context( + self, + mut context: OperationalAuditContext, + ) -> OperationalAuditContext { + context = context.outcome(self.outcome()); + match self.reason_code() { + Some(reason) => context.reason_code(reason), + None => context, + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs index 8b03009574..01180ee750 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs @@ -1277,7 +1277,8 @@ impl Node { state.dead_peers.insert(id, std::time::Instant::now()); state.connections.remove(&id); drop(state); - self.remove_peer(id).await; + self.remove_peer(id, MeshPeerRemovalReason::PeerDownProbeFailed) + .await; } pub(crate) async fn record_peer_down_rejection(&self, remote: EndpointId, dead_id: EndpointId) { diff --git a/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs b/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs index dcb727b9fe..b59cfe52d6 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/stage_transport.rs @@ -367,7 +367,11 @@ pub(crate) fn artifact_transfer_allowed_by_topology( /// Channels returned by Node::start for inbound tunnel streams. pub struct TunnelChannels { pub rpc: tokio::sync::mpsc::Receiver<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, - pub http: tokio::sync::mpsc::Receiver<(iroh::endpoint::SendStream, iroh::endpoint::RecvStream)>, + pub http: tokio::sync::mpsc::Receiver<( + EndpointId, + iroh::endpoint::SendStream, + iroh::endpoint::RecvStream, + )>, pub stage: tokio::sync::mpsc::Receiver<( EndpointId, iroh::endpoint::SendStream, diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs index 7ed8509cb0..08a1bad4a1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/connections.rs @@ -16,6 +16,110 @@ use tokio::sync::{mpsc, watch}; mod direct_path; +#[tokio::test] +async fn inbound_http_tunnel_channel_retains_authenticated_remote_endpoint() { + async fn start_node() -> (Node, TunnelChannels) { + let relay_urls = Vec::new(); + let relay_auths = HashMap::new(); + Node::start( + super::NodeRole::Client, + RelayConfig { + urls: &relay_urls, + auths: &relay_auths, + policy: RelayPolicy::Disabled, + }, + QuicBindSelection { + ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + port: Some(0), + }, + Some(0.0), + false, + None, + None, + crate::MeshRequirements::unrestricted(), + ) + .await + .expect("start tunnel identity test node") + } + + let (sender, _sender_channels) = start_node().await; + let (receiver, mut receiver_channels) = start_node().await; + sender.start_accepting(); + receiver.start_accepting(); + sender + .connect_to_peer(receiver.endpoint_addr_for_advertisement()) + .await + .expect("connect tunnel identity test nodes"); + + let _outbound = sender + .open_http_tunnel(receiver.id()) + .await + .expect("open authenticated HTTP tunnel"); + let (remote, _send, _recv) = tokio::time::timeout( + std::time::Duration::from_secs(5), + receiver_channels.http.recv(), + ) + .await + .expect("HTTP tunnel should reach receiver") + .expect("HTTP tunnel channel should remain open"); + + assert_eq!(remote, sender.id()); +} + +#[tokio::test] +#[serial] +async fn stage_alpn_emits_authenticated_direct_peer_audit_before_stream_dispatch() { + let (mut audits, _capture) = super::capture_mesh_operational_audits(); + let sender = make_test_node(super::NodeRole::Worker) + .await + .expect("sender node"); + let expected_sender_id = hex::encode(sender.id().as_bytes()); + let receiver = make_test_node(super::NodeRole::Worker) + .await + .expect("receiver node"); + receiver.start_accepting(); + record_mesh_operational_event_with_context( + MeshOperationalEvent::QuicInboundAccepted(MeshQuicInboundOutcome::Accepted), + mesh_peer_operational_context(receiver.id(), None), + ); + + let _connection = sender + .endpoint + .connect( + receiver.endpoint_addr_for_advertisement(), + skippy_protocol::STAGE_ALPN_V2, + ) + .await + .expect("authenticated stage connection"); + + let audit = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let record = audits.recv().await.expect("audit capture remains open"); + if record.code() == "mesh_quic_inbound_accepted" + && record.context().is_some_and(|context| { + let fields = context.fields(); + fields["subject_id"].as_str() == Some(expected_sender_id.as_str()) + }) + { + return record; + } + } + }) + .await + .expect("stage audit should be emitted before waiting for a stream"); + + let fields = audit.context().expect("stage audit context").fields(); + assert_eq!(fields["subject_kind"], "mesh_peer"); + assert_eq!(fields["subject_id"], expected_sender_id); + assert_eq!(fields["path_type"], "direct"); + assert!(fields["remote_addr"].as_str().is_some()); + assert_eq!(fields["outcome"], "accepted"); + assert_eq!( + fields["numeric_summaries"]["protocol_gen"], + skippy_protocol::STAGE_PROTOCOL_GENERATION + ); +} + #[tokio::test] async fn owner_control_stream_work_is_bounded_per_connection() { let permits = control_stream_semaphore(); diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs index c572fbad64..40808b8a96 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs @@ -4,6 +4,7 @@ use crate::mesh::connections::encode_endpoint_addr_token; use crate::mesh::now_secs; use iroh::SecretKey; use mesh_llm_types::mesh::DEMAND_TTL_SECS; +use serial_test::serial; use std::collections::HashMap; pub(crate) fn test_endpoint_id(seed: u8) -> EndpointId { @@ -72,853 +73,20 @@ pub(crate) fn test_peer(ts: Option) -> PeerInfo { ) } -#[test] -pub(crate) fn test_merge_none_to_some() { - let mut existing = test_peer(None); - let ann = test_announcement(Some(100)); +mod merge_and_refresh { + use super::*; - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_some_to_none_keeps_existing() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(None); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_earlier_incoming_wins() { - let mut existing = test_peer(Some(200)); - let ann = test_announcement(Some(100)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_later_incoming_loses() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(Some(200)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_merge_equal_values_unchanged() { - let mut existing = test_peer(Some(100)); - let ann = test_announcement(Some(100)); - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!(existing.first_joined_mesh_ts, Some(100)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_first_joined_mesh_ts() { - let old_peer = test_peer(Some(100)); - let new_peer = test_peer(Some(200)); - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_explicit_model_interests() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_stage_status_list_support() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.stage_status_list_supported = !old_peer.stage_status_list_supported; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_meaningfully_changed_stage_protocol_generation_support() { - let old_peer = test_peer(Some(100)); - let mut new_peer = test_peer(Some(100)); - new_peer.stage_protocol_generation_supported = !old_peer.stage_protocol_generation_supported; - - assert!(peer_meaningfully_changed(&old_peer, &new_peer)); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_explicit_model_interests() { - let mut existing = test_peer(Some(100)); - let mut ann = test_announcement(Some(100)); - ann.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!( - existing.explicit_model_interests, - vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] - ); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_stage_status_list_support() { - let mut existing = test_peer(Some(100)); - existing.stage_status_list_supported = false; - let mut ann = test_announcement(Some(100)); - ann.stage_status_list_supported = true; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert!(existing.stage_status_list_supported); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_stage_protocol_generation_support() { - let mut existing = test_peer(Some(100)); - existing.stage_protocol_generation_supported = false; - let mut ann = test_announcement(Some(100)); - ann.stage_protocol_generation_supported = true; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert!(existing.stage_protocol_generation_supported); -} - -#[test] -pub(crate) fn test_apply_transitive_ann_refreshes_advertised_model_throughput() { - let mut existing = test_peer(Some(100)); - let mut ann = test_announcement(Some(100)); - ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { - model_name: "qwen".to_string(), - avg_tokens_per_second_milli: 35_000, - throughput_samples: 4, - }]; - - apply_transitive_ann( - &mut existing, - &test_addr(0x33), - &ann, - test_endpoint_id(0xee), - ); - - assert_eq!( - existing.advertised_model_throughput, - ann.advertised_model_throughput - ); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_stage_status_list_support() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x44); - let addr = test_addr(0x44); - let mut ann = test_announcement(Some(100)); - ann.stage_status_list_supported = false; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.stage_status_list_supported = true; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert!(peer.stage_status_list_supported); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_stage_protocol_generation_support() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x45); - let addr = test_addr(0x45); - let mut ann = test_announcement(Some(100)); - ann.stage_protocol_generation_supported = false; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.stage_protocol_generation_supported = true; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert!(peer.stage_protocol_generation_supported); -} - -#[tokio::test] -pub(crate) async fn test_add_peer_refreshes_advertised_model_throughput() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x46); - let addr = test_addr(0x46); - let mut ann = test_announcement(Some(100)); - ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { - model_name: "qwen".to_string(), - avg_tokens_per_second_milli: 20_000, - throughput_samples: 2, - }]; - - node.add_peer(peer_id, addr.clone(), &ann, None).await; - ann.advertised_model_throughput[0].avg_tokens_per_second_milli = 48_000; - ann.advertised_model_throughput[0].throughput_samples = 9; - node.add_peer(peer_id, addr, &ann, None).await; - - let state = node.state.lock().await; - let peer = state.peers.get(&peer_id).expect("peer should be tracked"); - assert_eq!( - peer.advertised_model_throughput, - ann.advertised_model_throughput - ); -} - -#[tokio::test] -pub(crate) async fn test_collect_announcements_includes_self_explicit_model_interests() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - node.set_explicit_model_interests(vec![ - "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), - "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), - ]) - .await; - - let announcements = node.collect_announcements().await; - let self_announcement = announcements - .iter() - .find(|announcement| announcement.addr.id == node.id()) - .expect("self announcement must be present"); - - assert_eq!( - self_announcement.explicit_model_interests, - vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] - ); -} - -#[tokio::test] -pub(crate) async fn gossip_admission_advertisement_matrix() { - use crate::runtime::activity_policy::{ - activity_advertisement_decision, ActivityPolicyState, - }; - use mesh_llm_config::ActivityAdvertisement; - - let accepting = crate::proto::node::InferenceAdmissionState::Accepting; - let paused = crate::proto::node::InferenceAdmissionState::RemotePaused; - let cases = [ - (ActivityAdvertisement::None, false, None, false), - (ActivityAdvertisement::None, true, None, false), - (ActivityAdvertisement::AvailabilityOnly, false, None, false), - (ActivityAdvertisement::AvailabilityOnly, true, None, true), - ( - ActivityAdvertisement::CoarseState, - false, - Some(accepting), - false, - ), - ( - ActivityAdvertisement::CoarseState, - true, - Some(paused), - true, - ), - ]; - for (mode, blocking, expected_state, expected_withdrawal) in cases { - let state = if blocking { - ActivityPolicyState::RemotePaused - } else { - ActivityPolicyState::Accepting - }; - let decision = activity_advertisement_decision(true, mode, state, false); - assert_eq!(decision.admission_state, expected_state, "mode={mode:?}"); - assert_eq!( - decision.withdraw_model_availability, expected_withdrawal, - "mode={mode:?}" - ); - } - - let private = activity_advertisement_decision( - true, - ActivityAdvertisement::PrivateCoarseState, - ActivityPolicyState::AllPaused, - false, - ); - assert_eq!( - private.admission_state, - Some(crate::proto::node::InferenceAdmissionState::AllPaused) - ); - assert!(private.withdraw_model_availability); - - let public = activity_advertisement_decision( - true, - ActivityAdvertisement::PrivateCoarseState, - ActivityPolicyState::AllPaused, - true, - ); - assert_eq!(public.admission_state, None); - assert!(public.withdraw_model_availability); - - let disabled = activity_advertisement_decision( - false, - ActivityAdvertisement::CoarseState, - ActivityPolicyState::AllPaused, - false, - ); - assert_eq!(disabled.admission_state, None); - assert!(!disabled.withdraw_model_availability); -} - -#[tokio::test] -pub(crate) async fn legacy_peer_admission_and_known_empty_withdrawal() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let peer_id = test_endpoint_id(0x71); - let addr = test_addr(0x71); - - let mut legacy_add = test_announcement(Some(100)); - legacy_add.addr = addr.clone(); - legacy_add.role = NodeRole::Host { http_port: 9337 }; - legacy_add.version = None; - legacy_add.serving_models = vec!["legacy-model".to_string()]; - legacy_add.hosted_models = None; - legacy_add.inference_admission_state = None; - node.add_peer(peer_id, addr.clone(), &legacy_add, None).await; - - let state = node.state.lock().await; - let peer = state - .peers - .get(&peer_id) - .expect("legacy-style peer should be admitted"); - assert_eq!(peer.version, None); - assert!(!peer.hosted_models_known); - assert!(peer.routes_http_model("legacy-model")); - drop(state); - - let mut legacy_withdrawal = test_announcement(Some(100)); - legacy_withdrawal.addr = addr.clone(); - legacy_withdrawal.role = NodeRole::Host { http_port: 9337 }; - legacy_withdrawal.version = None; - legacy_withdrawal.serving_models = Vec::new(); - legacy_withdrawal.hosted_models = Some(Vec::new()); - legacy_withdrawal.inference_admission_state = None; - node.add_peer(peer_id, addr, &legacy_withdrawal, None).await; - - let state = node.state.lock().await; - let peer = state - .peers - .get(&peer_id) - .expect("legacy-withdrawal update should be applied"); - assert_eq!(peer.hosted_models, Vec::::new()); - assert!(peer.hosted_models_known); - assert!(!peer.routes_http_model("legacy-model")); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_handles_floor() { - // At or above the floor — allowed. - assert!(version_allowed_for_rebroadcast(Some("0.60.0"))); - assert!(version_allowed_for_rebroadcast(Some("0.60.2"))); - assert!(version_allowed_for_rebroadcast(Some("0.64.0"))); - assert!(version_allowed_for_rebroadcast(Some("0.65.1"))); - assert!(version_allowed_for_rebroadcast(Some("1.0.0"))); - // Below the floor — refused. - assert!(!version_allowed_for_rebroadcast(Some("0.57.0"))); - assert!(!version_allowed_for_rebroadcast(Some("0.55.1"))); - assert!(!version_allowed_for_rebroadcast(Some("0.58.0"))); - assert!(!version_allowed_for_rebroadcast(Some("0.59.99"))); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_handles_metadata_and_prerelease() { - // Build metadata is stripped. - assert!(version_allowed_for_rebroadcast(Some( - "0.65.1+skippy.20260504.kv.2" - ))); - assert!(!version_allowed_for_rebroadcast(Some("0.57.0+anything"))); - // Pre-release tags are stripped — 0.63.0-rc5 still passes. - assert!(version_allowed_for_rebroadcast(Some("0.63.0-rc5"))); - assert!(!version_allowed_for_rebroadcast(Some("0.58.0-beta"))); -} - -#[test] -pub(crate) fn version_allowed_for_rebroadcast_is_conservative_on_unknown() { - // Unparseable / missing / empty — preserved (don't drop legacy nodes - // that never advertised a version). - assert!(version_allowed_for_rebroadcast(None)); - assert!(version_allowed_for_rebroadcast(Some(""))); - assert!(version_allowed_for_rebroadcast(Some(" "))); - assert!(version_allowed_for_rebroadcast(Some("garbage"))); - assert!(version_allowed_for_rebroadcast(Some("0"))); - assert!(version_allowed_for_rebroadcast(Some("0.x"))); -} - -#[tokio::test] -pub(crate) async fn transitive_ingest_rejects_below_version_floor() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let old_addr = test_addr(0x57); - let new_addr = test_addr(0x65); - let old_id = old_addr.id; - let new_id = new_addr.id; - - let mut old_ann = test_announcement(None); - old_ann.addr = old_addr.clone(); - old_ann.role = NodeRole::Client; - old_ann.version = Some("0.57.0".to_string()); - let mut new_ann = test_announcement(None); - new_ann.addr = new_addr.clone(); - new_ann.role = NodeRole::Client; - new_ann.version = Some("0.65.0".to_string()); - // Give the v0.65.0 client a demand signal so the idle-transitive- - // client filter (a separate gate) doesn't drop it — this test - // exercises the version floor specifically. - new_ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - let bridge = test_endpoint_id(0xBB); - node.update_transitive_peer(old_id, &old_addr, &old_ann, bridge) - .await; - node.update_transitive_peer(new_id, &new_addr, &new_ann, bridge) - .await; - - // Old peer must NOT be in local state — it was rejected at ingest. - // New peer must be present. - { - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&old_id), - "v0.57.0 peer must be rejected at ingest, not appear in local state" - ); - assert!( - state.peers.contains_key(&new_id), - "v0.65.0 peer should be added to local state" - ); - } - - // Outbound gossip must also exclude the old peer. - let announcements = node.collect_announcements().await; - assert!( - !announcements.iter().any(|a| a.addr.id == old_id), - "v0.57.0 peer must not appear in outbound gossip" - ); - assert!( - announcements.iter().any(|a| a.addr.id == new_id), - "v0.65.0 peer should appear in outbound gossip" - ); -} - -#[test] -pub(crate) fn peer_is_idle_transitive_client_basic_shapes() { - // Empty idle client: no hostname, no direct measurement, no - // interests → caught. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - assert!(peer_is_idle_transitive_client(&ann)); - - // Real idle user with a hostname → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.hostname = Some("Sams-MacBook-Pro.local".into()); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Hostname-less client that someone directly measured → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.latency_source = Some(crate::proto::node::LatencySource::Direct); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Estimated latency (propagated guess, not direct) — still caught; - // only Direct counts as proof of contact. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.latency_source = Some(crate::proto::node::LatencySource::Estimated); - assert!(peer_is_idle_transitive_client(&ann)); - - // Client asking for a model → kept (demand signal). - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Client somehow advertising serving → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Client advertising hosted → kept. - let mut ann = test_announcement(None); - ann.role = NodeRole::Client; - ann.hosted_models = Some(vec!["Qwen3-8B-Q4_K_M".to_string()]); - assert!(!peer_is_idle_transitive_client(&ann)); - - // Host → never caught regardless of other fields. - let mut ann = test_announcement(None); - ann.role = NodeRole::Host { http_port: 9337 }; - assert!(!peer_is_idle_transitive_client(&ann)); - - // Worker → never caught. - let mut ann = test_announcement(None); - ann.role = NodeRole::Worker; - assert!(!peer_is_idle_transitive_client(&ann)); -} - -#[tokio::test] -pub(crate) async fn transitive_ingest_drops_idle_clients_but_keeps_clients_with_demand() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let idle_addr = test_addr(0xC1); - let demand_addr = test_addr(0xC2); - let host_addr = test_addr(0xC3); - let idle_id = idle_addr.id; - let demand_id = demand_addr.id; - let host_id = host_addr.id; - - // Idle client — should be dropped at transitive ingest. - let mut idle = test_announcement(None); - idle.addr = idle_addr.clone(); - idle.role = NodeRole::Client; - idle.version = Some("0.65.1".to_string()); - - // Client asking for a model — must be kept (demand signal). - let mut with_demand = test_announcement(None); - with_demand.addr = demand_addr.clone(); - with_demand.role = NodeRole::Client; - with_demand.version = Some("0.65.1".to_string()); - with_demand.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - // Host — must be kept (real compute). - let mut host = test_announcement(None); - host.addr = host_addr.clone(); - host.role = NodeRole::Host { http_port: 9337 }; - host.version = Some("0.65.1".to_string()); - host.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; - - let bridge = test_endpoint_id(0xBB); - node.update_transitive_peer(idle_id, &idle_addr, &idle, bridge) - .await; - node.update_transitive_peer(demand_id, &demand_addr, &with_demand, bridge) - .await; - node.update_transitive_peer(host_id, &host_addr, &host, bridge) - .await; - - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&idle_id), - "idle transitive client must be rejected" - ); - assert!( - state.peers.contains_key(&demand_id), - "client with requested_models must be kept (demand signal)" - ); - assert!( - state.peers.contains_key(&host_id), - "host must be kept (real compute)" - ); -} - -#[tokio::test] -pub(crate) async fn direct_add_peer_admits_idle_clients() { - // Idle clients we actually directly contact are still admitted. - // The predicate is for transitive ingest only — a direct connection - // is proof of life and the peer is observable. - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let addr = test_addr(0xC4); - let id = addr.id; - - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.role = NodeRole::Client; - ann.version = Some("0.65.1".to_string()); - // No requested, no serving, no hosted — pure idle client. - - node.add_peer(id, addr, &ann, None).await; - - let state = node.state.lock().await; - assert!( - state.peers.contains_key(&id), - "direct idle client must be admitted (direct contact is proof of life)" - ); -} - -#[tokio::test] -pub(crate) async fn direct_add_peer_rejects_below_version_floor() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - - let addr = test_addr(0x57); - let id = addr.id; - - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.role = NodeRole::Client; - ann.version = Some("0.57.0".to_string()); - - node.add_peer(id, addr, &ann, None).await; - - let state = node.state.lock().await; - assert!( - !state.peers.contains_key(&id), - "direct add of v0.57.0 peer must be rejected (no local state entry)" - ); -} - -#[tokio::test] -pub(crate) async fn rejected_direct_peer_does_not_apply_mesh_state_or_demand() { - // Given: a direct announcement that passes requirement validation but is - // rejected by the supported-version admission policy. - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let addr = test_addr(0x59); - let mut ann = test_announcement(None); - ann.addr = addr.clone(); - ann.version = Some("0.57.0".to_string()); - ann.mesh_id = Some("rejected-mesh".to_string()); - ann.model_demand.insert( - "rejected-demand".to_string(), - ModelDemand { - last_active: now_secs(), - request_count: 1, - }, - ); - - // When: the direct payload is applied through the production gossip path. - node.apply_announced_peers(addr.id, &[(addr.clone(), ann)], None, None, true) - .await - .expect("policy rejection should not fail the gossip frame"); - - // Then: admission rejection must happen before mesh-wide side effects. - assert_eq!(node.mesh_id().await, None); - assert!(node.get_demand().is_empty()); - assert!(!node.state.lock().await.peers.contains_key(&addr.id)); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub(crate) async fn inbound_gossip_rejection_preserves_dead_peer_state() -> Result<()> { - // Given: a host that still considers the direct sender dead, and a direct - // gossip payload whose sender announcement fails admission. - let host = Node::new_for_tests(NodeRole::Worker).await?; - let policy = crate::MeshGenesisPolicy::new( - "test-owner", - current_time_unix_ms(), - crate::MeshRequirements::default(), - ) - .expect("test policy should be valid"); - let mesh_id = policy.policy_derived_mesh_id().expect("mesh id"); - let policy_hash = policy.canonical_hash_hex().expect("policy hash"); - host.set_active_mesh_policy_for_tests(policy).await; - - let sender_id = test_endpoint_id(0x58); - host.state - .lock() - .await - .dead_peers - .insert(sender_id, std::time::Instant::now()); - - let mut sender_announcement = test_announcement(None); - sender_announcement.addr = EndpointAddr { - id: sender_id, - addrs: Default::default(), - }; - sender_announcement.role = NodeRole::Client; - sender_announcement.version = Some(crate::VERSION.to_string()); - sender_announcement.mesh_id = Some(mesh_id); - sender_announcement.mesh_policy_hash = Some(policy_hash); - let announcements = [(sender_announcement.addr.clone(), sender_announcement)]; - - // When: the same production phase used by `handle_gossip_stream` processes - // the rejected direct announcement. - host.validate_and_capture_inbound_gossip( - ControlProtocol::ProtoV1, - &announcements, - AnnouncedPeerContext::direct(sender_id, Some(NODE_PROTOCOL_GENERATION)), - ) - .await - .expect_err("rejected gossip must fail admission"); - - // Then: admission rejection is recorded, but liveness recovery has not - // cleared the sender from dead_peers. - let state = host.state.lock().await; - assert!( - state.dead_peers.contains_key(&sender_id), - "rejected inbound gossip must not clear dead-peer state" - ); - assert!( - state.requirement_rejected_peers.contains(&sender_id), - "rejected inbound gossip should still be tracked as an admission rejection" - ); - assert!( - !state.peers.contains_key(&sender_id), - "rejected inbound gossip must not admit the sender" - ); - - Ok(()) -} - -#[tokio::test] -pub(crate) async fn future_demand_timestamps_are_active_without_underflow() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let now = now_secs(); - node.merge_remote_demand(&HashMap::from([( - "future-demand".to_string(), - ModelDemand { - last_active: now + DEMAND_TTL_SECS + 60, - request_count: 3, - }, - )])); - - let active = node.active_demand().await; - assert!(active.contains_key("future-demand")); - - node.gc_demand().await; - assert!(node.get_demand().contains_key("future-demand")); + include!("gossip/merge_and_refresh.rs"); } -/// Regression test for the `--auto` startup wedge: when a transitive -/// gossip payload includes peers that would be rejected at ingest -/// (version-floor or idle-transitive-client), `maybe_connect_discovered_peer` -/// must skip the dial. Otherwise each unreachable ghost address triggers -/// a 30 s `connect_to_peer` timeout sequentially in the dial loop, -/// wedging the surrounding gossip exchange (and the `attempt_run_auto_join` -/// that initiated it) for tens of minutes. -/// -/// The function returns without panicking and without dialing within a -/// generous time bound — a real dial to a fake address would block on -/// the 30 s `PEER_CONNECT_AND_GOSSIP_TIMEOUT`. We assert the result is -/// reached well under that bound and that no connection entry was created. -#[tokio::test] -pub(crate) async fn maybe_connect_discovered_peer_skips_filtered_announcements() { - let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); - let my_role = NodeRole::Worker; +mod admission { + use super::*; - // Below-floor version — must be skipped without dialing. - let old_addr = test_addr(0x57); - let old_id = old_addr.id; - let mut old_ann = test_announcement(None); - old_ann.addr = old_addr.clone(); - old_ann.role = NodeRole::Client; - old_ann.version = Some("0.57.0".to_string()); - - // Idle transitive client (matching version, but no hostname / no - // direct measurement / no model interests) — must also be skipped. - let idle_addr = test_addr(0xC1); - let idle_id = idle_addr.id; - let mut idle_ann = test_announcement(None); - idle_ann.addr = idle_addr.clone(); - idle_ann.role = NodeRole::Client; - idle_ann.version = Some("0.65.1".to_string()); - - // Both calls together must return well under the 30 s connect - // timeout. If the dial-loop skip is missing, each call will block - // on PEER_CONNECT_AND_GOSSIP_TIMEOUT (30 s) attempting to dial the - // fake test address. - tokio::time::timeout(std::time::Duration::from_secs(5), async { - node.maybe_connect_discovered_peer(&my_role, old_addr, &old_ann, true, false) - .await; - node.maybe_connect_discovered_peer(&my_role, idle_addr, &idle_ann, true, false) - .await; - }) - .await - .expect("filtered peers must be skipped quickly, not dialed"); - - // No connection was attempted (no entry in state.connections), and - // no peer was added (the filtered announcements never reach add_peer - // or update_transitive_peer through this path). - let state = node.state.lock().await; - assert!( - !state.connections.contains_key(&old_id), - "below-floor peer must not be dialed" - ); - assert!( - !state.connections.contains_key(&idle_id), - "idle transitive client must not be dialed" - ); - assert!( - !state.peers.contains_key(&old_id), - "below-floor peer must not be added (this path is dial-only)" - ); - assert!( - !state.peers.contains_key(&idle_id), - "idle transitive client must not be added (this path is dial-only)" - ); + include!("gossip/admission.rs"); } -#[tokio::test] -pub(crate) async fn client_auto_join_probe_returns_none_for_single_candidate() { - let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); - let token = encode_endpoint_addr_token(&test_addr(0x42)); - - let selected = node - .join_first_responsive_candidate(&[(token, Some("single".to_string()))]) - .await - .unwrap(); - - assert!(selected.is_none()); -} - -#[tokio::test] -pub(crate) async fn client_auto_join_probe_candidate_collection_filters_unusable_tokens() { - let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); - let valid_addr = test_addr(0x42); - let dead_addr = test_addr(0x43); - let self_token = encode_endpoint_addr_token(&node.endpoint_addr_for_advertisement()); - let dead_token = encode_endpoint_addr_token(&dead_addr); - let valid_token = encode_endpoint_addr_token(&valid_addr); - - node.state - .lock() - .await - .dead_peers - .insert(dead_addr.id, std::time::Instant::now()); - - let candidates = node - .collect_join_probe_candidates(&[ - ("not-an-invite-token".to_string(), None), - (self_token, None), - (dead_token, None), - (valid_token, Some("usable".to_string())), - ]) - .await; +mod discovery { + use super::*; - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].addr.id, valid_addr.id); - assert_eq!(candidates[0].mesh_name.as_deref(), Some("usable")); + include!("gossip/discovery.rs"); } diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs new file mode 100644 index 0000000000..a39f66e790 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/admission.rs @@ -0,0 +1,537 @@ +#[tokio::test] +pub(crate) async fn gossip_admission_advertisement_matrix() { + use crate::runtime::activity_policy::{ + activity_advertisement_decision, ActivityPolicyState, + }; + use mesh_llm_config::ActivityAdvertisement; + + let accepting = crate::proto::node::InferenceAdmissionState::Accepting; + let paused = crate::proto::node::InferenceAdmissionState::RemotePaused; + let cases = [ + (ActivityAdvertisement::None, false, None, false), + (ActivityAdvertisement::None, true, None, false), + (ActivityAdvertisement::AvailabilityOnly, false, None, false), + (ActivityAdvertisement::AvailabilityOnly, true, None, true), + ( + ActivityAdvertisement::CoarseState, + false, + Some(accepting), + false, + ), + ( + ActivityAdvertisement::CoarseState, + true, + Some(paused), + true, + ), + ]; + for (mode, blocking, expected_state, expected_withdrawal) in cases { + let state = if blocking { + ActivityPolicyState::RemotePaused + } else { + ActivityPolicyState::Accepting + }; + let decision = activity_advertisement_decision(true, mode, state, false); + assert_eq!(decision.admission_state, expected_state, "mode={mode:?}"); + assert_eq!( + decision.withdraw_model_availability, expected_withdrawal, + "mode={mode:?}" + ); + } + + let private = activity_advertisement_decision( + true, + ActivityAdvertisement::PrivateCoarseState, + ActivityPolicyState::AllPaused, + false, + ); + assert_eq!( + private.admission_state, + Some(crate::proto::node::InferenceAdmissionState::AllPaused) + ); + assert!(private.withdraw_model_availability); + + let public = activity_advertisement_decision( + true, + ActivityAdvertisement::PrivateCoarseState, + ActivityPolicyState::AllPaused, + true, + ); + assert_eq!(public.admission_state, None); + assert!(public.withdraw_model_availability); + + let disabled = activity_advertisement_decision( + false, + ActivityAdvertisement::CoarseState, + ActivityPolicyState::AllPaused, + false, + ); + assert_eq!(disabled.admission_state, None); + assert!(!disabled.withdraw_model_availability); +} + +#[tokio::test] +pub(crate) async fn legacy_peer_admission_and_known_empty_withdrawal() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x71); + let addr = test_addr(0x71); + + let mut legacy_add = test_announcement(Some(100)); + legacy_add.addr = addr.clone(); + legacy_add.role = NodeRole::Host { http_port: 9337 }; + legacy_add.version = None; + legacy_add.serving_models = vec!["legacy-model".to_string()]; + legacy_add.hosted_models = None; + legacy_add.inference_admission_state = None; + node.add_peer(peer_id, addr.clone(), &legacy_add, None).await; + + let state = node.state.lock().await; + let peer = state + .peers + .get(&peer_id) + .expect("legacy-style peer should be admitted"); + assert_eq!(peer.version, None); + assert!(!peer.hosted_models_known); + assert!(peer.routes_http_model("legacy-model")); + drop(state); + + let mut legacy_withdrawal = test_announcement(Some(100)); + legacy_withdrawal.addr = addr.clone(); + legacy_withdrawal.role = NodeRole::Host { http_port: 9337 }; + legacy_withdrawal.version = None; + legacy_withdrawal.serving_models = Vec::new(); + legacy_withdrawal.hosted_models = Some(Vec::new()); + legacy_withdrawal.inference_admission_state = None; + node.add_peer(peer_id, addr, &legacy_withdrawal, None).await; + + let state = node.state.lock().await; + let peer = state + .peers + .get(&peer_id) + .expect("legacy-withdrawal update should be applied"); + assert_eq!(peer.hosted_models, Vec::::new()); + assert!(peer.hosted_models_known); + assert!(!peer.routes_http_model("legacy-model")); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_handles_floor() { + // At or above the floor — allowed. + assert!(version_allowed_for_rebroadcast(Some("0.60.0"))); + assert!(version_allowed_for_rebroadcast(Some("0.60.2"))); + assert!(version_allowed_for_rebroadcast(Some("0.64.0"))); + assert!(version_allowed_for_rebroadcast(Some("0.65.1"))); + assert!(version_allowed_for_rebroadcast(Some("1.0.0"))); + // Below the floor — refused. + assert!(!version_allowed_for_rebroadcast(Some("0.57.0"))); + assert!(!version_allowed_for_rebroadcast(Some("0.55.1"))); + assert!(!version_allowed_for_rebroadcast(Some("0.58.0"))); + assert!(!version_allowed_for_rebroadcast(Some("0.59.99"))); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_handles_metadata_and_prerelease() { + // Build metadata is stripped. + assert!(version_allowed_for_rebroadcast(Some( + "0.65.1+skippy.20260504.kv.2" + ))); + assert!(!version_allowed_for_rebroadcast(Some("0.57.0+anything"))); + // Pre-release tags are stripped — 0.63.0-rc5 still passes. + assert!(version_allowed_for_rebroadcast(Some("0.63.0-rc5"))); + assert!(!version_allowed_for_rebroadcast(Some("0.58.0-beta"))); +} + +#[test] +pub(crate) fn version_allowed_for_rebroadcast_is_conservative_on_unknown() { + // Unparseable / missing / empty — preserved (don't drop legacy nodes + // that never advertised a version). + assert!(version_allowed_for_rebroadcast(None)); + assert!(version_allowed_for_rebroadcast(Some(""))); + assert!(version_allowed_for_rebroadcast(Some(" "))); + assert!(version_allowed_for_rebroadcast(Some("garbage"))); + assert!(version_allowed_for_rebroadcast(Some("0"))); + assert!(version_allowed_for_rebroadcast(Some("0.x"))); +} + +#[tokio::test] +pub(crate) async fn transitive_ingest_rejects_below_version_floor() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let old_addr = test_addr(0x57); + let new_addr = test_addr(0x65); + let old_id = old_addr.id; + let new_id = new_addr.id; + + let mut old_ann = test_announcement(None); + old_ann.addr = old_addr.clone(); + old_ann.role = NodeRole::Client; + old_ann.version = Some("0.57.0".to_string()); + let mut new_ann = test_announcement(None); + new_ann.addr = new_addr.clone(); + new_ann.role = NodeRole::Client; + new_ann.version = Some("0.65.0".to_string()); + // Give the v0.65.0 client a demand signal so the idle-transitive- + // client filter (a separate gate) doesn't drop it — this test + // exercises the version floor specifically. + new_ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + let bridge = test_endpoint_id(0xBB); + node.update_transitive_peer(old_id, &old_addr, &old_ann, bridge) + .await; + node.update_transitive_peer(new_id, &new_addr, &new_ann, bridge) + .await; + + // Old peer must NOT be in local state — it was rejected at ingest. + // New peer must be present. + { + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&old_id), + "v0.57.0 peer must be rejected at ingest, not appear in local state" + ); + assert!( + state.peers.contains_key(&new_id), + "v0.65.0 peer should be added to local state" + ); + } + + // Outbound gossip must also exclude the old peer. + let announcements = node.collect_announcements().await; + assert!( + !announcements.iter().any(|a| a.addr.id == old_id), + "v0.57.0 peer must not appear in outbound gossip" + ); + assert!( + announcements.iter().any(|a| a.addr.id == new_id), + "v0.65.0 peer should appear in outbound gossip" + ); +} + +#[test] +pub(crate) fn peer_is_idle_transitive_client_basic_shapes() { + // Empty idle client: no hostname, no direct measurement, no + // interests → caught. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + assert!(peer_is_idle_transitive_client(&ann)); + + // Real idle user with a hostname → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.hostname = Some("Sams-MacBook-Pro.local".into()); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Hostname-less client that someone directly measured → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.latency_source = Some(crate::proto::node::LatencySource::Direct); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Estimated latency (propagated guess, not direct) — still caught; + // only Direct counts as proof of contact. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.latency_source = Some(crate::proto::node::LatencySource::Estimated); + assert!(peer_is_idle_transitive_client(&ann)); + + // Client asking for a model → kept (demand signal). + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Client somehow advertising serving → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Client advertising hosted → kept. + let mut ann = test_announcement(None); + ann.role = NodeRole::Client; + ann.hosted_models = Some(vec!["Qwen3-8B-Q4_K_M".to_string()]); + assert!(!peer_is_idle_transitive_client(&ann)); + + // Host → never caught regardless of other fields. + let mut ann = test_announcement(None); + ann.role = NodeRole::Host { http_port: 9337 }; + assert!(!peer_is_idle_transitive_client(&ann)); + + // Worker → never caught. + let mut ann = test_announcement(None); + ann.role = NodeRole::Worker; + assert!(!peer_is_idle_transitive_client(&ann)); +} + +#[tokio::test] +pub(crate) async fn transitive_ingest_drops_idle_clients_but_keeps_clients_with_demand() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let idle_addr = test_addr(0xC1); + let demand_addr = test_addr(0xC2); + let host_addr = test_addr(0xC3); + let idle_id = idle_addr.id; + let demand_id = demand_addr.id; + let host_id = host_addr.id; + + // Idle client — should be dropped at transitive ingest. + let mut idle = test_announcement(None); + idle.addr = idle_addr.clone(); + idle.role = NodeRole::Client; + idle.version = Some("0.65.1".to_string()); + + // Client asking for a model — must be kept (demand signal). + let mut with_demand = test_announcement(None); + with_demand.addr = demand_addr.clone(); + with_demand.role = NodeRole::Client; + with_demand.version = Some("0.65.1".to_string()); + with_demand.requested_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + // Host — must be kept (real compute). + let mut host = test_announcement(None); + host.addr = host_addr.clone(); + host.role = NodeRole::Host { http_port: 9337 }; + host.version = Some("0.65.1".to_string()); + host.serving_models = vec!["Qwen3-8B-Q4_K_M".to_string()]; + + let bridge = test_endpoint_id(0xBB); + node.update_transitive_peer(idle_id, &idle_addr, &idle, bridge) + .await; + node.update_transitive_peer(demand_id, &demand_addr, &with_demand, bridge) + .await; + node.update_transitive_peer(host_id, &host_addr, &host, bridge) + .await; + + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&idle_id), + "idle transitive client must be rejected" + ); + assert!( + state.peers.contains_key(&demand_id), + "client with requested_models must be kept (demand signal)" + ); + assert!( + state.peers.contains_key(&host_id), + "host must be kept (real compute)" + ); +} + +#[tokio::test] +pub(crate) async fn direct_add_peer_admits_idle_clients() { + // Idle clients we actually directly contact are still admitted. + // The predicate is for transitive ingest only — a direct connection + // is proof of life and the peer is observable. + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let addr = test_addr(0xC4); + let id = addr.id; + + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.role = NodeRole::Client; + ann.version = Some("0.65.1".to_string()); + // No requested, no serving, no hosted — pure idle client. + + node.add_peer(id, addr, &ann, None).await; + + let state = node.state.lock().await; + assert!( + state.peers.contains_key(&id), + "direct idle client must be admitted (direct contact is proof of life)" + ); +} + +#[tokio::test] +#[serial] +pub(crate) async fn direct_add_peer_rejects_below_version_floor() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + + let addr = test_addr(0x57); + let id = addr.id; + + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.role = NodeRole::Client; + ann.version = Some("0.57.0".to_string()); + + node.add_peer(id, addr, &ann, None).await; + + let state = node.state.lock().await; + assert!( + !state.peers.contains_key(&id), + "direct add of v0.57.0 peer must be rejected (no local state entry)" + ); +} + +#[tokio::test] +#[serial] +pub(crate) async fn below_version_floor_audit_is_direct_only() { + let direct_addr = test_addr(0x57); + let mut direct_ann = test_announcement(None); + direct_ann.addr = direct_addr.clone(); + direct_ann.role = NodeRole::Client; + direct_ann.version = Some("0.57.0".to_string()); + + let (mut direct_audits, direct_capture) = + crate::mesh::capture_mesh_operational_audits(); + let direct_node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + direct_node + .add_peer( + direct_addr.id, + direct_addr, + &direct_ann, + Some(crate::protocol::NODE_PROTOCOL_GENERATION), + ) + .await; + + let direct_subject_id = hex::encode(direct_ann.addr.id.as_bytes()); + let mut direct_records = Vec::new(); + while let Ok(record) = direct_audits.try_recv() { + if record.code() == "gossip_incompatible_version_rejected" + && record + .context() + .is_some_and(|context| context.fields()["subject_id"] == direct_subject_id) + { + direct_records.push(record); + } + } + assert_eq!(direct_records.len(), 1); + let direct_record = &direct_records[0]; + assert_eq!(direct_record.code(), "gossip_incompatible_version_rejected"); + let direct_context = direct_record.context().expect("direct peer context").fields(); + assert_eq!( + direct_context["reason_code"], + "protocol_version_unsupported" + ); + assert_eq!(direct_context["outcome"], "rejected"); + assert!(direct_context.get("numeric_summaries").is_none()); + assert_eq!(direct_context["subject_kind"], "mesh_peer"); + assert_eq!( + direct_context["subject_id"], + direct_subject_id + ); + drop(direct_capture); + + let transitive_addr = test_addr(0x58); + let mut transitive_ann = test_announcement(None); + transitive_ann.addr = transitive_addr.clone(); + transitive_ann.role = NodeRole::Host { http_port: 9337 }; + transitive_ann.version = Some("0.57.0".to_string()); + let bridge = test_endpoint_id(0xBB); + + let (mut transitive_audits, _transitive_capture) = + crate::mesh::capture_mesh_operational_audits(); + let transitive_node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + transitive_node + .update_transitive_peer( + transitive_addr.id, + &transitive_addr, + &transitive_ann, + bridge, + ) + .await; + + let transitive_subject_id = hex::encode(transitive_ann.addr.id.as_bytes()); + let mut transitive_records = 0; + while let Ok(record) = transitive_audits.try_recv() { + if record.code() == "gossip_incompatible_version_rejected" + && record + .context() + .is_some_and(|context| context.fields()["subject_id"] == transitive_subject_id) + { + transitive_records += 1; + } + } + assert_eq!(transitive_records, 0); +} + +#[tokio::test] +pub(crate) async fn rejected_direct_peer_does_not_apply_mesh_state_or_demand() { + // Given: a direct announcement that passes requirement validation but is + // rejected by the supported-version admission policy. + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let addr = test_addr(0x59); + let mut ann = test_announcement(None); + ann.addr = addr.clone(); + ann.version = Some("0.57.0".to_string()); + ann.mesh_id = Some("rejected-mesh".to_string()); + ann.model_demand.insert( + "rejected-demand".to_string(), + ModelDemand { + last_active: now_secs(), + request_count: 1, + }, + ); + + // When: the direct payload is applied through the production gossip path. + node.apply_announced_peers(addr.id, &[(addr.clone(), ann)], None, None, true) + .await + .expect("policy rejection should not fail the gossip frame"); + + // Then: admission rejection must happen before mesh-wide side effects. + assert_eq!(node.mesh_id().await, None); + assert!(node.get_demand().is_empty()); + assert!(!node.state.lock().await.peers.contains_key(&addr.id)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub(crate) async fn inbound_gossip_rejection_preserves_dead_peer_state() -> Result<()> { + // Given: a host that still considers the direct sender dead, and a direct + // gossip payload whose sender announcement fails admission. + let host = Node::new_for_tests(NodeRole::Worker).await?; + let policy = crate::MeshGenesisPolicy::new( + "test-owner", + current_time_unix_ms(), + crate::MeshRequirements::default(), + ) + .expect("test policy should be valid"); + let mesh_id = policy.policy_derived_mesh_id().expect("mesh id"); + let policy_hash = policy.canonical_hash_hex().expect("policy hash"); + host.set_active_mesh_policy_for_tests(policy).await; + + let sender_id = test_endpoint_id(0x58); + host.state + .lock() + .await + .dead_peers + .insert(sender_id, std::time::Instant::now()); + + let mut sender_announcement = test_announcement(None); + sender_announcement.addr = EndpointAddr { + id: sender_id, + addrs: Default::default(), + }; + sender_announcement.role = NodeRole::Client; + sender_announcement.version = Some(crate::VERSION.to_string()); + sender_announcement.mesh_id = Some(mesh_id); + sender_announcement.mesh_policy_hash = Some(policy_hash); + let announcements = [(sender_announcement.addr.clone(), sender_announcement)]; + + // When: the same production phase used by `handle_gossip_stream` processes + // the rejected direct announcement. + host.validate_and_capture_inbound_gossip( + ControlProtocol::ProtoV1, + &announcements, + AnnouncedPeerContext::direct(sender_id, Some(NODE_PROTOCOL_GENERATION)), + ) + .await + .expect_err("rejected gossip must fail admission"); + + // Then: admission rejection is recorded, but liveness recovery has not + // cleared the sender from dead_peers. + let state = host.state.lock().await; + assert!( + state.dead_peers.contains_key(&sender_id), + "rejected inbound gossip must not clear dead-peer state" + ); + assert!( + state.requirement_rejected_peers.contains(&sender_id), + "rejected inbound gossip should still be tracked as an admission rejection" + ); + assert!( + !state.peers.contains_key(&sender_id), + "rejected inbound gossip must not admit the sender" + ); + + Ok(()) +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs new file mode 100644 index 0000000000..024b4755c8 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/discovery.rs @@ -0,0 +1,129 @@ +#[tokio::test] +pub(crate) async fn future_demand_timestamps_are_active_without_underflow() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let now = now_secs(); + node.merge_remote_demand(&HashMap::from([( + "future-demand".to_string(), + ModelDemand { + last_active: now + DEMAND_TTL_SECS + 60, + request_count: 3, + }, + )])); + + let active = node.active_demand().await; + assert!(active.contains_key("future-demand")); + + node.gc_demand().await; + assert!(node.get_demand().contains_key("future-demand")); +} + +/// Regression test for the `--auto` startup wedge: when a transitive +/// gossip payload includes peers that would be rejected at ingest +/// (version-floor or idle-transitive-client), `maybe_connect_discovered_peer` +/// must skip the dial. Otherwise each unreachable ghost address triggers +/// a 30 s `connect_to_peer` timeout sequentially in the dial loop, +/// wedging the surrounding gossip exchange (and the `attempt_run_auto_join` +/// that initiated it) for tens of minutes. +/// +/// The function returns without panicking and without dialing within a +/// generous time bound — a real dial to a fake address would block on +/// the 30 s `PEER_CONNECT_AND_GOSSIP_TIMEOUT`. We assert the result is +/// reached well under that bound and that no connection entry was created. +#[tokio::test] +pub(crate) async fn maybe_connect_discovered_peer_skips_filtered_announcements() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let my_role = NodeRole::Worker; + + // Below-floor version — must be skipped without dialing. + let old_addr = test_addr(0x57); + let old_id = old_addr.id; + let mut old_ann = test_announcement(None); + old_ann.addr = old_addr.clone(); + old_ann.role = NodeRole::Client; + old_ann.version = Some("0.57.0".to_string()); + + // Idle transitive client (matching version, but no hostname / no + // direct measurement / no model interests) — must also be skipped. + let idle_addr = test_addr(0xC1); + let idle_id = idle_addr.id; + let mut idle_ann = test_announcement(None); + idle_ann.addr = idle_addr.clone(); + idle_ann.role = NodeRole::Client; + idle_ann.version = Some("0.65.1".to_string()); + + // Both calls together must return well under the 30 s connect + // timeout. If the dial-loop skip is missing, each call will block + // on PEER_CONNECT_AND_GOSSIP_TIMEOUT (30 s) attempting to dial the + // fake test address. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + node.maybe_connect_discovered_peer(&my_role, old_addr, &old_ann, true, false) + .await; + node.maybe_connect_discovered_peer(&my_role, idle_addr, &idle_ann, true, false) + .await; + }) + .await + .expect("filtered peers must be skipped quickly, not dialed"); + + // No connection was attempted (no entry in state.connections), and + // no peer was added (the filtered announcements never reach add_peer + // or update_transitive_peer through this path). + let state = node.state.lock().await; + assert!( + !state.connections.contains_key(&old_id), + "below-floor peer must not be dialed" + ); + assert!( + !state.connections.contains_key(&idle_id), + "idle transitive client must not be dialed" + ); + assert!( + !state.peers.contains_key(&old_id), + "below-floor peer must not be added (this path is dial-only)" + ); + assert!( + !state.peers.contains_key(&idle_id), + "idle transitive client must not be added (this path is dial-only)" + ); +} + +#[tokio::test] +pub(crate) async fn client_auto_join_probe_returns_none_for_single_candidate() { + let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); + let token = encode_endpoint_addr_token(&test_addr(0x42)); + + let selected = node + .join_first_responsive_candidate(&[(token, Some("single".to_string()))]) + .await + .unwrap(); + + assert!(selected.is_none()); +} + +#[tokio::test] +pub(crate) async fn client_auto_join_probe_candidate_collection_filters_unusable_tokens() { + let node = Node::new_for_tests(NodeRole::Client).await.unwrap(); + let valid_addr = test_addr(0x42); + let dead_addr = test_addr(0x43); + let self_token = encode_endpoint_addr_token(&node.endpoint_addr_for_advertisement()); + let dead_token = encode_endpoint_addr_token(&dead_addr); + let valid_token = encode_endpoint_addr_token(&valid_addr); + + node.state + .lock() + .await + .dead_peers + .insert(dead_addr.id, std::time::Instant::now()); + + let candidates = node + .collect_join_probe_candidates(&[ + ("not-an-invite-token".to_string(), None), + (self_token, None), + (dead_token, None), + (valid_token, Some("usable".to_string())), + ]) + .await; + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].addr.id, valid_addr.id); + assert_eq!(candidates[0].mesh_name.as_deref(), Some("usable")); +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs new file mode 100644 index 0000000000..27ca0dcaef --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs @@ -0,0 +1,265 @@ +#[test] +pub(crate) fn test_merge_none_to_some() { + let mut existing = test_peer(None); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_some_to_none_keeps_existing() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(None); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_earlier_incoming_wins() { + let mut existing = test_peer(Some(200)); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_later_incoming_loses() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(Some(200)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_merge_equal_values_unchanged() { + let mut existing = test_peer(Some(100)); + let ann = test_announcement(Some(100)); + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!(existing.first_joined_mesh_ts, Some(100)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_first_joined_mesh_ts() { + let old_peer = test_peer(Some(100)); + let new_peer = test_peer(Some(200)); + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_explicit_model_interests() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_stage_status_list_support() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.stage_status_list_supported = !old_peer.stage_status_list_supported; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_meaningfully_changed_stage_protocol_generation_support() { + let old_peer = test_peer(Some(100)); + let mut new_peer = test_peer(Some(100)); + new_peer.stage_protocol_generation_supported = !old_peer.stage_protocol_generation_supported; + + assert!(peer_meaningfully_changed(&old_peer, &new_peer)); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_explicit_model_interests() { + let mut existing = test_peer(Some(100)); + let mut ann = test_announcement(Some(100)); + ann.explicit_model_interests = vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into()]; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!( + existing.explicit_model_interests, + vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] + ); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_stage_status_list_support() { + let mut existing = test_peer(Some(100)); + existing.stage_status_list_supported = false; + let mut ann = test_announcement(Some(100)); + ann.stage_status_list_supported = true; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert!(existing.stage_status_list_supported); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_stage_protocol_generation_support() { + let mut existing = test_peer(Some(100)); + existing.stage_protocol_generation_supported = false; + let mut ann = test_announcement(Some(100)); + ann.stage_protocol_generation_supported = true; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert!(existing.stage_protocol_generation_supported); +} + +#[test] +pub(crate) fn test_apply_transitive_ann_refreshes_advertised_model_throughput() { + let mut existing = test_peer(Some(100)); + let mut ann = test_announcement(Some(100)); + ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { + model_name: "qwen".to_string(), + avg_tokens_per_second_milli: 35_000, + throughput_samples: 4, + }]; + + apply_transitive_ann( + &mut existing, + &test_addr(0x33), + &ann, + test_endpoint_id(0xee), + ); + + assert_eq!( + existing.advertised_model_throughput, + ann.advertised_model_throughput + ); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_stage_status_list_support() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x44); + let addr = test_addr(0x44); + let mut ann = test_announcement(Some(100)); + ann.stage_status_list_supported = false; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.stage_status_list_supported = true; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert!(peer.stage_status_list_supported); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_stage_protocol_generation_support() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x45); + let addr = test_addr(0x45); + let mut ann = test_announcement(Some(100)); + ann.stage_protocol_generation_supported = false; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.stage_protocol_generation_supported = true; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert!(peer.stage_protocol_generation_supported); +} + +#[tokio::test] +pub(crate) async fn test_add_peer_refreshes_advertised_model_throughput() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + let peer_id = test_endpoint_id(0x46); + let addr = test_addr(0x46); + let mut ann = test_announcement(Some(100)); + ann.advertised_model_throughput = vec![crate::network::metrics::ModelThroughputHint { + model_name: "qwen".to_string(), + avg_tokens_per_second_milli: 20_000, + throughput_samples: 2, + }]; + + node.add_peer(peer_id, addr.clone(), &ann, None).await; + ann.advertised_model_throughput[0].avg_tokens_per_second_milli = 48_000; + ann.advertised_model_throughput[0].throughput_samples = 9; + node.add_peer(peer_id, addr, &ann, None).await; + + let state = node.state.lock().await; + let peer = state.peers.get(&peer_id).expect("peer should be tracked"); + assert_eq!( + peer.advertised_model_throughput, + ann.advertised_model_throughput + ); +} + +#[tokio::test] +pub(crate) async fn test_collect_announcements_includes_self_explicit_model_interests() { + let node = Node::new_for_tests(NodeRole::Worker).await.unwrap(); + node.set_explicit_model_interests(vec![ + "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), + "Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".into(), + ]) + .await; + + let announcements = node.collect_announcements().await; + let self_announcement = announcements + .iter() + .find(|announcement| announcement.addr.id == node.id()) + .expect("self announcement must be present"); + + assert_eq!( + self_announcement.explicit_model_interests, + vec!["Qwen/Qwen3-Coder-Next-GGUF@main:Q4_K_M".to_string()] + ); +} diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs index 73a971c85e..ea344b5ee1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/protocol_frames.rs @@ -914,6 +914,11 @@ async fn reconnect_gossip_failure_removes_zombie_peer() { let node = make_test_node(super::NodeRole::Worker) .await .expect("test node must start"); + let capture_dir = tempfile::tempdir().expect("swarm capture tempdir"); + let recorder = crate::capture::SwarmCaptureRecorder::new(capture_dir.path().join("capture")) + .expect("swarm capture recorder"); + let capture_path = recorder.path().to_path_buf(); + node.set_swarm_capture_recorder(Some(recorder)); node.insert_test_peer(make_test_peer_info(peer_id)).await; @@ -930,6 +935,25 @@ async fn reconnect_gossip_failure_removes_zombie_peer() { "zombie peer must be removed when reconnect gossip fails (relay-connected but process dead)" ); + let capture_line = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if let Ok(contents) = std::fs::read_to_string(&capture_path) + && let Some(line) = contents.lines().find(|line| line.contains("peer_removed")) + { + break line.to_string(); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("peer removal should reach swarm capture"); + let capture: serde_json::Value = + serde_json::from_str(&capture_line).expect("peer removal capture JSON"); + assert_eq!(capture["fields"]["reason"], "recovered_gossip_failed"); + assert!(capture["fields"]["last_seen_age_ms"].is_u64()); + assert!(capture["fields"]["last_mentioned_age_ms"].is_u64()); + assert_eq!(capture["fields"]["had_connection"], false); + let peer_key2 = SecretKey::from_bytes(&[0xF1; 32]); let peer_id2 = EndpointId::from(peer_key2.public()); node.insert_test_peer(make_test_peer_info(peer_id2)).await; diff --git a/crates/mesh-llm-host-runtime/src/models/catalog.rs b/crates/mesh-llm-host-runtime/src/models/catalog.rs index 5e8fc9746e..8a1e0834d7 100644 --- a/crates/mesh-llm-host-runtime/src/models/catalog.rs +++ b/crates/mesh-llm-host-runtime/src/models/catalog.rs @@ -942,20 +942,64 @@ pub async fn download_hf_repo_file_with_progress_label( revision: revision.clone(), file: file.to_string(), }; - let HfAssetsDownload { - mut paths, - transfer_stats, - } = download_hf_assets(label, vec![asset.clone()], progress).await?; - paths.sort(); - let path = paths - .iter() - .find(|path| path_suffix_matches_ignore_case(path, &asset.file)) - .cloned() - .ok_or_else(|| { - anyhow::anyhow!( - "Downloaded Hugging Face asset not found in cache: {repo}/{file}@{revision}" + let mut retried_dangling_symlink = false; + let (path, paths, transfer_stats) = loop { + let HfAssetsDownload { + mut paths, + transfer_stats, + } = download_hf_assets(label, vec![asset.clone()], progress).await?; + paths.sort(); + let path = paths + .iter() + .find(|path| path_suffix_matches_ignore_case(path, &asset.file)) + .cloned() + .ok_or_else(|| { + anyhow::anyhow!( + "Downloaded Hugging Face asset not found in cache: {repo}/{file}@{revision}" + ) + })?; + let resolution_error = match std::fs::canonicalize(&path) { + Ok(_) => break (path, paths, transfer_stats), + Err(err) => err, + }; + if resolution_error.kind() != std::io::ErrorKind::NotFound { + return Err(resolution_error).with_context(|| { + format!( + "Resolve downloaded Hugging Face asset {repo}/{file}@{revision} at {}", + path.display() + ) + }); + } + let is_symlink = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata.file_type().is_symlink(), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, + Err(err) => { + return Err(err).with_context(|| { + format!("Inspect downloaded Hugging Face asset {}", path.display()) + }); + } + }; + if !is_symlink { + return Err(resolution_error).with_context(|| { + format!( + "Resolve downloaded Hugging Face asset {repo}/{file}@{revision} at {}", + path.display() + ) + }); + } + std::fs::remove_file(&path).with_context(|| { + format!( + "Remove dangling Hugging Face snapshot pointer {}", + path.display() ) })?; + if retried_dangling_symlink { + anyhow::bail!( + "Downloaded Hugging Face asset remained a dangling snapshot symlink after one retry: {repo}/{file}@{revision}" + ); + } + retried_dangling_symlink = true; + }; let display_name = Path::new(&asset.file) .file_name() .and_then(|value| value.to_str()) @@ -1523,6 +1567,167 @@ mod tests { assert_eq!(download.paths, vec![shard_one, shard_two, shard_three]); } + #[cfg(unix)] + #[tokio::test] + async fn download_hf_repo_file_retries_dangling_snapshot_symlink_once() { + use std::os::unix::fs::symlink; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let snapshot_dir = cache.path().join("snapshots/commit"); + let blob_dir = cache.path().join("blobs"); + let snapshot = snapshot_dir.join("model.gguf"); + let blob = blob_dir.join("etag"); + std::fs::create_dir_all(&snapshot_dir).unwrap(); + std::fs::create_dir_all(&blob_dir).unwrap(); + symlink("../../blobs/etag", &snapshot).unwrap(); + assert!( + std::fs::symlink_metadata(&snapshot) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("dangling-snapshot-retry-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let snapshot = snapshot.clone(); + let blob = blob.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![snapshot.clone()]), + 1 => { + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + std::fs::write(&blob, b"gguf").unwrap(); + symlink("../../blobs/etag", &snapshot).unwrap(); + Ok(vec![snapshot.clone()]) + } + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let download = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await + .unwrap(); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a dangling Ok(path) must not be trusted" + ); + assert_eq!(download.path, snapshot); + assert!(download.path.exists()); + assert_eq!(std::fs::read(&download.path).unwrap(), b"gguf"); + } + + #[cfg(unix)] + #[tokio::test] + async fn download_hf_repo_file_removes_second_dangling_snapshot_symlink() { + use std::os::unix::fs::symlink; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let snapshot_dir = cache.path().join("snapshots/commit"); + let snapshot = snapshot_dir.join("model.gguf"); + std::fs::create_dir_all(&snapshot_dir).unwrap(); + symlink("../../blobs/first-etag", &snapshot).unwrap(); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("double-dangling-snapshot-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let snapshot = snapshot.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![snapshot.clone()]), + 1 => { + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + symlink("../../blobs/second-etag", &snapshot).unwrap(); + Ok(vec![snapshot.clone()]) + } + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let result = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await; + + assert!(result.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!( + std::fs::symlink_metadata(&snapshot).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + } + + #[tokio::test] + async fn download_hf_repo_file_does_not_retry_missing_non_symlink_path() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let cache = tempfile::tempdir().unwrap(); + let fixture = cache.path().join("fixture"); + let missing = fixture.join("model.gguf"); + let sentinel = fixture.join("sentinel"); + std::fs::create_dir_all(&fixture).unwrap(); + std::fs::write(&sentinel, b"keep").unwrap(); + + let calls = Arc::new(AtomicUsize::new(0)); + let label = format!("missing-non-symlink-{}", cache.path().display()); + let _guard = DownloadHfAssetsOverrideGuard::set( + label.clone(), + Arc::new({ + let calls = Arc::clone(&calls); + let missing = missing.clone(); + move |_, _| match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(vec![missing.clone()]), + attempt => panic!("unexpected download attempt {attempt}"), + } + }), + ); + + let result = download_hf_repo_file_with_progress_label( + "org/repo", + Some("commit"), + "model.gguf", + &label, + false, + ) + .await; + + assert!(result.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(fixture.is_dir()); + assert_eq!(std::fs::read(sentinel).unwrap(), b"keep"); + assert!(!missing.exists()); + } + #[test] fn is_mlx_primary_asset_includes_first_shard() { assert!(is_mlx_primary_asset("model.safetensors")); diff --git a/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs b/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs index 59e5824a99..fc0678fd95 100644 --- a/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs +++ b/crates/mesh-llm-host-runtime/src/models/resolve/tests.rs @@ -167,6 +167,9 @@ fn synthetic_local_gguf_ref_for_test(path: &Path) -> String { async fn bare_name_resolves_from_remote_catalog() { let query = "RemoteOnlyResolverFallbackModel-Q4_K_M"; let source_file = "RemoteOnlyResolverFallbackModel-Q4_K_M.gguf"; + let cache = tempfile::tempdir().unwrap(); + let downloaded = cache.path().join(source_file); + std::fs::write(&downloaded, b"gguf").unwrap(); let _catalog_guard = crate::models::remote_catalog::set_catalog_entries_for_test(vec![remote_catalog_entry( @@ -177,14 +180,17 @@ async fn bare_name_resolves_from_remote_catalog() { )]); let _download_guard = catalog::set_download_hf_assets_label_override( query.to_string(), - Arc::new(move |_| Ok(vec![PathBuf::from(format!("/tmp/{source_file}"))])), + Arc::new({ + let downloaded = downloaded.clone(); + move |_| Ok(vec![downloaded.clone()]) + }), ); let resolved = resolve_model_spec_with_progress(Path::new(query), false) .await .unwrap(); - assert_eq!(resolved, PathBuf::from(format!("/tmp/{source_file}"))); + assert_eq!(resolved, downloaded); } #[tokio::test] @@ -192,6 +198,9 @@ async fn bare_name_resolves_from_remote_catalog() { async fn bare_name_resolution_prefers_remote_catalog_over_baked_catalog() { let query = "Qwen3-8B-Q4_K_M"; let source_file = "RemotePreferred-Q4_K_M.gguf"; + let cache = tempfile::tempdir().unwrap(); + let downloaded = cache.path().join(source_file); + std::fs::write(&downloaded, b"gguf").unwrap(); let _catalog_guard = crate::models::remote_catalog::set_catalog_entries_for_test(vec![remote_catalog_entry( query, @@ -201,14 +210,17 @@ async fn bare_name_resolution_prefers_remote_catalog_over_baked_catalog() { )]); let _download_guard = catalog::set_download_hf_assets_label_override( "Remote Preferred Catalog Model".to_string(), - Arc::new(move |_| Ok(vec![PathBuf::from(format!("/tmp/{source_file}"))])), + Arc::new({ + let downloaded = downloaded.clone(); + move |_| Ok(vec![downloaded.clone()]) + }), ); let resolved = resolve_model_spec_with_progress(Path::new(query), false) .await .unwrap(); - assert_eq!(resolved, PathBuf::from(format!("/tmp/{source_file}"))); + assert_eq!(resolved, downloaded); } #[test] @@ -805,6 +817,13 @@ fn format_huggingface_display_ref_uses_selector_for_split_gguf() { #[serial] async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { let fixture = load_gemma_live_fixture(); + let cache = tempfile::tempdir().unwrap(); + let bf16_dir = cache.path().join("BF16"); + let shard_one = bf16_dir.join("gemma-4-31B-it-BF16-00001-of-00002.gguf"); + let shard_two = bf16_dir.join("gemma-4-31B-it-BF16-00002-of-00002.gguf"); + std::fs::create_dir_all(&bf16_dir).unwrap(); + std::fs::write(&shard_one, b"gguf-one").unwrap(); + std::fs::write(&shard_two, b"gguf-two").unwrap(); let _siblings_guard = RepoSiblingEntriesOverrideGuard::set(Arc::new({ let repo = fixture.repo.clone(); let siblings = fixture @@ -832,11 +851,10 @@ async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { })); let _download_guard = catalog::set_download_hf_assets_label_override( "unsloth/gemma-4-31B-it-GGUF:BF16".to_string(), - Arc::new(|_| { - Ok(vec![ - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00001-of-00002.gguf"), - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00002-of-00002.gguf"), - ]) + Arc::new({ + let shard_one = shard_one.clone(); + let shard_two = shard_two.clone(); + move |_| Ok(vec![shard_one.clone(), shard_two.clone()]) }), ); @@ -844,10 +862,7 @@ async fn download_exact_ref_bf16_shorthand_downloads_full_split_model() { .await .unwrap(); - assert_eq!( - resolved, - PathBuf::from("/tmp/BF16/gemma-4-31B-it-BF16-00001-of-00002.gguf") - ); + assert_eq!(resolved, shard_one); assert_eq!( *planned.lock().unwrap(), vec![ diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index 72ac27463a..08c84ec00b 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -1,5 +1,5 @@ use crate::inference::{election, pipeline}; -use crate::logging::{OpenAiLifecycleAttachment, OpenAiRouteObserver}; +use crate::logging::{CallerPathType, OpenAiLifecycleAttachment, OpenAiRouteObserver}; use crate::mesh; use crate::network::affinity; use crate::network::openai::auto_route; @@ -927,13 +927,20 @@ async fn handle_buffered_api_request( tcp_stream: tokio::net::TcpStream, mut request: proxy::BufferedHttpRequest, ctx: ProxyConnectionContext<'_>, + source_addr: Option, ) { // Claim the parent at host OpenAI ingress. All downstream dispatch sees // only a metadata observer; this scope remains the sole terminal owner. + let caller_addr = source_addr.map(|addr| addr.to_string()); let request_metadata = crate::logging::RequestSummaryMetadata::from_openai_ingress_path(&request.client_path) .with_source(Some("direct_http")) - .with_method(Some(&request.method)); + .with_method(Some(&request.method)) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let mut lifecycle = crate::logging_runtime_state() .map(|state| state.openai_ingress_attachment(request.request_id, request_metadata)) .unwrap_or_else(OpenAiLifecycleAttachment::unowned); @@ -1058,6 +1065,7 @@ async fn handle_api_proxy_connection( targets: election::ModelTargets, affinity: affinity::AffinityRouter, ) { + let source_addr = tcp_stream.peer_addr().ok(); let plugin_manager = node.plugin_manager().await; match proxy::read_http_request_with_plugin_manager_with_context( &mut tcp_stream, @@ -1072,8 +1080,13 @@ async fn handle_api_proxy_connection( affinity: &affinity, plugin_manager: plugin_manager.as_ref(), }; - handle_buffered_api_request(tcp_stream, request, ProxyConnectionContext { route }) - .await; + handle_buffered_api_request( + tcp_stream, + request, + ProxyConnectionContext { route }, + source_addr, + ) + .await; } Err(error) => { let _ = super::parse_failure::send_read_failure(tcp_stream, &error).await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs index cc12d300b6..66d7f23499 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/durable_artifacts.rs @@ -44,6 +44,10 @@ async fn parsed_missing_model_error_persists_the_client_visible_response_artifac let mut client = tokio::net::TcpStream::connect(address) .await .expect("connect ingress client"); + let caller_addr = client + .local_addr() + .expect("ingress caller address") + .to_string(); client .write_all(request.as_bytes()) .await @@ -60,8 +64,25 @@ async fn parsed_missing_model_error_persists_the_client_visible_response_artifac ); let state = crate::logging_runtime_state().expect("installed logging runtime"); + let active = state + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); state.pump_persistence_for_test().await; let request_key = request_id.as_uuid().to_string(); + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_key) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); let artifacts = state .store() .expect("metadata store") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index bfc2009dfa..a67173ed69 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -24,6 +24,10 @@ const MAX_OBJECT_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024; const MAX_CHUNKED_WIRE_BYTES: usize = MAX_BODY_BYTES * 6 + 64 * 1024; const MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_OBJECT_UPLOAD_BODY_BYTES * 6 + 64 * 1024; pub(super) const MAX_HEADERS: usize = 64; +const CRLF: &[u8] = b"\r\n"; +const LF: &[u8] = b"\n"; +const CRLF_HEADER_TERMINATOR: &[u8] = b"\r\n\r\n"; +const LF_HEADER_TERMINATOR: &[u8] = b"\n\n"; #[derive(Debug, Clone, Copy)] pub(super) struct HttpReadLimits { @@ -650,6 +654,65 @@ pub(crate) fn canonical_request_id_from_header_prefix(prefix: &[u8]) -> Option Option<(usize, &'static [u8])> { + let crlf = prefix + .windows(CRLF_HEADER_TERMINATOR.len()) + .position(|window| window == CRLF_HEADER_TERMINATOR) + .map(|offset| (offset + CRLF_HEADER_TERMINATOR.len(), CRLF)); + let lf = prefix + .windows(LF_HEADER_TERMINATOR.len()) + .position(|window| window == LF_HEADER_TERMINATOR) + .map(|offset| (offset + LF_HEADER_TERMINATOR.len(), LF)); + + match (crlf, lf) { + (Some(crlf), Some(lf)) => Some(if crlf.0 <= lf.0 { crlf } else { lf }), + (Some(terminator), None) | (None, Some(terminator)) => Some(terminator), + (None, None) => None, + } +} + +pub(crate) fn ensure_canonical_request_id_in_header_prefix( + mut prefix: Vec, +) -> (Vec, Option) { + let mut headers_buf = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut request = httparse::Request::new(&mut headers_buf); + let Ok(httparse::Status::Complete(header_end)) = request.parse(&prefix) else { + return (prefix, None); + }; + if header_end > MAX_HEADER_BYTES { + return (prefix, None); + } + + let request_id_header_count = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")) + .count(); + if let Some(request_id) = canonical_request_id_from_headers(request.headers) { + return (prefix, Some(request_id)); + } + if request_id_header_count != 0 || request.headers.len() >= MAX_HEADERS { + return (prefix, None); + } + + let Some((terminator_end, line_ending)) = http_header_terminator(&prefix[..header_end]) else { + return (prefix, None); + }; + if terminator_end != header_end { + return (prefix, None); + } + + let request_id = RequestId::new(); + let mut header = format!("x-request-id: {}", request_id.as_uuid()).into_bytes(); + header.extend_from_slice(line_ending); + if header_end.saturating_add(header.len()) > MAX_HEADER_BYTES { + return (prefix, None); + } + let insertion_offset = header_end - line_ending.len(); + prefix.splice(insertion_offset..insertion_offset, header); + (prefix, Some(request_id)) +} + /// Parse the private raw-lifecycle assertion from a complete, bounded HTTP /// header prefix. The marker is accepted only once and only when its UUID /// exactly matches the one canonical `x-request-id` header. diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index 2075101129..6f006c29ef 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -5,7 +5,8 @@ use crate::inference::election; use crate::logging::{ - OpenAiLifecycleAttachment, OpenAiRouteAttempt, OpenAiRouteObserver, ProxyAttemptFinish, + CallerPathType, OpenAiLifecycleAttachment, OpenAiRouteAttempt, OpenAiRouteObserver, + ProxyAttemptFinish, }; use crate::mesh; use crate::network::affinity::{ @@ -234,11 +235,20 @@ fn capture_path_for_request(request: &BufferedHttpRequest) -> &str { &request.client_path } -fn attach_request_logging(request: &mut BufferedHttpRequest) -> OpenAiLifecycleAttachment { +fn attach_request_logging( + request: &mut BufferedHttpRequest, + source_addr: Option, +) -> OpenAiLifecycleAttachment { + let caller_addr = source_addr.map(|addr| addr.to_string()); let metadata = crate::logging::RequestSummaryMetadata::from_openai_ingress_path(&request.client_path) .with_source(Some("mesh_forwarded")) - .with_method(Some(&request.method)); + .with_method(Some(&request.method)) + .with_caller_identity( + None, + caller_addr.as_deref(), + caller_addr.as_ref().map(|_| CallerPathType::LocalHttp), + ); let lifecycle = crate::logging_runtime_state() .map(|state| state.openai_ingress_attachment(request.request_id, metadata)) .unwrap_or_else(OpenAiLifecycleAttachment::unowned); @@ -310,7 +320,7 @@ pub async fn handle_mesh_request( }; // The parsed host ingress owns the parent. Downstream route code receives // only the attachment's metadata observer and cannot terminalize it. - let mut lifecycle = attach_request_logging(&mut request); + let mut lifecycle = attach_request_logging(&mut request, source_addr); if node.swarm_capture_enabled() { node.capture_http_request(crate::mesh::HttpCaptureEvent { event: "openai_ingress_http_request", diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs index 584ba0c84f..9af95db2a7 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/durable_artifacts.rs @@ -197,6 +197,10 @@ async fn passive_missing_model_error_persists_the_client_visible_response_artifa let mut client = tokio::net::TcpStream::connect(address) .await .expect("connect passive client"); + let caller_addr = client + .local_addr() + .expect("passive caller address") + .to_string(); client .write_all(request.as_bytes()) .await @@ -213,7 +217,24 @@ async fn passive_missing_model_error_persists_the_client_visible_response_artifa ); let state = crate::logging_runtime_state().expect("installed logging runtime"); + let active = state + .service_for_test() + .expect("logging service") + .registry_ref() + .get_recent(&request_id.as_uuid().to_string()) + .expect("active request summary"); + assert_eq!(active.metadata.caller_addr(), Some(caller_addr.as_str())); + assert_eq!(active.metadata.caller_path_type(), Some("local_http")); state.pump_persistence_for_test().await; + let durable = state + .store() + .expect("metadata store") + .query_request_with_caller(&request_id.as_uuid().to_string()) + .expect("durable request query") + .expect("durable request summary"); + assert_eq!(durable.caller_addr.as_deref(), Some(caller_addr.as_str())); + assert_eq!(durable.caller_path_type.as_deref(), Some("local_http")); + assert!(durable.caller_endpoint_id.is_none()); let artifacts = state .store() .expect("metadata store") diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel.rs b/crates/mesh-llm-host-runtime/src/network/tunnel.rs index 5a8e370eb0..e1e4d137c0 100644 --- a/crates/mesh-llm-host-runtime/src/network/tunnel.rs +++ b/crates/mesh-llm-host-runtime/src/network/tunnel.rs @@ -12,7 +12,7 @@ use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; -const TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES: usize = 8 * 1024; +mod inbound_http; /// Global byte counter for tunnel traffic static BYTES_TRANSFERRED: AtomicU64 = AtomicU64::new(0); @@ -39,6 +39,7 @@ impl Manager { iroh::endpoint::RecvStream, )>, mut tunnel_http_rx: tokio::sync::mpsc::Receiver<( + EndpointId, iroh::endpoint::SendStream, iroh::endpoint::RecvStream, )>, @@ -58,7 +59,7 @@ impl Manager { let http_port_ref = mgr.http_port.clone(); let http_node = mgr.node.clone(); tokio::spawn(async move { - while let Some((send, recv)) = tunnel_http_rx.recv().await { + while let Some((remote, send, recv)) = tunnel_http_rx.recv().await { let port = http_port_ref.load(Ordering::Relaxed); if port == 0 { tracing::warn!("Inbound HTTP tunnel but no OpenAI surface running, dropping"); @@ -66,7 +67,10 @@ impl Manager { } let node = http_node.clone(); tokio::spawn(async move { - if let Err(e) = handle_inbound_http_stream(node, send, recv, port).await { + if let Err(e) = + inbound_http::handle_inbound_http_stream(node, remote, send, recv, port) + .await + { tracing::warn!("Inbound HTTP tunnel stream error: {e}"); } }); @@ -99,73 +103,6 @@ impl Manager { } } -/// Handle an inbound HTTP tunnel bi-stream: connect to the local API proxy and relay. -async fn handle_inbound_http_stream( - node: Node, - quic_send: iroh::endpoint::SendStream, - mut quic_recv: iroh::endpoint::RecvStream, - http_port: u16, -) -> Result<()> { - // Admission check for remote QUIC HTTP ingress. - match node - .activity_policy_guard - .check_admission(crate::runtime::IngressType::RemoteQuicHttp) - { - crate::runtime::AdmissionResult::Allowed => {} - crate::runtime::AdmissionResult::Paused { reason, .. } => { - tracing::debug!(reason, "Inbound HTTP tunnel rejected by activity policy"); - anyhow::bail!("remote inference paused: {reason}"); - } - } - - tracing::info!("Inbound HTTP tunnel stream -> API proxy :{http_port}"); - let mut tcp_stream = TcpStream::connect(format!("127.0.0.1:{http_port}")).await?; - tcp_stream.set_nodelay(true)?; - let _inflight = node.begin_inflight_request(); - - // Only raw mesh ingress that successfully claimed a parent emits the - // private assertion. Direct API requests have it stripped before they can - // reach this tunnel, so they retain normal target frontend ownership. - let prefix = read_tunneled_http_header_prefix(&mut quic_recv).await?; - let _remote_suppression = - crate::network::openai::request_parse::raw_lifecycle_owner_from_header_prefix(&prefix) - .and_then(|request_id| { - crate::logging_runtime_state() - .and_then(|state| state.suppress_remote_tunneled_request(request_id)) - }); - tcp_stream.write_all(&prefix).await?; - - let (tcp_read, tcp_write) = tokio::io::split(tcp_stream); - relay_bidirectional(tcp_read, tcp_write, quic_send, quic_recv).await -} - -/// Read at most one bounded HTTP header prefix without changing the bytes that -/// the existing relay will subsequently forward. A read may include a small -/// amount of body data from the same transport chunk; it is forwarded directly -/// to the local API proxy and is never retained by logging state. -async fn read_tunneled_http_header_prefix(reader: &mut R) -> Result> -where - R: AsyncRead + Unpin, -{ - let mut prefix = Vec::with_capacity(TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES); - let mut chunk = [0u8; TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES]; - let max_header_bytes = crate::network::openai::request_parse::MAX_HEADER_BYTES; - - while prefix.len() < max_header_bytes { - if prefix.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - let read_cap = (max_header_bytes - prefix.len()).min(chunk.len()); - let bytes_read = reader.read(&mut chunk[..read_cap]).await?; - if bytes_read == 0 { - break; - } - prefix.extend_from_slice(&chunk[..bytes_read]); - } - - Ok(prefix) -} - async fn handle_inbound_stage_transport( node: Node, remote: EndpointId, @@ -428,18 +365,6 @@ fn relay_remaining_chunks_error(total: u64, err: std::io::Error) -> Result<()> { mod tests { use super::*; - #[tokio::test] - async fn tunnel_prefetch_forwards_a_complete_bounded_header_prefix() { - let (mut writer, mut reader) = tokio::io::duplex(4096); - let request = b"POST /v1/chat/completions HTTP/1.1\r\nx-request-id: 550e8400-e29b-41d4-a716-446655440000\r\n\r\n"; - writer.write_all(request).await.unwrap(); - - let prefix = read_tunneled_http_header_prefix(&mut reader).await.unwrap(); - - assert_eq!(prefix, request); - assert!(prefix.len() <= crate::network::openai::request_parse::MAX_HEADER_BYTES); - } - /// Simulate relay_bidirectional behavior when one direction finishes /// before the other — the scenario that caused the remote proxy bug. /// diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs new file mode 100644 index 0000000000..8a40ac0e2b --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http.rs @@ -0,0 +1,130 @@ +use crate::mesh::Node; +use anyhow::{Context, Result}; +use iroh::EndpointId; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +const TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES: usize = 8 * 1024; +const TUNNELED_HTTP_HEADER_READ_TIMEOUT: Duration = Duration::from_secs(5); + +#[cfg(test)] +mod tests; + +/// Handle an inbound HTTP tunnel bi-stream: connect to the local API proxy and relay. +pub(super) async fn handle_inbound_http_stream( + node: Node, + remote: EndpointId, + quic_send: iroh::endpoint::SendStream, + mut quic_recv: iroh::endpoint::RecvStream, + http_port: u16, +) -> Result<()> { + // Admission check for remote QUIC HTTP ingress. + match node + .activity_policy_guard + .check_admission(crate::runtime::IngressType::RemoteQuicHttp) + { + crate::runtime::AdmissionResult::Allowed => {} + crate::runtime::AdmissionResult::Paused { reason, .. } => { + tracing::debug!(reason, "Inbound HTTP tunnel rejected by activity policy"); + anyhow::bail!("remote inference paused: {reason}"); + } + } + + tracing::info!("Inbound HTTP tunnel stream -> API proxy :{http_port}"); + let mut tcp_stream = TcpStream::connect(format!("127.0.0.1:{http_port}")).await?; + tcp_stream.set_nodelay(true)?; + let _inflight = node.begin_inflight_request(); + + // Only raw mesh ingress that successfully claimed a parent emits the + // private assertion. Direct API requests have it stripped before they can + // reach this tunnel, so they retain normal target frontend ownership. + let prefix = read_tunneled_http_header_prefix(&mut quic_recv).await?; + let (prefix, _) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix(prefix); + let caller_metadata = + remote_tunnel_request_metadata(remote, node.authenticated_peer_path(remote).await); + let (attribution_request_id, suppression_request_id) = remote_tunnel_request_ids(&prefix); + let logging_state = crate::logging_runtime_state(); + let _remote_attribution = attribution_request_id.and_then(|request_id| { + logging_state + .as_ref() + .and_then(|state| state.attribute_remote_tunneled_request(request_id, caller_metadata)) + }); + let _remote_suppression = suppression_request_id.and_then(|request_id| { + logging_state + .as_ref() + .and_then(|state| state.suppress_remote_tunneled_request(request_id)) + }); + tcp_stream.write_all(&prefix).await?; + + let (tcp_read, tcp_write) = tokio::io::split(tcp_stream); + super::relay_bidirectional(tcp_read, tcp_write, quic_send, quic_recv).await +} + +fn remote_tunnel_request_ids( + prefix: &[u8], +) -> ( + Option, + Option, +) { + ( + crate::network::openai::request_parse::canonical_request_id_from_header_prefix(prefix), + crate::network::openai::request_parse::raw_lifecycle_owner_from_header_prefix(prefix), + ) +} + +fn remote_tunnel_request_metadata( + remote: EndpointId, + path: Option, +) -> crate::logging::RequestSummaryMetadata { + let endpoint_id = hex::encode(remote.as_bytes()); + let (addr, path_type) = match path { + Some(observation) if observation.path_type == "direct" => ( + observation + .observed_direct_remote_addr + .map(|addr| addr.to_string()), + Some(crate::logging::CallerPathType::RemoteQuicHttp), + ), + Some(observation) if observation.path_type == "relay" => { + (None, Some(crate::logging::CallerPathType::Relay)) + } + Some(_) | None => (None, None), + }; + crate::logging::RequestSummaryMetadata::default().with_caller_identity( + Some(&endpoint_id), + addr.as_deref(), + path_type, + ) +} + +/// Read at most one bounded HTTP header prefix without changing the bytes that +/// the existing relay will subsequently forward. A read may include a small +/// amount of body data from the same transport chunk; it is forwarded directly +/// to the local API proxy and is never retained by logging state. +async fn read_tunneled_http_header_prefix(reader: &mut R) -> Result> +where + R: AsyncRead + Unpin, +{ + tokio::time::timeout(TUNNELED_HTTP_HEADER_READ_TIMEOUT, async { + let mut prefix = Vec::with_capacity(TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES); + let mut chunk = [0u8; TUNNELED_HTTP_HEADER_READ_CHUNK_BYTES]; + let max_header_bytes = crate::network::openai::request_parse::MAX_HEADER_BYTES; + + while prefix.len() < max_header_bytes { + if crate::network::openai::request_parse::http_header_terminator(&prefix).is_some() { + break; + } + let read_cap = (max_header_bytes - prefix.len()).min(chunk.len()); + let bytes_read = reader.read(&mut chunk[..read_cap]).await?; + if bytes_read == 0 { + break; + } + prefix.extend_from_slice(&chunk[..bytes_read]); + } + + Ok(prefix) + }) + .await + .context("timed out reading tunneled HTTP header prefix")? +} diff --git a/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs new file mode 100644 index 0000000000..0ef57c37d4 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/tunnel/inbound_http/tests.rs @@ -0,0 +1,387 @@ +use super::*; +use std::time::Duration; +use tokio::io::AsyncWriteExt; + +#[test] +fn tunneled_request_metadata_uses_authenticated_peer_and_selected_direct_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x61; 32]).public()); + let direct_addr = "192.0.2.61:11204".parse().expect("direct address"); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "direct", + rtt_ms: Some(9), + observed_direct_remote_addr: Some(direct_addr), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert_eq!( + metadata.caller_addr(), + Some(direct_addr.to_string().as_str()) + ); + assert_eq!(metadata.caller_path_type(), Some("remote_quic_http")); +} + +#[test] +fn tunneled_request_metadata_omits_address_for_relay_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x62; 32]).public()); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "relay", + rtt_ms: Some(27), + observed_direct_remote_addr: Some( + "203.0.113.62:443".parse().expect("relay-shaped address"), + ), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert_eq!(metadata.caller_path_type(), Some("relay")); +} + +#[test] +fn tunneled_request_metadata_preserves_endpoint_when_path_is_missing() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x63; 32]).public()); + let metadata = remote_tunnel_request_metadata(remote, None); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert!(metadata.caller_path_type().is_none()); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[test] +fn tunneled_request_metadata_preserves_endpoint_for_unrecognized_path() { + let remote = EndpointId::from(iroh::SecretKey::from_bytes(&[0x64; 32]).public()); + let metadata = remote_tunnel_request_metadata( + remote, + Some(crate::mesh::SelectedPathObservation { + path_type: "unknown", + rtt_ms: None, + observed_direct_remote_addr: Some( + "198.51.100.64:443" + .parse() + .expect("unrecognized path address"), + ), + }), + ); + + assert_eq!( + metadata.caller_endpoint_id(), + Some(hex::encode(remote.as_bytes()).as_str()) + ); + assert!(metadata.caller_addr().is_none()); + assert!(metadata.caller_path_type().is_none()); + assert!(metadata.has_authenticated_remote_caller()); +} + +#[tokio::test] +async fn tunnel_prefetch_forwards_a_complete_bounded_header_prefix() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let request = b"POST /v1/chat/completions HTTP/1.1\r\nx-request-id: 550e8400-e29b-41d4-a716-446655440000\r\n\r\n"; + writer.write_all(request).await.unwrap(); + + let prefix = read_tunneled_http_header_prefix(&mut reader).await.unwrap(); + + assert_eq!(prefix, request); + assert!(prefix.len() <= crate::network::openai::request_parse::MAX_HEADER_BYTES); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_completes_on_lf_only_header_terminator() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let request = b"GET /v1/models HTTP/1.1\nHost: localhost\n\n"; + writer.write_all(request).await.unwrap(); + + let prefix = tokio::time::timeout( + Duration::from_secs(1), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("LF-only header pre-read must complete while the stream remains open") + .unwrap(); + + assert_eq!(prefix, request); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_times_out_when_peer_is_silent() { + let (_writer, mut reader) = tokio::io::duplex(4096); + + let result = tokio::time::timeout( + Duration::from_secs(6), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("header pre-read must finish within the regression guard"); + + let error = result.expect_err("a silent peer must exceed the header deadline"); + assert!( + error + .to_string() + .contains("timed out reading tunneled HTTP header prefix") + ); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_slow_partial_progress_does_not_reset_deadline() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let writer_task = tokio::spawn(async move { + for fragment in [ + b"POST /v1/responses HTTP/1.1\r\n".as_slice(), + b"Host: localhost\r\n".as_slice(), + b"Content-Length: 2\r\n".as_slice(), + b"\r\n{}".as_slice(), + ] { + writer.write_all(fragment).await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + } + }); + + let result = tokio::time::timeout( + Duration::from_secs(7), + read_tunneled_http_header_prefix(&mut reader), + ) + .await + .expect("header pre-read must finish within the regression guard"); + + let error = result.expect_err("partial progress must not reset the header deadline"); + assert!( + error + .to_string() + .contains("timed out reading tunneled HTTP header prefix") + ); + writer_task.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn tunnel_prefetch_preserves_fragmented_header_and_body_overread() { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let fragments: [&[u8]; 3] = [ + b"POST /v1/responses HTTP/1.1\r\n", + b"Host: localhost\r\nContent-Length: 2\r\n", + b"\r\n{}", + ]; + let writer_task = tokio::spawn(async move { + for fragment in fragments { + writer.write_all(fragment).await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); + + let prefix = read_tunneled_http_header_prefix(&mut reader) + .await + .expect("fragmented header must complete before the deadline"); + + assert_eq!( + prefix, + b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}" + ); + writer_task.await.unwrap(); +} + +#[test] +fn tunnel_prefix_without_request_id_gets_one_canonical_id() { + let original = b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}"; + + let (rewritten, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.to_vec(), + ); + + let request_id = request_id.expect("generated canonical request ID"); + assert_eq!( + remote_tunnel_request_ids(&rewritten), + (Some(request_id), None) + ); + assert_eq!( + rewritten + .windows(b"x-request-id:".len()) + .filter(|window| window.eq_ignore_ascii_case(b"x-request-id:")) + .count(), + 1 + ); + assert!(rewritten.ends_with(b"\r\n\r\n{}")); +} + +#[test] +fn lf_only_tunnel_prefix_inserts_request_id_on_its_own_header_line() { + let original = b"GET /v1/models HTTP/1.1\nHost: localhost\n\n"; + + let (rewritten, request_id) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.to_vec(), + ); + + let request_id = request_id.expect("generated canonical request ID"); + let mut headers = [httparse::EMPTY_HEADER; 8]; + let mut request = httparse::Request::new(&mut headers); + assert_eq!( + request + .parse(&rewritten) + .expect("rewritten request must parse"), + httparse::Status::Complete(rewritten.len()) + ); + assert_eq!( + request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("x-request-id")) + .count(), + 1 + ); + let request_id_line = format!("\nx-request-id: {}\n\n", request_id.as_uuid()); + assert!( + rewritten + .windows(request_id_line.len()) + .any(|window| window == request_id_line.as_bytes()) + ); + assert!(!rewritten.contains(&b'\r')); +} + +#[test] +fn tunnel_prefix_preserves_existing_canonical_request_id_byte_for_byte() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let original = format!( + "POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ) + .into_bytes(); + + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + + assert_eq!(selected, Some(request_id)); + assert_eq!(rewritten, original); +} + +#[test] +fn duplicate_or_malformed_tunnel_request_id_fails_open_without_rewrite() { + let first = mesh_llm_events::logging::identifiers::RequestId::new(); + let second = mesh_llm_events::logging::identifiers::RequestId::new(); + let duplicate = format!( + "GET /v1/models HTTP/1.1\r\nx-request-id: {}\r\nx-request-id: {}\r\n\r\n", + first.as_uuid(), + second.as_uuid() + ) + .into_bytes(); + let malformed = b"GET /v1/models HTTP/1.1\r\nx-request-id: client-controlled\r\n\r\n".to_vec(); + + for original in [duplicate, malformed] { + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + assert_eq!(selected, None); + assert_eq!(rewritten, original); + assert_eq!(remote_tunnel_request_ids(&rewritten), (None, None)); + } +} + +#[test] +fn full_bounded_tunnel_header_without_request_id_fails_open() { + let start = b"GET /v1/models HTTP/1.1\r\nx-padding: "; + let ending = b"\r\n\r\n"; + let padding_length = + crate::network::openai::request_parse::MAX_HEADER_BYTES - start.len() - ending.len(); + let mut original = start.to_vec(); + original.extend(std::iter::repeat_n(b'x', padding_length)); + original.extend_from_slice(ending); + + let (rewritten, selected) = + crate::network::openai::request_parse::ensure_canonical_request_id_in_header_prefix( + original.clone(), + ); + + assert_eq!( + original.len(), + crate::network::openai::request_parse::MAX_HEADER_BYTES + ); + assert_eq!(selected, None); + assert_eq!(rewritten, original); + assert_eq!(remote_tunnel_request_ids(&rewritten), (None, None)); +} + +#[test] +fn markerless_canonical_request_id_allows_attribution_without_suppression() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let prefix = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\n\r\n", + request_id.as_uuid() + ); + + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), None) + ); +} + +#[test] +fn near_limit_markerless_headers_keep_canonical_attribution() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let start = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\nx-padding: ", + request_id.as_uuid() + ); + let ending = "\r\n\r\n"; + let padding_length = + crate::network::openai::request_parse::MAX_HEADER_BYTES - start.len() - ending.len(); + let prefix = format!("{start}{}{ending}", "x".repeat(padding_length)); + + assert_eq!( + prefix.len(), + crate::network::openai::request_parse::MAX_HEADER_BYTES + ); + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), None) + ); +} + +#[test] +fn matching_private_marker_enables_single_parent_suppression() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let prefix = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {0}\r\n{1}: {0}\r\n\r\n", + request_id.as_uuid(), + crate::network::openai::request_parse::RAW_LIFECYCLE_OWNER_HEADER, + ); + + assert_eq!( + remote_tunnel_request_ids(prefix.as_bytes()), + (Some(request_id), Some(request_id)) + ); +} + +#[test] +fn malformed_incomplete_and_maximal_header_prefixes_fail_open() { + let request_id = mesh_llm_events::logging::identifiers::RequestId::new(); + let malformed = b"POST /v1/responses HTTP/1.1\r\nx-request-id: not-a-uuid\r\n\r\n"; + let incomplete = format!( + "POST /v1/responses HTTP/1.1\r\nx-request-id: {}\r\n", + request_id.as_uuid() + ); + let maximal = vec![b'x'; crate::network::openai::request_parse::MAX_HEADER_BYTES]; + + for prefix in [ + malformed.as_slice(), + incomplete.as_bytes(), + maximal.as_slice(), + ] { + assert_eq!(remote_tunnel_request_ids(prefix), (None, None)); + } +} diff --git a/crates/mesh-llm-host-runtime/tests/audit_test.rs b/crates/mesh-llm-host-runtime/tests/audit_test.rs index 48fd5b30c2..2ad00a1f75 100644 --- a/crates/mesh-llm-host-runtime/tests/audit_test.rs +++ b/crates/mesh-llm-host-runtime/tests/audit_test.rs @@ -1,8 +1,33 @@ use mesh_llm_events::audit::AuditLevel; use mesh_llm_events::audit::AuditLogFormat; +use mesh_llm_host_runtime::OperationalAuditSubjectKind; #[test] fn test_audit_types_exported() { let _ = AuditLogFormat::JsonLines; let _ = AuditLevel::Info; } + +#[test] +fn operational_audit_subject_kind_is_source_compatible() { + let subjects = [ + (OperationalAuditSubjectKind::Runtime, "runtime"), + (OperationalAuditSubjectKind::Model, "model"), + ( + OperationalAuditSubjectKind::RuntimeInstance, + "runtime_instance", + ), + (OperationalAuditSubjectKind::CliCommand, "cli_command"), + ]; + + for (subject, expected) in subjects { + // This exhaustive match forces the table above to change when the enum grows. + match subject { + OperationalAuditSubjectKind::Runtime + | OperationalAuditSubjectKind::Model + | OperationalAuditSubjectKind::RuntimeInstance + | OperationalAuditSubjectKind::CliCommand => {} + } + assert_eq!(subject.as_str(), expected); + } +} diff --git a/crates/mesh-llm-log-store/Cargo.toml b/crates/mesh-llm-log-store/Cargo.toml index f64074187a..7171f3fbbb 100644 --- a/crates/mesh-llm-log-store/Cargo.toml +++ b/crates/mesh-llm-log-store/Cargo.toml @@ -18,7 +18,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } data-encoding = "2.6" hex = "0.4" mesh-llm-events = { version = "0.76.0-rc7", path = "../mesh-llm-events" } -rusqlite = { version = "0.37", features = ["bundled"] } +rusqlite = { version = "0.37", features = ["bundled", "functions"] } serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/mesh-llm-log-store/README.md b/crates/mesh-llm-log-store/README.md index 81cbd6d8c3..f89ac5de75 100644 --- a/crates/mesh-llm-log-store/README.md +++ b/crates/mesh-llm-log-store/README.md @@ -8,3 +8,24 @@ artifact storage, and the audited maintenance operations used by the host runtime's trusted local logging APIs. The crate keeps persistence policy and storage details below the runtime and transport layers so callers can query, retain, and clean up logs without coupling those APIs to SQLite internals. + +## Schema lifecycle + +New empty databases are initialized atomically to the complete current schema. +Reopening a current database performs no schema work and preserves its data. A +database with unrecognized objects at version 0 is rejected without being reset +or migrated. + +The only public-schema compatibility exception is the exact released physical +schema with source `user_version` 3 or 11. Its tables, columns, indexes, foreign +keys, checks, partial-index predicates, and `AUTOINCREMENT` semantics must match +the released structure exactly, its `application_id` must be zero, and it must +not contain private lineage objects. The store then adds the three nullable +caller identity columns in place and atomically adopts the database into the +private lineage at epoch 1 without rebuilding tables or copying rows. + +Lookalikes, partial schemas, and the same physical schema under any source marker +other than 3 or 11 fail closed without mutation. Future private schema changes +must be registered as contiguous, forward-only steps; each step commits its +schema, data, and `user_version` together so a later failure can resume from the +last committed version. diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs index 462e2061d1..0768f4e6c3 100644 --- a/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/schema_lifecycle.rs @@ -1,11 +1,7 @@ use super::*; -// ════════════════════════════════════ -// MIGRATION TESTS -// ════════════════════════════════════ - #[test] -fn fresh_db_migrates_to_latest() { +fn fresh_db_initializes_to_current_version() { let (store, _, _tmp) = open_store(); assert_eq!(store.schema_version(), CURRENT_VERSION); } @@ -137,12 +133,12 @@ fn sqlite_root_database_and_sidecars_have_only_current_user_acl() { } #[test] -fn reopen_preserves_data_and_skips_migrations() { +fn current_schema_reopen_preserves_data_without_schema_work() { let tmp = tempfile::tempdir().expect("create temp dir"); let clock: Arc = Arc::new(TestClock::default()); // Insert data. - let store1 = LogStore::open(tmp.path(), clock.clone()).expect("open v1"); + let store1 = LogStore::open(tmp.path(), clock.clone()).expect("open current schema"); store1 .insert_summary( "s-001", @@ -173,7 +169,7 @@ fn reopen_preserves_data_and_skips_migrations() { } #[test] -fn migrations_are_idempotent_on_reopen() { +fn current_schema_is_a_noop_on_reopen() { let tmp = tempfile::tempdir().expect("create temp dir"); let clock: Arc = Arc::new(TestClock::default()); @@ -184,6 +180,123 @@ fn migrations_are_idempotent_on_reopen() { } } +#[test] +fn unknown_nonempty_version_zero_is_rejected_before_database_mutation() { + assert_unknown_schema_rejected_before_mutation(0); +} + +#[test] +fn unknown_future_version_is_rejected_before_database_mutation() { + assert_unknown_schema_rejected_before_mutation(2); +} + +#[test] +fn markerless_identified_version_one_is_rejected_before_wal_or_schema_mutation() { + let root = tempfile::tempdir().expect("database root"); + let database = root.path().join("log_store.db"); + let connection = rusqlite::Connection::open(&database).expect("open fixture database"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;", + ) + .expect("seed forged schema"); + drop(connection); + + let clock: Arc = Arc::new(TestClock::default()); + let error = match LogStore::open(root.path(), clock) { + Ok(_) => panic!("markerless schema must fail closed"), + Err(error) => error, + }; + + assert!(matches!( + error, + LogStoreError::SchemaIncompatible { + found: 1, + supported: 1 + } + )); + assert!(!root.path().join("log_store.db-wal").exists()); + assert!(!root.path().join("log_store.db-shm").exists()); + let connection = rusqlite::Connection::open(database).expect("reopen fixture database"); + assert_eq!( + connection + .pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) + .expect("journal mode"), + "delete" + ); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"), + 1 + ); +} + +fn assert_unknown_schema_rejected_before_mutation(version: u32) { + let root = tempfile::tempdir().expect("database root"); + let database = root.path().join("log_store.db"); + let connection = rusqlite::Connection::open(&database).expect("open fixture database"); + connection + .execute_batch(&format!( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); PRAGMA user_version = {version};" + )) + .expect("seed unknown schema"); + drop(connection); + + let clock: Arc = Arc::new(TestClock::default()); + let error = match LogStore::open(root.path(), clock) { + Ok(_) => panic!("unknown schema must fail closed"), + Err(error) => error, + }; + + assert!(matches!( + error, + LogStoreError::SchemaIncompatible { found, supported: 1 } if found == version + )); + let connection = rusqlite::Connection::open(database).expect("reopen fixture database"); + assert_eq!( + connection + .pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .expect("schema version"), + version + ); + assert_eq!( + connection + .pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) + .expect("journal mode"), + "delete" + ); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('summaries', 'audit_entries')", + [], + |row| row.get::<_, i64>(0), + ) + .expect("log table count"), + 0 + ); +} + // ════════════════════════════════════ // TERMINAL EVENT TRANSACTION TESTS // ════════════════════════════════════ diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs index 775d745a4e..258907b9dc 100644 --- a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs @@ -1,573 +1,5 @@ use super::*; -// ════════════════════════════════════ -// EMPTY / SINGLE ITEM PAGINATION TESTS -// ════════════════════════════════════ - -#[test] -fn empty_table_pagination() { - let (store, _, _tmp) = open_store(); - - let page = store.list_summaries(10, None).unwrap(); - assert!(page.items.is_empty()); - assert!(page.next_cursor.is_none()); - - // Also test lifecycle events and artifacts. - let ev_page = store.list_lifecycle_events(10, None).unwrap(); - assert!(ev_page.items.is_empty()); - - let art_page = store.list_artifact_pointers(10, None).unwrap(); - assert!(art_page.items.is_empty()); -} - -#[test] -fn single_item_pagination() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "only-one", - Some("llama3"), - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - - let page = store.list_summaries(10, None).unwrap(); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].request_id, "only-one"); - assert!(page.next_cursor.is_none()); -} - -// ════════════════════════════════════ -// SUMMARY STATUS COUNTS TEST -// ════════════════════════════════════ - -#[test] -fn summary_status_counts() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "s-active-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - - // Insert + terminal update. - store - .insert_summary( - "s-completed-1", - None, - Some("route-a"), - Some("provider-x"), - Some("engine-y"), - &clock.now(), - None, - None, - None, - ) - .unwrap(); - let payload = r#"{"type":"completed","status_code":200}"#; - store - .write_terminal_event( - "s-completed-1", - "evt-c1", - payload, - "completed", - Some(200), - &clock.now(), - ) - .unwrap(); - - // Failed terminal. - store - .insert_summary( - "s-failed-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - let failed_payload = r#"{"type":"failed","error":"timeout"}"#; - store - .write_terminal_event( - "s-failed-1", - "evt-f1", - failed_payload, - "failed", - None, - &clock.now(), - ) - .unwrap(); - - let counts = store.count_summaries_by_status().unwrap(); - assert_eq!(counts.len(), 3); // active, completed, failed states - - // Verify specific counts. - for (state, count) in &counts { - match state.as_str() { - "active" => assert_eq!(*count, 1), - "completed" => assert_eq!(*count, 1), - "failed" => assert_eq!(*count, 1), - _ => panic!("unexpected state: {}", state), - } - } -} - -// ════════════════════════════════════ -// HAPPY PATH INSERT + COUNT TESTS -// ════════════════════════════════════ - -#[test] -fn artifact_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_artifact_pointer( - "art-1", - "req-1", - &clock.now(), - "log", - Some(r#"{"size": 42}"#), - ) - .expect("insert artifact"); - - assert_eq!(store.count_table("artifact_pointers").unwrap(), 1); - - // Duplicate PK should fail with AlreadyExists. - let err = store - .insert_artifact_pointer("art-1", "req-1", &clock.now(), "log", None) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); -} - -#[test] -fn proxy_record_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_proxy_record( - "att-1", - "req-1", - &clock.now(), - "http://target.api", - Some("provider-x"), - Some("engine-y"), - Some(&clock.now()), - Some(&clock.now()), - Some(200), - None, - ) - .expect("insert proxy record"); - - assert_eq!(store.count_table("proxy_records").unwrap(), 1); - - // Duplicate PK fails. - let err = store - .insert_proxy_record( - "att-1", - "req-1", - &clock.now(), - "http://other.api", - None, - None, - None, - None, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); -} - -#[test] -fn audit_entry_insert_and_count() { - let (store, clock, _tmp) = open_store(); - - // With request_id. - store - .insert_summary( - "req-1", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_summary( - "req-2", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .unwrap(); - store - .insert_audit_entry( - "aud-1", - Some("req-1"), - &clock.now(), - "user-alice", - "model_added", - Some(r#"{"model":"llama3"}"#), - ) - .expect("insert audit with request_id"); - - // Without request_id (standalone). - store - .insert_audit_entry("aud-2", None, &clock.now(), "system", "startup", None) - .expect("insert audit without request_id"); - - assert_eq!(store.count_table("audit_entries").unwrap(), 2); - - // Duplicate PK fails. - let err = store - .insert_audit_entry( - "aud-1", - Some("req-1"), - &clock.now(), - "user-bob", - "other_action", - None, - ) - .unwrap_err(); - assert!(matches!(err, LogStoreError::AlreadyExists { .. })); - - // UNIQUE(request_id, entry_id) — same request_id + different entry_id should work. - store - .insert_audit_entry( - "aud-3", - Some("req-1"), - &clock.now(), - "user-carol", - "action_3", - None, - ) - .expect("different entry_id with same request_id is fine"); - - // UNIQUE(request_id, entry_id) — different request_id + different entry_id works. - store - .insert_audit_entry( - "aud-5", - Some("req-2"), - &clock.now(), - "user-carol", - "action_3", - None, - ) - .expect("different request and entry are fine"); - - // Different entry_id always works (entry_id is PK). - store - .insert_audit_entry( - "aud-4", - Some("req-1"), - &clock.now(), - "user-dave", - "action_4", - None, - ) - .expect("another unique entry_id with same request_id is fine"); - - assert_eq!(store.count_table("audit_entries").unwrap(), 5); -} - -#[test] -fn audit_entries_page_by_occurred_at_and_entry_id_without_detail_leakage() { - let (store, _, _tmp) = open_store(); - let timestamp = "2025-06-15T12:00:00Z"; - for entry_id in ["audit-0001", "audit-0003", "audit-0002"] { - store - .insert_audit_entry( - entry_id, - None, - timestamp, - "runtime", - "runtime_ready", - Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), - ) - .unwrap(); - } - - let first_page = store - .list_audit_entries(Some(2), None, AuditEntryFilters::default()) - .unwrap(); - assert_eq!( - first_page - .items - .iter() - .map(|entry| entry.entry_id.as_str()) - .collect::>(), - vec!["audit-0003", "audit-0002"] - ); - assert_eq!(first_page.items[0].severity, Some(AuditEntrySeverity::Info)); - assert!( - !format!("{:?}", first_page.items).contains("SENTINEL-AUDIT-SECRET"), - "detail_json must never cross the AuditEntryRow boundary" - ); - - let second_page = store - .list_audit_entries( - Some(2), - first_page.next_cursor.as_deref(), - AuditEntryFilters::default(), - ) - .unwrap(); - assert_eq!(second_page.items.len(), 1); - assert_eq!(second_page.items[0].entry_id, "audit-0001"); - assert!(second_page.next_cursor.is_none()); -} - -#[test] -fn audit_entries_reconcile_forward_by_durable_sequence() { - let (store, clock, tmp) = open_store(); - let external_writer = LogStore::open(tmp.path(), clock).expect("open independent writer"); - for entry_id in ["audit-sequence-1", "audit-sequence-2", "audit-sequence-3"] { - external_writer - .insert_audit_entry( - entry_id, - None, - "2026-08-12T12:00:00Z", - "runtime", - "runtime_ready", - Some(r#"{"severity":"info"}"#), - ) - .unwrap(); - } - - let first = store - .list_audit_entries_after_sequence(0, 2, AuditEntryFilters::default()) - .unwrap(); - assert_eq!( - first - .iter() - .map(|entry| entry.entry_id.as_str()) - .collect::>(), - ["audit-sequence-1", "audit-sequence-2"] - ); - let second = store - .list_audit_entries_after_sequence( - u64::try_from(first[1].sequence).unwrap(), - 2, - AuditEntryFilters::default(), - ) - .unwrap(); - assert_eq!(second[0].entry_id, "audit-sequence-3"); -} - -#[test] -fn audit_entries_project_only_versioned_bounded_context() { - let (store, _, _tmp) = open_store(); - store - .insert_audit_entry( - "audit-context-0001", - None, - "2026-08-12T12:00:00Z", - "runtime", - "runtime_model_ready", - Some( - r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"numeric_summaries":{"bytes":4096},"secret":"SENTINEL-AUDIT-SECRET"}"#, - ), - ) - .unwrap(); - - let page = store - .list_audit_entries(Some(1), None, AuditEntryFilters::default()) - .unwrap(); - let row = &page.items[0]; - assert_eq!(row.context_version, Some(1)); - assert_eq!(row.subject_kind.as_deref(), Some("model")); - assert_eq!(row.subject_id.as_deref(), Some("local-gguf/sha256-safe")); - assert_eq!(row.operation_id.as_deref(), Some("runtime-7")); - assert_eq!(row.outcome.as_deref(), Some("ready")); - assert_eq!(row.duration_ms, Some(42)); - assert_eq!(row.numeric_summaries.get("bytes"), Some(&4096)); - assert!(!format!("{row:?}").contains("SENTINEL-AUDIT-SECRET")); -} - -#[test] -fn audit_entry_query_rejects_malformed_cursor_and_out_of_range_limit() { - let (store, _, _tmp) = open_store(); - - let cursor_error = store - .list_audit_entries(Some(1), Some("not-a-cursor"), AuditEntryFilters::default()) - .unwrap_err(); - assert!(matches!(cursor_error, LogStoreError::CursorMalformed(_))); - - for limit in [Some(0), Some(101)] { - let limit_error = store - .list_audit_entries(limit, None, AuditEntryFilters::default()) - .unwrap_err(); - assert!(matches!(limit_error, LogStoreError::InvalidQuery(_))); - } -} - -#[test] -fn audit_entry_query_handles_nullable_request_empty_page_and_allowlisted_filters() { - let (store, _, _tmp) = open_store(); - let empty_page = store - .list_audit_entries(None, None, AuditEntryFilters::default()) - .unwrap(); - assert!(empty_page.items.is_empty()); - assert!(empty_page.next_cursor.is_none()); - - store - .insert_audit_entry( - "audit-null-request", - None, - "2025-06-15T12:00:00Z", - "runtime", - "runtime_ready", - Some(r#"{"severity":"info"}"#), - ) - .unwrap(); - store - .insert_audit_entry( - "audit-mesh-warning", - None, - "2025-06-15T12:00:01Z", - "mesh", - "mesh_quic_handler_failed", - Some(r#"{"severity":"warning"}"#), - ) - .unwrap(); - - let page = store - .list_audit_entries( - None, - None, - AuditEntryFilters { - source: Some(AuditEntrySource::Mesh), - severity: Some(AuditEntrySeverity::Warning), - ..AuditEntryFilters::default() - }, - ) - .unwrap(); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].entry_id, "audit-mesh-warning"); - assert!(page.items[0].request_id.is_none()); -} - -#[test] -fn nullable_column_conversion_errors_are_not_silenced() { - let (store, clock, _tmp) = open_store(); - store - .insert_summary( - "req-invalid-column", - None, - None, - None, - None, - &clock.now(), - None, - None, - None, - ) - .expect("insert summary"); - store - .conn() - .execute( - "UPDATE summaries SET status_code = 'not-an-integer' WHERE request_id = 'req-invalid-column'", - [], - ) - .expect("store invalid status code"); - - assert!(matches!( - store.get_summary("req-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_summaries(10, None), - Err(LogStoreError::QueryFailed(_)) - )); - - store - .conn() - .execute( - "UPDATE summaries SET status_code = NULL WHERE request_id = 'req-invalid-column'", - [], - ) - .expect("restore nullable summary column"); - store - .insert_artifact_pointer( - "art-invalid-column", - "req-invalid-column", - &clock.now(), - "log", - None, - ) - .expect("insert artifact pointer"); - store - .conn() - .execute( - "UPDATE artifact_pointers SET checksum = x'00' WHERE artifact_id = 'art-invalid-column'", - [], - ) - .expect("store invalid checksum type"); - - assert!(matches!( - store.get_artifact_pointer("art-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_artifact_pointers_for_request("req-invalid-column"), - Err(LogStoreError::QueryFailed(_)) - )); - assert!(matches!( - store.list_artifact_pointers(10, None), - Err(LogStoreError::QueryFailed(_)) - )); -} +mod basic; +mod query; +mod sanitization; diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs new file mode 100644 index 0000000000..e404aeb122 --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/basic.rs @@ -0,0 +1,326 @@ +use super::*; + +// ════════════════════════════════════ +// EMPTY / SINGLE ITEM PAGINATION TESTS +// ════════════════════════════════════ + +#[test] +fn empty_table_pagination() { + let (store, _, _tmp) = open_store(); + + let page = store.list_summaries(10, None).unwrap(); + assert!(page.items.is_empty()); + assert!(page.next_cursor.is_none()); + + // Also test lifecycle events and artifacts. + let ev_page = store.list_lifecycle_events(10, None).unwrap(); + assert!(ev_page.items.is_empty()); + + let art_page = store.list_artifact_pointers(10, None).unwrap(); + assert!(art_page.items.is_empty()); +} + +#[test] +fn single_item_pagination() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "only-one", + Some("llama3"), + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + + let page = store.list_summaries(10, None).unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].request_id, "only-one"); + assert!(page.next_cursor.is_none()); +} + +// ════════════════════════════════════ +// SUMMARY STATUS COUNTS TEST +// ════════════════════════════════════ + +#[test] +fn summary_status_counts() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "s-active-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + + // Insert + terminal update. + store + .insert_summary( + "s-completed-1", + None, + Some("route-a"), + Some("provider-x"), + Some("engine-y"), + &clock.now(), + None, + None, + None, + ) + .unwrap(); + let payload = r#"{"type":"completed","status_code":200}"#; + store + .write_terminal_event( + "s-completed-1", + "evt-c1", + payload, + "completed", + Some(200), + &clock.now(), + ) + .unwrap(); + + // Failed terminal. + store + .insert_summary( + "s-failed-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + let failed_payload = r#"{"type":"failed","error":"timeout"}"#; + store + .write_terminal_event( + "s-failed-1", + "evt-f1", + failed_payload, + "failed", + None, + &clock.now(), + ) + .unwrap(); + + let counts = store.count_summaries_by_status().unwrap(); + assert_eq!(counts.len(), 3); // active, completed, failed states + + // Verify specific counts. + for (state, count) in &counts { + match state.as_str() { + "active" => assert_eq!(*count, 1), + "completed" => assert_eq!(*count, 1), + "failed" => assert_eq!(*count, 1), + _ => panic!("unexpected state: {}", state), + } + } +} + +// ════════════════════════════════════ +// HAPPY PATH INSERT + COUNT TESTS +// ════════════════════════════════════ + +#[test] +fn artifact_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_artifact_pointer( + "art-1", + "req-1", + &clock.now(), + "log", + Some(r#"{"size": 42}"#), + ) + .expect("insert artifact"); + + assert_eq!(store.count_table("artifact_pointers").unwrap(), 1); + + // Duplicate PK should fail with AlreadyExists. + let err = store + .insert_artifact_pointer("art-1", "req-1", &clock.now(), "log", None) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); +} + +#[test] +fn proxy_record_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_proxy_record( + "att-1", + "req-1", + &clock.now(), + "http://target.api", + Some("provider-x"), + Some("engine-y"), + Some(&clock.now()), + Some(&clock.now()), + Some(200), + None, + ) + .expect("insert proxy record"); + + assert_eq!(store.count_table("proxy_records").unwrap(), 1); + + // Duplicate PK fails. + let err = store + .insert_proxy_record( + "att-1", + "req-1", + &clock.now(), + "http://other.api", + None, + None, + None, + None, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); +} + +#[test] +fn audit_entry_insert_and_count() { + let (store, clock, _tmp) = open_store(); + + // With request_id. + store + .insert_summary( + "req-1", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_summary( + "req-2", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .unwrap(); + store + .insert_audit_entry( + "aud-1", + Some("req-1"), + &clock.now(), + "user-alice", + "model_added", + Some(r#"{"model":"llama3"}"#), + ) + .expect("insert audit with request_id"); + + // Without request_id (standalone). + store + .insert_audit_entry("aud-2", None, &clock.now(), "system", "startup", None) + .expect("insert audit without request_id"); + + assert_eq!(store.count_table("audit_entries").unwrap(), 2); + + // Duplicate PK fails. + let err = store + .insert_audit_entry( + "aud-1", + Some("req-1"), + &clock.now(), + "user-bob", + "other_action", + None, + ) + .unwrap_err(); + assert!(matches!(err, LogStoreError::AlreadyExists { .. })); + + // UNIQUE(request_id, entry_id) — same request_id + different entry_id should work. + store + .insert_audit_entry( + "aud-3", + Some("req-1"), + &clock.now(), + "user-carol", + "action_3", + None, + ) + .expect("different entry_id with same request_id is fine"); + + // UNIQUE(request_id, entry_id) — different request_id + different entry_id works. + store + .insert_audit_entry( + "aud-5", + Some("req-2"), + &clock.now(), + "user-carol", + "action_3", + None, + ) + .expect("different request and entry are fine"); + + // Different entry_id always works (entry_id is PK). + store + .insert_audit_entry( + "aud-4", + Some("req-1"), + &clock.now(), + "user-dave", + "action_4", + None, + ) + .expect("another unique entry_id with same request_id is fine"); + + assert_eq!(store.count_table("audit_entries").unwrap(), 5); +} diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs new file mode 100644 index 0000000000..1695ad8f2b --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/query.rs @@ -0,0 +1,478 @@ +use super::*; + +#[test] +fn audit_entries_page_by_occurred_at_and_entry_id_without_detail_leakage() { + let (store, _, _tmp) = open_store(); + let timestamp = "2025-06-15T12:00:00Z"; + for entry_id in ["audit-0001", "audit-0003", "audit-0002"] { + store + .insert_audit_entry( + entry_id, + None, + timestamp, + "runtime", + "runtime_ready", + Some(r#"{"severity":"info","secret":"SENTINEL-AUDIT-SECRET"}"#), + ) + .unwrap(); + } + + let first_page = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + first_page + .items + .iter() + .map(|entry| entry.entry_id.as_str()) + .collect::>(), + vec!["audit-0003", "audit-0002"] + ); + assert_eq!(first_page.items[0].severity, Some(AuditEntrySeverity::Info)); + assert!( + !format!("{:?}", first_page.items).contains("SENTINEL-AUDIT-SECRET"), + "detail_json must never cross the AuditEntryRow boundary" + ); + + let second_page = store + .list_audit_entries( + Some(2), + first_page.next_cursor.as_deref(), + AuditEntryFilters::default(), + ) + .unwrap(); + assert_eq!(second_page.items.len(), 1); + assert_eq!(second_page.items[0].entry_id, "audit-0001"); + assert!(second_page.next_cursor.is_none()); +} + +#[test] +fn audit_entries_reconcile_forward_by_durable_sequence() { + let (store, clock, tmp) = open_store(); + let external_writer = LogStore::open(tmp.path(), clock).expect("open independent writer"); + for entry_id in ["audit-sequence-1", "audit-sequence-2", "audit-sequence-3"] { + external_writer + .insert_audit_entry( + entry_id, + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"severity":"info"}"#), + ) + .unwrap(); + } + + let first = store + .list_audit_entries_after_sequence(0, 2, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + first + .iter() + .map(|entry| entry.entry_id.as_str()) + .collect::>(), + ["audit-sequence-1", "audit-sequence-2"] + ); + let second = store + .list_audit_entries_after_sequence( + u64::try_from(first[1].sequence).unwrap(), + 2, + AuditEntryFilters::default(), + ) + .unwrap(); + assert_eq!(second[0].entry_id, "audit-sequence-3"); +} + +#[test] +fn audit_entries_project_only_versioned_bounded_context() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-context-0001", + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_model_ready", + Some( + r#"{"severity":"info","context_version":1,"subject_kind":"model","subject_id":"local-gguf/sha256-safe","operation_id":"runtime-7","outcome":"ready","duration_ms":42,"numeric_summaries":{"bytes":4096},"secret":"SENTINEL-AUDIT-SECRET"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entries(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + let row = &page.items[0]; + assert_eq!(row.context_version, Some(1)); + assert_eq!(row.subject_kind.as_deref(), Some("model")); + assert_eq!(row.subject_id.as_deref(), Some("local-gguf/sha256-safe")); + assert_eq!(row.operation_id.as_deref(), Some("runtime-7")); + assert_eq!(row.outcome.as_deref(), Some("ready")); + assert_eq!(row.duration_ms, Some(42)); + assert_eq!(row.numeric_summaries.get("bytes"), Some(&4096)); + assert!(!format!("{row:?}").contains("SENTINEL-AUDIT-SECRET")); +} + +#[test] +fn audit_entries_drop_malformed_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-malformed", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm gpus --draft run-benchmark --backend cuda"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_duplicate_command_summary_flags_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-duplicate", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(r#"{"context_version":1,"command_summary":"mesh-llm models list --json --json"}"#), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_deep_malformed_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-deep", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm load unload status discover rotate-key setup --port 1234"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items[0].command_summary.is_none()); +} + +#[test] +fn audit_entries_drop_noncanonical_and_raw_relay_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + for (request_id, summary) in [ + ( + "audit-command-summary-whitespace", + "mesh-llm models list --json", + ), + ( + "audit-command-summary-raw-relay", + "mesh-llm load name [REDACTED] --relay private-relay", + ), + ] { + store + .insert_audit_entry( + request_id, + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some(&format!( + r#"{{"context_version":1,"command_summary":"{summary}"}}"# + )), + ) + .unwrap(); + } + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .unwrap(); + assert!(page.items.iter().all(|item| item.command_summary.is_none())); +} + +#[test] +fn audit_entries_preserve_valid_command_summaries_at_durable_boundary() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-command-summary-valid", + None, + "2026-08-12T12:00:00Z", + "cli", + "command_completed", + Some( + r#"{"context_version":1,"command_summary":"mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]"}"#, + ), + ) + .unwrap(); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .unwrap(); + assert_eq!( + page.items[0].command_summary.as_deref(), + Some("mesh-llm runtime guardrails --mode metrics --port 41731 --root-relay [REDACTED]") + ); +} + +#[test] +fn audit_entries_preserve_direct_mesh_peer_identity_and_path() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-direct", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","remote_addr":"192.168.1.44:11204","path_type":"direct"}"#, + ), + ) + .expect("insert direct peer audit"); + + let page = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .expect("list direct peer audit"); + let row = &page.items[0]; + + assert_eq!(row.entry.subject_kind.as_deref(), Some("mesh_peer")); + assert_eq!( + row.entry.subject_id.as_deref(), + Some("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ); + assert_eq!(row.remote_addr.as_deref(), Some("192.168.1.44:11204")); + assert_eq!(row.path_type.as_deref(), Some("direct")); +} + +#[test] +fn audit_entries_omit_relay_address_and_parse_legacy_path_as_absent() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-relay", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-relay","remote_addr":"203.0.113.10:443","path_type":"relay"}"#, + ), + ) + .expect("insert relay peer audit"); + store + .insert_audit_entry( + "audit-legacy", + None, + "2026-08-12T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"context_version":1,"subject_kind":"runtime","subject_id":"host"}"#), + ) + .expect("insert legacy audit"); + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list relay and legacy audits"); + let relay = &page.items[0]; + let legacy = &page.items[1]; + + assert_eq!(relay.path_type.as_deref(), Some("relay")); + assert!(relay.remote_addr.is_none()); + assert!(legacy.path_type.is_none()); + assert!(legacy.remote_addr.is_none()); +} + +#[test] +fn audit_entries_drop_unknown_paths_and_invalid_direct_addresses() { + let (store, _, _tmp) = open_store(); + store + .insert_audit_entry( + "audit-peer-unknown-path", + None, + "2026-08-12T12:00:01Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-unknown","remote_addr":"192.168.1.44:11204","path_type":"proxy"}"#, + ), + ) + .expect("insert unknown peer path audit"); + store + .insert_audit_entry( + "audit-peer-invalid-address", + None, + "2026-08-12T12:00:00Z", + "mesh", + "mesh_quic_inbound_accepted", + Some( + r#"{"context_version":1,"subject_kind":"mesh_peer","subject_id":"peer-direct","remote_addr":"not-a-socket-address","path_type":"direct"}"#, + ), + ) + .expect("insert invalid direct address audit"); + + let page = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list bounded peer path audits"); + let unknown_path = &page.items[0]; + let invalid_address = &page.items[1]; + + assert!(unknown_path.path_type.is_none()); + assert!(unknown_path.remote_addr.is_none()); + assert_eq!(invalid_address.path_type.as_deref(), Some("direct")); + assert!(invalid_address.remote_addr.is_none()); +} + +#[test] +fn audit_entry_query_rejects_malformed_cursor_and_out_of_range_limit() { + let (store, _, _tmp) = open_store(); + + let cursor_error = store + .list_audit_entries(Some(1), Some("not-a-cursor"), AuditEntryFilters::default()) + .unwrap_err(); + assert!(matches!(cursor_error, LogStoreError::CursorMalformed(_))); + + for limit in [Some(0), Some(101)] { + let limit_error = store + .list_audit_entries(limit, None, AuditEntryFilters::default()) + .unwrap_err(); + assert!(matches!(limit_error, LogStoreError::InvalidQuery(_))); + } +} + +#[test] +fn audit_entry_query_handles_nullable_request_empty_page_and_allowlisted_filters() { + let (store, _, _tmp) = open_store(); + let empty_page = store + .list_audit_entries(None, None, AuditEntryFilters::default()) + .unwrap(); + assert!(empty_page.items.is_empty()); + assert!(empty_page.next_cursor.is_none()); + + store + .insert_audit_entry( + "audit-null-request", + None, + "2025-06-15T12:00:00Z", + "runtime", + "runtime_ready", + Some(r#"{"severity":"info"}"#), + ) + .unwrap(); + store + .insert_audit_entry( + "audit-mesh-warning", + None, + "2025-06-15T12:00:01Z", + "mesh", + "mesh_quic_handler_failed", + Some(r#"{"severity":"warning"}"#), + ) + .unwrap(); + + let page = store + .list_audit_entries( + None, + None, + AuditEntryFilters { + source: Some(AuditEntrySource::Mesh), + severity: Some(AuditEntrySeverity::Warning), + ..AuditEntryFilters::default() + }, + ) + .unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].entry_id, "audit-mesh-warning"); + assert!(page.items[0].request_id.is_none()); +} + +#[test] +fn nullable_column_conversion_errors_are_not_silenced() { + let (store, clock, _tmp) = open_store(); + store + .insert_summary( + "req-invalid-column", + None, + None, + None, + None, + &clock.now(), + None, + None, + None, + ) + .expect("insert summary"); + store + .conn() + .execute( + "UPDATE summaries SET status_code = 'not-an-integer' WHERE request_id = 'req-invalid-column'", + [], + ) + .expect("store invalid status code"); + + assert!(matches!( + store.get_summary("req-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_summaries(10, None), + Err(LogStoreError::QueryFailed(_)) + )); + + store + .conn() + .execute( + "UPDATE summaries SET status_code = NULL WHERE request_id = 'req-invalid-column'", + [], + ) + .expect("restore nullable summary column"); + store + .insert_artifact_pointer( + "art-invalid-column", + "req-invalid-column", + &clock.now(), + "log", + None, + ) + .expect("insert artifact pointer"); + store + .conn() + .execute( + "UPDATE artifact_pointers SET checksum = x'00' WHERE artifact_id = 'art-invalid-column'", + [], + ) + .expect("store invalid checksum type"); + + assert!(matches!( + store.get_artifact_pointer("art-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_artifact_pointers_for_request("req-invalid-column"), + Err(LogStoreError::QueryFailed(_)) + )); + assert!(matches!( + store.list_artifact_pointers(10, None), + Err(LogStoreError::QueryFailed(_)) + )); +} diff --git a/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs new file mode 100644 index 0000000000..289b5110e0 --- /dev/null +++ b/crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit/sanitization.rs @@ -0,0 +1,223 @@ +use super::*; + +fn home_directory() -> String { + home_directory_from(|variable| std::env::var(variable).ok()) + .expect("HOME or USERPROFILE should be available to the test") +} + +fn home_directory_from(mut variable_value: impl FnMut(&str) -> Option) -> Option { + ["HOME", "USERPROFILE"] + .into_iter() + .find_map(|variable| variable_value(variable).filter(|value| !value.is_empty())) +} + +#[test] +fn home_directory_ignores_empty_home_before_userprofile() { + let selected = home_directory_from(|variable| match variable { + "HOME" => Some(String::new()), + "USERPROFILE" => Some(r"C:\Users\operator".to_owned()), + _ => None, + }); + + assert_eq!(selected.as_deref(), Some(r"C:\Users\operator")); +} + +#[test] +fn insert_audit_entry_sanitizes_detail_before_sqlite_persistence() { + // Given: valid structured detail containing every sensitive value class. + let (store, clock, _tmp) = open_store(); + let home = home_directory(); + let detail = serde_json::json!({ + "safe": {"outcome": "completed", "count": 3}, + "nested": {"password": "store-secret-value"}, + "note": "request failed with Bearer store-credential-value", + "path": format!("{home}/private/audit.json"), + }) + .to_string(); + + // When: the public repository method persists the detail. + store + .insert_audit_entry( + "audit-sanitized-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&detail), + ) + .expect("insert audit detail"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-sanitized-detail'", + [], + |row| row.get(0), + ) + .expect("read raw audit detail"); + + // Then: the raw column retains safe structure without sensitive originals. + assert!(!raw_detail.contains("store-secret-value")); + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("store-credential-value")); + assert!(!raw_detail.contains(&home)); + let stored: serde_json::Value = + serde_json::from_str(&raw_detail).expect("stored detail remains valid JSON"); + assert_eq!(stored["safe"]["outcome"], "completed"); + assert_eq!(stored["safe"]["count"], 3); +} + +#[test] +fn insert_audit_entry_sanitizes_actor_and_action_before_sqlite_persistence() { + let (store, clock, _tmp) = open_store(); + let home = home_directory(); + let source = format!("{home}/runtime\nsource"); + let code = format!("Bearer {}", "x".repeat(1_100)); + + store + .insert_audit_entry( + "audit-sanitized-scalars", + None, + &clock.now(), + &source, + &code, + None, + ) + .expect("insert audit scalars"); + store + .insert_audit_entry( + "audit-bounded-scalars", + None, + &clock.now(), + "runtime", + &"x".repeat(1_100), + None, + ) + .expect("insert oversized audit scalar"); + let stored: (String, String) = store + .conn() + .query_row( + "SELECT actor, action FROM audit_entries WHERE entry_id = 'audit-sanitized-scalars'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("read raw audit scalars"); + + assert_eq!(stored.0, "~/runtimesource"); + assert_eq!(stored.1, "[REDACTED]"); + assert!(!stored.0.contains(&home)); + assert!(stored.0.chars().count() <= 1_024); + assert!(stored.1.chars().count() <= 1_024); + let bounded: String = store + .conn() + .query_row( + "SELECT action FROM audit_entries WHERE entry_id = 'audit-bounded-scalars'", + [], + |row| row.get(0), + ) + .expect("read bounded audit scalar"); + assert!(bounded.ends_with("... [TRUNCATED]")); + assert!(bounded.chars().count() <= 1_024); +} + +#[test] +fn insert_audit_entry_replaces_malformed_sensitive_detail_before_sqlite_persistence() { + // Given: malformed JSON containing a credential marker and value. + let (store, clock, _tmp) = open_store(); + let malformed = r#"{"note":"Bearer malformed-store-credential""#; + + // When: the public repository method persists the detail. + store + .insert_audit_entry( + "audit-malformed-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(malformed), + ) + .expect("insert malformed audit detail"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-malformed-detail'", + [], + |row| row.get(0), + ) + .expect("read raw malformed audit detail"); + + // Then: SQLite contains only the fixed valid-JSON replacement. + assert_eq!(raw_detail, r#""[REDACTED]""#); + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("malformed-store-credential")); +} + +#[test] +fn insert_audit_entry_bounds_raw_detail_and_sanitizes_colliding_keys_before_persistence() { + // Given: oversized raw detail plus valid detail with unsafe and colliding oversized keys. + let (store, clock, _tmp) = open_store(); + let home = home_directory(); + let oversized_raw = serde_json::json!({ + "sentinel": "RAW-SQLITE-OVERSIZED-SENTINEL", + "padding": "x".repeat(256 * 1024), + }) + .to_string(); + let collision_prefix = "k".repeat(1_024); + let unsafe_keys = serde_json::json!({ + format!("{home}/private/audit\nkey"): "path-value", + "Bearer raw-store-key-private": "sensitive-field-value", + format!("{collision_prefix}a"): "collision-a", + format!("{collision_prefix}b"): "collision-b", + }) + .to_string(); + + // When: the public repository method persists both raw detail documents. + store + .insert_audit_entry( + "audit-oversized-raw-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&oversized_raw), + ) + .expect("insert oversized raw detail"); + store + .insert_audit_entry( + "audit-unsafe-key-detail", + None, + &clock.now(), + "runtime", + "runtime_ready", + Some(&unsafe_keys), + ) + .expect("insert unsafe key detail"); + let oversized_stored: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-oversized-raw-detail'", + [], + |row| row.get(0), + ) + .expect("read oversized raw detail"); + let keys_stored: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = 'audit-unsafe-key-detail'", + [], + |row| row.get(0), + ) + .expect("read unsafe key detail"); + + // Then: SQLite retains no oversized sentinel or unsafe key text and keeps both safe values. + assert_eq!(oversized_stored, r#""[TRUNCATED]""#); + assert!(!oversized_stored.contains("RAW-SQLITE-OVERSIZED-SENTINEL")); + assert!(!keys_stored.contains(&home)); + assert!(!keys_stored.contains("Bearer")); + assert!(!keys_stored.contains("raw-store-key-private")); + let stored: serde_json::Value = + serde_json::from_str(&keys_stored).expect("stored detail remains valid JSON"); + let object = stored.as_object().expect("stored detail remains an object"); + assert!(object.keys().all(|key| key.chars().count() <= 1_024)); + assert!(object.values().any(|value| value == "collision-a")); + assert!(object.values().any(|value| value == "collision-b")); +} diff --git a/crates/mesh-llm-log-store/src/audit_detail.rs b/crates/mesh-llm-log-store/src/audit_detail.rs new file mode 100644 index 0000000000..fa11137950 --- /dev/null +++ b/crates/mesh-llm-log-store/src/audit_detail.rs @@ -0,0 +1,16 @@ +use crate::repositories::AuditEntryRow; + +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AuditEntryDetail { + pub entry: AuditEntryRow, + pub remote_addr: Option, + pub path_type: Option, + pub command_summary: Option, +} + +impl From for AuditEntryRow { + fn from(detail: AuditEntryDetail) -> Self { + detail.entry + } +} diff --git a/crates/mesh-llm-log-store/src/lib.rs b/crates/mesh-llm-log-store/src/lib.rs index 9879897af4..e01edbc985 100644 --- a/crates/mesh-llm-log-store/src/lib.rs +++ b/crates/mesh-llm-log-store/src/lib.rs @@ -18,6 +18,7 @@ mod space_maintenance_tests; mod artifact_privacy; mod artifact_repository; mod artifacts; +mod audit_detail; mod capture; mod cursor; mod error; @@ -25,6 +26,7 @@ mod maintenance; mod migrations; mod query; mod repositories; +mod schema; mod store; mod timestamps; @@ -35,6 +37,7 @@ pub use artifact_repository::UnavailableArtifactPointer; pub use artifacts::{ ArtifactContent, ArtifactFileStore, ArtifactRedactor, ArtifactStatus, ArtifactWriteReceipt, }; +pub use audit_detail::AuditEntryDetail; pub use capture::{ ARTIFACT_CAPTURE_DISABLED_PRIVACY_UNAVAILABLE, ArtifactCaptureDisabledReason, ArtifactCaptureHealthMarker, ArtifactCaptureOutcome, FailOpenArtifactCapture, @@ -50,7 +53,7 @@ pub use maintenance::{ pub use migrations::CURRENT_VERSION as LOG_STORE_SCHEMA_VERSION; pub use query::{ ArtifactRecord, EventRecord, MAX_QUERY_LIMIT, PageQuery, ProxyQuery, ProxyRecord, QueryPage, - QuerySort, RequestOutcome, RequestQuery, RequestRecord, + QuerySort, RequestOutcome, RequestQuery, RequestRecord, RequestRecordWithCaller, }; pub use repositories::{ AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, CascadeArtifactPointer, diff --git a/crates/mesh-llm-log-store/src/maintenance/execution.rs b/crates/mesh-llm-log-store/src/maintenance/execution.rs index d02f424bc8..dd784910dc 100644 --- a/crates/mesh-llm-log-store/src/maintenance/execution.rs +++ b/crates/mesh-llm-log-store/src/maintenance/execution.rs @@ -922,18 +922,20 @@ fn write_audit( reason: &str, ) -> Result<(), LogStoreError> { let occurred_at = canonical_persisted_timestamp(occurred_at)?; - let detail = serde_json::json!({ - "actor": "trusted_local_operator", - "source": "logs_api", - "result": result, - "reason": reason, - "operationId": operation_id.to_string(), - }) - .to_string(); + let detail = mesh_llm_events::audit::SanitizedAuditDetailJson::sanitize( + &serde_json::json!({ + "actor": "trusted_local_operator", + "source": "logs_api", + "result": result, + "reason": reason, + "operationId": operation_id.to_string(), + }) + .to_string(), + ); transaction .execute( "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?1, NULL, ?2, ?3, ?4, ?5)", - rusqlite::params![entry_id, occurred_at, "logs_api", action, detail], + rusqlite::params![entry_id, occurred_at, "logs_api", action, detail.as_str()], ) .map_err(LogStoreError::Sqlite)?; Ok(()) diff --git a/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs b/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs index c05c589d82..22fcd33b88 100644 --- a/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs +++ b/crates/mesh-llm-log-store/src/maintenance/tests/cleanup.rs @@ -32,6 +32,48 @@ fn preview_snapshots_bounded_targets_and_rejects_invalid_typed_input() { ); } +#[test] +fn preview_cleanup_sanitizes_audit_detail_inside_transaction() { + // Given: a valid maintenance request whose reason contains a credential and home path. + let (_root, artifacts) = fixture(); + let store = artifacts.store_ref(); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let request = CleanupPreviewRequest { + reason: MaintenanceReason::try_from( + format!("Bearer maintenance-credential at {home}/private/audit.json").as_str(), + ) + .expect("valid maintenance reason"), + ..request(32, "2025-02-01T00:00:00Z") + }; + + // When: preview writes its audit row in the receipt transaction. + let receipt = store + .preview_cleanup(&request, &NeverCancelled) + .expect("preview cleanup"); + let raw_detail: String = store + .conn() + .query_row( + "SELECT detail_json FROM audit_entries WHERE entry_id = ?1", + [receipt + .preview_audit_id + .as_deref() + .expect("preview audit ID")], + |row| row.get(0), + ) + .expect("raw maintenance audit detail"); + + // Then: sensitive originals are absent and safe transaction fields remain structured. + assert!(!raw_detail.contains("Bearer")); + assert!(!raw_detail.contains("maintenance-credential")); + assert!(!raw_detail.contains(&home)); + let stored: serde_json::Value = + serde_json::from_str(&raw_detail).expect("maintenance detail remains valid JSON"); + assert_eq!(stored["actor"], "trusted_local_operator"); + assert_eq!(stored["source"], "logs_api"); + assert_eq!(stored["result"], "previewed"); + assert_eq!(stored["operationId"], request.operation_id.to_string()); +} + #[test] fn cleanup_targets_visible_requests_instead_of_hidden_management_traffic() { let (_root, artifacts) = fixture(); diff --git a/crates/mesh-llm-log-store/src/migrations.rs b/crates/mesh-llm-log-store/src/migrations.rs index 68586634f0..de207bcc45 100644 --- a/crates/mesh-llm-log-store/src/migrations.rs +++ b/crates/mesh-llm-log-store/src/migrations.rs @@ -1,903 +1,180 @@ -//! Forward-only schema migrations for log_store. +//! Initial schema installation and future forward migrations for log_store. -use rusqlite::Connection; +use rusqlite::{Connection, Transaction, TransactionBehavior}; -mod legacy_v10; +mod lineage; +mod released_schema; -/// Current forward-only schema version for the integrated local logging feature. -/// -/// Versions 4 through 10 were used by pre-integration development builds. The -/// version remains monotonic so a v10 development store can be identified and -/// upgraded without mistaking it for an unknown future schema. -pub const CURRENT_VERSION: u32 = 11; +type MigrationFn = fn(&Connection) -> Result<(), rusqlite::Error>; -const MIGRATIONS_V1: &str = r#" -CREATE TABLE IF NOT EXISTS summaries ( - request_id TEXT PRIMARY KEY, - state TEXT NOT NULL DEFAULT 'active' - CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), - created_at TEXT NOT NULL, - terminal_at TEXT, - route TEXT, - model TEXT, - provider TEXT, - engine TEXT, - status_code INTEGER, - error_msg TEXT, - tenant_id TEXT, - account_id TEXT, - user_id TEXT -); - -CREATE INDEX IF NOT EXISTS idx_summaries_created ON summaries (created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_state ON summaries (state); - -CREATE TABLE IF NOT EXISTS lifecycle_events ( - event_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - payload_json TEXT NOT NULL DEFAULT '{}', - event_type TEXT NOT NULL DEFAULT 'unknown', - is_terminal INTEGER NOT NULL DEFAULT 0 - CHECK (is_terminal IN (0, 1)), - - UNIQUE(request_id, event_id) -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_terminal_event_one_per_request -ON lifecycle_events (request_id) -WHERE is_terminal = 1; - -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request ON lifecycle_events (request_id); -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); - -CREATE TABLE IF NOT EXISTS artifact_pointers ( - artifact_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - kind TEXT NOT NULL, - metadata_json TEXT, - media_kind TEXT, - checksum TEXT, - bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), - version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), - redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), - truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), - stored_at TEXT, - missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), - corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), - - UNIQUE(request_id, artifact_id) -); - -CREATE INDEX IF NOT EXISTS idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); - -CREATE TABLE IF NOT EXISTS proxy_records ( - attempt_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, - target TEXT NOT NULL, - provider TEXT, - engine TEXT, - started_at TEXT, - completed_at TEXT, - status_code INTEGER, - error_msg TEXT, - - UNIQUE(request_id, attempt_id) -); - -CREATE INDEX IF NOT EXISTS idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); - -CREATE TABLE IF NOT EXISTS audit_entries ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), - entry_id TEXT NOT NULL UNIQUE, - request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, - actor TEXT NOT NULL, - action TEXT NOT NULL, - detail_json TEXT, - - UNIQUE(request_id, entry_id) -); - -CREATE INDEX IF NOT EXISTS idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); - -CREATE TABLE IF NOT EXISTS webhook_deliveries ( - delivery_id TEXT PRIMARY KEY, - request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - terminal_outcome TEXT NOT NULL - CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), - terminal_status_code INTEGER - CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), - occurred_at TEXT NOT NULL, - target_url TEXT NOT NULL, - attempt_number INTEGER NOT NULL DEFAULT 0 - CHECK (attempt_number BETWEEN 0 AND 20), - status_code INTEGER - CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), - response_body TEXT, - error_msg TEXT, - state TEXT NOT NULL DEFAULT 'succeeded' - CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), - created_at TEXT NOT NULL DEFAULT '', - updated_at TEXT NOT NULL DEFAULT '', - next_attempt_at TEXT, - lease_expires_at TEXT, - claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), - max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), - last_error_code TEXT - CHECK (last_error_code IS NULL OR last_error_code IN - ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), - - UNIQUE(request_id, delivery_id) -); - -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_eligible -ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); - -CREATE TABLE IF NOT EXISTS cleanup_runs ( - run_id TEXT PRIMARY KEY, - occurred_at TEXT NOT NULL, - policy_name TEXT NOT NULL, - cutoff_before TEXT NOT NULL, - deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), - duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) -); - -CREATE INDEX IF NOT EXISTS idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); - -CREATE TABLE IF NOT EXISTS maintenance_operations ( - operation_id TEXT PRIMARY KEY, - action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), - cutoff_before TEXT NOT NULL, - request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), - reason TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), - planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), - planned_events INTEGER NOT NULL CHECK (planned_events >= 0), - planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), - planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), - planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), - executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), - executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), - executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), - executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), - executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), - has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), - created_at TEXT NOT NULL, - completed_at TEXT, - selection_fingerprint TEXT NOT NULL DEFAULT '', - artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), - artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), - artifact_file_failure_class TEXT - CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), - preview_audit_id TEXT, - execution_audit_id TEXT, - cleanup_filters_json TEXT NOT NULL DEFAULT '{}' -); - -CREATE TABLE IF NOT EXISTS maintenance_operation_targets ( - operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - request_id TEXT NOT NULL, - PRIMARY KEY (operation_id, request_id), - UNIQUE (operation_id, ordinal) -); - -CREATE INDEX IF NOT EXISTS idx_maintenance_operation_targets_operation -ON maintenance_operation_targets (operation_id, ordinal); - -CREATE TABLE IF NOT EXISTS pending_artifact_deletions ( - artifact_id TEXT PRIMARY KEY, - request_id TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_pending_artifact_deletions_request -ON pending_artifact_deletions (request_id, artifact_id); -"#; - -// Query and retention indexes are intentionally added in a second migration: -// V1 is already durable in the field, and these indexes must be available to -// upgraded stores without rebuilding their logging history. -const MIGRATIONS_V2: &str = r#" -CREATE INDEX IF NOT EXISTS idx_summaries_terminal_order -ON summaries (state, COALESCE(terminal_at, created_at), request_id); -CREATE INDEX IF NOT EXISTS idx_summaries_route_created -ON summaries (route, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_model_created -ON summaries (model, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_provider_created -ON summaries (provider, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_engine_created -ON summaries (engine, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_status_created -ON summaries (status_code, created_at DESC, request_id DESC); -CREATE INDEX IF NOT EXISTS idx_summaries_state_created -ON summaries (state, created_at DESC, request_id DESC); - -CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_occurred -ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); -CREATE INDEX IF NOT EXISTS idx_artifact_pointers_request_occurred -ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); - -CREATE INDEX IF NOT EXISTS idx_proxy_records_request_occurred -ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_provider_occurred -ON proxy_records (provider, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_engine_occurred -ON proxy_records (engine, occurred_at DESC, attempt_id DESC); -CREATE INDEX IF NOT EXISTS idx_proxy_records_status_occurred -ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); - -CREATE INDEX IF NOT EXISTS idx_audit_entries_actor_occurred -ON audit_entries (actor, occurred_at DESC, entry_id DESC); -CREATE INDEX IF NOT EXISTS idx_audit_entries_severity_occurred -ON audit_entries ( - CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, - occurred_at DESC, - entry_id DESC -); +#[derive(Clone, Copy)] +struct Migration { + version: u32, + apply: MigrationFn, +} -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_ready -ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) -WHERE state IN ('pending', 'retry', 'manual_retry'); -CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_expired_lease -ON webhook_deliveries (lease_expires_at, created_at, delivery_id) -WHERE state = 'in_flight'; -"#; +struct MigrationPlan<'a> { + target: u32, + initialize: MigrationFn, + migrations: &'a [Migration], +} -// Artifact bodies can be intentionally unavailable even though their -// metadata record is healthy. Keep that state separate from the existing -// missing/corrupt file-health flags so API consumers never have to infer an -// intentional omission from a null checksum. -const MIGRATIONS_V3: &str = r#" -ALTER TABLE artifact_pointers ADD COLUMN unavailable_reason TEXT - CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( - 'streaming_response_not_assembled', - 'response_body_not_bounded', - 'capture_content_limit_exceeded', - 'capture_memory_budget_exceeded', - 'artifact_capture_disabled', - 'artifact_capture_failed' - )); -"#; +const MIGRATIONS: &[Migration] = &[]; +const APPLICATION_ID: u32 = 0x4D4C4F47; -/// Install the complete V1 schema and its version marker atomically. -pub fn apply_migrations(conn: &Connection) -> Result<(), rusqlite::Error> { - let current_ver: i32 = conn.query_row("PRAGMA user_version", [], |r| r.get(0))?; +/// Current schema version for the integrated local logging feature. +pub const CURRENT_VERSION: u32 = 1; - if !accepted_schema_version(conn, current_ver as u32)? { - return Err(rusqlite::Error::InvalidQuery); - } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SchemaClassification { + Fresh, + Private { version: u32 }, + ReleasedSchema { source_version: u32 }, + Incompatible { found: u32 }, +} - match current_ver { - 0 => { - apply_migration_transactionally(conn, 1, MIGRATIONS_V1)?; - apply_migration_transactionally(conn, 2, MIGRATIONS_V2)?; - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 1 => { - apply_migration_transactionally(conn, 2, MIGRATIONS_V2)?; - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 2 => { - apply_migration_transactionally(conn, 3, MIGRATIONS_V3)?; - mark_current(conn) - } - 3 => mark_current(conn), - legacy_v10::LEGACY_VERSION => legacy_v10::migrate(conn), - _ => Ok(()), - } +pub fn apply_migrations(connection: &Connection) -> Result<(), rusqlite::Error> { + run_migrations( + connection, + MigrationPlan { + target: CURRENT_VERSION, + initialize: crate::schema::initialize, + migrations: MIGRATIONS, + }, + ) } -/// Return the version pair when the database cannot be migrated safely. -/// -/// The v10 development lineage is accepted only when its structural -/// fingerprint matches. This check is read-only so callers can surface an -/// actionable compatibility state before any migration is attempted. pub(crate) fn incompatible_schema( - conn: &Connection, + connection: &Connection, ) -> Result, rusqlite::Error> { - let found: u32 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?; - let supported = CURRENT_VERSION; - let compatible = accepted_schema_version(conn, found)?; - Ok((!compatible).then_some((found, supported))) -} - -/// Whether `version` is a schema the store can open: a blank-slate version -/// the migration steps can upgrade, the current version, or the -/// fingerprint-guarded v10 development lineage. -fn accepted_schema_version(conn: &Connection, version: u32) -> Result { - Ok(matches!(version, 0..=3) - || version == CURRENT_VERSION - || (version == legacy_v10::LEGACY_VERSION as u32 && legacy_v10::matches(conn)?)) -} - -fn mark_current(conn: &Connection) -> Result<(), rusqlite::Error> { - apply_migration_transactionally(conn, CURRENT_VERSION as i32, "") + incompatible_schema_for_target(connection, CURRENT_VERSION) } -fn apply_migration_transactionally( - conn: &Connection, - version: i32, - migration: &str, -) -> Result<(), rusqlite::Error> { - let transaction = conn.unchecked_transaction()?; - transaction.execute_batch(migration)?; - transaction.execute_batch(&format!("PRAGMA user_version = {version}"))?; - transaction.commit() +fn incompatible_schema_for_target( + connection: &Connection, + target: u32, +) -> Result, rusqlite::Error> { + let classification = classify_schema(connection)?; + let compatible = match classification { + SchemaClassification::Fresh => true, + SchemaClassification::Private { version } => version <= target, + SchemaClassification::ReleasedSchema { .. } => target == CURRENT_VERSION, + SchemaClassification::Incompatible { .. } => false, + }; + Ok((!compatible).then_some((schema_version(connection)?, target))) } -#[cfg(test)] -mod tests { - use super::*; - - fn table_columns(connection: &Connection, table: &str) -> Vec { - let mut statement = connection - .prepare(&format!("PRAGMA table_info('{table}')")) - .expect("prepare table info"); - statement - .query_map([], |row| row.get(1)) - .expect("query table info") - .collect::, _>>() - .expect("collect table columns") +fn classify_schema(connection: &Connection) -> Result { + if let Some(source_version) = released_schema::source_version(connection)? { + return Ok(SchemaClassification::ReleasedSchema { source_version }); } - - fn schema_object_names(connection: &Connection, object_type: &str) -> Vec { - let mut statement = connection - .prepare( - "SELECT name FROM sqlite_master \ - WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", - ) - .expect("prepare schema object query"); - statement - .query_map([object_type], |row| row.get(0)) - .expect("query schema objects") - .collect::, _>>() - .expect("collect schema objects") + let found = schema_version(connection)?; + let identity = application_id(connection)?; + if found == 0 && identity == 0 && !has_user_objects(connection)? { + return Ok(SchemaClassification::Fresh); } - - fn schema_sql(connection: &Connection, object_type: &str, name: &str) -> String { - connection - .query_row( - "SELECT sql FROM sqlite_master WHERE type = ?1 AND name = ?2", - [object_type, name], - |row| row.get(0), - ) - .expect("schema SQL") - } - - const EXPECTED_TABLES: &[&str] = &[ - "artifact_pointers", - "audit_entries", - "cleanup_runs", - "lifecycle_events", - "maintenance_operation_targets", - "maintenance_operations", - "pending_artifact_deletions", - "proxy_records", - "summaries", - "webhook_deliveries", - ]; - - const EXPECTED_TABLE_COLUMNS: &[(&str, &[&str])] = &[ - ( - "summaries", - &[ - "request_id", - "state", - "created_at", - "terminal_at", - "route", - "model", - "provider", - "engine", - "status_code", - "error_msg", - "tenant_id", - "account_id", - "user_id", - ], - ), - ( - "lifecycle_events", - &[ - "event_id", - "request_id", - "occurred_at", - "payload_json", - "event_type", - "is_terminal", - ], - ), - ( - "artifact_pointers", - &[ - "artifact_id", - "request_id", - "occurred_at", - "kind", - "metadata_json", - "media_kind", - "checksum", - "bytes", - "version", - "redacted", - "truncated", - "stored_at", - "missing", - "corrupt", - "unavailable_reason", - ], - ), - ( - "proxy_records", - &[ - "attempt_id", - "request_id", - "occurred_at", - "target", - "provider", - "engine", - "started_at", - "completed_at", - "status_code", - "error_msg", - ], - ), - ( - "audit_entries", - &[ - "sequence", - "entry_id", - "request_id", - "occurred_at", - "actor", - "action", - "detail_json", - ], - ), - ( - "webhook_deliveries", - &[ - "delivery_id", - "request_id", - "terminal_outcome", - "terminal_status_code", - "occurred_at", - "target_url", - "attempt_number", - "status_code", - "response_body", - "error_msg", - "state", - "created_at", - "updated_at", - "next_attempt_at", - "lease_expires_at", - "claim_generation", - "max_attempts", - "last_error_code", - ], - ), - ( - "cleanup_runs", - &[ - "run_id", - "occurred_at", - "policy_name", - "cutoff_before", - "deleted_count", - "duration_ms", - ], - ), - ( - "maintenance_operations", - &[ - "operation_id", - "action", - "cutoff_before", - "request_limit", - "reason", - "state", - "planned_requests", - "planned_events", - "planned_artifacts", - "planned_proxy_records", - "planned_database_rows", - "executed_requests", - "executed_events", - "executed_artifacts", - "executed_proxy_records", - "executed_database_rows", - "has_more", - "created_at", - "completed_at", - "selection_fingerprint", - "artifact_files_removed", - "artifact_files_failed", - "artifact_file_failure_class", - "preview_audit_id", - "execution_audit_id", - "cleanup_filters_json", - ], - ), - ( - "maintenance_operation_targets", - &["operation_id", "ordinal", "request_id"], - ), - ("pending_artifact_deletions", &["artifact_id", "request_id"]), - ]; - - const EXPECTED_INDEXES: &[&str] = &[ - "idx_artifact_pointers_occurred", - "idx_artifact_pointers_request_occurred", - "idx_audit_entries_actor_occurred", - "idx_audit_entries_occurred", - "idx_audit_entries_severity_occurred", - "idx_cleanup_runs_occurred", - "idx_lifecycle_events_occurred", - "idx_lifecycle_events_request", - "idx_lifecycle_events_request_occurred", - "idx_lifecycle_events_request_terminal", - "idx_maintenance_operation_targets_operation", - "idx_pending_artifact_deletions_request", - "idx_proxy_records_engine_occurred", - "idx_proxy_records_occurred", - "idx_proxy_records_provider_occurred", - "idx_proxy_records_request_occurred", - "idx_proxy_records_status_occurred", - "idx_summaries_created", - "idx_summaries_engine_created", - "idx_summaries_model_created", - "idx_summaries_provider_created", - "idx_summaries_route_created", - "idx_summaries_state", - "idx_summaries_state_created", - "idx_summaries_status_created", - "idx_summaries_terminal_order", - "idx_terminal_event_one_per_request", - "idx_webhook_deliveries_eligible", - "idx_webhook_deliveries_expired_lease", - "idx_webhook_deliveries_occurred", - "idx_webhook_deliveries_ready", - ]; - - fn assert_schema_identity(connection: &Connection) { - assert_eq!( - CURRENT_VERSION, 11, - "the canonical schema follows the pre-integration v10 lineage" - ); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, u32>(0)) - .expect("schema version"), - 11 - ); - assert_eq!(schema_object_names(connection, "table"), EXPECTED_TABLES); - assert_eq!(schema_object_names(connection, "index"), EXPECTED_INDEXES); - } - - fn assert_schema_columns(connection: &Connection) { - for (table, expected) in EXPECTED_TABLE_COLUMNS { - assert_eq!(table_columns(connection, table), *expected, "{table}"); - } + if found > 0 && identity == APPLICATION_ID && lineage::is_valid(connection)? { + return Ok(SchemaClassification::Private { version: found }); } + Ok(SchemaClassification::Incompatible { found }) +} - fn assert_schema_constraints(connection: &Connection) { - let lifecycle = schema_sql(connection, "table", "lifecycle_events"); - assert!(lifecycle.contains("CHECK (is_terminal IN (0, 1))")); - assert!( - schema_sql(connection, "index", "idx_terminal_event_one_per_request") - .contains("WHERE is_terminal = 1") - ); - let webhook = schema_sql(connection, "table", "webhook_deliveries"); - for constraint in [ - "terminal_outcome IN", - "terminal_status_code BETWEEN 100 AND 599", - "status_code BETWEEN 100 AND 599", - "state IN", - "attempt_number BETWEEN 0 AND 20", - "max_attempts BETWEEN 1 AND 20", - "claim_generation >= 0", - "last_error_code IN", - ] { - assert!( - webhook.contains(constraint), - "missing webhook constraint {constraint}" - ); +fn run_migrations(connection: &Connection, plan: MigrationPlan<'_>) -> Result<(), rusqlite::Error> { + validate_registry(plan.target, plan.migrations)?; + let classification = classify_schema(connection)?; + let mut current = match classification { + SchemaClassification::Fresh => 0, + SchemaClassification::Private { version } if version <= plan.target => version, + SchemaClassification::ReleasedSchema { source_version } + if plan.target == CURRENT_VERSION => + { + released_schema::import(connection, source_version)?; + CURRENT_VERSION } - assert!( - schema_sql(connection, "table", "summaries") - .contains("state IN ('active', 'completed', 'failed'") - ); - let artifacts = schema_sql(connection, "table", "artifact_pointers"); - for constraint in [ - "bytes >= 0", - "version >= 1", - "redacted IN (0, 1)", - "truncated IN (0, 1)", - "missing IN (0, 1)", - "corrupt IN (0, 1)", - ] { - assert!( - artifacts.contains(constraint), - "missing artifact constraint {constraint}" - ); + SchemaClassification::Private { .. } + | SchemaClassification::ReleasedSchema { .. } + | SchemaClassification::Incompatible { .. } => return Err(rusqlite::Error::InvalidQuery), + }; + if matches!(classification, SchemaClassification::Fresh) { + let transaction = immediate_transaction(connection)?; + if schema_version(&transaction)? != 0 + || application_id(&transaction)? != 0 + || has_user_objects(&transaction)? + { + return Err(rusqlite::Error::InvalidQuery); } - let maintenance = schema_sql(connection, "table", "maintenance_operations"); - for constraint in [ - "action IN ('cleanup', 'delete_one')", - "request_limit BETWEEN 1 AND 100", - "state IN ('previewed', 'completed', 'partial')", - "has_more IN (0, 1)", - "artifact_file_failure_class IN ('io', 'unsafe_path')", - ] { - assert!( - maintenance.contains(constraint), - "missing maintenance constraint {constraint}" - ); + set_application_id(&transaction, APPLICATION_ID)?; + (plan.initialize)(&transaction)?; + lineage::install(&transaction)?; + set_schema_version(&transaction, 1)?; + transaction.commit()?; + current = 1; + } + for migration in plan + .migrations + .iter() + .filter(|migration| migration.version > current) + { + let transaction = immediate_transaction(connection)?; + if schema_version(&transaction)? != migration.version - 1 + || application_id(&transaction)? != APPLICATION_ID + || !lineage::is_valid(&transaction)? + { + return Err(rusqlite::Error::InvalidQuery); } - assert!( - !schema_sql(connection, "table", "pending_artifact_deletions").contains("REFERENCES"), - "pending work must outlive deleted pointer and summary owners" - ); - } - - fn assert_fresh_only_migration_sql() { - assert!(!MIGRATIONS_V1.contains("ALTER TABLE")); - assert!(!MIGRATIONS_V1.contains("UPDATE ")); - assert!(!MIGRATIONS_V1.contains("DROP INDEX")); - } - - #[test] - fn fresh_database_is_exactly_complete_current_schema() { - let connection = Connection::open_in_memory().expect("open database"); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .expect("enable foreign keys"); - apply_migrations(&connection).expect("apply fresh schema"); - - assert_schema_identity(&connection); - - assert_schema_columns(&connection); - assert_schema_constraints(&connection); - assert_fresh_only_migration_sql(); - } - - #[test] - fn rejects_future_schema_versions_without_modifying_them() { - let connection = Connection::open_in_memory().unwrap(); - connection.pragma_update(None, "user_version", 99).unwrap(); - assert!(matches!( - apply_migrations(&connection), - Err(rusqlite::Error::InvalidQuery) - )); - let version: i32 = connection - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .unwrap(); - assert_eq!(version, 99); - } - - #[test] - fn rejects_unrecognized_v10_schema_without_modifying_it() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch( - "CREATE TABLE sentinel (value TEXT); \ - INSERT INTO sentinel VALUES ('unchanged'); \ - PRAGMA user_version = 10;", - ) - .unwrap(); - - assert!(matches!( - apply_migrations(&connection), - Err(rusqlite::Error::InvalidQuery) - )); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, i32>(0)) - .unwrap(), - 10 - ); - assert_eq!( - connection - .query_row("SELECT value FROM sentinel", [], |row| row - .get::<_, String>(0)) - .unwrap(), - "unchanged" - ); + (migration.apply)(&transaction)?; + set_schema_version(&transaction, migration.version)?; + transaction.commit()?; } + Ok(()) +} - #[test] - fn failed_migration_does_not_advance_schema_version() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute("CREATE TABLE summaries (request_id TEXT PRIMARY KEY)", []) - .unwrap(); - assert!(apply_migrations(&connection).is_err()); - let version: i32 = connection - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .unwrap(); - assert_eq!(version, 0); +fn validate_registry(target: u32, migrations: &[Migration]) -> Result<(), rusqlite::Error> { + let terminal = target.checked_add(1).ok_or(rusqlite::Error::InvalidQuery)?; + let mut expected = 2_u32; + for migration in migrations { + if migration.version != expected { + return Err(rusqlite::Error::InvalidQuery); + } + expected = expected + .checked_add(1) + .ok_or(rusqlite::Error::InvalidQuery)?; } - - #[test] - fn forced_v1_failure_rolls_back_partial_schema_and_user_version() { - let connection = Connection::open_in_memory().expect("open database"); - let forced_failure = r#" - CREATE TABLE must_rollback (id INTEGER PRIMARY KEY); - CREATE TABLE invalid_sql ( - "#; - - assert!(apply_migration_transactionally(&connection, 1, forced_failure).is_err()); - assert_eq!( - connection - .query_row("PRAGMA user_version", [], |row| row.get::<_, u32>(0)) - .expect("schema version"), - 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'must_rollback'", - [], - |row| row.get::<_, u32>(0), - ) - .expect("partial table count"), - 0 - ); + if expected != terminal { + return Err(rusqlite::Error::InvalidQuery); } + Ok(()) +} - #[test] - fn schema_v1_upgrades_to_current_and_current_schema_is_a_noop() { - let connection = Connection::open_in_memory().expect("open database"); - connection - .execute_batch(MIGRATIONS_V1) - .expect("install V1 schema"); - connection.pragma_update(None, "user_version", 1).unwrap(); - apply_migrations(&connection).expect("upgrade V1 to current"); - assert_schema_identity(&connection); +fn immediate_transaction(connection: &Connection) -> Result, rusqlite::Error> { + Transaction::new_unchecked(connection, TransactionBehavior::Immediate) +} - connection - .execute_batch( - "CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('unchanged');", - ) - .expect("seed sentinel"); - apply_migrations(&connection).expect("current schema is a no-op"); - assert_eq!( - connection - .query_row("SELECT value FROM sentinel", [], |row| row - .get::<_, String>(0)) - .expect("sentinel"), - "unchanged" - ); - } +fn schema_version(connection: &Connection) -> Result { + let value: i32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + Ok(u32::from_ne_bytes(value.to_ne_bytes())) +} - #[test] - fn seeded_v2_artifact_row_survives_v3_with_nullable_unavailable_reason() { - let root = tempfile::tempdir().expect("database root"); - let connection = Connection::open(root.path().join("log_store.db")).expect("open database"); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .expect("enable foreign keys"); - connection - .execute_batch(MIGRATIONS_V1) - .expect("install V1 schema"); - connection - .execute_batch(MIGRATIONS_V2) - .expect("install V2 schema"); - connection.pragma_update(None, "user_version", 2).unwrap(); - connection - .execute( - "INSERT INTO summaries (request_id, created_at) \ - VALUES (?1, ?2)", - [ - "00000000-0000-4000-8000-000000000401", - "2025-01-01T00:00:00.000000000Z", - ], - ) - .expect("seed V2 summary"); - connection - .execute( - "INSERT INTO artifact_pointers \ - (artifact_id, request_id, occurred_at, kind, media_kind, checksum, bytes, version, redacted) \ - VALUES (?1, ?2, ?3, 'response', 'application/json', ?4, 2, 1, 1)", - rusqlite::params![ - "00000000-0000-4000-8000-000000000402", - "00000000-0000-4000-8000-000000000401", - "2025-01-01T00:00:00.100000000Z", - "ab" - ], - ) - .expect("seed V2 artifact"); +fn set_schema_version(connection: &Connection, version: u32) -> Result<(), rusqlite::Error> { + connection.pragma_update(None, "user_version", version) +} - apply_migrations(&connection).expect("upgrade V2 to V3"); - assert_schema_identity(&connection); - let row = connection - .query_row( - "SELECT request_id, kind, media_kind, checksum, bytes, redacted, unavailable_reason \ - FROM artifact_pointers WHERE artifact_id = ?1", - ["00000000-0000-4000-8000-000000000402"], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, i64>(4)?, - row.get::<_, i32>(5)?, - row.get::<_, Option>(6)?, - )) - }, - ) - .expect("read migrated artifact row"); - assert_eq!( - row, - ( - "00000000-0000-4000-8000-000000000401".to_string(), - "response".to_string(), - Some("application/json".to_string()), - Some("ab".to_string()), - 2, - 1, - None, - ) - ); - drop(connection); - let store = crate::LogStore::reopen_at(root.path(), std::sync::Arc::new(crate::RealClock)) - .expect("reopen migrated store through the production path"); - let page = store - .query_artifacts( - "00000000-0000-4000-8000-000000000401", - &crate::PageQuery { - limit: 10, - cursor: None, - sort: crate::QuerySort::Ascending, - }, - ) - .expect("read migrated artifact through repository query"); - assert_eq!(page.items.len(), 1); - assert_eq!( - page.items[0].artifact_id, - "00000000-0000-4000-8000-000000000402" - ); - assert_eq!(page.items[0].unavailable_reason, None); - assert!(!page.items[0].missing); - assert!(!page.items[0].corrupt); - } +fn application_id(connection: &Connection) -> Result { + let value: i32 = connection.pragma_query_value(None, "application_id", |row| row.get(0))?; + Ok(u32::from_ne_bytes(value.to_ne_bytes())) +} - #[test] - fn initial_webhook_schema_bounds_terminal_status_codes() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch("PRAGMA foreign_keys = ON;") - .unwrap(); - apply_migrations(&connection).unwrap(); - connection - .execute( - "INSERT INTO summaries (request_id, created_at) VALUES ('request-1', '2026-08-04T12:00:00Z')", - [], - ) - .unwrap(); +fn set_application_id(connection: &Connection, application_id: u32) -> Result<(), rusqlite::Error> { + connection.pragma_update(None, "application_id", application_id) +} - let error = connection - .execute( - "INSERT INTO webhook_deliveries \ - (delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, target_url, attempt_number) \ - VALUES ('delivery-1', 'request-1', 'completed', 99, '2026-08-04T12:00:00Z', 'configured_webhook', 0)", - [], - ) - .expect_err("invalid terminal status must violate the V1 schema check"); - assert!(matches!( - error, - rusqlite::Error::SqliteFailure(_, Some(message)) - if message.contains("terminal_status_code") - )); - } +fn has_user_objects(connection: &Connection) -> Result { + connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type IN ('table', 'index', 'view', 'trigger') AND name NOT LIKE 'sqlite_%')", + [], + |row| row.get(0), + ) } + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs b/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs deleted file mode 100644 index 57b4cd7566..0000000000 --- a/crates/mesh-llm-log-store/src/migrations/legacy_v10.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Compatibility migration for local stores created by pre-integration builds. - -use rusqlite::Connection; - -pub(super) const LEGACY_VERSION: i32 = 10; - -/// Match the abandoned v10 development schema by structure, not only by its -/// version marker. This keeps an unrelated future schema at version 10 from -/// being rewritten. -pub(super) fn matches(conn: &Connection) -> Result { - conn.query_row( - r#" - SELECT - EXISTS(SELECT 1 FROM pragma_table_info('lifecycle_events') WHERE name = 'payload_json') - AND NOT EXISTS(SELECT 1 FROM pragma_table_info('lifecycle_events') WHERE name = 'event_type') - AND EXISTS(SELECT 1 FROM pragma_table_info('audit_entries') WHERE name = 'source') - AND NOT EXISTS(SELECT 1 FROM pragma_table_info('audit_entries') WHERE name = 'sequence') - AND EXISTS(SELECT 1 FROM pragma_table_info('maintenance_operations') WHERE name = 'intent_hash') - AND EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'maintenance_previews') - "#, - [], - |row| row.get(0), - ) -} - -/// Replace the pre-integration schema transactionally while retaining request, -/// lifecycle, artifact, proxy, audit, webhook, and cleanup records. Obsolete -/// maintenance receipts are intentionally retired; their audit entries remain. -pub(super) fn migrate(conn: &Connection) -> Result<(), rusqlite::Error> { - let transaction = conn.unchecked_transaction()?; - transaction.execute_batch(LEGACY_PREPARE_SQL)?; - transaction.execute_batch(super::MIGRATIONS_V1)?; - transaction.execute_batch(super::MIGRATIONS_V2)?; - transaction.execute_batch(super::MIGRATIONS_V3)?; - transaction.execute_batch(LEGACY_COPY_SQL)?; - transaction.execute_batch(&format!("PRAGMA user_version = {}", super::CURRENT_VERSION))?; - transaction.commit() -} - -const LEGACY_PREPARE_SQL: &str = r#" -PRAGMA defer_foreign_keys = ON; - -DROP TRIGGER IF EXISTS maintenance_operation_targets_resolution_insert; -DROP TRIGGER IF EXISTS maintenance_operation_targets_resolution_update; - -DROP TABLE IF EXISTS maintenance_operation_remaining_requests; -DROP TABLE IF EXISTS maintenance_preview_targets; -DROP TABLE IF EXISTS maintenance_operation_targets; -DROP TABLE IF EXISTS maintenance_operations; -DROP TABLE IF EXISTS maintenance_previews; -DROP TABLE IF EXISTS pending_artifact_deletions; - -ALTER TABLE summaries RENAME TO legacy_v10_summaries; -ALTER TABLE lifecycle_events RENAME TO legacy_v10_lifecycle_events; -ALTER TABLE artifact_pointers RENAME TO legacy_v10_artifact_pointers; -ALTER TABLE proxy_records RENAME TO legacy_v10_proxy_records; -ALTER TABLE audit_entries RENAME TO legacy_v10_audit_entries; -ALTER TABLE webhook_deliveries RENAME TO legacy_v10_webhook_deliveries; -ALTER TABLE cleanup_runs RENAME TO legacy_v10_cleanup_runs; - -DROP INDEX IF EXISTS idx_artifact_pointers_occurred; -DROP INDEX IF EXISTS idx_audit_entries_occurred; -DROP INDEX IF EXISTS idx_audit_entries_operation_id; -DROP INDEX IF EXISTS idx_cleanup_runs_occurred; -DROP INDEX IF EXISTS idx_lifecycle_events_occurred; -DROP INDEX IF EXISTS idx_lifecycle_events_request; -DROP INDEX IF EXISTS idx_maintenance_operation_remaining_requests; -DROP INDEX IF EXISTS idx_maintenance_operation_targets_operation; -DROP INDEX IF EXISTS idx_maintenance_preview_targets_operation; -DROP INDEX IF EXISTS idx_proxy_records_occurred; -DROP INDEX IF EXISTS idx_proxy_records_request_attempt; -DROP INDEX IF EXISTS idx_summaries_created; -DROP INDEX IF EXISTS idx_summaries_state; -DROP INDEX IF EXISTS idx_terminal_event_one_per_request; -DROP INDEX IF EXISTS idx_webhook_deliveries_occurred; -"#; - -const LEGACY_COPY_SQL: &str = r#" -INSERT INTO summaries ( - request_id, state, created_at, terminal_at, route, model, provider, engine, - status_code, error_msg, tenant_id, account_id, user_id -) -SELECT - request_id, - CASE - WHEN state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped') THEN state - ELSE 'failed' - END, - created_at, terminal_at, route, model, provider, engine, status_code, - error_msg, tenant_id, account_id, user_id -FROM legacy_v10_summaries; - -INSERT INTO lifecycle_events ( - event_id, request_id, occurred_at, payload_json, event_type, is_terminal -) -SELECT - event_id, - request_id, - occurred_at, - payload_json, - CASE - WHEN json_valid(payload_json) THEN COALESCE(json_extract(payload_json, '$.type'), 'unknown') - ELSE 'unknown' - END, - CASE - WHEN json_valid(payload_json) - AND json_extract(payload_json, '$.type') IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped') - THEN 1 - ELSE 0 - END -FROM legacy_v10_lifecycle_events; - -INSERT INTO artifact_pointers ( - artifact_id, request_id, occurred_at, kind, metadata_json, media_kind, - checksum, bytes, version, redacted, truncated, stored_at, missing, corrupt, - unavailable_reason -) -SELECT - artifact_id, request_id, occurred_at, kind, metadata_json, media_kind, - checksum, MAX(bytes, 0), MAX(version, 1), - CASE WHEN redacted = 1 THEN 1 ELSE 0 END, - CASE WHEN truncated = 1 THEN 1 ELSE 0 END, - stored_at, - CASE WHEN missing = 1 THEN 1 ELSE 0 END, - CASE WHEN corrupt = 1 THEN 1 ELSE 0 END, - NULL -FROM legacy_v10_artifact_pointers; - -INSERT INTO proxy_records ( - attempt_id, request_id, occurred_at, target, provider, engine, started_at, - completed_at, status_code, error_msg -) -SELECT - attempt_id, request_id, occurred_at, target, provider, engine, started_at, - completed_at, status_code, error_msg -FROM legacy_v10_proxy_records; - -INSERT INTO audit_entries ( - sequence, entry_id, request_id, occurred_at, actor, action, detail_json -) -SELECT - ROW_NUMBER() OVER (ORDER BY occurred_at, entry_id), - entry_id, - request_id, - occurred_at, - CASE - WHEN actor IN ('logging_service', 'runtime', 'mesh', 'cli', 'logs_api') THEN actor - ELSE 'logging_service' - END, - action, - json_patch( - json_patch( - json_patch( - json_patch( - CASE WHEN json_valid(detail_json) THEN detail_json ELSE '{}' END, - CASE WHEN result IS NOT NULL THEN json_object('outcome', result) ELSE '{}' END - ), - CASE WHEN source IS NOT NULL THEN json_object('source', source) ELSE '{}' END - ), - CASE WHEN reason IS NOT NULL THEN json_object('reason_code', reason) ELSE '{}' END - ), - CASE WHEN operation_id IS NOT NULL THEN json_object('operation_id', operation_id) ELSE '{}' END - ) -FROM legacy_v10_audit_entries; - -INSERT INTO webhook_deliveries ( - delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, - target_url, attempt_number, status_code, state, created_at, updated_at, - max_attempts -) -SELECT - delivery.delivery_id, - delivery.request_id, - CASE - WHEN summary.state IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped') THEN summary.state - ELSE 'failed' - END, - CASE WHEN summary.status_code BETWEEN 100 AND 599 THEN summary.status_code ELSE NULL END, - delivery.occurred_at, - 'configured_webhook', - MIN(MAX(delivery.attempt_number, 0), 20), - CASE WHEN delivery.status_code BETWEEN 100 AND 599 THEN delivery.status_code ELSE NULL END, - CASE WHEN delivery.status_code BETWEEN 200 AND 299 THEN 'succeeded' ELSE 'dead_letter' END, - delivery.occurred_at, - delivery.occurred_at, - MIN(MAX(delivery.attempt_number, 1), 20) -FROM legacy_v10_webhook_deliveries AS delivery -LEFT JOIN legacy_v10_summaries AS summary ON summary.request_id = delivery.request_id; - -INSERT INTO cleanup_runs ( - run_id, occurred_at, policy_name, cutoff_before, deleted_count, duration_ms -) -SELECT - run_id, occurred_at, policy_name, cutoff_before, MAX(deleted_count, 0), - CASE WHEN duration_ms >= 0 THEN duration_ms ELSE NULL END -FROM legacy_v10_cleanup_runs; - -DROP TABLE legacy_v10_lifecycle_events; -DROP TABLE legacy_v10_artifact_pointers; -DROP TABLE legacy_v10_proxy_records; -DROP TABLE legacy_v10_webhook_deliveries; -DROP TABLE legacy_v10_audit_entries; -DROP TABLE legacy_v10_cleanup_runs; -DROP TABLE legacy_v10_summaries; -"#; - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use rusqlite::Connection; - - use crate::{LogStore, RealClock}; - - #[test] - fn fingerprinted_v10_store_upgrades_and_preserves_ledger_rows() { - let root = tempfile::tempdir().expect("legacy database root"); - let database = root.path().join("log_store.db"); - let connection = Connection::open(&database).expect("open legacy database"); - connection - .execute_batch(LEGACY_FIXTURE_SQL) - .expect("seed legacy database"); - drop(connection); - - let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("upgrade legacy store"); - - assert_eq!(store.schema_version(), super::super::CURRENT_VERSION); - assert!(store.get_summary("legacy-request").unwrap().is_some()); - let connection = store.conn(); - let event: (String, i64) = connection - .query_row( - "SELECT event_type, is_terminal FROM lifecycle_events WHERE event_id = 'legacy-event'", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read migrated event"); - assert_eq!(event, ("completed".to_string(), 1)); - let audit: (i64, String) = connection - .query_row( - "SELECT sequence, actor FROM audit_entries WHERE entry_id = 'legacy-audit'", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read migrated audit"); - assert_eq!(audit, (1, "logging_service".to_string())); - let default_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit'", - [], - |row| row.get(0), - ) - .expect("read migrated audit detail"); - let default_detail: serde_json::Value = - serde_json::from_str(&default_detail).expect("migrated detail is valid JSON"); - assert_eq!(default_detail["outcome"], "succeeded"); - assert_eq!(default_detail["source"], "system"); - let explicit_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit-explicit'", - [], - |row| row.get(0), - ) - .expect("read migrated explicit audit detail"); - let explicit_detail: serde_json::Value = - serde_json::from_str(&explicit_detail).expect("migrated explicit detail is valid JSON"); - assert_eq!(explicit_detail["custom"], "kept"); - assert_eq!(explicit_detail["outcome"], "failed"); - assert_eq!(explicit_detail["source"], "mesh"); - assert_eq!(explicit_detail["reason_code"], "model_load_failed"); - assert_eq!(explicit_detail["operation_id"], "op-42"); - let partial_null_detail: String = connection - .query_row( - "SELECT detail_json FROM audit_entries WHERE entry_id = 'legacy-audit-partial-null'", - [], - |row| row.get(0), - ) - .expect("read migrated partial-null audit detail"); - let partial_null_detail: serde_json::Value = serde_json::from_str(&partial_null_detail) - .expect("migrated partial-null detail is valid JSON"); - // `reason` and `operation_id` are NULL on the legacy row; the merge - // must not clobber the pre-existing detail_json keys with nulls. - assert_eq!(partial_null_detail["outcome"], "succeeded"); - assert_eq!(partial_null_detail["source"], "mesh"); - assert_eq!(partial_null_detail["reason_code"], "pre-existing-reason"); - assert_eq!(partial_null_detail["operation_id"], "pre-existing-op"); - let obsolete_tables: i64 = connection - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name LIKE 'legacy_v10_%'", - [], - |row| row.get(0), - ) - .expect("count legacy tables"); - assert_eq!(obsolete_tables, 0); - } - - const LEGACY_FIXTURE_SQL: &str = r#" - PRAGMA foreign_keys = ON; - CREATE TABLE summaries ( - request_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active', created_at TEXT NOT NULL, - terminal_at TEXT, route TEXT, model TEXT, provider TEXT, engine TEXT, status_code INTEGER, - error_msg TEXT, tenant_id TEXT, account_id TEXT, user_id TEXT - ); - CREATE TABLE lifecycle_events ( - event_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}', UNIQUE(request_id, event_id) - ); - CREATE TABLE artifact_pointers ( - artifact_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, kind TEXT NOT NULL, metadata_json TEXT, media_kind TEXT, checksum TEXT, - bytes INTEGER NOT NULL DEFAULT 0, version INTEGER NOT NULL DEFAULT 1, redacted INTEGER NOT NULL DEFAULT 0, - truncated INTEGER NOT NULL DEFAULT 0, stored_at TEXT, missing INTEGER NOT NULL DEFAULT 0, - corrupt INTEGER NOT NULL DEFAULT 0, UNIQUE(request_id, artifact_id) - ); - CREATE TABLE proxy_records ( - attempt_id TEXT PRIMARY KEY, request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, - occurred_at TEXT NOT NULL, target TEXT NOT NULL, provider TEXT, engine TEXT, started_at TEXT, - completed_at TEXT, status_code INTEGER, error_msg TEXT, attempt_number INTEGER NOT NULL DEFAULT 0, - UNIQUE(request_id, attempt_id) - ); - CREATE TABLE audit_entries ( - entry_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, actor TEXT NOT NULL, action TEXT NOT NULL, detail_json TEXT, - source TEXT NOT NULL DEFAULT 'system', reason TEXT, result TEXT NOT NULL DEFAULT 'succeeded', - operation_id TEXT, UNIQUE(request_id, entry_id) - ); - CREATE TABLE webhook_deliveries ( - delivery_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, - occurred_at TEXT NOT NULL, target_url TEXT NOT NULL, attempt_number INTEGER NOT NULL, - status_code INTEGER, response_body TEXT, error_msg TEXT, UNIQUE(request_id, delivery_id) - ); - CREATE TABLE cleanup_runs ( - run_id TEXT PRIMARY KEY, occurred_at TEXT NOT NULL, policy_name TEXT NOT NULL, - cutoff_before TEXT NOT NULL, deleted_count INTEGER NOT NULL DEFAULT 0, duration_ms INTEGER - ); - CREATE TABLE maintenance_previews (operation_id TEXT PRIMARY KEY); - CREATE TABLE maintenance_operations (operation_id TEXT PRIMARY KEY, intent_hash TEXT NOT NULL); - INSERT INTO summaries ( - request_id, state, created_at, terminal_at, route, model, provider, engine, status_code - ) VALUES ( - 'legacy-request', 'completed', '2026-08-03T03:50:26.386Z', '2026-08-03T03:50:26.387Z', - 'chat_completions', 'legacy-model', 'local', 'skippy', 200 - ); - INSERT INTO lifecycle_events (event_id, request_id, occurred_at, payload_json) - VALUES ( - 'legacy-event', 'legacy-request', '2026-08-03T03:50:26.387Z', - '{"type":"completed","status_code":200}' - ); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action) - VALUES ('legacy-audit', '2026-08-03T03:50:27Z', 'system', 'cleanup_preview'); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action, source, reason, result, operation_id, detail_json) - VALUES ( - 'legacy-audit-explicit', '2026-08-03T03:50:28Z', 'runtime', 'model_start', - 'mesh', 'model_load_failed', 'failed', 'op-42', - '{"custom":"kept"}' - ); - INSERT INTO audit_entries (entry_id, occurred_at, actor, action, source, reason, result, operation_id, detail_json) - VALUES ( - 'legacy-audit-partial-null', '2026-08-03T03:50:29Z', 'runtime', 'model_start', - 'mesh', NULL, 'succeeded', NULL, - '{"reason_code":"pre-existing-reason","operation_id":"pre-existing-op"}' - ); - PRAGMA user_version = 10; - "#; -} diff --git a/crates/mesh-llm-log-store/src/migrations/lineage.rs b/crates/mesh-llm-log-store/src/migrations/lineage.rs new file mode 100644 index 0000000000..87d89f8e3d --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/lineage.rs @@ -0,0 +1,81 @@ +use rusqlite::Connection; +use rusqlite::types::ValueRef; + +const TABLE_NAME: &str = "_mesh_llm_log_store_lineage"; + +pub(super) fn install(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE _mesh_llm_log_store_lineage ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + epoch INTEGER NOT NULL CHECK (epoch = 1) + ) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage (id, epoch) VALUES (1, 1);", + ) +} + +pub(super) fn is_valid(connection: &Connection) -> Result { + if !has_exact_private_object(connection)? + || !has_expected_table_shape(connection)? + || !has_expected_columns(connection)? + { + return Ok(false); + } + has_expected_row(connection) +} + +fn has_exact_private_object(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 1 + AND COUNT(*) FILTER ( + WHERE type = 'table' AND name = ?1 AND tbl_name = ?1 AND sql IS NOT NULL + ) = 1 + FROM sqlite_schema + WHERE name GLOB '_mesh_llm_log_store_*'", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_table_shape(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 1 + FROM pragma_table_list + WHERE schema = 'main' AND name = ?1 AND type = 'table' + AND ncol = 2 AND wr = 1 AND strict = 0", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_columns(connection: &Connection) -> Result { + connection.query_row( + "SELECT COUNT(*) = 2 + AND COUNT(*) FILTER ( + WHERE cid = 0 AND name = 'id' AND type = 'INTEGER' + AND \"notnull\" = 1 AND dflt_value IS NULL AND pk = 1 AND hidden = 0 + ) = 1 + AND COUNT(*) FILTER ( + WHERE cid = 1 AND name = 'epoch' AND type = 'INTEGER' + AND \"notnull\" = 1 AND dflt_value IS NULL AND pk = 0 AND hidden = 0 + ) = 1 + FROM pragma_table_xinfo(?1)", + [TABLE_NAME], + |row| row.get(0), + ) +} + +fn has_expected_row(connection: &Connection) -> Result { + let mut statement = connection.prepare("SELECT * FROM _mesh_llm_log_store_lineage")?; + if statement.column_count() != 2 { + return Ok(false); + } + let mut rows = statement.query([])?; + let Some(row) = rows.next()? else { + return Ok(false); + }; + let row_is_valid = matches!( + (row.get_ref(0)?, row.get_ref(1)?), + (ValueRef::Integer(1), ValueRef::Integer(1)) + ); + Ok(row_is_valid && rows.next()?.is_none()) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema.rs b/crates/mesh-llm-log-store/src/migrations/released_schema.rs new file mode 100644 index 0000000000..63414bfa2e --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema.rs @@ -0,0 +1,53 @@ +use rusqlite::Connection; + +mod fingerprint; + +const SOURCE_VERSIONS: [u32; 2] = [3, 11]; + +pub(super) fn source_version(connection: &Connection) -> Result, rusqlite::Error> { + let source_version = super::schema_version(connection)?; + Ok(matches_source(connection, source_version)?.then_some(source_version)) +} + +fn matches_source(connection: &Connection, source_version: u32) -> Result { + Ok(SOURCE_VERSIONS.contains(&source_version) + && super::schema_version(connection)? == source_version + && super::application_id(connection)? == 0 + && !fingerprint::has_private_objects(connection)? + && fingerprint::matches(connection)?) +} + +pub(super) fn import(connection: &Connection, source_version: u32) -> Result<(), rusqlite::Error> { + import_with_hook(connection, source_version, |_| Ok(())) +} + +fn import_with_hook( + connection: &Connection, + source_version: u32, + before_commit: impl FnOnce(&Connection) -> Result<(), rusqlite::Error>, +) -> Result<(), rusqlite::Error> { + let transaction = super::immediate_transaction(connection)?; + if !matches_source(&transaction, source_version)? { + return Err(rusqlite::Error::InvalidQuery); + } + transaction.execute_batch( + "ALTER TABLE summaries ADD COLUMN caller_endpoint_id TEXT; + ALTER TABLE summaries ADD COLUMN caller_addr TEXT; + ALTER TABLE summaries ADD COLUMN caller_path_type TEXT;", + )?; + super::lineage::install(&transaction)?; + super::set_application_id(&transaction, super::APPLICATION_ID)?; + super::set_schema_version(&transaction, super::CURRENT_VERSION)?; + before_commit(&transaction)?; + transaction.commit() +} + +#[cfg(test)] +pub(super) fn import_with_before_commit_failure( + connection: &Connection, + source_version: u32, +) -> Result<(), rusqlite::Error> { + import_with_hook(connection, source_version, |_| { + Err(rusqlite::Error::InvalidQuery) + }) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs new file mode 100644 index 0000000000..073ad6f506 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint.rs @@ -0,0 +1,206 @@ +use rusqlite::Connection; + +mod contract; +mod create_table; +mod predicate; +mod semantic_contract; + +pub(super) fn has_private_objects(connection: &Connection) -> Result { + connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name GLOB '_mesh_llm_log_store_*')", + [], + |row| row.get(0), + ) +} + +pub(super) fn matches(connection: &Connection) -> Result { + Ok(has_exact_objects(connection)? + && has_exact_tables(connection)? + && has_exact_indexes(connection)? + && has_exact_foreign_keys(connection)? + && has_exact_create_table_semantics(connection)? + && predicate::matches(connection)?) +} + +fn has_exact_objects(connection: &Connection) -> Result { + let unexpected: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema + WHERE type IN ('view', 'trigger') AND name NOT LIKE 'sqlite_%')", + [], + |row| row.get(0), + )?; + if unexpected { + return Ok(false); + } + let tables = object_names(connection, "table")?; + let indexes = object_names(connection, "index")?; + Ok( + names_match(&tables, contract::TABLES.iter().map(|table| table.name)) + && names_match(&indexes, contract::INDEXES.iter().map(|index| index.name)), + ) +} + +fn object_names( + connection: &Connection, + object_type: &str, +) -> Result, rusqlite::Error> { + let mut statement = connection.prepare( + "SELECT name FROM sqlite_schema + WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + statement + .query_map([object_type], |row| row.get(0))? + .collect() +} + +fn names_match<'a>(actual: &[String], expected: impl Iterator) -> bool { + actual.iter().map(String::as_str).eq(expected) +} + +fn has_exact_tables(connection: &Connection) -> Result { + for table in contract::TABLES { + let shape = connection.query_row( + "SELECT ncol, wr, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name = ?1", + [table.name], + |row| { + Ok(( + row.get::<_, usize>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ); + let Ok(shape) = shape else { + return Ok(false); + }; + let columns = column_signature(connection, table.name)?; + if shape.1 + || shape.2 + || columns.split('|').count() != shape.0 + || columns != table.columns + || implicit_index_signature(connection, table.name)? != table.implicit_indexes + { + return Ok(false); + } + } + Ok(true) +} + +fn column_signature(connection: &Connection, table: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%s:%s:%d:%s:%d:%d', name, type, \"notnull\", + COALESCE(dflt_value, '-'), pk, hidden) AS signature + FROM pragma_table_xinfo(?1) ORDER BY cid + )", + [table], + |row| row.get(0), + ) +} + +fn implicit_index_signature( + connection: &Connection, + table: &str, +) -> Result { + let mut statement = connection.prepare( + "SELECT name, \"unique\", origin, partial FROM pragma_index_list(?1) + WHERE origin != 'c' ORDER BY name", + )?; + let indexes = statement + .query_map([table], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + )) + })? + .collect::, _>>()?; + indexes + .into_iter() + .map(|(name, unique, origin, partial)| { + Ok(format!( + "{table}:{name}:{}:{origin}:{}:{}", + u8::from(unique), + u8::from(partial), + index_column_signature(connection, &name)? + )) + }) + .collect::, rusqlite::Error>>() + .map(|signatures| signatures.join("\n")) +} + +fn has_exact_indexes(connection: &Connection) -> Result { + for index in contract::INDEXES { + let metadata = connection.query_row( + "SELECT \"unique\", origin, partial FROM pragma_index_list(?1) WHERE name = ?2", + [index.table, index.name], + |row| { + Ok(( + row.get::<_, bool>(0)?, + row.get::<_, String>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ); + let Ok(metadata) = metadata else { + return Ok(false); + }; + if metadata != (index.unique, "c".to_owned(), index.partial) + || index_column_signature(connection, index.name)? != index.columns + { + return Ok(false); + } + } + Ok(true) +} + +fn index_column_signature(connection: &Connection, index: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%d:%s:%d:%s', cid, COALESCE(name, '-'), \"desc\", coll) AS signature + FROM pragma_index_xinfo(?1) WHERE key = 1 ORDER BY seqno + )", + [index], + |row| row.get(0), + ) +} + +fn has_exact_foreign_keys(connection: &Connection) -> Result { + for table in contract::TABLES { + if foreign_key_signature(connection, table.name)? != table.foreign_keys { + return Ok(false); + } + } + Ok(true) +} + +fn foreign_key_signature(connection: &Connection, table: &str) -> Result { + connection.query_row( + "SELECT COALESCE(group_concat(signature, '|'), '') FROM ( + SELECT printf('%s:%s:%s:%s:%s:%s', \"table\", \"from\", \"to\", + on_update, on_delete, match) AS signature + FROM pragma_foreign_key_list(?1) ORDER BY id, seq + )", + [table], + |row| row.get(0), + ) +} + +fn has_exact_create_table_semantics(connection: &Connection) -> Result { + for expected in semantic_contract::TABLES { + let sql = connection.query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [expected.name], + |row| row.get::<_, String>(0), + )?; + let Some(actual) = create_table::parse(&sql) else { + return Ok(false); + }; + if !expected.matches(&actual) { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs new file mode 100644 index 0000000000..931e2a8a37 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/contract.rs @@ -0,0 +1,264 @@ +pub(super) struct Table { + pub(super) name: &'static str, + pub(super) columns: &'static str, + pub(super) foreign_keys: &'static str, + pub(super) implicit_indexes: &'static str, +} + +pub(super) const TABLES: &[Table] = &[ + Table { + name: "artifact_pointers", + columns: "artifact_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|kind:TEXT:1:-:0:0|metadata_json:TEXT:0:-:0:0|media_kind:TEXT:0:-:0:0|checksum:TEXT:0:-:0:0|bytes:INTEGER:1:0:0:0|version:INTEGER:1:1:0:0|redacted:INTEGER:1:0:0:0|truncated:INTEGER:1:0:0:0|stored_at:TEXT:0:-:0:0|missing:INTEGER:1:0:0:0|corrupt:INTEGER:1:0:0:0|unavailable_reason:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "artifact_pointers:sqlite_autoindex_artifact_pointers_1:1:pk:0:0:artifact_id:0:BINARY\nartifact_pointers:sqlite_autoindex_artifact_pointers_2:1:u:0:1:request_id:0:BINARY|0:artifact_id:0:BINARY", + }, + Table { + name: "audit_entries", + columns: "sequence:INTEGER:0:-:1:0|entry_id:TEXT:1:-:0:0|request_id:TEXT:0:-:0:0|occurred_at:TEXT:1:-:0:0|actor:TEXT:1:-:0:0|action:TEXT:1:-:0:0|detail_json:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:SET NULL:NONE", + implicit_indexes: "audit_entries:sqlite_autoindex_audit_entries_1:1:u:0:1:entry_id:0:BINARY\naudit_entries:sqlite_autoindex_audit_entries_2:1:u:0:2:request_id:0:BINARY|1:entry_id:0:BINARY", + }, + Table { + name: "cleanup_runs", + columns: "run_id:TEXT:0:-:1:0|occurred_at:TEXT:1:-:0:0|policy_name:TEXT:1:-:0:0|cutoff_before:TEXT:1:-:0:0|deleted_count:INTEGER:1:0:0:0|duration_ms:INTEGER:0:-:0:0", + foreign_keys: "", + implicit_indexes: "cleanup_runs:sqlite_autoindex_cleanup_runs_1:1:pk:0:0:run_id:0:BINARY", + }, + Table { + name: "lifecycle_events", + columns: "event_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|payload_json:TEXT:1:'{}':0:0|event_type:TEXT:1:'unknown':0:0|is_terminal:INTEGER:1:0:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "lifecycle_events:sqlite_autoindex_lifecycle_events_1:1:pk:0:0:event_id:0:BINARY\nlifecycle_events:sqlite_autoindex_lifecycle_events_2:1:u:0:1:request_id:0:BINARY|0:event_id:0:BINARY", + }, + Table { + name: "maintenance_operation_targets", + columns: "operation_id:TEXT:1:-:1:0|ordinal:INTEGER:1:-:0:0|request_id:TEXT:1:-:2:0", + foreign_keys: "maintenance_operations:operation_id:operation_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "maintenance_operation_targets:sqlite_autoindex_maintenance_operation_targets_1:1:pk:0:0:operation_id:0:BINARY|2:request_id:0:BINARY\nmaintenance_operation_targets:sqlite_autoindex_maintenance_operation_targets_2:1:u:0:0:operation_id:0:BINARY|1:ordinal:0:BINARY", + }, + Table { + name: "maintenance_operations", + columns: "operation_id:TEXT:0:-:1:0|action:TEXT:1:-:0:0|cutoff_before:TEXT:1:-:0:0|request_limit:INTEGER:1:-:0:0|reason:TEXT:1:-:0:0|state:TEXT:1:-:0:0|planned_requests:INTEGER:1:-:0:0|planned_events:INTEGER:1:-:0:0|planned_artifacts:INTEGER:1:-:0:0|planned_proxy_records:INTEGER:1:-:0:0|planned_database_rows:INTEGER:1:-:0:0|executed_requests:INTEGER:1:0:0:0|executed_events:INTEGER:1:0:0:0|executed_artifacts:INTEGER:1:0:0:0|executed_proxy_records:INTEGER:1:0:0:0|executed_database_rows:INTEGER:1:0:0:0|has_more:INTEGER:1:-:0:0|created_at:TEXT:1:-:0:0|completed_at:TEXT:0:-:0:0|selection_fingerprint:TEXT:1:'':0:0|artifact_files_removed:INTEGER:1:0:0:0|artifact_files_failed:INTEGER:1:0:0:0|artifact_file_failure_class:TEXT:0:-:0:0|preview_audit_id:TEXT:0:-:0:0|execution_audit_id:TEXT:0:-:0:0|cleanup_filters_json:TEXT:1:'{}':0:0", + foreign_keys: "", + implicit_indexes: "maintenance_operations:sqlite_autoindex_maintenance_operations_1:1:pk:0:0:operation_id:0:BINARY", + }, + Table { + name: "pending_artifact_deletions", + columns: "artifact_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0", + foreign_keys: "", + implicit_indexes: "pending_artifact_deletions:sqlite_autoindex_pending_artifact_deletions_1:1:pk:0:0:artifact_id:0:BINARY", + }, + Table { + name: "proxy_records", + columns: "attempt_id:TEXT:0:-:1:0|request_id:TEXT:1:-:0:0|occurred_at:TEXT:1:-:0:0|target:TEXT:1:-:0:0|provider:TEXT:0:-:0:0|engine:TEXT:0:-:0:0|started_at:TEXT:0:-:0:0|completed_at:TEXT:0:-:0:0|status_code:INTEGER:0:-:0:0|error_msg:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:CASCADE:NONE", + implicit_indexes: "proxy_records:sqlite_autoindex_proxy_records_1:1:pk:0:0:attempt_id:0:BINARY\nproxy_records:sqlite_autoindex_proxy_records_2:1:u:0:1:request_id:0:BINARY|0:attempt_id:0:BINARY", + }, + Table { + name: "summaries", + columns: "request_id:TEXT:0:-:1:0|state:TEXT:1:'active':0:0|created_at:TEXT:1:-:0:0|terminal_at:TEXT:0:-:0:0|route:TEXT:0:-:0:0|model:TEXT:0:-:0:0|provider:TEXT:0:-:0:0|engine:TEXT:0:-:0:0|status_code:INTEGER:0:-:0:0|error_msg:TEXT:0:-:0:0|tenant_id:TEXT:0:-:0:0|account_id:TEXT:0:-:0:0|user_id:TEXT:0:-:0:0", + foreign_keys: "", + implicit_indexes: "summaries:sqlite_autoindex_summaries_1:1:pk:0:0:request_id:0:BINARY", + }, + Table { + name: "webhook_deliveries", + columns: "delivery_id:TEXT:0:-:1:0|request_id:TEXT:0:-:0:0|terminal_outcome:TEXT:1:-:0:0|terminal_status_code:INTEGER:0:-:0:0|occurred_at:TEXT:1:-:0:0|target_url:TEXT:1:-:0:0|attempt_number:INTEGER:1:0:0:0|status_code:INTEGER:0:-:0:0|response_body:TEXT:0:-:0:0|error_msg:TEXT:0:-:0:0|state:TEXT:1:'succeeded':0:0|created_at:TEXT:1:'':0:0|updated_at:TEXT:1:'':0:0|next_attempt_at:TEXT:0:-:0:0|lease_expires_at:TEXT:0:-:0:0|claim_generation:INTEGER:1:0:0:0|max_attempts:INTEGER:1:1:0:0|last_error_code:TEXT:0:-:0:0", + foreign_keys: "summaries:request_id:request_id:NO ACTION:SET NULL:NONE", + implicit_indexes: "webhook_deliveries:sqlite_autoindex_webhook_deliveries_1:1:pk:0:0:delivery_id:0:BINARY\nwebhook_deliveries:sqlite_autoindex_webhook_deliveries_2:1:u:0:1:request_id:0:BINARY|0:delivery_id:0:BINARY", + }, +]; + +pub(super) struct Index { + pub(super) name: &'static str, + pub(super) table: &'static str, + pub(super) unique: bool, + pub(super) partial: bool, + pub(super) columns: &'static str, +} + +macro_rules! index { + ($name:literal, $table:literal, $columns:literal) => { + Index { + name: $name, + table: $table, + unique: false, + partial: false, + columns: $columns, + } + }; +} + +pub(super) const INDEXES: &[Index] = &[ + index!( + "idx_artifact_pointers_occurred", + "artifact_pointers", + "2:occurred_at:1:BINARY|0:artifact_id:1:BINARY" + ), + index!( + "idx_artifact_pointers_request_occurred", + "artifact_pointers", + "1:request_id:0:BINARY|2:occurred_at:0:BINARY|0:artifact_id:0:BINARY" + ), + index!( + "idx_audit_entries_actor_occurred", + "audit_entries", + "4:actor:0:BINARY|3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_audit_entries_occurred", + "audit_entries", + "3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_audit_entries_severity_occurred", + "audit_entries", + "-2:-:0:BINARY|3:occurred_at:1:BINARY|1:entry_id:1:BINARY" + ), + index!( + "idx_cleanup_runs_occurred", + "cleanup_runs", + "1:occurred_at:1:BINARY|0:run_id:1:BINARY" + ), + index!( + "idx_lifecycle_events_occurred", + "lifecycle_events", + "2:occurred_at:1:BINARY|0:event_id:1:BINARY" + ), + index!( + "idx_lifecycle_events_request", + "lifecycle_events", + "1:request_id:0:BINARY" + ), + index!( + "idx_lifecycle_events_request_occurred", + "lifecycle_events", + "1:request_id:0:BINARY|2:occurred_at:0:BINARY|0:event_id:0:BINARY" + ), + index!( + "idx_lifecycle_events_request_terminal", + "lifecycle_events", + "1:request_id:0:BINARY|5:is_terminal:0:BINARY" + ), + index!( + "idx_maintenance_operation_targets_operation", + "maintenance_operation_targets", + "0:operation_id:0:BINARY|1:ordinal:0:BINARY" + ), + index!( + "idx_pending_artifact_deletions_request", + "pending_artifact_deletions", + "1:request_id:0:BINARY|0:artifact_id:0:BINARY" + ), + index!( + "idx_proxy_records_engine_occurred", + "proxy_records", + "5:engine:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_occurred", + "proxy_records", + "2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_provider_occurred", + "proxy_records", + "4:provider:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_request_occurred", + "proxy_records", + "1:request_id:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_proxy_records_status_occurred", + "proxy_records", + "8:status_code:0:BINARY|2:occurred_at:1:BINARY|0:attempt_id:1:BINARY" + ), + index!( + "idx_summaries_created", + "summaries", + "2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_engine_created", + "summaries", + "7:engine:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_model_created", + "summaries", + "5:model:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_provider_created", + "summaries", + "6:provider:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_route_created", + "summaries", + "4:route:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!("idx_summaries_state", "summaries", "1:state:0:BINARY"), + index!( + "idx_summaries_state_created", + "summaries", + "1:state:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_status_created", + "summaries", + "8:status_code:0:BINARY|2:created_at:1:BINARY|0:request_id:1:BINARY" + ), + index!( + "idx_summaries_terminal_order", + "summaries", + "1:state:0:BINARY|-2:-:0:BINARY|0:request_id:0:BINARY" + ), + Index { + name: "idx_terminal_event_one_per_request", + table: "lifecycle_events", + unique: true, + partial: true, + columns: "1:request_id:0:BINARY", + }, + index!( + "idx_webhook_deliveries_eligible", + "webhook_deliveries", + "10:state:0:BINARY|13:next_attempt_at:0:BINARY|14:lease_expires_at:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY" + ), + Index { + name: "idx_webhook_deliveries_expired_lease", + table: "webhook_deliveries", + unique: false, + partial: true, + columns: "14:lease_expires_at:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY", + }, + index!( + "idx_webhook_deliveries_occurred", + "webhook_deliveries", + "4:occurred_at:1:BINARY|0:delivery_id:1:BINARY" + ), + Index { + name: "idx_webhook_deliveries_ready", + table: "webhook_deliveries", + unique: false, + partial: true, + columns: "-2:-:0:BINARY|11:created_at:0:BINARY|0:delivery_id:0:BINARY", + }, +]; + +#[cfg(test)] +mod tests { + use super::{INDEXES, TABLES}; + + #[test] + fn tables_are_lexicographically_ordered() { + assert!(TABLES.windows(2).all(|pair| pair[0].name < pair[1].name)); + } + + #[test] + fn indexes_are_lexicographically_ordered() { + assert!(INDEXES.windows(2).all(|pair| pair[0].name < pair[1].name)); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs new file mode 100644 index 0000000000..90836e7ab3 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table.rs @@ -0,0 +1,212 @@ +use super::predicate::token; + +mod autoincrement; + +#[derive(Debug, Eq, PartialEq)] +pub(super) enum Collation { + Binary, + Named(String), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum Deferrability { + NotDeferrable, + Deferrable, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum InitialTiming { + Immediate, + Deferred, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct Column { + pub(super) collation: Collation, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct ForeignKey { + pub(super) columns: Vec, + pub(super) deferrability: Deferrability, + pub(super) initial_timing: InitialTiming, +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct Table { + pub(super) columns: Vec, + pub(super) foreign_keys: Vec, + pub(super) autoincrement_column: Option, +} + +pub(super) fn parse(sql: &str) -> Option { + let clauses = top_level_clauses(sql)?; + let mut columns = Vec::new(); + let mut foreign_keys = Vec::new(); + let mut autoincrement_column = None; + for clause in clauses { + let tokens = token::tokenize(clause); + let first = tokens.first()?.as_str(); + if is_table_constraint(first) { + if autoincrement::contains_keyword(&tokens) { + return None; + } + if let Some(foreign_key) = table_foreign_key(&tokens)? { + foreign_keys.push(foreign_key); + } + continue; + } + + let column_name = tokens.first()?.clone(); + if let autoincrement::Clause::Column(column) = autoincrement::parse(&tokens)? + && autoincrement_column.replace(column).is_some() + { + return None; + } + columns.push(Column { + collation: column_collation(&tokens)?, + }); + if top_level_position(&tokens, "REFERENCES").is_some() { + foreign_keys.push(foreign_key(&tokens, vec![column_name])?); + } + } + Some(Table { + columns, + foreign_keys, + autoincrement_column, + }) +} + +fn is_table_constraint(first: &str) -> bool { + matches!( + first, + "CONSTRAINT" | "PRIMARY" | "UNIQUE" | "CHECK" | "FOREIGN" + ) +} + +fn column_collation(tokens: &[String]) -> Option { + let Some(position) = top_level_position(tokens, "COLLATE") else { + return Some(Collation::Binary); + }; + match tokens.get(position + 1)?.as_str() { + "BINARY" => Some(Collation::Binary), + name => Some(Collation::Named(name.to_owned())), + } +} + +fn table_foreign_key(tokens: &[String]) -> Option> { + let Some(foreign) = top_level_position(tokens, "FOREIGN") else { + return Some(None); + }; + if tokens.get(foreign + 1).map(String::as_str) != Some("KEY") + || tokens.get(foreign + 2).map(String::as_str) != Some("(") + { + return None; + } + let close = tokens[foreign + 3..] + .iter() + .position(|value| value == ")")? + + foreign + + 3; + let columns = tokens[foreign + 3..close].to_vec(); + if columns.is_empty() { + return None; + } + foreign_key(tokens, columns).map(Some) +} + +fn foreign_key(tokens: &[String], columns: Vec) -> Option { + let references = top_level_position(tokens, "REFERENCES")?; + let (deferrability, initial_timing) = foreign_key_timing(&tokens[references + 1..])?; + Some(ForeignKey { + columns, + deferrability, + initial_timing, + }) +} + +fn foreign_key_timing(tokens: &[String]) -> Option<(Deferrability, InitialTiming)> { + let mut deferrability = Deferrability::NotDeferrable; + let mut initial_timing = InitialTiming::Immediate; + let mut depth = 0_u32; + for (position, value) in tokens.iter().enumerate() { + match value.as_str() { + "(" => depth += 1, + ")" => depth = depth.checked_sub(1)?, + "DEFERRABLE" if depth == 0 => { + deferrability = if position > 0 && tokens[position - 1] == "NOT" { + Deferrability::NotDeferrable + } else { + Deferrability::Deferrable + }; + } + "INITIALLY" if depth == 0 => { + initial_timing = match tokens.get(position + 1)?.as_str() { + "IMMEDIATE" => InitialTiming::Immediate, + "DEFERRED" => InitialTiming::Deferred, + _ => return None, + }; + } + _ => {} + } + } + Some((deferrability, initial_timing)) +} + +fn top_level_position(tokens: &[String], expected: &str) -> Option { + let mut depth = 0_u32; + for (position, value) in tokens.iter().enumerate() { + match value.as_str() { + "(" => depth += 1, + ")" => depth = depth.checked_sub(1)?, + _ if depth == 0 && value == expected => return Some(position), + _ => {} + } + } + None +} + +fn top_level_clauses(sql: &str) -> Option> { + let bytes = sql.as_bytes(); + let body_start = bytes.iter().position(|value| *value == b'(')? + 1; + let mut clauses = Vec::new(); + let mut clause_start = body_start; + let mut depth = 1_u32; + let mut quote = None; + let mut position = body_start; + while position < bytes.len() { + if let Some(closing) = quote { + if bytes[position] == closing { + if closing != b']' && bytes.get(position + 1) == Some(&closing) { + position += 2; + continue; + } + quote = None; + } + position += 1; + continue; + } + match bytes[position] { + b'\'' | b'"' | b'`' => quote = Some(bytes[position]), + b'[' => quote = Some(b']'), + b'(' => depth += 1, + b')' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + clauses.push(sql[clause_start..position].trim()); + return Some(clauses); + } + } + b',' if depth == 1 => { + clauses.push(sql[clause_start..position].trim()); + clause_start = position + 1; + } + _ => {} + } + position += 1; + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs new file mode 100644 index 0000000000..86eb9537ea --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/autoincrement.rs @@ -0,0 +1,27 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) enum Clause { + Absent, + Column(String), +} + +pub(super) fn parse(tokens: &[String]) -> Option { + let positions = tokens + .iter() + .enumerate() + .filter_map(|(position, token)| (token == "AUTOINCREMENT").then_some(position)) + .collect::>(); + match positions.as_slice() { + [] => Some(Clause::Absent), + [4] if tokens.get(1).map(String::as_str) == Some("INTEGER") + && tokens.get(2).map(String::as_str) == Some("PRIMARY") + && tokens.get(3).map(String::as_str) == Some("KEY") => + { + Some(Clause::Column(tokens.first()?.clone())) + } + _ => None, + } +} + +pub(super) fn contains_keyword(tokens: &[String]) -> bool { + tokens.iter().any(|token| token == "AUTOINCREMENT") +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs new file mode 100644 index 0000000000..d67ae3820a --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/create_table/tests.rs @@ -0,0 +1,99 @@ +use super::*; + +#[test] +fn parser_treats_omitted_and_explicit_binary_column_collation_equally() { + // Given + let omitted = "CREATE TABLE sample (value TEXT)"; + let explicit = "CREATE TABLE sample (value TEXT COLLATE binary)"; + + // When + let omitted = parse(omitted).expect("parse omitted collation"); + let explicit = parse(explicit).expect("parse explicit collation"); + + // Then + assert_eq!(omitted.columns, explicit.columns); + assert_eq!(omitted.columns[0].collation, Collation::Binary); +} + +#[test] +fn parser_reads_inline_and_table_foreign_key_timing() { + // Given + let sql = "CREATE TABLE sample ( + parent_id TEXT REFERENCES parents(id) NOT DEFERRABLE INITIALLY IMMEDIATE, + backup_id TEXT, + FOREIGN KEY (backup_id) REFERENCES parents(id) DEFERRABLE INITIALLY DEFERRED + )"; + + // When + let table = parse(sql).expect("parse foreign keys"); + + // Then + assert_eq!( + table.foreign_keys, + [ + ForeignKey { + columns: vec!["PARENT_ID".to_owned()], + deferrability: Deferrability::NotDeferrable, + initial_timing: InitialTiming::Immediate, + }, + ForeignKey { + columns: vec!["BACKUP_ID".to_owned()], + deferrability: Deferrability::Deferrable, + initial_timing: InitialTiming::Deferred, + }, + ] + ); +} + +#[test] +fn parser_ignores_commas_and_parentheses_inside_nested_or_quoted_regions() { + // Given + let sql = "CREATE TABLE sample ( + value TEXT DEFAULT 'kept, with (punctuation)', + state TEXT CHECK (state IN ('one', 'two')) + )"; + + // When + let table = parse(sql).expect("parse nested and quoted regions"); + + // Then + assert_eq!(table.columns.len(), 2); +} + +#[test] +fn parser_records_integer_primary_key_autoincrement_semantics() { + // Given + let autoincrement = + "CREATE TABLE sample (sequence INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)"; + let rowid_reuse = "CREATE TABLE sample (sequence INTEGER PRIMARY KEY, value TEXT)"; + + // When + let autoincrement = parse(autoincrement).expect("parse autoincrement primary key"); + let rowid_reuse = parse(rowid_reuse).expect("parse reusable rowid primary key"); + + // Then + assert_eq!( + autoincrement.autoincrement_column.as_deref(), + Some("SEQUENCE") + ); + assert_eq!(rowid_reuse.autoincrement_column, None); +} + +#[test] +fn parser_rejects_impossible_or_misplaced_autoincrement_semantics() { + const INVALID_TABLES: &[&str] = &[ + "CREATE TABLE sample (sequence TEXT PRIMARY KEY AUTOINCREMENT)", + "CREATE TABLE sample (sequence INTEGER AUTOINCREMENT PRIMARY KEY)", + "CREATE TABLE sample (sequence INTEGER, PRIMARY KEY (sequence) AUTOINCREMENT)", + "CREATE TABLE sample (sequence INTEGER PRIMARY KEY AUTOINCREMENT AUTOINCREMENT)", + "CREATE TABLE sample (first INTEGER PRIMARY KEY AUTOINCREMENT, second INTEGER PRIMARY KEY AUTOINCREMENT)", + ]; + + for sql in INVALID_TABLES { + // When + let parsed = parse(sql); + + // Then + assert_eq!(parsed, None, "accepted invalid AUTOINCREMENT clause: {sql}"); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs new file mode 100644 index 0000000000..fb6b25e2b0 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate.rs @@ -0,0 +1,178 @@ +use rusqlite::Connection; + +pub(super) mod token; + +struct Predicate { + object_type: &'static str, + name: &'static str, + checks: &'static [&'static str], + required: &'static [&'static str], +} + +pub(super) fn matches(connection: &Connection) -> Result { + for object in PREDICATES { + let sql = connection.query_row( + "SELECT sql FROM sqlite_schema WHERE type = ?1 AND name = ?2", + [object.object_type, object.name], + |row| row.get::<_, String>(0), + )?; + let tokens = token::tokenize(&sql); + let expected_checks = object + .checks + .iter() + .map(|predicate| token::tokenize(predicate)) + .collect::>(); + let exact_partial = object + .required + .iter() + .find(|predicate| predicate.trim_start().starts_with("WHERE ")) + .is_none_or(|predicate| token::tail(&tokens, "WHERE") == token::tokenize(predicate)); + if token::checks(&tokens) != Some(expected_checks) + || !exact_partial + || object + .checks + .iter() + .chain(object.required) + .any(|predicate| !token::contains(&tokens, &token::tokenize(predicate))) + { + return Ok(false); + } + } + Ok(true) +} + +const PREDICATES: &[Predicate] = &[ + Predicate { + object_type: "table", + name: "summaries", + checks: &["state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')"], + required: &[], + }, + Predicate { + object_type: "table", + name: "lifecycle_events", + checks: &["is_terminal IN (0, 1)"], + required: &["UNIQUE(request_id, event_id)"], + }, + Predicate { + object_type: "table", + name: "artifact_pointers", + checks: &[ + "bytes >= 0", + "version >= 1", + "redacted IN (0, 1)", + "truncated IN (0, 1)", + "missing IN (0, 1)", + "corrupt IN (0, 1)", + "unavailable_reason IS NULL OR unavailable_reason IN ('streaming_response_not_assembled', 'response_body_not_bounded', 'capture_content_limit_exceeded', 'capture_memory_budget_exceeded', 'artifact_capture_disabled', 'artifact_capture_failed')", + ], + required: &["UNIQUE(request_id, artifact_id)"], + }, + Predicate { + object_type: "table", + name: "proxy_records", + checks: &[], + required: &["UNIQUE(request_id, attempt_id)"], + }, + Predicate { + object_type: "table", + name: "audit_entries", + checks: &["sequence > 0"], + required: &[ + "entry_id TEXT NOT NULL UNIQUE", + "UNIQUE(request_id, entry_id)", + ], + }, + Predicate { + object_type: "table", + name: "webhook_deliveries", + checks: &[ + "terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')", + "terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599", + "attempt_number BETWEEN 0 AND 20", + "status_code IS NULL OR status_code BETWEEN 100 AND 599", + "state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')", + "claim_generation >= 0", + "max_attempts BETWEEN 1 AND 20", + "last_error_code IS NULL OR last_error_code IN ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')", + ], + required: &["UNIQUE(request_id, delivery_id)"], + }, + Predicate { + object_type: "table", + name: "cleanup_runs", + checks: &[ + "deleted_count >= 0", + "duration_ms IS NULL OR duration_ms >= 0", + ], + required: &[], + }, + Predicate { + object_type: "table", + name: "maintenance_operations", + checks: &[ + "action IN ('cleanup', 'delete_one')", + "request_limit BETWEEN 1 AND 100", + "state IN ('previewed', 'completed', 'partial')", + "planned_requests >= 0", + "planned_events >= 0", + "planned_artifacts >= 0", + "planned_proxy_records >= 0", + "planned_database_rows >= 0", + "executed_requests >= 0", + "executed_events >= 0", + "executed_artifacts >= 0", + "executed_proxy_records >= 0", + "executed_database_rows >= 0", + "has_more IN (0, 1)", + "artifact_files_removed >= 0", + "artifact_files_failed >= 0", + "artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')", + ], + required: &[], + }, + Predicate { + object_type: "table", + name: "maintenance_operation_targets", + checks: &["ordinal >= 0"], + required: &[ + "PRIMARY KEY (operation_id, request_id)", + "UNIQUE (operation_id, ordinal)", + ], + }, + Predicate { + object_type: "index", + name: "idx_summaries_terminal_order", + checks: &[], + required: &["COALESCE(terminal_at, created_at)"], + }, + Predicate { + object_type: "index", + name: "idx_audit_entries_severity_occurred", + checks: &[], + required: &[ + "CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END", + ], + }, + Predicate { + object_type: "index", + name: "idx_terminal_event_one_per_request", + checks: &[], + required: &["WHERE is_terminal = 1"], + }, + Predicate { + object_type: "index", + name: "idx_webhook_deliveries_ready", + checks: &[], + required: &[ + "COALESCE(next_attempt_at, created_at)", + "WHERE state IN ('pending', 'retry', 'manual_retry')", + ], + }, + Predicate { + object_type: "index", + name: "idx_webhook_deliveries_expired_lease", + checks: &[], + required: &["WHERE state = 'in_flight'"], + }, +]; diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs new file mode 100644 index 0000000000..bbb641c1f6 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/predicate/token.rs @@ -0,0 +1,120 @@ +pub(in super::super) fn tokenize(sql: &str) -> Vec { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut start = 0; + while start < bytes.len() { + if bytes[start].is_ascii_whitespace() || bytes[start] == b',' { + start += 1; + continue; + } + if matches!(bytes[start], b'(' | b')') { + tokens.push(char::from(bytes[start]).to_string()); + start += 1; + continue; + } + if matches!(bytes[start], b'\'' | b'"' | b'`' | b'[') { + let end = quoted_end(bytes, start); + tokens.push(sql[start..end].to_owned()); + start = end; + continue; + } + let mut end = start + 1; + if matches!(bytes[start], b'<' | b'>' | b'=' | b'!') { + if end < bytes.len() && matches!(bytes[end], b'=' | b'>') { + end += 1; + } + } else { + while end < bytes.len() + && !bytes[end].is_ascii_whitespace() + && !matches!(bytes[end], b',' | b'(' | b')' | b'<' | b'>' | b'=' | b'!') + { + end += 1; + } + } + tokens.push(sql[start..end].to_ascii_uppercase()); + start = end; + } + tokens +} + +fn quoted_end(bytes: &[u8], start: usize) -> usize { + let opening = bytes[start]; + let closing = if opening == b'[' { b']' } else { opening }; + let mut end = start + 1; + while end < bytes.len() { + if bytes[end] != closing { + end += 1; + continue; + } + if closing != b']' && bytes.get(end + 1) == Some(&closing) { + end += 2; + continue; + } + return end + 1; + } + bytes.len() +} + +pub(super) fn contains(actual: &[String], expected: &[String]) -> bool { + actual + .windows(expected.len()) + .any(|window| window == expected) +} + +pub(super) fn tail(tokens: &[String], start: &str) -> Vec { + tokens + .iter() + .position(|token| token == start) + .map_or_else(Vec::new, |position| tokens[position..].to_vec()) +} + +pub(super) fn checks(tokens: &[String]) -> Option>> { + let mut expressions = Vec::new(); + let mut cursor = 0; + while let Some(offset) = tokens[cursor..].iter().position(|token| token == "CHECK") { + let check = cursor + offset; + if tokens.get(check + 1).map(String::as_str) != Some("(") { + return None; + } + let mut depth = 1_u32; + let mut end = check + 2; + while depth > 0 { + match tokens.get(end).map(String::as_str) { + Some("(") => depth += 1, + Some(")") => depth -= 1, + Some(_) => {} + None => return None, + } + end += 1; + } + expressions.push(tokens[check + 2..end - 1].to_vec()); + cursor = end; + } + Some(expressions) +} + +#[cfg(test)] +mod tests { + use super::tokenize; + + #[test] + fn quoted_sql_string_is_one_case_preserving_token_when_it_contains_sql_punctuation() { + // Given + let sql = "state IN ('Active value, (pending)! It''s exact.')"; + + // When + let tokens = tokenize(sql); + + // Then + assert_eq!( + tokens, + [ + "STATE", + "IN", + "(", + "'Active value, (pending)! It''s exact.'", + ")" + ] + ); + } +} diff --git a/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs new file mode 100644 index 0000000000..caf095c261 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/released_schema/fingerprint/semantic_contract.rs @@ -0,0 +1,84 @@ +use super::create_table::{Collation, Deferrability, ForeignKey, InitialTiming, Table}; + +pub(super) struct TableContract { + pub(super) name: &'static str, + column_collation: Collation, + foreign_keys: &'static [ForeignKeyContract], + autoincrement_column: Option<&'static str>, +} + +struct ForeignKeyContract { + columns: &'static [&'static str], + deferrability: Deferrability, + initial_timing: InitialTiming, +} + +impl TableContract { + pub(super) fn matches(&self, table: &Table) -> bool { + table + .columns + .iter() + .all(|column| column.collation == self.column_collation) + && table.foreign_keys.len() == self.foreign_keys.len() + && table + .foreign_keys + .iter() + .zip(self.foreign_keys) + .all(|(actual, expected)| expected.matches(actual)) + && table.autoincrement_column.as_deref() == self.autoincrement_column + } +} + +impl ForeignKeyContract { + fn matches(&self, foreign_key: &ForeignKey) -> bool { + foreign_key + .columns + .iter() + .map(String::as_str) + .eq(self.columns.iter().copied()) + && foreign_key.deferrability == self.deferrability + && foreign_key.initial_timing == self.initial_timing + } +} + +macro_rules! foreign_key { + ($column:literal) => { + ForeignKeyContract { + columns: &[$column], + deferrability: Deferrability::NotDeferrable, + initial_timing: InitialTiming::Immediate, + } + }; +} + +macro_rules! table { + ($name:literal, $autoincrement_column:expr, $foreign_keys:expr) => { + TableContract { + name: $name, + column_collation: Collation::Binary, + foreign_keys: $foreign_keys, + autoincrement_column: $autoincrement_column, + } + }; +} + +pub(super) const TABLES: &[TableContract] = &[ + table!("artifact_pointers", None, &[foreign_key!("REQUEST_ID")]), + table!( + "audit_entries", + Some("SEQUENCE"), + &[foreign_key!("REQUEST_ID")] + ), + table!("cleanup_runs", None, &[]), + table!("lifecycle_events", None, &[foreign_key!("REQUEST_ID")]), + table!( + "maintenance_operation_targets", + None, + &[foreign_key!("OPERATION_ID")] + ), + table!("maintenance_operations", None, &[]), + table!("pending_artifact_deletions", None, &[]), + table!("proxy_records", None, &[foreign_key!("REQUEST_ID")]), + table!("summaries", None, &[]), + table!("webhook_deliveries", None, &[foreign_key!("REQUEST_ID")]), +]; diff --git a/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs b/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs new file mode 100644 index 0000000000..068032cdbb --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/compatibility.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::{LogStore, RealClock}; +use std::sync::Arc; + +fn initialized_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept');", + ) +} + +fn no_op_migration(_: &Connection) -> Result<(), rusqlite::Error> { + Ok(()) +} + +fn initialize_to_version(connection: &Connection, version: u32) { + let migrations = [ + Migration { + version: 2, + apply: no_op_migration, + }, + Migration { + version: 3, + apply: no_op_migration, + }, + ]; + let migration_count = usize::try_from(version.saturating_sub(1)).expect("migration count"); + run_migrations( + connection, + MigrationPlan { + target: version, + initialize: initialized_schema, + migrations: &migrations[..migration_count], + }, + ) + .expect("initialize synthetic schema"); +} + +fn assert_inspection_preserves_database( + connection: &Connection, + target: u32, + expected: Option<(u32, u32)>, +) { + let identity = application_id(connection).expect("application identity"); + let found = schema_version(connection).expect("schema version"); + let object_count = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"); + + assert_eq!( + incompatible_schema_for_target(connection, target).expect("inspect schema"), + expected + ); + assert_eq!( + application_id(connection).expect("application identity"), + identity + ); + assert_eq!(schema_version(connection).expect("schema version"), found); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("schema object count"), + object_count + ); +} + +#[test] +fn initialized_version_one_is_compatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + assert_inspection_preserves_database(&connection, 2, None); +} + +#[test] +fn initialized_version_two_is_compatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 2); + assert_inspection_preserves_database(&connection, 2, None); +} + +#[test] +fn initialized_version_three_is_incompatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 3); + assert_inspection_preserves_database(&connection, 2, Some((3, 2))); +} + +#[test] +fn nonempty_version_zero_is_incompatible_with_target_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialized_schema(&connection).expect("seed unknown schema"); + assert_inspection_preserves_database(&connection, 2, Some((0, 2))); +} + +#[test] +fn missing_identity_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + connection + .pragma_update(None, "application_id", 0_u32) + .expect("clear application identity"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn wrong_identity_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialize_to_version(&connection, 1); + connection + .pragma_update(None, "application_id", 0x4E4F5045_u32) + .expect("replace application identity"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn markerless_identified_version_one_is_incompatible_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + initialized_schema(&connection).expect("seed unknown schema"); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("seed application identity"); + connection + .pragma_update(None, "user_version", 1_u32) + .expect("seed schema version"); + assert_inspection_preserves_database(&connection, 2, Some((1, 2))); +} + +#[test] +fn high_bit_header_values_are_classified_as_incompatible() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .pragma_update(None, "user_version", -2_147_483_647_i32) + .expect("seed high-bit schema version"); + connection + .pragma_update(None, "application_id", -2_147_483_646_i32) + .expect("seed high-bit application identity"); + + assert_eq!( + incompatible_schema_for_target(&connection, 2).expect("inspect high-bit schema"), + Some((0x8000_0001, 2)) + ); +} + +#[test] +fn released_schema_marker_eleven_store_upgrades_without_losing_rows() { + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + { + let connection = Connection::open(&database_path).expect("open released database"); + super::released_schema_fixture::install(&connection); + } + + let store = LogStore::open(root.path(), Arc::new(RealClock)) + .expect("upgrade released version eleven store"); + + assert_eq!(store.schema_version(), CURRENT_VERSION); + super::released_schema_fixture::assert_seeded_history(&store.conn()); + drop(store); + + let reopened = LogStore::open(root.path(), Arc::new(RealClock)) + .expect("reopen upgraded version eleven store"); + assert_eq!(reopened.schema_version(), CURRENT_VERSION); + assert_eq!( + table_columns(&reopened.conn(), "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + "caller_endpoint_id", + "caller_addr", + "caller_path_type", + ] + ); + super::released_schema_fixture::assert_seeded_history(&reopened.conn()); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/mod.rs b/crates/mesh-llm-log-store/src/migrations/tests/mod.rs new file mode 100644 index 0000000000..d188f76141 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/mod.rs @@ -0,0 +1,78 @@ +use super::*; +use rusqlite::Connection; + +mod compatibility; +mod released_schema_fixture; +mod released_schema_import; +mod runner; +mod schema_contract; +mod sqlite_autoincrement; + +pub(super) const EXPECTED_INDEXES: &[&str] = &[ + "idx_artifact_pointers_occurred", + "idx_artifact_pointers_request_occurred", + "idx_audit_entries_actor_occurred", + "idx_audit_entries_occurred", + "idx_audit_entries_severity_occurred", + "idx_cleanup_runs_occurred", + "idx_lifecycle_events_occurred", + "idx_lifecycle_events_request", + "idx_lifecycle_events_request_occurred", + "idx_lifecycle_events_request_terminal", + "idx_maintenance_operation_targets_operation", + "idx_pending_artifact_deletions_request", + "idx_proxy_records_engine_occurred", + "idx_proxy_records_occurred", + "idx_proxy_records_provider_occurred", + "idx_proxy_records_request_occurred", + "idx_proxy_records_status_occurred", + "idx_summaries_created", + "idx_summaries_engine_created", + "idx_summaries_model_created", + "idx_summaries_provider_created", + "idx_summaries_route_created", + "idx_summaries_state", + "idx_summaries_state_created", + "idx_summaries_status_created", + "idx_summaries_terminal_order", + "idx_terminal_event_one_per_request", + "idx_webhook_deliveries_eligible", + "idx_webhook_deliveries_expired_lease", + "idx_webhook_deliveries_occurred", + "idx_webhook_deliveries_ready", +]; + +fn table_columns(connection: &Connection, table: &str) -> Vec { + let mut statement = connection + .prepare(&format!("PRAGMA table_info('{table}')")) + .expect("prepare table info"); + statement + .query_map([], |row| row.get(1)) + .expect("query table info") + .collect::, _>>() + .expect("collect table columns") +} + +pub(super) fn schema_object_names(connection: &Connection, object_type: &str) -> Vec { + let mut statement = connection + .prepare( + "SELECT name FROM sqlite_master \ + WHERE type = ?1 AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .expect("prepare schema objects"); + statement + .query_map([object_type], |row| row.get(0)) + .expect("query schema objects") + .collect::, _>>() + .expect("collect schema objects") +} + +fn schema_sql(connection: &Connection, object_type: &str, name: &str) -> String { + connection + .query_row( + "SELECT sql FROM sqlite_master WHERE type = ?1 AND name = ?2", + [object_type, name], + |row| row.get(0), + ) + .expect("schema SQL") +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs new file mode 100644 index 0000000000..9888737687 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_fixture.rs @@ -0,0 +1,356 @@ +use rusqlite::Connection; + +pub(super) const SOURCE_VERSION_THREE: u32 = 3; +pub(super) const SOURCE_VERSION_ELEVEN: u32 = 11; +pub(super) const SOURCE_VERSIONS: [u32; 2] = [SOURCE_VERSION_THREE, SOURCE_VERSION_ELEVEN]; + +#[derive(Debug, Eq, PartialEq)] +pub(super) struct SeededHistory { + summary: String, + lifecycle_event: String, + audit_entry: String, +} + +pub(super) fn connection() -> Connection { + let connection = Connection::open_in_memory().expect("open released database"); + install(&connection); + connection +} + +pub(super) fn install(connection: &Connection) { + install_at_marker(connection, SOURCE_VERSION_ELEVEN); +} + +pub(super) fn install_at_marker(connection: &Connection, marker: u32) { + assert!(SOURCE_VERSIONS.contains(&marker)); + connection + .execute_batch(RELEASED_SCHEMA) + .expect("install frozen released schema"); + connection + .pragma_update(None, "user_version", marker) + .expect("set frozen released schema marker"); +} + +pub(super) fn install_without_audit_autoincrement(connection: &Connection) { + let schema = + RELEASED_SCHEMA.replace("INTEGER PRIMARY KEY AUTOINCREMENT", "INTEGER PRIMARY KEY"); + assert_ne!( + schema, RELEASED_SCHEMA, + "fixture mutation must change the schema" + ); + connection + .execute_batch(&schema) + .expect("install frozen released schema without audit autoincrement"); + connection + .pragma_update(None, "user_version", SOURCE_VERSION_ELEVEN) + .expect("set frozen released schema marker"); +} + +pub(super) fn seeded_history(connection: &Connection) -> SeededHistory { + SeededHistory { + summary: connection + .query_row( + "SELECT json_array(request_id, state, created_at, terminal_at, route, model, + provider, engine, status_code, error_msg, tenant_id, account_id, + user_id) + FROM summaries WHERE request_id = 'released-request'", + [], + |row| row.get(0), + ) + .expect("snapshot released summary"), + lifecycle_event: connection + .query_row( + "SELECT json_array(event_id, request_id, occurred_at, payload_json, event_type, + is_terminal) + FROM lifecycle_events WHERE event_id = 'released-event'", + [], + |row| row.get(0), + ) + .expect("snapshot released lifecycle event"), + audit_entry: connection + .query_row( + "SELECT json_array(sequence, entry_id, request_id, occurred_at, actor, action, + detail_json) + FROM audit_entries WHERE entry_id = 'released-audit'", + [], + |row| row.get(0), + ) + .expect("snapshot released audit entry"), + } +} + +pub(super) fn assert_seeded_history(connection: &Connection) { + assert_eq!( + connection + .query_row( + "SELECT state FROM summaries WHERE request_id = 'released-request'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released summary"), + "completed" + ); + assert_eq!( + connection + .query_row( + "SELECT event_type FROM lifecycle_events WHERE event_id = 'released-event'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released lifecycle event"), + "completed" + ); + assert_eq!( + connection + .query_row( + "SELECT action FROM audit_entries WHERE entry_id = 'released-audit'", + [], + |row| row.get::<_, String>(0), + ) + .expect("released audit entry"), + "request.completed" + ); +} + +const RELEASED_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS summaries ( + request_id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'active' + CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), + created_at TEXT NOT NULL, + terminal_at TEXT, + route TEXT, + model TEXT, + provider TEXT, + engine TEXT, + status_code INTEGER, + error_msg TEXT, + tenant_id TEXT, + account_id TEXT, + user_id TEXT +); + +CREATE INDEX IF NOT EXISTS idx_summaries_created ON summaries (created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_state ON summaries (state); + +CREATE TABLE IF NOT EXISTS lifecycle_events ( + event_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + event_type TEXT NOT NULL DEFAULT 'unknown', + is_terminal INTEGER NOT NULL DEFAULT 0 + CHECK (is_terminal IN (0, 1)), + UNIQUE(request_id, event_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_terminal_event_one_per_request +ON lifecycle_events (request_id) +WHERE is_terminal = 1; +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request ON lifecycle_events (request_id); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); + +CREATE TABLE IF NOT EXISTS artifact_pointers ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + kind TEXT NOT NULL, + metadata_json TEXT, + media_kind TEXT, + checksum TEXT, + bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), + version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), + redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), + truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), + stored_at TEXT, + missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), + corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), + UNIQUE(request_id, artifact_id) +); + +CREATE INDEX IF NOT EXISTS idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); + +CREATE TABLE IF NOT EXISTS proxy_records ( + attempt_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + target TEXT NOT NULL, + provider TEXT, + engine TEXT, + started_at TEXT, + completed_at TEXT, + status_code INTEGER, + error_msg TEXT, + UNIQUE(request_id, attempt_id) +); + +CREATE INDEX IF NOT EXISTS idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); + +CREATE TABLE IF NOT EXISTS audit_entries ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), + entry_id TEXT NOT NULL UNIQUE, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + detail_json TEXT, + UNIQUE(request_id, entry_id) +); + +CREATE INDEX IF NOT EXISTS idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); + +CREATE TABLE IF NOT EXISTS webhook_deliveries ( + delivery_id TEXT PRIMARY KEY, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + terminal_outcome TEXT NOT NULL + CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), + terminal_status_code INTEGER + CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), + occurred_at TEXT NOT NULL, + target_url TEXT NOT NULL, + attempt_number INTEGER NOT NULL DEFAULT 0 + CHECK (attempt_number BETWEEN 0 AND 20), + status_code INTEGER + CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), + response_body TEXT, + error_msg TEXT, + state TEXT NOT NULL DEFAULT 'succeeded' + CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), + created_at TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT '', + next_attempt_at TEXT, + lease_expires_at TEXT, + claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), + max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), + last_error_code TEXT + CHECK (last_error_code IS NULL OR last_error_code IN + ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), + UNIQUE(request_id, delivery_id) +); + +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_eligible +ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); + +CREATE TABLE IF NOT EXISTS cleanup_runs ( + run_id TEXT PRIMARY KEY, + occurred_at TEXT NOT NULL, + policy_name TEXT NOT NULL, + cutoff_before TEXT NOT NULL, + deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), + duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) +); + +CREATE INDEX IF NOT EXISTS idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); + +CREATE TABLE IF NOT EXISTS maintenance_operations ( + operation_id TEXT PRIMARY KEY, + action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), + cutoff_before TEXT NOT NULL, + request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), + reason TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), + planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), + planned_events INTEGER NOT NULL CHECK (planned_events >= 0), + planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), + planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), + planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), + executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), + executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), + executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), + executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), + executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), + has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), + created_at TEXT NOT NULL, + completed_at TEXT, + selection_fingerprint TEXT NOT NULL DEFAULT '', + artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), + artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), + artifact_file_failure_class TEXT + CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), + preview_audit_id TEXT, + execution_audit_id TEXT, + cleanup_filters_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS maintenance_operation_targets ( + operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + request_id TEXT NOT NULL, + PRIMARY KEY (operation_id, request_id), + UNIQUE (operation_id, ordinal) +); + +CREATE INDEX IF NOT EXISTS idx_maintenance_operation_targets_operation +ON maintenance_operation_targets (operation_id, ordinal); + +CREATE TABLE IF NOT EXISTS pending_artifact_deletions ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_pending_artifact_deletions_request +ON pending_artifact_deletions (request_id, artifact_id); + +CREATE INDEX IF NOT EXISTS idx_summaries_terminal_order +ON summaries (state, COALESCE(terminal_at, created_at), request_id); +CREATE INDEX IF NOT EXISTS idx_summaries_route_created +ON summaries (route, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_model_created +ON summaries (model, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_provider_created +ON summaries (provider, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_engine_created +ON summaries (engine, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_status_created +ON summaries (status_code, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_summaries_state_created +ON summaries (state, created_at DESC, request_id DESC); +CREATE INDEX IF NOT EXISTS idx_lifecycle_events_request_occurred +ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); +CREATE INDEX IF NOT EXISTS idx_artifact_pointers_request_occurred +ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_request_occurred +ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_provider_occurred +ON proxy_records (provider, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_engine_occurred +ON proxy_records (engine, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_proxy_records_status_occurred +ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_entries_actor_occurred +ON audit_entries (actor, occurred_at DESC, entry_id DESC); +CREATE INDEX IF NOT EXISTS idx_audit_entries_severity_occurred +ON audit_entries ( + CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, + occurred_at DESC, + entry_id DESC +); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_ready +ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) +WHERE state IN ('pending', 'retry', 'manual_retry'); +CREATE INDEX IF NOT EXISTS idx_webhook_deliveries_expired_lease +ON webhook_deliveries (lease_expires_at, created_at, delivery_id) +WHERE state = 'in_flight'; + +ALTER TABLE artifact_pointers ADD COLUMN unavailable_reason TEXT + CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( + 'streaming_response_not_assembled', + 'response_body_not_bounded', + 'capture_content_limit_exceeded', + 'capture_memory_budget_exceeded', + 'artifact_capture_disabled', + 'artifact_capture_failed' + )); + +INSERT INTO summaries (request_id, state, created_at, terminal_at) +VALUES ('released-request', 'completed', '2026-08-01T00:00:00Z', '2026-08-01T00:00:01Z'); +INSERT INTO lifecycle_events + (event_id, request_id, occurred_at, payload_json, event_type, is_terminal) +VALUES + ('released-event', 'released-request', '2026-08-01T00:00:01Z', '{}', 'completed', 1); +INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) +VALUES + ('released-audit', 'released-request', '2026-08-01T00:00:02Z', 'system', 'request.completed', '{}'); +"#; diff --git a/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs new file mode 100644 index 0000000000..58971de28e --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/released_schema_import.rs @@ -0,0 +1,574 @@ +use super::released_schema_fixture::{ + self, SOURCE_VERSION_ELEVEN, SOURCE_VERSION_THREE, SOURCE_VERSIONS, +}; +use super::*; + +fn released_connection() -> Connection { + released_schema_fixture::connection() +} + +fn assert_released_state_is_unchanged(connection: &Connection, source_version: u32) { + assert_eq!( + schema_version(connection).expect("schema version"), + source_version + ); + assert_eq!(application_id(connection).expect("application identity"), 0); + released_schema_fixture::assert_seeded_history(connection); + assert!(!lineage::is_valid(connection).expect("inspect lineage")); +} + +fn assert_rejected_without_mutation(connection: &Connection) { + let objects_before = schema_object_names(connection, "table"); + let source_version = schema_version(connection).expect("source version"); + + // When + let compatibility = incompatible_schema(connection).expect("inspect compatibility"); + + // Then + assert_eq!(compatibility, Some((source_version, CURRENT_VERSION))); + assert_eq!(schema_object_names(connection, "table"), objects_before); + assert_released_state_is_unchanged(connection, source_version); +} + +fn assert_imported_identity_and_caller_columns(connection: &Connection) { + assert_eq!( + schema_version(connection).expect("schema version"), + CURRENT_VERSION + ); + assert_eq!( + application_id(connection).expect("application identity"), + APPLICATION_ID + ); + assert!(lineage::is_valid(connection).expect("inspect lineage")); + let caller_values = connection + .query_row( + "SELECT caller_endpoint_id, caller_addr, caller_path_type + FROM summaries WHERE request_id = 'released-request'", + [], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .expect("read imported caller columns"); + assert_eq!(caller_values, (None, None, None)); +} + +#[test] +fn released_schema_source_versions_are_classified_for_import_without_mutation() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { source_version } + ); + assert_eq!(compatibility, None); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn released_schema_is_accepted_only_for_the_current_production_target() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + + // When + let compatibility = incompatible_schema_for_target(&connection, CURRENT_VERSION + 1) + .expect("inspect non-production target compatibility"); + + // Then + assert_eq!(compatibility, Some((source_version, CURRENT_VERSION + 1))); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn released_schema_without_if_not_exists_is_classified_for_import_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, ' IF NOT EXISTS', '') + WHERE sql IS NOT NULL; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("remove optional creation clause"); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_predicates_match_independently_of_sql_whitespace() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >=\n0') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + ) + .expect("reformat released predicate"); + + // When + let classification = classify_schema(&connection).expect("classify reformatted schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_with_extra_missing_or_altered_objects_is_rejected() { + const MUTATIONS: &[&str] = &[ + "CREATE TABLE unrelated (value TEXT);", + "DROP TABLE cleanup_runs;", + "DROP INDEX idx_summaries_state; + CREATE INDEX idx_summaries_state ON summaries (created_at);", + "CREATE VIEW released_view AS SELECT request_id FROM summaries;", + "CREATE TRIGGER released_trigger AFTER INSERT ON summaries BEGIN SELECT 1; END;", + "ALTER TABLE cleanup_runs ADD COLUMN lookalike TEXT;", + "CREATE TABLE _mesh_llm_log_store_lookalike (value TEXT);", + ]; + + for mutation in MUTATIONS { + // Given + let connection = released_connection(); + connection + .execute_batch(mutation) + .expect("mutate released schema"); + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn released_schema_without_audit_autoincrement_is_rejected_without_mutation() { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_without_audit_autoincrement(&connection); + let tables_before = schema_object_names(&connection, "table"); + let summary_columns_before = table_columns(&connection, "summaries"); + let history_before = released_schema_fixture::seeded_history(&connection); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + compatibility, + Some((SOURCE_VERSION_ELEVEN, CURRENT_VERSION)) + ); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_eq!( + table_columns(&connection, "summaries"), + summary_columns_before + ); + assert_eq!( + schema_version(&connection).expect("schema version"), + SOURCE_VERSION_ELEVEN + ); + assert_eq!( + application_id(&connection).expect("application identity"), + 0 + ); + assert!(!lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + history_before + ); +} + +#[test] +fn released_schema_with_altered_foreign_key_metadata_is_rejected() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema + SET sql = replace(sql, 'ON DELETE CASCADE', 'ON DELETE SET NULL') + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("alter released foreign key"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_changed_quoted_check_literal_case_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'CHECK (state IN (''active'',', + 'CHECK (state IN (''ACTIVE'',' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("change released check literal case"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_deferrable_foreign_key_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'ON DELETE CASCADE', + 'ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED' + ) + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("defer released foreign key"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_collated_nonindexed_column_is_rejected_without_mutation() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'error_msg TEXT,', + 'error_msg TEXT COLLATE NOCASE,' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("collate released nonindexed column"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_explicit_binary_nonindexed_column_collation_is_accepted() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'error_msg TEXT,', + 'error_msg TEXT COLLATE BINARY,' + ) + WHERE type = 'table' AND name = 'summaries'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("make released binary collation explicit"); + + // When + let classification = classify_schema(&connection).expect("classify released schema"); + + // Then + assert_eq!( + classification, + SchemaClassification::ReleasedSchema { + source_version: SOURCE_VERSION_ELEVEN + } + ); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn released_schema_with_near_match_implicit_unique_index_is_rejected() { + // Given + let connection = released_connection(); + connection + .execute_batch( + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'UNIQUE(request_id, event_id)', + 'UNIQUE(request_id, event_id, occurred_at) /* UNIQUE(request_id, event_id) */' + ) + WHERE type = 'table' AND name = 'lifecycle_events'; + PRAGMA writable_schema = OFF; + PRAGMA schema_version = 2;", + ) + .expect("alter released implicit unique index"); + + // When / Then + assert_rejected_without_mutation(&connection); +} + +#[test] +fn released_schema_with_altered_check_or_partial_index_predicate_is_rejected() { + const MUTATIONS: &[&str] = &[ + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >= -1') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'bytes >= 0', 'bytes >= 0 OR bytes = -1') + WHERE type = 'table' AND name = 'artifact_pointers'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace(sql, 'WHERE is_terminal = 1', 'WHERE is_terminal >= 0') + WHERE type = 'index' AND name = 'idx_terminal_event_one_per_request'; + PRAGMA writable_schema = OFF;", + "PRAGMA writable_schema = ON; + UPDATE sqlite_schema SET sql = replace( + sql, + 'WHERE is_terminal = 1', + 'WHERE is_terminal = 1 OR request_id IS NULL' + ) + WHERE type = 'index' AND name = 'idx_terminal_event_one_per_request'; + PRAGMA writable_schema = OFF;", + ]; + + for mutation in MUTATIONS { + // Given + let connection = released_connection(); + connection + .execute_batch(mutation) + .expect("alter released predicate"); + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn released_schema_with_nonzero_application_identity_is_rejected() { + // Given + let connection = released_connection(); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("set private identity"); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!( + compatibility, + Some((SOURCE_VERSION_ELEVEN, CURRENT_VERSION)) + ); + assert_eq!( + application_id(&connection).expect("application identity"), + APPLICATION_ID + ); + assert_eq!( + schema_version(&connection).expect("schema version"), + SOURCE_VERSION_ELEVEN + ); +} + +#[test] +fn released_schema_with_any_partial_caller_column_state_is_rejected() { + const CALLER_COLUMNS: &[&str] = &[ + "caller_endpoint_id TEXT", + "caller_addr TEXT", + "caller_path_type TEXT", + ]; + + for count in 1..=CALLER_COLUMNS.len() { + // Given + let connection = released_connection(); + for column in &CALLER_COLUMNS[..count] { + connection + .execute_batch(&format!("ALTER TABLE summaries ADD COLUMN {column};")) + .expect("add caller column"); + } + + // When / Then + assert_rejected_without_mutation(&connection); + } +} + +#[test] +fn all_other_source_versions_are_rejected_without_mutation() { + for version in [0_u32, 1, 2, 4, 10, 12, 99] { + // Given + let connection = released_connection(); + connection + .pragma_update(None, "user_version", version) + .expect("set unsupported version"); + + // When + let compatibility = incompatible_schema(&connection).expect("inspect compatibility"); + + // Then + assert_eq!(compatibility, Some((version, CURRENT_VERSION))); + assert_eq!( + schema_version(&connection).expect("schema version"), + version + ); + assert_eq!( + application_id(&connection).expect("application identity"), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM summaries", [], |row| row + .get::<_, u32>(0)) + .expect("summary count"), + 1 + ); + } +} + +#[test] +fn forced_released_schema_import_failure_rolls_back_both_source_versions() { + for source_version in SOURCE_VERSIONS { + // Given + let connection = Connection::open_in_memory().expect("open released database"); + released_schema_fixture::install_at_marker(&connection, source_version); + let tables_before = schema_object_names(&connection, "table"); + + // When + let error = released_schema::import_with_before_commit_failure(&connection, source_version) + .expect_err("force failure before import commit"); + + // Then + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_eq!( + table_columns(&connection, "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + ] + ); + assert_released_state_is_unchanged(&connection, source_version); + } +} + +#[test] +fn importer_revalidates_the_classified_source_version_before_writing() { + // Given + let connection = released_connection(); + let tables_before = schema_object_names(&connection, "table"); + + // When + let error = released_schema::import(&connection, SOURCE_VERSION_THREE) + .expect_err("reject mismatched classified source version"); + + // Then + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(schema_object_names(&connection, "table"), tables_before); + assert_released_state_is_unchanged(&connection, SOURCE_VERSION_ELEVEN); +} + +#[test] +fn production_import_installs_private_identity_lineage_and_nullable_columns() { + // Given + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + let connection = Connection::open(&database_path).expect("open released database"); + released_schema_fixture::install(&connection); + drop(connection); + + // When + let store = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("import released store"); + + // Then + let connection = store.conn(); + assert_imported_identity_and_caller_columns(&connection); + released_schema_fixture::assert_seeded_history(&connection); +} + +#[test] +fn production_import_accepts_published_marker_three_and_preserves_rows_after_reopen() { + // Given + let root = tempfile::tempdir().expect("temporary store root"); + let database_path = root.path().join("log_store.db"); + let released_history = { + let connection = Connection::open(&database_path).expect("open released database"); + released_schema_fixture::install_at_marker(&connection, SOURCE_VERSION_THREE); + released_schema_fixture::seeded_history(&connection) + }; + + // When + let store = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("import published marker-three store"); + + // Then + { + let connection = store.conn(); + assert_imported_identity_and_caller_columns(&connection); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + released_history + ); + } + drop(store); + + let reopened = crate::LogStore::open(root.path(), std::sync::Arc::new(crate::RealClock)) + .expect("reopen imported marker-three store"); + let connection = reopened.conn(); + assert_imported_identity_and_caller_columns(&connection); + assert_eq!( + released_schema_fixture::seeded_history(&connection), + released_history + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/runner.rs b/crates/mesh-llm-log-store/src/migrations/tests/runner.rs new file mode 100644 index 0000000000..a20955249d --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/runner.rs @@ -0,0 +1,325 @@ +use super::*; + +mod identity; + +fn synthetic_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch("CREATE TABLE baseline (value TEXT NOT NULL);") +} + +fn failing_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE initialization_partial (value TEXT); INSERT INTO missing_table VALUES (1);", + ) +} + +fn conflicting_initial_schema(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE baseline (value TEXT NOT NULL); + CREATE TABLE _mesh_llm_log_store_lineage ( + id INTEGER NOT NULL PRIMARY KEY, + epoch INTEGER NOT NULL + ) WITHOUT ROWID;", + ) +} + +fn migration_v2(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v2 (value TEXT NOT NULL); INSERT INTO future_v2 VALUES ('v2');", + ) +} + +fn failing_migration_v2(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v2_partial (value TEXT); INSERT INTO missing_table VALUES (2);", + ) +} + +fn failing_migration_v3(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v3_partial (value TEXT); INSERT INTO missing_table VALUES (3);", + ) +} + +fn migration_v3(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch( + "CREATE TABLE future_v3 (value TEXT NOT NULL); INSERT INTO future_v3 VALUES ('v3');", + ) +} + +fn version(connection: &Connection) -> u32 { + connection + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("schema version") +} + +fn application_id(connection: &Connection) -> u32 { + connection + .pragma_query_value(None, "application_id", |row| row.get(0)) + .expect("application identity") +} + +fn user_objects(connection: &Connection) -> i64 { + connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'index', 'view', 'trigger') AND name NOT LIKE 'sqlite_%'", + [], + |row| row.get(0), + ) + .expect("user object count") +} + +fn plan<'a>( + target: u32, + initialize: MigrationFn, + migrations: &'a [Migration], +) -> MigrationPlan<'a> { + MigrationPlan { + target, + initialize, + migrations, + } +} + +#[test] +fn empty_version_zero_bootstraps_atomically() { + let connection = Connection::open_in_memory().expect("open database"); + + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 2); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn nonempty_version_zero_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch("CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('kept');") + .expect("seed unknown schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(user_objects(&connection), 1); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn preexisting_lineage_marker_makes_version_zero_nonfresh() { + let connection = Connection::open_in_memory().expect("open database"); + lineage::install(&connection).expect("seed lineage marker"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 1); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn production_rejects_version_two_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + lineage::install(&connection).expect("seed lineage marker"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); PRAGMA application_id = 0x4D4C4F47; \ + PRAGMA user_version = 2;", + ) + .expect("seed future schema"); + + assert!(apply_migrations(&connection).is_err()); + + assert_eq!(version(&connection), 2); + assert_eq!(user_objects(&connection), 2); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn invalid_registries_are_rejected_before_initialization() { + let invalid_registries: &[(u32, &[Migration])] = &[ + (2, &[]), + ( + 3, + &[Migration { + version: 2, + apply: migration_v2, + }], + ), + ( + 3, + &[ + Migration { + version: 3, + apply: migration_v3, + }, + Migration { + version: 2, + apply: migration_v2, + }, + ], + ), + ( + 3, + &[ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 2, + apply: migration_v2, + }, + ], + ), + ( + 2, + &[Migration { + version: 1, + apply: migration_v2, + }], + ), + ( + 3, + &[ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: migration_v3, + }, + Migration { + version: 4, + apply: migration_v3, + }, + ], + ), + ]; + + for (target, registry) in invalid_registries { + let connection = Connection::open_in_memory().expect("open database"); + assert!( + run_migrations( + &connection, + plan(*target, synthetic_initial_schema, registry), + ) + .is_err() + ); + assert_eq!(version(&connection), 0); + assert_eq!(user_objects(&connection), 0); + } +} + +#[test] +fn synthetic_version_two_step_commits_schema_data_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + + run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)) + .expect("migrate to v2"); + + assert_eq!(version(&connection), 2); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 3); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v2", [], |row| row + .get::<_, String>(0)) + .expect("v2 row"), + "v2" + ); +} + +#[test] +fn failed_version_two_step_rolls_back_its_schema_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: failing_migration_v2, + }]; + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 2); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); +} + +#[test] +fn committed_version_two_resumes_with_corrected_version_three() { + let connection = Connection::open_in_memory().expect("open database"); + let failing = [ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: failing_migration_v3, + }, + ]; + assert!(run_migrations(&connection, plan(3, synthetic_initial_schema, &failing)).is_err()); + assert_eq!(version(&connection), 2); + assert_eq!(user_objects(&connection), 3); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + + let corrected = [ + Migration { + version: 2, + apply: migration_v2, + }, + Migration { + version: 3, + apply: migration_v3, + }, + ]; + run_migrations(&connection, plan(3, synthetic_initial_schema, &corrected)).expect("resume v3"); + + assert_eq!(version(&connection), 3); + assert_eq!(user_objects(&connection), 4); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v3", [], |row| row + .get::<_, String>(0)) + .expect("v3 row"), + "v3" + ); +} + +#[test] +fn failing_initialization_rolls_back_schema_and_version() { + let connection = Connection::open_in_memory().expect("open database"); + + assert!(run_migrations(&connection, plan(1, failing_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 0); +} + +#[test] +fn initialization_lineage_conflict_rolls_back_every_object_and_pragma() { + let connection = Connection::open_in_memory().expect("open database"); + + assert!(run_migrations(&connection, plan(1, conflicting_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), 0); + assert_eq!(user_objects(&connection), 0); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs b/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs new file mode 100644 index 0000000000..241d164725 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/runner/identity.rs @@ -0,0 +1,192 @@ +use super::*; + +#[test] +fn initialized_version_one_is_a_data_preserving_noop() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + connection + .execute("INSERT INTO baseline VALUES ('kept')", []) + .expect("seed current schema data"); + + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("reopen schema"); + + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM baseline", [], |row| row + .get::<_, String>(0)) + .expect("preserved row"), + "kept" + ); +} + +#[test] +fn markerless_identified_version_one_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;", + ) + .expect("seed forged schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 1); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn nonzero_version_without_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); INSERT INTO sentinel VALUES ('kept'); \ + PRAGMA user_version = 1;", + ) + .expect("seed foreign schema"); + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), 0); + assert!(!schema_object_names(&connection, "table").contains(&"future_v2".to_owned())); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); +} + +#[test] +fn nonzero_version_with_wrong_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + connection + .execute_batch( + "CREATE TABLE sentinel (value TEXT); PRAGMA application_id = 0x4E4F5045; \ + PRAGMA user_version = 1;", + ) + .expect("seed foreign schema"); + + assert!(run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)).is_err()); + + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), 0x4E4F5045); + assert_eq!(user_objects(&connection), 1); + assert!(!schema_object_names(&connection, "table").contains(&"future_v2".to_owned())); +} + +#[test] +fn empty_version_zero_with_identity_is_rejected_without_mutation() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .pragma_update(None, "application_id", APPLICATION_ID) + .expect("seed identity without schema"); + + assert!(run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).is_err()); + + assert_eq!(version(&connection), 0); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), 0); +} + +#[test] +fn identified_version_one_advances_to_future_target() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + let registry = [Migration { + version: 2, + apply: migration_v2, + }]; + + run_migrations(&connection, plan(2, synthetic_initial_schema, ®istry)) + .expect("migrate identified schema"); + + assert_eq!(version(&connection), 2); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert!(lineage::is_valid(&connection).expect("inspect lineage")); + assert_eq!( + connection + .query_row("SELECT value FROM future_v2", [], |row| row + .get::<_, String>(0)) + .expect("v2 row"), + "v2" + ); +} + +#[test] +fn malformed_lineage_states_are_rejected_without_migration() { + const CASES: &[&str] = &[ + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 2);", + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL, extra INTEGER) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 1, 0);", + "CREATE TABLE _mesh_llm_log_store_lineage (id INTEGER NOT NULL PRIMARY KEY, epoch INTEGER NOT NULL) WITHOUT ROWID; + INSERT INTO _mesh_llm_log_store_lineage VALUES (1, 1); + INSERT INTO _mesh_llm_log_store_lineage VALUES (2, 1);", + "CREATE VIEW _mesh_llm_log_store_lineage AS SELECT 1 AS id, 1 AS epoch;", + ]; + for marker_sql in CASES { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch(&format!( + "CREATE TABLE sentinel (value TEXT NOT NULL); INSERT INTO sentinel VALUES ('kept'); + {marker_sql} + PRAGMA application_id = 0x4D4C4F47; PRAGMA user_version = 1;" + )) + .expect("seed malformed lineage"); + let objects_before = user_objects(&connection); + + let error = run_migrations(&connection, plan(1, synthetic_initial_schema, &[])) + .expect_err("reject malformed lineage"); + + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert_eq!(user_objects(&connection), objects_before); + assert_eq!( + connection + .query_row("SELECT value FROM sentinel", [], |row| row + .get::<_, String>(0)) + .expect("sentinel row"), + "kept" + ); + } +} + +#[test] +fn extra_private_object_invalidates_an_initialized_lineage() { + let connection = Connection::open_in_memory().expect("open database"); + run_migrations(&connection, plan(1, synthetic_initial_schema, &[])).expect("initialize schema"); + connection + .execute_batch("CREATE VIEW _mesh_llm_log_store_extra AS SELECT 1 AS value;") + .expect("seed extra private object"); + + let error = run_migrations(&connection, plan(1, synthetic_initial_schema, &[])) + .expect_err("reject extra private object"); + + assert!(matches!(error, rusqlite::Error::InvalidQuery)); + assert_eq!(version(&connection), 1); + assert_eq!(application_id(&connection), APPLICATION_ID); + assert!( + schema_object_names(&connection, "view").contains(&"_mesh_llm_log_store_extra".to_owned()) + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs b/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs new file mode 100644 index 0000000000..cf0403a206 --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/schema_contract.rs @@ -0,0 +1,129 @@ +use super::*; + +const EXPECTED_TABLES: &[&str] = &[ + "artifact_pointers", + "audit_entries", + "cleanup_runs", + "lifecycle_events", + "maintenance_operation_targets", + "maintenance_operations", + "pending_artifact_deletions", + "proxy_records", + "summaries", + "webhook_deliveries", +]; + +#[test] +fn fresh_database_is_exactly_complete_current_schema() { + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch("PRAGMA foreign_keys = ON;") + .expect("enable foreign keys"); + + apply_migrations(&connection).expect("apply fresh schema"); + + assert_eq!(CURRENT_VERSION, 1); + assert!(MIGRATIONS.is_empty()); + assert_eq!( + connection + .pragma_query_value(None, "application_id", |row| row.get::<_, u32>(0)) + .expect("application identity"), + APPLICATION_ID + ); + assert_eq!( + connection + .pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .expect("schema version"), + 1 + ); + let application_tables = schema_object_names(&connection, "table") + .into_iter() + .filter(|name| name != "_mesh_llm_log_store_lineage") + .collect::>(); + assert_eq!(application_tables, EXPECTED_TABLES); + assert_eq!( + private_schema_object_names(&connection), + ["_mesh_llm_log_store_lineage"] + ); + assert!(lineage::is_valid(&connection).expect("inspect lineage marker")); + assert!( + schema_sql(&connection, "table", "_mesh_llm_log_store_lineage").contains("WITHOUT ROWID") + ); + assert_eq!(schema_object_names(&connection, "index"), EXPECTED_INDEXES); + assert_eq!( + table_columns(&connection, "summaries"), + [ + "request_id", + "state", + "created_at", + "terminal_at", + "route", + "model", + "provider", + "engine", + "status_code", + "error_msg", + "tenant_id", + "account_id", + "user_id", + "caller_endpoint_id", + "caller_addr", + "caller_path_type", + ] + ); + assert_eq!( + table_columns(&connection, "artifact_pointers") + .last() + .map(String::as_str), + Some("unavailable_reason") + ); + assert!(schema_sql(&connection, "table", "lifecycle_events").contains("is_terminal IN (0, 1)")); + assert!( + schema_sql(&connection, "index", "idx_terminal_event_one_per_request") + .contains("WHERE is_terminal = 1") + ); + assert!( + schema_sql(&connection, "table", "webhook_deliveries") + .contains("terminal_status_code BETWEEN 100 AND 599") + ); + assert!(!schema_sql(&connection, "table", "pending_artifact_deletions").contains("REFERENCES")); +} + +fn private_schema_object_names(connection: &Connection) -> Vec { + let mut statement = connection + .prepare( + "SELECT name FROM sqlite_schema + WHERE name GLOB '_mesh_llm_log_store_*' ORDER BY name", + ) + .expect("prepare private schema objects"); + statement + .query_map([], |row| row.get(0)) + .expect("query private schema objects") + .collect::, _>>() + .expect("collect private schema objects") +} + +#[test] +fn initial_webhook_schema_bounds_terminal_status_codes() { + let connection = Connection::open_in_memory().expect("open database"); + apply_migrations(&connection).expect("apply schema"); + connection + .execute( + "INSERT INTO summaries (request_id, created_at) VALUES ('request-1', '2026-08-04T12:00:00Z')", + [], + ) + .expect("seed request"); + + let error = connection + .execute( + "INSERT INTO webhook_deliveries \ + (delivery_id, request_id, terminal_outcome, terminal_status_code, occurred_at, target_url, attempt_number) \ + VALUES ('delivery-1', 'request-1', 'completed', 99, '2026-08-04T12:00:00Z', 'configured_webhook', 0)", + [], + ) + .expect_err("reject status below 100"); + + assert!( + matches!(error, rusqlite::Error::SqliteFailure(_, Some(message)) if message.contains("terminal_status_code")) + ); +} diff --git a/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs b/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs new file mode 100644 index 0000000000..79419f934d --- /dev/null +++ b/crates/mesh-llm-log-store/src/migrations/tests/sqlite_autoincrement.rs @@ -0,0 +1,42 @@ +use rusqlite::Connection; + +#[test] +fn deleting_maximum_rowid_reuses_it_without_autoincrement_but_not_with_autoincrement() { + // Given + let connection = Connection::open_in_memory().expect("open database"); + connection + .execute_batch( + "CREATE TABLE reusable (sequence INTEGER PRIMARY KEY); + CREATE TABLE monotonic (sequence INTEGER PRIMARY KEY AUTOINCREMENT); + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO reusable DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES; + DELETE FROM reusable WHERE sequence = 3; + DELETE FROM monotonic WHERE sequence = 3;", + ) + .expect("seed and delete maximum rowids"); + + // When + connection + .execute_batch( + "INSERT INTO reusable DEFAULT VALUES; + INSERT INTO monotonic DEFAULT VALUES;", + ) + .expect("insert replacement rows"); + + // Then + let reusable = connection + .query_row("SELECT MAX(sequence) FROM reusable", [], |row| { + row.get::<_, u64>(0) + }) + .expect("read reusable sequence"); + let monotonic = connection + .query_row("SELECT MAX(sequence) FROM monotonic", [], |row| { + row.get::<_, u64>(0) + }) + .expect("read monotonic sequence"); + assert_eq!((reusable, monotonic), (3, 4)); +} diff --git a/crates/mesh-llm-log-store/src/query/mod.rs b/crates/mesh-llm-log-store/src/query/mod.rs index 9aa9610d3b..4edabe0f66 100644 --- a/crates/mesh-llm-log-store/src/query/mod.rs +++ b/crates/mesh-llm-log-store/src/query/mod.rs @@ -4,9 +4,12 @@ //! parsing, active-registry merging, and artifact-content authorization belong //! to later host/API layers. +mod records; mod related; mod requests; +pub use records::{RequestRecord, RequestRecordWithCaller}; + use chrono::DateTime; use crate::LogStoreError; @@ -67,6 +70,8 @@ pub struct RequestQuery { pub from: Option, pub to: Option, pub route: Option, + pub exclude_route: Option, + pub exclude_route_prefix: Option, pub model: Option, pub provider: Option, pub engine: Option, @@ -79,7 +84,14 @@ impl RequestQuery { pub fn validate(&self) -> Result<(), LogStoreError> { validate_limit(self.limit)?; validate_time_range(self.from.as_deref(), self.to.as_deref())?; - for value in [&self.route, &self.model, &self.provider, &self.engine] { + for value in [ + &self.route, + &self.exclude_route, + &self.exclude_route_prefix, + &self.model, + &self.provider, + &self.engine, + ] { validate_filter(value.as_deref())?; } Ok(()) @@ -118,19 +130,6 @@ impl ProxyQuery { } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RequestRecord { - pub request_id: String, - pub outcome: String, - pub created_at: String, - pub terminal_at: Option, - pub route: Option, - pub model: Option, - pub provider: Option, - pub engine: Option, - pub status_code: Option, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub struct EventRecord { pub event_id: String, diff --git a/crates/mesh-llm-log-store/src/query/records.rs b/crates/mesh-llm-log-store/src/query/records.rs new file mode 100644 index 0000000000..46638be485 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/records.rs @@ -0,0 +1,27 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RequestRecord { + pub request_id: String, + pub outcome: String, + pub created_at: String, + pub terminal_at: Option, + pub route: Option, + pub model: Option, + pub provider: Option, + pub engine: Option, + pub status_code: Option, +} + +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RequestRecordWithCaller { + pub request: RequestRecord, + pub caller_endpoint_id: Option, + pub caller_addr: Option, + pub caller_path_type: Option, +} + +impl From for RequestRecord { + fn from(detailed: RequestRecordWithCaller) -> Self { + detailed.request + } +} diff --git a/crates/mesh-llm-log-store/src/query/related.rs b/crates/mesh-llm-log-store/src/query/related.rs index cb93f16876..54e91d57b1 100644 --- a/crates/mesh-llm-log-store/src/query/related.rs +++ b/crates/mesh-llm-log-store/src/query/related.rs @@ -2,13 +2,20 @@ use std::collections::BTreeMap; use rusqlite::types::Value; -use super::requests::{Conditions, CursorPosition, query_page, validate_cursor_position}; +mod projection; +mod selection; + +use super::requests::selection::{ + Conditions, CursorPosition, query_page, validate_cursor_position, +}; use super::{ ArtifactRecord, EventRecord, PageQuery, ProxyQuery, ProxyRecord, QueryPage, validate_identifier, }; use crate::cursor::decode_ordering_cursor; use crate::{LogStore, LogStoreError}; +use projection::{artifact_record, event_record, proxy_record}; + impl LogStore { /// Return lifecycle records only for the named durable request. pub fn query_events( @@ -176,130 +183,9 @@ impl LogStore { |record| (&record.occurred_at, &record.attempt_id), ) } - - fn query_related_page( - &self, - related: RelatedQuery<'_>, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - cursor_fields: impl Fn(&T) -> (&str, &str), - ) -> Result, LogStoreError> { - validate_identifier(related.request_id)?; - related.page.validate()?; - let conditions = Conditions { - sql: " WHERE request_id = ?".to_string(), - }; - let mut values = vec![Value::Text(related.request_id.to_string())]; - let connection = self.conn(); - if let Some(cursor) = &related.page.cursor { - let (timestamp, id) = decode_ordering_cursor(cursor)?; - validate_cursor_position( - &connection, - CursorPosition { - table: related.table, - timestamp_column: "occurred_at", - id_column: related.id_column, - timestamp: ×tamp, - id: &id, - conditions: &conditions, - values: &values, - }, - )?; - values.push(Value::Text(timestamp)); - values.push(Value::Text(id)); - } - let mut sql = format!("SELECT {} FROM {}", related.columns, related.table); - conditions.append_to_sql(&mut sql); - if related.page.cursor.is_some() { - sql.push_str(&format!( - " AND (occurred_at, {}) {} (?, ?)", - related.id_column, - related.page.sort.cursor_operator() - )); - } - sql.push_str(&format!( - " ORDER BY occurred_at {}, {} {} LIMIT ?", - related.page.sort.sql_order(), - related.id_column, - related.page.sort.sql_order() - )); - values.push(Value::Integer( - i64::try_from(related.page.limit + 1) - .map_err(|_| LogStoreError::InvalidQuery("limit is out of range".to_string()))?, - )); - query_page( - &connection, - sql, - values, - related.page.limit, - map, - cursor_fields, - ) - } - - fn query_related_for_requests( - &self, - request_ids: &[String], - per_request_limit: usize, - related: RelatedBatchQuery, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - request_id: impl Fn(&T) -> &str, - ) -> Result>, LogStoreError> { - if request_ids.is_empty() { - return Ok(BTreeMap::new()); - } - if request_ids.len() > super::MAX_QUERY_LIMIT { - return Err(LogStoreError::InvalidQuery(format!( - "request owner count must be at most {}", - super::MAX_QUERY_LIMIT - ))); - } - super::validate_limit(per_request_limit)?; - for owner in request_ids { - validate_identifier(owner)?; - } - - let placeholders = std::iter::repeat_n("?", request_ids.len()) - .collect::>() - .join(", "); - let sql = format!( - "SELECT {columns} FROM (\ - SELECT {columns}, ROW_NUMBER() OVER (\ - PARTITION BY request_id ORDER BY occurred_at ASC, {id_column} ASC\ - ) AS owner_row_number \ - FROM {table} WHERE request_id IN ({placeholders})\ - ) WHERE owner_row_number <= ? \ - ORDER BY request_id ASC, occurred_at ASC, {id_column} ASC", - columns = related.columns, - id_column = related.id_column, - table = related.table, - ); - let mut values = request_ids - .iter() - .cloned() - .map(Value::Text) - .collect::>(); - values.push(Value::Integer(i64::try_from(per_request_limit).map_err( - |_| LogStoreError::InvalidQuery("limit is out of range".to_string()), - )?)); - let connection = self.conn(); - let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let rows = statement - .query_map(rusqlite::params_from_iter(values), map) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - let mut grouped: BTreeMap> = BTreeMap::new(); - for row in rows { - grouped - .entry(request_id(&row).to_owned()) - .or_default() - .push(row); - } - Ok(grouped) - } } -struct RelatedQuery<'a> { +pub(super) struct RelatedQuery<'a> { table: &'static str, columns: &'static str, id_column: &'static str, @@ -307,49 +193,8 @@ struct RelatedQuery<'a> { page: &'a PageQuery, } -struct RelatedBatchQuery { +pub(super) struct RelatedBatchQuery { table: &'static str, columns: &'static str, id_column: &'static str, } - -fn event_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(EventRecord { - event_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - payload_json: row.get(3)?, - }) -} - -fn artifact_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(ArtifactRecord { - artifact_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - kind: row.get(3)?, - media_kind: row.get(4)?, - checksum: row.get(5)?, - bytes: row.get(6)?, - version: row.get(7)?, - redacted: row.get::<_, i32>(8)? != 0, - truncated: row.get::<_, i32>(9)? != 0, - missing: row.get::<_, i32>(10)? != 0, - corrupt: row.get::<_, i32>(11)? != 0, - unavailable_reason: row.get(12)?, - }) -} - -fn proxy_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(ProxyRecord { - attempt_id: row.get(0)?, - request_id: row.get(1)?, - occurred_at: row.get(2)?, - target: row.get(3)?, - provider: row.get(4)?, - engine: row.get(5)?, - started_at: row.get(6)?, - completed_at: row.get(7)?, - status_code: row.get(8)?, - }) -} diff --git a/crates/mesh-llm-log-store/src/query/related/projection.rs b/crates/mesh-llm-log-store/src/query/related/projection.rs new file mode 100644 index 0000000000..83f54713d7 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/related/projection.rs @@ -0,0 +1,42 @@ +use super::super::{ArtifactRecord, EventRecord, ProxyRecord}; + +pub(super) fn event_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(EventRecord { + event_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + payload_json: row.get(3)?, + }) +} + +pub(super) fn artifact_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ArtifactRecord { + artifact_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + kind: row.get(3)?, + media_kind: row.get(4)?, + checksum: row.get(5)?, + bytes: row.get(6)?, + version: row.get(7)?, + redacted: row.get::<_, i32>(8)? != 0, + truncated: row.get::<_, i32>(9)? != 0, + missing: row.get::<_, i32>(10)? != 0, + corrupt: row.get::<_, i32>(11)? != 0, + unavailable_reason: row.get(12)?, + }) +} + +pub(super) fn proxy_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ProxyRecord { + attempt_id: row.get(0)?, + request_id: row.get(1)?, + occurred_at: row.get(2)?, + target: row.get(3)?, + provider: row.get(4)?, + engine: row.get(5)?, + started_at: row.get(6)?, + completed_at: row.get(7)?, + status_code: row.get(8)?, + }) +} diff --git a/crates/mesh-llm-log-store/src/query/related/selection.rs b/crates/mesh-llm-log-store/src/query/related/selection.rs new file mode 100644 index 0000000000..9eb2b6e551 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/related/selection.rs @@ -0,0 +1,133 @@ +use std::collections::BTreeMap; + +use rusqlite::types::Value; + +use super::{RelatedBatchQuery, RelatedQuery}; +use crate::cursor::decode_ordering_cursor; +use crate::query::requests::selection::{ + Conditions, CursorPosition, query_page, validate_cursor_position, +}; +use crate::query::{QueryPage, validate_identifier}; +use crate::{LogStore, LogStoreError}; + +impl LogStore { + pub(super) fn query_related_page( + &self, + related: RelatedQuery<'_>, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + cursor_fields: impl Fn(&T) -> (&str, &str), + ) -> Result, LogStoreError> { + validate_identifier(related.request_id)?; + related.page.validate()?; + let conditions = Conditions { + sql: " WHERE request_id = ?".to_string(), + }; + let mut values = vec![Value::Text(related.request_id.to_string())]; + let connection = self.conn(); + if let Some(cursor) = &related.page.cursor { + let (timestamp, id) = decode_ordering_cursor(cursor)?; + validate_cursor_position( + &connection, + CursorPosition { + table: related.table, + timestamp_column: "occurred_at", + id_column: related.id_column, + timestamp: ×tamp, + id: &id, + conditions: &conditions, + values: &values, + }, + )?; + values.push(Value::Text(timestamp)); + values.push(Value::Text(id)); + } + let mut sql = format!("SELECT {} FROM {}", related.columns, related.table); + conditions.append_to_sql(&mut sql); + if related.page.cursor.is_some() { + sql.push_str(&format!( + " AND (occurred_at, {}) {} (?, ?)", + related.id_column, + related.page.sort.cursor_operator() + )); + } + sql.push_str(&format!( + " ORDER BY occurred_at {}, {} {} LIMIT ?", + related.page.sort.sql_order(), + related.id_column, + related.page.sort.sql_order() + )); + values.push(Value::Integer( + i64::try_from(related.page.limit + 1) + .map_err(|_| LogStoreError::InvalidQuery("limit is out of range".to_string()))?, + )); + query_page( + &connection, + sql, + values, + related.page.limit, + map, + cursor_fields, + ) + } + + pub(super) fn query_related_for_requests( + &self, + request_ids: &[String], + per_request_limit: usize, + related: RelatedBatchQuery, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + request_id: impl Fn(&T) -> &str, + ) -> Result>, LogStoreError> { + if request_ids.is_empty() { + return Ok(BTreeMap::new()); + } + if request_ids.len() > super::super::MAX_QUERY_LIMIT { + return Err(LogStoreError::InvalidQuery(format!( + "request owner count must be at most {}", + super::super::MAX_QUERY_LIMIT + ))); + } + super::super::validate_limit(per_request_limit)?; + for owner in request_ids { + validate_identifier(owner)?; + } + let placeholders = std::iter::repeat_n("?", request_ids.len()) + .collect::>() + .join(", "); + let sql = format!( + "SELECT {columns} FROM (\ + SELECT {columns}, ROW_NUMBER() OVER (\ + PARTITION BY request_id ORDER BY occurred_at ASC, {id_column} ASC\ + ) AS owner_row_number \ + FROM {table} WHERE request_id IN ({placeholders})\ + ) WHERE owner_row_number <= ? \ + ORDER BY request_id ASC, occurred_at ASC, {id_column} ASC", + columns = related.columns, + id_column = related.id_column, + table = related.table, + ); + let mut values = request_ids + .iter() + .cloned() + .map(Value::Text) + .collect::>(); + values.push(Value::Integer(i64::try_from(per_request_limit).map_err( + |_| LogStoreError::InvalidQuery("limit is out of range".to_string()), + )?)); + let connection = self.conn(); + let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let rows = statement + .query_map(rusqlite::params_from_iter(values), map) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + let mut grouped = BTreeMap::new(); + for row in rows { + grouped + .entry(request_id(&row).to_owned()) + .or_insert_with(Vec::new) + .push(row); + } + Ok(grouped) + } +} diff --git a/crates/mesh-llm-log-store/src/query/requests.rs b/crates/mesh-llm-log-store/src/query/requests.rs index bf3ab98e6a..b05c73f489 100644 --- a/crates/mesh-llm-log-store/src/query/requests.rs +++ b/crates/mesh-llm-log-store/src/query/requests.rs @@ -1,41 +1,51 @@ -use rusqlite::types::Value; +mod projection; +pub(super) mod selection; -use mesh_llm_events::logging::timestamp::canonical_logging_timestamp; +use rusqlite::types::Value; -use super::{QueryPage, RequestQuery, RequestRecord}; -use crate::cursor::{decode_ordering_cursor, encode_cursor}; +use super::{QueryPage, RequestQuery, RequestRecord, RequestRecordWithCaller}; +use crate::cursor::decode_ordering_cursor; use crate::{LogStore, LogStoreError}; -const REQUEST_COLUMNS: &str = - "request_id, state, created_at, terminal_at, route, model, provider, engine, status_code"; +use projection::{REQUEST_COLUMNS, request_record_with_caller}; +use selection::{CursorPosition, query_page, validate_cursor_position}; + const REQUEST_ID_BATCH_SIZE: usize = 100; impl LogStore { - /// Return one durable request summary without exposing internal identity or - /// error columns. Active-state merging is intentionally owned by the host. pub fn query_request(&self, request_id: &str) -> Result, LogStoreError> { + self.query_request_with_caller(request_id) + .map(|record| record.map(Into::into)) + } + + pub fn query_request_with_caller( + &self, + request_id: &str, + ) -> Result, LogStoreError> { let connection = self.conn(); let sql = format!("SELECT {REQUEST_COLUMNS} FROM summaries WHERE request_id = ?"); - match connection.query_row(&sql, [request_id], request_record) { + match connection.query_row(&sql, [request_id], request_record_with_caller) { Ok(record) => Ok(Some(record)), Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), } } - /// Return durable summaries for a bounded set of request IDs. - /// - /// The caller may supply more IDs than SQLite's conservative parameter - /// budget because this method splits them into fixed-size statements while - /// holding the connection only once. Missing IDs are omitted. pub fn query_requests_by_ids( &self, request_ids: &[String], ) -> Result, LogStoreError> { + self.query_requests_by_ids_with_caller(request_ids) + .map(|records| records.into_iter().map(Into::into).collect()) + } + + pub fn query_requests_by_ids_with_caller( + &self, + request_ids: &[String], + ) -> Result, LogStoreError> { if request_ids.is_empty() { return Ok(Vec::new()); } - let connection = self.conn(); let mut records = Vec::with_capacity(request_ids.len()); for request_ids in request_ids.chunks(REQUEST_ID_BATCH_SIZE) { @@ -47,7 +57,10 @@ impl LogStore { ); let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; let rows = statement - .query_map(rusqlite::params_from_iter(request_ids), request_record) + .query_map( + rusqlite::params_from_iter(request_ids), + request_record_with_caller, + ) .map_err(LogStoreError::Sqlite)?; records.extend( rows.collect::, _>>() @@ -57,17 +70,24 @@ impl LogStore { Ok(records) } - /// List durable summaries by a stable `(created_at, request_id)` keyset. - /// A valid cursor must name a row in the exact current filter scope, so a - /// forged pair and a cursor retained away both fail deterministically. pub fn query_requests( &self, query: &RequestQuery, ) -> Result, LogStoreError> { + self.query_requests_with_caller(query) + .map(|page| QueryPage { + items: page.items.into_iter().map(Into::into).collect(), + next_cursor: page.next_cursor, + }) + } + + pub fn query_requests_with_caller( + &self, + query: &RequestQuery, + ) -> Result, LogStoreError> { query.validate()?; - let (conditions, mut values) = request_conditions(query); + let (conditions, mut values) = selection::request_conditions(query); let connection = self.conn(); - if let Some(cursor) = &query.cursor { let (timestamp, request_id) = decode_ordering_cursor(cursor)?; validate_cursor_position( @@ -85,7 +105,6 @@ impl LogStore { values.push(Value::Text(timestamp)); values.push(Value::Text(request_id)); } - let mut sql = format!("SELECT {REQUEST_COLUMNS} FROM summaries"); conditions.append_to_sql(&mut sql); if query.cursor.is_some() { @@ -107,138 +126,8 @@ impl LogStore { sql, values, query.limit, - request_record, - |record| (&record.created_at, &record.request_id), + request_record_with_caller, + |record| (&record.request.created_at, &record.request.request_id), ) } } - -pub(super) struct Conditions { - pub(super) sql: String, -} - -impl Conditions { - pub(super) fn append_to_sql(&self, sql: &mut String) { - sql.push_str(&self.sql); - } -} - -fn request_conditions(query: &RequestQuery) -> (Conditions, Vec) { - let mut sql = String::new(); - let mut values = Vec::new(); - if let Some(from) = &query.from { - sql.push_str(" WHERE created_at >= ?"); - values.push(Value::Text(normalize_timestamp(from))); - } else { - sql.push_str(" WHERE 1 = 1"); - } - if let Some(to) = &query.to { - sql.push_str(" AND created_at <= ?"); - values.push(Value::Text(normalize_timestamp(to))); - } - for (column, value) in [ - ("route", &query.route), - ("model", &query.model), - ("provider", &query.provider), - ("engine", &query.engine), - ] { - if let Some(value) = value { - sql.push_str(&format!(" AND {column} = ?")); - values.push(Value::Text(value.clone())); - } - } - if let Some(status_code) = query.status_code { - sql.push_str(" AND status_code = ?"); - values.push(Value::Integer(i64::from(status_code))); - } - if let Some(outcome) = query.outcome { - sql.push_str(" AND state = ?"); - values.push(Value::Text(outcome.as_str().to_string())); - } - let conditions = Conditions { sql }; - (conditions, values) -} - -pub(super) struct CursorPosition<'a> { - pub(super) table: &'static str, - pub(super) timestamp_column: &'static str, - pub(super) id_column: &'static str, - pub(super) timestamp: &'a str, - pub(super) id: &'a str, - pub(super) conditions: &'a Conditions, - pub(super) values: &'a [Value], -} - -pub(super) fn validate_cursor_position( - connection: &rusqlite::Connection, - position: CursorPosition<'_>, -) -> Result<(), LogStoreError> { - let mut sql = format!( - "SELECT 1 FROM {} WHERE {} = ? AND {} = ?", - position.table, position.timestamp_column, position.id_column - ); - let mut parameters = vec![ - Value::Text(position.timestamp.to_string()), - Value::Text(position.id.to_string()), - ]; - if position.conditions.sql.starts_with(" WHERE ") { - sql.push_str(" AND "); - sql.push_str(&position.conditions.sql[7..]); - } - // `values` is supplied separately so callers cannot accidentally validate - // the cursor with a different set of filters than their page query. - parameters.extend(position.values.iter().cloned()); - match connection.query_row(&sql, rusqlite::params_from_iter(parameters.iter()), |_| { - Ok(()) - }) { - Ok(()) => Ok(()), - Err(rusqlite::Error::QueryReturnedNoRows) => Err(LogStoreError::CursorInvalid), - Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), - } -} - -pub(super) fn query_page( - connection: &rusqlite::Connection, - sql: String, - values: Vec, - limit: usize, - map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, - cursor_fields: impl Fn(&T) -> (&str, &str), -) -> Result, LogStoreError> { - let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let rows = statement - .query_map(rusqlite::params_from_iter(values.iter()), map) - .map_err(LogStoreError::Sqlite)?; - let mut items = rows - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - let has_more = items.len() > limit; - items.truncate(limit); - let next_cursor = if has_more { - items.last().map(|item| { - let (timestamp, id) = cursor_fields(item); - encode_cursor(timestamp, id) - }) - } else { - None - }; - Ok(QueryPage { items, next_cursor }) -} - -fn normalize_timestamp(value: &str) -> String { - canonical_logging_timestamp(value).expect("RequestQuery::validate parses time bounds") -} - -fn request_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(RequestRecord { - request_id: row.get(0)?, - outcome: row.get(1)?, - created_at: row.get(2)?, - terminal_at: row.get(3)?, - route: row.get(4)?, - model: row.get(5)?, - provider: row.get(6)?, - engine: row.get(7)?, - status_code: row.get(8)?, - }) -} diff --git a/crates/mesh-llm-log-store/src/query/requests/projection.rs b/crates/mesh-llm-log-store/src/query/requests/projection.rs new file mode 100644 index 0000000000..e1b0ca6494 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/requests/projection.rs @@ -0,0 +1,25 @@ +use super::super::{RequestRecord, RequestRecordWithCaller}; + +pub(super) const REQUEST_COLUMNS: &str = "request_id, state, created_at, terminal_at, route, model, provider, engine, status_code, \ + caller_endpoint_id, caller_addr, caller_path_type"; + +pub(super) fn request_record_with_caller( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(RequestRecordWithCaller { + request: RequestRecord { + request_id: row.get(0)?, + outcome: row.get(1)?, + created_at: row.get(2)?, + terminal_at: row.get(3)?, + route: row.get(4)?, + model: row.get(5)?, + provider: row.get(6)?, + engine: row.get(7)?, + status_code: row.get(8)?, + }, + caller_endpoint_id: row.get(9)?, + caller_addr: row.get(10)?, + caller_path_type: row.get(11)?, + }) +} diff --git a/crates/mesh-llm-log-store/src/query/requests/selection.rs b/crates/mesh-llm-log-store/src/query/requests/selection.rs new file mode 100644 index 0000000000..44b4a34e46 --- /dev/null +++ b/crates/mesh-llm-log-store/src/query/requests/selection.rs @@ -0,0 +1,128 @@ +use mesh_llm_events::logging::timestamp::canonical_logging_timestamp; +use rusqlite::types::Value; + +use super::super::{QueryPage, RequestQuery}; +use crate::LogStoreError; +use crate::cursor::encode_cursor; + +pub(crate) struct Conditions { + pub(crate) sql: String, +} + +impl Conditions { + pub(crate) fn append_to_sql(&self, sql: &mut String) { + sql.push_str(&self.sql); + } +} + +pub(super) fn request_conditions(query: &RequestQuery) -> (Conditions, Vec) { + let mut sql = String::new(); + let mut values = Vec::new(); + if let Some(from) = &query.from { + sql.push_str(" WHERE created_at >= ?"); + values.push(Value::Text(normalize_timestamp(from))); + } else { + sql.push_str(" WHERE 1 = 1"); + } + if let Some(to) = &query.to { + sql.push_str(" AND created_at <= ?"); + values.push(Value::Text(normalize_timestamp(to))); + } + for (column, value) in [ + ("route", &query.route), + ("model", &query.model), + ("provider", &query.provider), + ("engine", &query.engine), + ] { + if let Some(value) = value { + sql.push_str(&format!(" AND {column} = ?")); + values.push(Value::Text(value.clone())); + } + } + if let Some(route) = &query.exclude_route { + sql.push_str(" AND (route IS NULL OR route != ?)"); + values.push(Value::Text(route.clone())); + } + if let Some(prefix) = &query.exclude_route_prefix { + sql.push_str(" AND (route IS NULL OR substr(route, 1, length(?)) != ?)"); + values.push(Value::Text(prefix.clone())); + values.push(Value::Text(prefix.clone())); + } + if let Some(status_code) = query.status_code { + sql.push_str(" AND status_code = ?"); + values.push(Value::Integer(i64::from(status_code))); + } + if let Some(outcome) = query.outcome { + sql.push_str(" AND state = ?"); + values.push(Value::Text(outcome.as_str().to_string())); + } + (Conditions { sql }, values) +} + +pub(crate) struct CursorPosition<'a> { + pub(crate) table: &'static str, + pub(crate) timestamp_column: &'static str, + pub(crate) id_column: &'static str, + pub(crate) timestamp: &'a str, + pub(crate) id: &'a str, + pub(crate) conditions: &'a Conditions, + pub(crate) values: &'a [Value], +} + +pub(crate) fn validate_cursor_position( + connection: &rusqlite::Connection, + position: CursorPosition<'_>, +) -> Result<(), LogStoreError> { + let mut sql = format!( + "SELECT 1 FROM {} WHERE {} = ? AND {} = ?", + position.table, position.timestamp_column, position.id_column + ); + let mut parameters = vec![ + Value::Text(position.timestamp.to_string()), + Value::Text(position.id.to_string()), + ]; + if position.conditions.sql.starts_with(" WHERE ") { + sql.push_str(" AND "); + sql.push_str(&position.conditions.sql[7..]); + } + parameters.extend(position.values.iter().cloned()); + match connection.query_row(&sql, rusqlite::params_from_iter(parameters.iter()), |_| { + Ok(()) + }) { + Ok(()) => Ok(()), + Err(rusqlite::Error::QueryReturnedNoRows) => Err(LogStoreError::CursorInvalid), + Err(error) => Err(LogStoreError::QueryFailed(error.to_string())), + } +} + +pub(crate) fn query_page( + connection: &rusqlite::Connection, + sql: String, + values: Vec, + limit: usize, + map: fn(&rusqlite::Row<'_>) -> rusqlite::Result, + cursor_fields: impl Fn(&T) -> (&str, &str), +) -> Result, LogStoreError> { + let mut statement = connection.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let rows = statement + .query_map(rusqlite::params_from_iter(values.iter()), map) + .map_err(LogStoreError::Sqlite)?; + let mut items = rows + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + let has_more = items.len() > limit; + items.truncate(limit); + let next_cursor = if has_more { + items.last().map(|item| { + let (timestamp, id) = cursor_fields(item); + encode_cursor(timestamp, id) + }) + } else { + None + }; + Ok(QueryPage { items, next_cursor }) +} + +fn normalize_timestamp(value: &str) -> String { + canonical_logging_timestamp(value).expect("RequestQuery::validate parses time bounds") +} diff --git a/crates/mesh-llm-log-store/src/query_pagination_tests.rs b/crates/mesh-llm-log-store/src/query_pagination_tests.rs index 9220708cc8..da6cb27dd1 100644 --- a/crates/mesh-llm-log-store/src/query_pagination_tests.rs +++ b/crates/mesh-llm-log-store/src/query_pagination_tests.rs @@ -26,6 +26,8 @@ fn descending_page(cursor: Option) -> RequestQuery { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, diff --git a/crates/mesh-llm-log-store/src/query_tests.rs b/crates/mesh-llm-log-store/src/query_tests.rs index 34f4a090fd..d8b9201b5c 100644 --- a/crates/mesh-llm-log-store/src/query_tests.rs +++ b/crates/mesh-llm-log-store/src/query_tests.rs @@ -27,6 +27,8 @@ fn request_query() -> RequestQuery { from: None, to: None, route: None, + exclude_route: None, + exclude_route_prefix: None, model: None, provider: None, engine: None, @@ -83,6 +85,8 @@ fn request_query_applies_all_filters_and_normalizes_time_bounds() { from: Some("2026-08-02T20:00:00-04:00".to_string()), to: Some("2026-08-03T00:01:00Z".to_string()), route: Some("chat".to_string()), + exclude_route: None, + exclude_route_prefix: None, model: Some("model-a".to_string()), provider: Some("provider-a".to_string()), engine: Some("engine-a".to_string()), @@ -96,6 +100,527 @@ fn request_query_applies_all_filters_and_normalizes_time_bounds() { assert_eq!(page.items[0].request_id, "matching-request"); } +#[test] +fn request_route_exclusions_apply_before_pagination_and_preserve_null_routes() { + // Given + let (_root, store) = open_store(); + for (request_id, route, created_at) in [ + ("visible-null-route", None, "2026-08-03T00:00:01Z"), + ( + "visible-chat-route", + Some("chat_completions"), + "2026-08-03T00:00:02Z", + ), + ( + "hidden-management-route", + Some("management_get_status"), + "2026-08-03T00:00:03Z", + ), + ( + "hidden-models-route", + Some("models"), + "2026-08-03T00:00:04Z", + ), + ] { + store + .insert_summary( + request_id, None, route, None, None, created_at, None, None, None, + ) + .expect("insert request summary"); + } + + // When + let page = store + .query_requests(&RequestQuery { + limit: 2, + exclude_route: Some("models".to_string()), + exclude_route_prefix: Some("management_".to_string()), + ..request_query() + }) + .expect("query visible request summaries"); + + // Then + assert_eq!( + page.items + .iter() + .map(|record| record.request_id.as_str()) + .collect::>(), + ["visible-chat-route", "visible-null-route"] + ); + assert!(page.next_cursor.is_none()); +} + +#[test] +fn caller_identity_round_trips_without_replacing_principal_identity() { + let (_root, store) = open_store(); + let endpoint_id = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + store + .conn() + .execute( + "INSERT INTO summaries \ + (request_id, created_at, tenant_id, account_id, user_id, caller_endpoint_id, caller_addr, caller_path_type) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + rusqlite::params![ + "request-with-caller", + "2026-08-20T12:00:00Z", + "tenant-a", + "account-a", + "user-a", + endpoint_id, + "192.0.2.42:11204", + "remote_quic_http", + ], + ) + .expect("insert caller summary"); + + let repository_row = store + .get_summary("request-with-caller") + .expect("query repository summary") + .expect("repository summary"); + let query_row = store + .query_request_with_caller("request-with-caller") + .expect("query request record") + .expect("request record"); + + assert_eq!(repository_row.tenant_id.as_deref(), Some("tenant-a")); + assert_eq!(repository_row.account_id.as_deref(), Some("account-a")); + assert_eq!(repository_row.user_id.as_deref(), Some("user-a")); + assert_eq!(query_row.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(query_row.caller_addr.as_deref(), Some("192.0.2.42:11204")); + assert_eq!( + query_row.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + +#[test] +fn authenticated_relay_upsert_replaces_and_protects_the_complete_caller_tuple() { + let (_root, store) = open_store(); + let request_id = "request-with-authenticated-relay"; + let endpoint_id = "7272727272727272727272727272727272727272727272727272727272727272"; + + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + Some("127.0.0.1:40123"), + Some("local_http"), + "2026-08-20T12:00:00Z", + ) + .expect("persist provisional local caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + None, + Some("relay"), + "2026-08-20T12:00:01Z", + ) + .expect("replace with authenticated relay caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + None, + Some("127.0.0.1:49999"), + Some("local_http"), + "2026-08-20T12:00:02Z", + ) + .expect("ignore later provisional local caller"); + + let repository_row = store + .get_summary(request_id) + .expect("query summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(query_row.caller_addr, None); + assert_eq!(query_row.caller_path_type.as_deref(), Some("relay")); +} + +#[test] +fn authenticated_endpoint_only_caller_round_trips_through_public_query_api() { + let (_root, store) = open_store(); + let request_id = "request-with-endpoint-only-caller"; + let endpoint_id = "7373737373737373737373737373737373737373737373737373737373737373"; + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + Some(endpoint_id), + None, + None, + "2026-08-20T12:00:00Z", + ) + .expect("persist endpoint-only caller"); + + let direct = store + .query_request_with_caller(request_id) + .expect("query request") + .expect("request record"); + let page = store + .query_requests_with_caller(&request_query()) + .expect("query request page"); + let listed = page + .items + .iter() + .find(|record| record.request.request_id == request_id) + .expect("listed request"); + + for record in [&direct, listed] { + assert_eq!(record.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(record.caller_addr, None); + assert_eq!(record.caller_path_type, None); + } +} + +#[test] +fn endpoint_only_upsert_replaces_local_or_empty_and_protects_first_authenticated_caller() { + let (_root, store) = open_store(); + let endpoint_id = "7474747474747474747474747474747474747474747474747474747474747474"; + let later_endpoint_id = "7575757575757575757575757575757575757575757575757575757575757575"; + + for (request_id, caller_addr, caller_path_type) in [ + ("endpoint-only-replaces-empty", None, None), + ( + "endpoint-only-replaces-local", + Some("127.0.0.1:40123"), + Some("local_http"), + ), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("persist initial caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + None, + None, + "2026-08-20T12:00:01Z", + ) + .expect("replace with endpoint-only caller"); + + for (later_endpoint, later_addr, later_path) in [ + (None, Some("127.0.0.1:49999"), Some("local_http")), + ( + Some(later_endpoint_id), + Some("192.0.2.75:11204"), + Some("remote_quic_http"), + ), + (Some(later_endpoint_id), None, Some("relay")), + (Some(later_endpoint_id), None, None), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + later_endpoint, + later_addr, + later_path, + "2026-08-20T12:00:02Z", + ) + .expect("ignore later caller"); + } + + let record = store + .query_request_with_caller(request_id) + .expect("query request") + .expect("request record"); + assert_eq!(record.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(record.caller_addr, None); + assert_eq!(record.caller_path_type, None); + assert_eq!(record.request.model.as_deref(), Some("model-a")); + assert_eq!(record.request.route.as_deref(), Some("responses")); + assert_eq!(record.request.provider.as_deref(), Some("provider-a")); + assert_eq!(record.request.engine.as_deref(), Some("engine-a")); + } +} + +#[test] +fn unrecognized_stage_caller_does_not_replace_local_request_metadata() { + let (_root, store) = open_store(); + let request_id = "request-with-stage-second"; + let endpoint_id = "7575757575757575757575757575757575757575757575757575757575757575"; + + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + None, + Some("127.0.0.1:40123"), + Some("local_http"), + "2026-08-20T12:00:00Z", + ) + .expect("persist provisional local caller"); + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + Some(endpoint_id), + Some("192.0.2.75:11204"), + Some("remote_quic_stage"), + "2026-08-20T12:00:01Z", + ) + .expect("ignore unrecognized stage caller"); + + let repository_row = store + .get_summary(request_id) + .expect("query summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id, None); + assert_eq!(query_row.caller_addr.as_deref(), Some("127.0.0.1:40123")); + assert_eq!(query_row.caller_path_type.as_deref(), Some("local_http")); +} + +#[test] +fn unrecognized_stage_caller_does_not_enter_empty_summary_metadata() { + let (_root, store) = open_store(); + let endpoint_id = "7676767676767676767676767676767676767676767676767676767676767676"; + + for (request_id, path_type) in [ + ("request-with-stage-first", "remote_quic_stage"), + ("request-with-empty-path", ""), + ("request-with-partial-path", "remote_quic_http/"), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("model-a"), + Some("responses"), + Some("provider-a"), + Some("engine-a"), + Some(endpoint_id), + Some("192.0.2.76:11204"), + Some(path_type), + "2026-08-20T12:00:00Z", + ) + .expect("ignore unrecognized caller on empty summary"); + + let repository_row = store + .get_summary(request_id) + .expect("query repository summary") + .expect("summary"); + let query_row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(repository_row.model.as_deref(), Some("model-a")); + assert_eq!(repository_row.route.as_deref(), Some("responses")); + assert_eq!(repository_row.provider.as_deref(), Some("provider-a")); + assert_eq!(repository_row.engine.as_deref(), Some("engine-a")); + assert_eq!(query_row.caller_endpoint_id, None); + assert_eq!(query_row.caller_addr, None); + assert_eq!(query_row.caller_path_type, None); + } +} + +#[test] +fn partial_caller_tuples_do_not_enter_empty_summary_metadata() { + let (_root, store) = open_store(); + let endpoint_id = "7777777777777777777777777777777777777777777777777777777777777777"; + let partial_tuples = [ + ( + "request-local-without-address", + None, + None, + Some("local_http"), + ), + ( + "request-remote-without-endpoint", + None, + Some("192.0.2.77:11204"), + Some("remote_quic_http"), + ), + ("request-relay-without-endpoint", None, None, Some("relay")), + ( + "request-values-without-path", + Some(endpoint_id), + Some("192.0.2.77:11204"), + None, + ), + ( + "request-invalid-endpoint-only", + Some("not-an-authenticated-endpoint"), + None, + None, + ), + ( + "request-remote-invalid-endpoint", + Some("not-an-authenticated-endpoint"), + Some("192.0.2.77:11204"), + Some("remote_quic_http"), + ), + ( + "request-local-invalid-address", + None, + Some("not-a-socket-address"), + Some("local_http"), + ), + ]; + + for (request_id, caller_endpoint_id, caller_addr, caller_path_type) in partial_tuples { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + caller_endpoint_id, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("ignore partial caller tuple"); + + let row = store + .query_request_with_caller(request_id) + .expect("query request record") + .expect("request record"); + assert_eq!(row.caller_endpoint_id, None); + assert_eq!(row.caller_addr, None); + assert_eq!(row.caller_path_type, None); + } +} + +#[test] +fn supported_caller_tuples_are_canonicalized_before_persistence() { + let (_root, store) = open_store(); + let endpoint_id = "7878787878787878787878787878787878787878787878787878787878787878"; + + for (request_id, caller_endpoint_id, caller_addr, caller_path_type) in [ + ( + "request-local-canonical", + Some("ignored-endpoint"), + Some("[2001:0db8:0:0::1]:40123"), + Some("local_http"), + ), + ( + "request-remote-canonical", + Some(endpoint_id), + Some("[2001:0db8:0:0::2]:11204"), + Some("remote_quic_http"), + ), + ( + "request-relay-clears-address", + Some(endpoint_id), + Some("192.0.2.78:11204"), + Some("relay"), + ), + ( + "request-remote-clears-invalid-address", + Some(endpoint_id), + Some("not-a-socket-address"), + Some("remote_quic_http"), + ), + ] { + store + .upsert_summary_metadata_with_caller( + request_id, + None, + None, + None, + None, + caller_endpoint_id, + caller_addr, + caller_path_type, + "2026-08-20T12:00:00Z", + ) + .expect("persist supported caller tuple"); + } + + let local = store + .query_request_with_caller("request-local-canonical") + .expect("query local caller") + .expect("local caller"); + assert_eq!(local.caller_endpoint_id, None); + assert_eq!(local.caller_addr.as_deref(), Some("[2001:db8::1]:40123")); + assert_eq!(local.caller_path_type.as_deref(), Some("local_http")); + + let remote = store + .query_request_with_caller("request-remote-canonical") + .expect("query remote caller") + .expect("remote caller"); + assert_eq!(remote.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(remote.caller_addr.as_deref(), Some("[2001:db8::2]:11204")); + assert_eq!(remote.caller_path_type.as_deref(), Some("remote_quic_http")); + + let relay = store + .query_request_with_caller("request-relay-clears-address") + .expect("query relay caller") + .expect("relay caller"); + assert_eq!(relay.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(relay.caller_addr, None); + assert_eq!(relay.caller_path_type.as_deref(), Some("relay")); + + let remote_without_addr = store + .query_request_with_caller("request-remote-clears-invalid-address") + .expect("query remote caller with invalid address") + .expect("remote caller with invalid address"); + assert_eq!( + remote_without_addr.caller_endpoint_id.as_deref(), + Some(endpoint_id) + ); + assert_eq!(remote_without_addr.caller_addr, None); + assert_eq!( + remote_without_addr.caller_path_type.as_deref(), + Some("remote_quic_http") + ); +} + #[test] fn request_time_bounds_compare_instants_within_the_boundary_second() { let (_root, store) = open_store(); @@ -386,6 +911,13 @@ fn forged_or_scope_mismatched_request_cursor_is_rejected() { store.query_requests(&query), Err(LogStoreError::CursorInvalid) )); + + query.route = None; + query.exclude_route = Some("chat".to_string()); + assert!(matches!( + store.query_requests(&query), + Err(LogStoreError::CursorInvalid) + )); } #[test] diff --git a/crates/mesh-llm-log-store/src/repositories.rs b/crates/mesh-llm-log-store/src/repositories.rs index eac4a54fe4..c9cc06b6b8 100644 --- a/crates/mesh-llm-log-store/src/repositories.rs +++ b/crates/mesh-llm-log-store/src/repositories.rs @@ -4,15 +4,20 @@ use crate::cursor::{decode_ordering_cursor, encode_cursor}; use crate::error::LogStoreError; use crate::store::LogStore; use crate::timestamps::{ - canonical_comparison_timestamp, canonical_optional_persisted_timestamp, - canonical_persisted_timestamp, canonical_timestamp_metadata, + canonical_optional_persisted_timestamp, canonical_persisted_timestamp, + canonical_timestamp_metadata, }; -use rusqlite::{OptionalExtension, Row, Transaction, types::Value}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use rusqlite::{OptionalExtension, Row, Transaction}; +use serde::Serialize; +mod audit; +mod caller_metadata; mod cleanup; +pub use audit::{ + AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, + DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT, +}; pub use cleanup::{ RetentionCleanupResult, RetentionPolicy, RetentionTable, RetentionTablePolicy, RetentionTableResult, @@ -24,8 +29,6 @@ const MAX_WEBHOOK_TIMESTAMP_BYTES: usize = 64; const MIN_HTTP_STATUS_CODE: u16 = 100; const MAX_HTTP_STATUS_CODE: u16 = 599; const CONFIGURED_WEBHOOK_TARGET: &str = "configured_webhook"; -pub const DEFAULT_AUDIT_ENTRY_LIMIT: usize = 50; -pub const MAX_AUDIT_ENTRY_LIMIT: usize = 100; // ─── Row types returned by queries ────────────────────── @@ -63,236 +66,6 @@ pub struct LifecycleEventRow { pub occurred_at: String, } -/// Public, privacy-safe projection of a durable operational audit entry. -/// -/// `detail_json` remains private to durable audit storage. Only the versioned, -/// bounded fields below are projected from it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AuditEntryRow { - pub sequence: i64, - pub entry_id: String, - pub request_id: Option, - pub occurred_at: String, - pub source: String, - pub code: String, - pub severity: Option, - pub context_version: Option, - pub subject_kind: Option, - pub subject_id: Option, - pub operation_id: Option, - pub correlation_request_id: Option, - pub reason_code: Option, - pub outcome: Option, - pub duration_ms: Option, - pub numeric_summaries: BTreeMap, -} - -#[derive(Default, Deserialize)] -struct StoredAuditDetail { - severity: Option, - context_version: Option, - subject_kind: Option, - subject_id: Option, - operation_id: Option, - request_id: Option, - reason_code: Option, - outcome: Option, - duration_ms: Option, - #[serde(default)] - numeric_summaries: BTreeMap, -} - -impl StoredAuditDetail { - fn parse(raw: Option) -> Self { - raw.and_then(|raw| serde_json::from_str(&raw).ok()) - .unwrap_or_default() - } - - fn bounded(mut self) -> Self { - if self.context_version != Some(1) { - return Self { - severity: self.severity, - ..Self::default() - }; - } - self.subject_kind = self.subject_kind.filter(|value| { - matches!( - value.as_str(), - "runtime" | "model" | "runtime_instance" | "cli_command" - ) - }); - self.subject_id = bounded_audit_value(self.subject_id); - self.operation_id = bounded_audit_value(self.operation_id); - self.request_id = bounded_audit_value(self.request_id); - self.reason_code = bounded_audit_code(self.reason_code); - self.outcome = bounded_audit_code(self.outcome); - self.numeric_summaries = self - .numeric_summaries - .into_iter() - .filter(|(key, _)| bounded_code(key)) - .take(8) - .collect(); - self - } -} - -fn bounded_audit_value(value: Option) -> Option { - value.filter(|value| !value.is_empty() && value.chars().count() <= 256) -} - -fn bounded_audit_code(value: Option) -> Option { - value.filter(|value| bounded_code(value)) -} - -fn bounded_code(value: &str) -> bool { - !value.is_empty() - && value.len() <= 64 - && value - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') -} - -/// Finite source vocabulary accepted by durable audit queries. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AuditEntrySource { - LoggingService, - Runtime, - Mesh, - Cli, - LogsApi, -} - -impl AuditEntrySource { - const fn as_str(self) -> &'static str { - match self { - Self::LoggingService => "logging_service", - Self::Runtime => "runtime", - Self::Mesh => "mesh", - Self::Cli => "cli", - Self::LogsApi => "logs_api", - } - } -} - -/// Finite severity vocabulary accepted by durable audit queries. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AuditEntrySeverity { - Info, - Warning, - Error, -} - -impl AuditEntrySeverity { - const fn as_str(self) -> &'static str { - match self { - Self::Info => "info", - Self::Warning => "warning", - Self::Error => "error", - } - } - - fn parse(value: Option) -> Option { - match value.as_deref() { - Some("info") => Some(Self::Info), - Some("warning") => Some(Self::Warning), - Some("error") => Some(Self::Error), - _ => None, - } - } -} - -/// Typed allowlisted filters for the operational audit read model. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AuditEntryFilters { - pub source: Option, - pub severity: Option, - pub from: Option, - pub to: Option, -} - -fn validate_audit_entry_limit(limit: Option) -> Result { - let limit = limit.unwrap_or(DEFAULT_AUDIT_ENTRY_LIMIT); - if (1..=MAX_AUDIT_ENTRY_LIMIT).contains(&limit) { - Ok(limit) - } else { - Err(LogStoreError::InvalidQuery(format!( - "audit entry limit must be within 1..={MAX_AUDIT_ENTRY_LIMIT}" - ))) - } -} - -fn audit_entry_row(row: &Row<'_>) -> rusqlite::Result { - let detail = StoredAuditDetail::parse(row.get(6)?).bounded(); - Ok(AuditEntryRow { - sequence: row.get(0)?, - entry_id: row.get(1)?, - request_id: row.get(2)?, - occurred_at: row.get(3)?, - source: row.get(4)?, - code: row.get(5)?, - severity: AuditEntrySeverity::parse(detail.severity), - context_version: detail.context_version, - subject_kind: detail.subject_kind, - subject_id: detail.subject_id, - operation_id: detail.operation_id, - correlation_request_id: detail.request_id, - reason_code: detail.reason_code, - outcome: detail.outcome, - duration_ms: detail.duration_ms, - numeric_summaries: detail.numeric_summaries, - }) -} - -fn audit_entry_query_parts( - cursor: Option<&(String, String)>, - filters: AuditEntryFilters, -) -> Result<(String, Vec), LogStoreError> { - let mut clauses = Vec::new(); - let mut parameters = Vec::new(); - if let Some((occurred_at, entry_id)) = cursor { - clauses.push("(occurred_at, entry_id) < (?, ?)".to_string()); - parameters.push(Value::Text(occurred_at.clone())); - parameters.push(Value::Text(entry_id.clone())); - } - if let Some(source) = filters.source { - clauses.push("actor = ?".to_string()); - parameters.push(Value::Text(source.as_str().to_string())); - } - if let Some(severity) = filters.severity { - clauses.push("CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END = ?".to_string()); - parameters.push(Value::Text(severity.as_str().to_string())); - } - let from = filters - .from - .as_deref() - .map(canonical_comparison_timestamp) - .transpose()?; - let to = filters - .to - .as_deref() - .map(canonical_comparison_timestamp) - .transpose()?; - if from > to && to.is_some() { - return Err(LogStoreError::InvalidQuery( - "audit from must not be after to".to_string(), - )); - } - if let Some(from) = from { - clauses.push("occurred_at >= ?".to_string()); - parameters.push(Value::Text(from)); - } - if let Some(to) = to { - clauses.push("occurred_at <= ?".to_string()); - parameters.push(Value::Text(to)); - } - let where_clause = if clauses.is_empty() { - String::new() - } else { - format!("WHERE {}", clauses.join(" AND ")) - }; - Ok((where_clause, parameters)) -} - /// Closed terminal request classification safe to retain for webhook delivery. /// /// The fixed vocabulary excludes raw errors, endpoints, paths, prompts, @@ -992,35 +765,6 @@ impl LogStore { } } - /// Insert a summary when absent and otherwise fill only metadata fields - /// that have not yet been recorded. Lifecycle state, timestamps, and - /// identity fields remain untouched. - #[allow(clippy::too_many_arguments)] - pub fn upsert_summary_metadata( - &self, - request_id: &str, - model: Option<&str>, - route: Option<&str>, - provider: Option<&str>, - engine: Option<&str>, - occurred_at: &str, - ) -> Result<(), LogStoreError> { - let occurred_at = canonical_persisted_timestamp(occurred_at)?; - self.conn() - .execute( - "INSERT INTO summaries (request_id, created_at, model, route, provider, engine) \ - VALUES (?, ?, ?, ?, ?, ?) \ - ON CONFLICT(request_id) DO UPDATE SET \ - model = COALESCE(summaries.model, excluded.model), \ - route = COALESCE(summaries.route, excluded.route), \ - provider = COALESCE(summaries.provider, excluded.provider), \ - engine = COALESCE(summaries.engine, excluded.engine)", - rusqlite::params![request_id, occurred_at, model, route, provider, engine], - ) - .map(|_| ()) - .map_err(|error| LogStoreError::InsertFailed(error.to_string())) - } - /// Get a summary by request_id. Returns None if not found (no-op style). pub fn get_summary(&self, request_id: &str) -> Result, LogStoreError> { let conn = self.conn(); @@ -1361,106 +1105,6 @@ impl LogStore { } } - // ════════════════════════════ - // Audit Entries - // ════════════════════════════ - - pub fn insert_audit_entry( - &self, - entry_id: &str, - request_id: Option<&str>, - occurred_at: &str, - source: &str, - code: &str, - detail_json: Option<&str>, - ) -> Result<(), LogStoreError> { - let occurred_at = canonical_persisted_timestamp(occurred_at)?; - let conn = self.conn(); - match conn.execute( - "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?, ?, ?, ?, ?, ?)", - rusqlite::params![entry_id, request_id, occurred_at, source, code, detail_json], - ) { - Ok(_) => Ok(()), - Err(ref e) => match map_insert_constraint_error(e, format!("audit_entry {entry_id}")) { - Some(error) => Err(error), - None => Err(LogStoreError::InsertFailed(e.to_string())), - }, - } - } - - /// List privacy-safe operational audit rows with cursor pagination. - /// - /// The query mirrors lifecycle ordering on `(occurred_at, entry_id)` while - /// fetching one extra row to determine the next cursor. `detail_json` is - /// only used internally for the bounded severity projection. - pub fn list_audit_entries( - &self, - limit: Option, - after_cursor: Option<&str>, - filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - let limit = validate_audit_entry_limit(limit)?; - let cursor = after_cursor.map(decode_ordering_cursor).transpose()?; - let (where_clause, parameters) = audit_entry_query_parts(cursor.as_ref(), filters)?; - let sql = format!( - "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ - FROM audit_entries {where_clause} \ - ORDER BY occurred_at DESC, entry_id DESC LIMIT {}", - limit + 1 - ); - let conn = self.conn(); - let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; - let mut items: Vec = statement - .query_map(rusqlite::params_from_iter(parameters), audit_entry_row) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; - - let has_more = items.len() > limit; - if has_more { - items.pop(); - } - let next_cursor = has_more.then(|| { - let last = &items[items.len() - 1]; - encode_cursor(&last.occurred_at, &last.entry_id) - }); - Ok(Page { items, next_cursor }) - } - - /// Read newly committed audit rows in database sequence order for the - /// trusted-local event stream. Unlike the public keyset listing, this is a - /// forward-only reconciliation cursor shared by every process writing the - /// durable store. - pub fn list_audit_entries_after_sequence( - &self, - after_sequence: u64, - limit: usize, - filters: AuditEntryFilters, - ) -> Result, LogStoreError> { - let limit = validate_audit_entry_limit(Some(limit))?; - let after_sequence = i64::try_from(after_sequence).map_err(|_| { - LogStoreError::InvalidQuery("audit sequence is outside the supported range".to_string()) - })?; - let (where_clause, mut parameters) = audit_entry_query_parts(None, filters)?; - let predicate = if where_clause.is_empty() { - "WHERE sequence > ?".to_string() - } else { - format!("{where_clause} AND sequence > ?") - }; - parameters.push(Value::Integer(after_sequence)); - let sql = format!( - "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ - FROM audit_entries {predicate} ORDER BY sequence ASC LIMIT {limit}" - ); - let conn = self.conn(); - let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; - statement - .query_map(rusqlite::params_from_iter(parameters), audit_entry_row) - .map_err(LogStoreError::Sqlite)? - .collect::, _>>() - .map_err(|error| LogStoreError::QueryFailed(error.to_string())) - } - // ════════════════════════════ // Webhook Deliveries // ════════════════════════════ diff --git a/crates/mesh-llm-log-store/src/repositories/audit.rs b/crates/mesh-llm-log-store/src/repositories/audit.rs new file mode 100644 index 0000000000..73061f42fb --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit.rs @@ -0,0 +1,155 @@ +use mesh_llm_events::audit::{SanitizedAuditDetailJson, SanitizedAuditScalar}; +use rusqlite::types::Value; + +use super::{Page, map_insert_constraint_error}; +use crate::cursor::{decode_ordering_cursor, encode_cursor}; +use crate::error::LogStoreError; +use crate::store::LogStore; +use crate::timestamps::canonical_persisted_timestamp; + +mod detail; +mod model; +mod paging; +mod projection; + +pub use model::{ + AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, AuditEntrySource, + DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT, +}; + +use crate::AuditEntryDetail; +use paging::{ + query_parts as audit_entry_query_parts, validate_limit as validate_audit_entry_limit, +}; +use projection::audit_entry_detail; + +impl LogStore { + pub fn insert_audit_entry( + &self, + entry_id: &str, + request_id: Option<&str>, + occurred_at: &str, + source: &str, + code: &str, + detail_json: Option<&str>, + ) -> Result<(), LogStoreError> { + let occurred_at = canonical_persisted_timestamp(occurred_at)?; + let source = SanitizedAuditScalar::sanitize(source); + let code = SanitizedAuditScalar::sanitize(code); + let detail_json = detail_json.map(SanitizedAuditDetailJson::sanitize); + let conn = self.conn(); + match conn.execute( + "INSERT INTO audit_entries (entry_id, request_id, occurred_at, actor, action, detail_json) VALUES (?, ?, ?, ?, ?, ?)", + rusqlite::params![ + entry_id, + request_id, + occurred_at, + source.as_str(), + code.as_str(), + detail_json.as_ref().map(SanitizedAuditDetailJson::as_str) + ], + ) { + Ok(_) => Ok(()), + Err(ref e) => match map_insert_constraint_error(e, format!("audit_entry {entry_id}")) { + Some(error) => Err(error), + None => Err(LogStoreError::InsertFailed(e.to_string())), + }, + } + } + + /// List privacy-safe operational audit rows with cursor pagination. + /// + /// The query mirrors lifecycle ordering on `(occurred_at, entry_id)` while + /// fetching one extra row to determine the next cursor. `detail_json` is + /// only used internally for the bounded severity projection. + pub fn list_audit_entries( + &self, + limit: Option, + after_cursor: Option<&str>, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + self.list_audit_entry_details(limit, after_cursor, filters) + .map(|page| Page { + items: page.items.into_iter().map(Into::into).collect(), + next_cursor: page.next_cursor, + }) + } + + pub fn list_audit_entry_details( + &self, + limit: Option, + after_cursor: Option<&str>, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + let limit = validate_audit_entry_limit(limit)?; + let cursor = after_cursor.map(decode_ordering_cursor).transpose()?; + let (where_clause, parameters) = audit_entry_query_parts(cursor.as_ref(), filters)?; + let sql = format!( + "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ + FROM audit_entries {where_clause} \ + ORDER BY occurred_at DESC, entry_id DESC LIMIT {}", + limit + 1 + ); + let conn = self.conn(); + let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; + let mut items: Vec = statement + .query_map(rusqlite::params_from_iter(parameters), audit_entry_detail) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string()))?; + + let has_more = items.len() > limit; + if has_more { + items.pop(); + } + let next_cursor = has_more.then(|| { + let last = &items[items.len() - 1]; + encode_cursor(&last.entry.occurred_at, &last.entry.entry_id) + }); + Ok(Page { items, next_cursor }) + } + + /// Read newly committed audit rows in database sequence order for the + /// trusted-local event stream. Unlike the public keyset listing, this is a + /// forward-only reconciliation cursor shared by every process writing the + /// durable store. + pub fn list_audit_entries_after_sequence( + &self, + after_sequence: u64, + limit: usize, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + self.list_audit_entry_details_after_sequence(after_sequence, limit, filters) + .map(|entries| entries.into_iter().map(Into::into).collect()) + } + + pub fn list_audit_entry_details_after_sequence( + &self, + after_sequence: u64, + limit: usize, + filters: AuditEntryFilters, + ) -> Result, LogStoreError> { + let limit = validate_audit_entry_limit(Some(limit))?; + let after_sequence = i64::try_from(after_sequence).map_err(|_| { + LogStoreError::InvalidQuery("audit sequence is outside the supported range".to_string()) + })?; + let (where_clause, mut parameters) = audit_entry_query_parts(None, filters)?; + let predicate = if where_clause.is_empty() { + "WHERE sequence > ?".to_string() + } else { + format!("{where_clause} AND sequence > ?") + }; + parameters.push(Value::Integer(after_sequence)); + let sql = format!( + "SELECT sequence, entry_id, request_id, occurred_at, actor, action, detail_json \ + FROM audit_entries {predicate} ORDER BY sequence ASC LIMIT {limit}" + ); + let conn = self.conn(); + let mut statement = conn.prepare(&sql).map_err(LogStoreError::Sqlite)?; + statement + .query_map(rusqlite::params_from_iter(parameters), audit_entry_detail) + .map_err(LogStoreError::Sqlite)? + .collect::, _>>() + .map_err(|error| LogStoreError::QueryFailed(error.to_string())) + } +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/detail.rs b/crates/mesh-llm-log-store/src/repositories/audit/detail.rs new file mode 100644 index 0000000000..8e1e930cd4 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/detail.rs @@ -0,0 +1,90 @@ +use std::{collections::BTreeMap, net::SocketAddr}; + +use mesh_llm_events::CliCommandSummary; +use serde::Deserialize; + +#[derive(Default, Deserialize)] +pub(super) struct StoredAuditDetail { + pub(super) severity: Option, + pub(super) context_version: Option, + pub(super) subject_kind: Option, + pub(super) subject_id: Option, + pub(super) remote_addr: Option, + pub(super) path_type: Option, + pub(super) operation_id: Option, + pub(super) request_id: Option, + pub(super) reason_code: Option, + pub(super) outcome: Option, + pub(super) command_summary: Option, + pub(super) duration_ms: Option, + #[serde(default)] + pub(super) numeric_summaries: BTreeMap, +} + +impl StoredAuditDetail { + pub(super) fn parse(raw: Option) -> Self { + raw.and_then(|raw| serde_json::from_str(&raw).ok()) + .unwrap_or_default() + } + + pub(super) fn bounded(mut self) -> Self { + if self.context_version != Some(1) { + return Self { + severity: self.severity, + ..Self::default() + }; + } + self.subject_kind = self.subject_kind.filter(|value| { + matches!( + value.as_str(), + "runtime" | "model" | "runtime_instance" | "cli_command" | "mesh_peer" + ) + }); + self.subject_id = bounded_audit_value(self.subject_id); + self.path_type = self + .path_type + .filter(|value| matches!(value.as_str(), "direct" | "relay")); + self.remote_addr = match self.path_type.as_deref() { + Some("direct") => self + .remote_addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + Some("relay") | None => None, + Some(_) => None, + }; + self.operation_id = bounded_audit_value(self.operation_id); + self.request_id = bounded_audit_value(self.request_id); + self.reason_code = bounded_audit_code(self.reason_code); + self.outcome = bounded_audit_code(self.outcome); + self.command_summary = bounded_command_summary(self.command_summary); + self.numeric_summaries = self + .numeric_summaries + .into_iter() + .filter(|(key, _)| bounded_code(key)) + .take(8) + .collect(); + self + } +} + +fn bounded_audit_value(value: Option) -> Option { + value.filter(|value| !value.is_empty() && value.chars().count() <= 256) +} + +fn bounded_command_summary(value: Option) -> Option { + value + .and_then(|value| CliCommandSummary::sanitize(&value)) + .map(|summary| summary.as_str().to_owned()) +} + +fn bounded_audit_code(value: Option) -> Option { + value.filter(|value| bounded_code(value)) +} + +fn bounded_code(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/model.rs b/crates/mesh-llm-log-store/src/repositories/audit/model.rs new file mode 100644 index 0000000000..664c1a96fe --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/model.rs @@ -0,0 +1,79 @@ +use std::collections::BTreeMap; + +pub const DEFAULT_AUDIT_ENTRY_LIMIT: usize = 50; +pub const MAX_AUDIT_ENTRY_LIMIT: usize = 100; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditEntryRow { + pub sequence: i64, + pub entry_id: String, + pub request_id: Option, + pub occurred_at: String, + pub source: String, + pub code: String, + pub severity: Option, + pub context_version: Option, + pub subject_kind: Option, + pub subject_id: Option, + pub operation_id: Option, + pub correlation_request_id: Option, + pub reason_code: Option, + pub outcome: Option, + pub duration_ms: Option, + pub numeric_summaries: BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditEntrySource { + LoggingService, + Runtime, + Mesh, + Cli, + LogsApi, +} + +impl AuditEntrySource { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::LoggingService => "logging_service", + Self::Runtime => "runtime", + Self::Mesh => "mesh", + Self::Cli => "cli", + Self::LogsApi => "logs_api", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditEntrySeverity { + Info, + Warning, + Error, +} + +impl AuditEntrySeverity { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Info => "info", + Self::Warning => "warning", + Self::Error => "error", + } + } + + pub(super) fn parse(value: Option) -> Option { + match value.as_deref() { + Some("info") => Some(Self::Info), + Some("warning") => Some(Self::Warning), + Some("error") => Some(Self::Error), + _ => None, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditEntryFilters { + pub source: Option, + pub severity: Option, + pub from: Option, + pub to: Option, +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/paging.rs b/crates/mesh-llm-log-store/src/repositories/audit/paging.rs new file mode 100644 index 0000000000..adc264c1f8 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/paging.rs @@ -0,0 +1,66 @@ +use rusqlite::types::Value; + +use super::{AuditEntryFilters, DEFAULT_AUDIT_ENTRY_LIMIT, MAX_AUDIT_ENTRY_LIMIT}; +use crate::error::LogStoreError; +use crate::timestamps::canonical_comparison_timestamp; + +pub(super) fn validate_limit(limit: Option) -> Result { + let limit = limit.unwrap_or(DEFAULT_AUDIT_ENTRY_LIMIT); + if (1..=MAX_AUDIT_ENTRY_LIMIT).contains(&limit) { + Ok(limit) + } else { + Err(LogStoreError::InvalidQuery(format!( + "audit entry limit must be within 1..={MAX_AUDIT_ENTRY_LIMIT}" + ))) + } +} + +pub(super) fn query_parts( + cursor: Option<&(String, String)>, + filters: AuditEntryFilters, +) -> Result<(String, Vec), LogStoreError> { + let mut clauses = Vec::new(); + let mut parameters = Vec::new(); + if let Some((occurred_at, entry_id)) = cursor { + clauses.push("(occurred_at, entry_id) < (?, ?)".to_string()); + parameters.push(Value::Text(occurred_at.clone())); + parameters.push(Value::Text(entry_id.clone())); + } + if let Some(source) = filters.source { + clauses.push("actor = ?".to_string()); + parameters.push(Value::Text(source.as_str().to_string())); + } + if let Some(severity) = filters.severity { + clauses.push("CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END = ?".to_string()); + parameters.push(Value::Text(severity.as_str().to_string())); + } + let from = filters + .from + .as_deref() + .map(canonical_comparison_timestamp) + .transpose()?; + let to = filters + .to + .as_deref() + .map(canonical_comparison_timestamp) + .transpose()?; + if from > to && to.is_some() { + return Err(LogStoreError::InvalidQuery( + "audit from must not be after to".to_string(), + )); + } + if let Some(from) = from { + clauses.push("occurred_at >= ?".to_string()); + parameters.push(Value::Text(from)); + } + if let Some(to) = to { + clauses.push("occurred_at <= ?".to_string()); + parameters.push(Value::Text(to)); + } + let where_clause = if clauses.is_empty() { + String::new() + } else { + format!("WHERE {}", clauses.join(" AND ")) + }; + Ok((where_clause, parameters)) +} diff --git a/crates/mesh-llm-log-store/src/repositories/audit/projection.rs b/crates/mesh-llm-log-store/src/repositories/audit/projection.rs new file mode 100644 index 0000000000..62caad00db --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/audit/projection.rs @@ -0,0 +1,36 @@ +use mesh_llm_events::audit::SanitizedAuditScalar; + +use super::detail::StoredAuditDetail; +use super::{AuditEntryRow, AuditEntrySeverity}; +use crate::AuditEntryDetail; + +pub(super) fn audit_entry_detail(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let detail = StoredAuditDetail::parse(row.get(6)?).bounded(); + Ok(AuditEntryDetail { + entry: AuditEntryRow { + sequence: row.get(0)?, + entry_id: row.get(1)?, + request_id: row.get(2)?, + occurred_at: row.get(3)?, + source: SanitizedAuditScalar::sanitize(&row.get::<_, String>(4)?) + .as_str() + .to_owned(), + code: SanitizedAuditScalar::sanitize(&row.get::<_, String>(5)?) + .as_str() + .to_owned(), + severity: AuditEntrySeverity::parse(detail.severity), + context_version: detail.context_version, + subject_kind: detail.subject_kind, + subject_id: detail.subject_id, + operation_id: detail.operation_id, + correlation_request_id: detail.request_id, + reason_code: detail.reason_code, + outcome: detail.outcome, + duration_ms: detail.duration_ms, + numeric_summaries: detail.numeric_summaries, + }, + remote_addr: detail.remote_addr, + path_type: detail.path_type, + command_summary: detail.command_summary, + }) +} diff --git a/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs b/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs new file mode 100644 index 0000000000..c4a96c4b66 --- /dev/null +++ b/crates/mesh-llm-log-store/src/repositories/caller_metadata.rs @@ -0,0 +1,162 @@ +use std::net::SocketAddr; + +use crate::error::LogStoreError; +use crate::store::LogStore; +use crate::timestamps::canonical_persisted_timestamp; + +#[derive(Default)] +pub(super) struct CallerMetadata<'a> { + pub(super) endpoint_id: Option<&'a str>, + pub(super) addr: Option, + pub(super) path_type: Option<&'static str>, +} + +pub(super) fn canonical_caller_metadata<'a>( + endpoint_id: Option<&'a str>, + addr: Option<&str>, + path_type: Option<&str>, +) -> CallerMetadata<'a> { + match path_type { + Some("local_http") => addr + .and_then(|value| value.parse::().ok()) + .map(|value| CallerMetadata { + endpoint_id: None, + addr: Some(value.to_string()), + path_type: Some("local_http"), + }) + .unwrap_or_default(), + Some("remote_quic_http") if endpoint_id.is_some_and(authenticated_endpoint_id) => { + CallerMetadata { + endpoint_id, + addr: addr + .and_then(|value| value.parse::().ok()) + .map(|value| value.to_string()), + path_type: Some("remote_quic_http"), + } + } + Some("relay") if endpoint_id.is_some_and(authenticated_endpoint_id) => CallerMetadata { + endpoint_id, + addr: None, + path_type: Some("relay"), + }, + None if endpoint_id.is_some_and(authenticated_endpoint_id) && addr.is_none() => { + CallerMetadata { + endpoint_id, + addr: None, + path_type: None, + } + } + Some("remote_quic_http" | "relay") | Some(_) | None => CallerMetadata::default(), + } +} + +fn authenticated_endpoint_id(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +impl LogStore { + /// Insert a summary when absent and otherwise fill only metadata fields + /// that have not yet been recorded. + #[allow(clippy::too_many_arguments)] + pub fn upsert_summary_metadata( + &self, + request_id: &str, + model: Option<&str>, + route: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + occurred_at: &str, + ) -> Result<(), LogStoreError> { + self.upsert_summary_metadata_with_caller( + request_id, + model, + route, + provider, + engine, + None, + None, + None, + occurred_at, + ) + } + + /// Insert a summary when absent and otherwise fill only metadata fields + /// that have not yet been recorded. Lifecycle state, timestamps, and + /// identity fields remain untouched. + #[allow(clippy::too_many_arguments)] + pub fn upsert_summary_metadata_with_caller( + &self, + request_id: &str, + model: Option<&str>, + route: Option<&str>, + provider: Option<&str>, + engine: Option<&str>, + caller_endpoint_id: Option<&str>, + caller_addr: Option<&str>, + caller_path_type: Option<&str>, + occurred_at: &str, + ) -> Result<(), LogStoreError> { + let occurred_at = canonical_persisted_timestamp(occurred_at)?; + let caller = canonical_caller_metadata(caller_endpoint_id, caller_addr, caller_path_type); + self.conn() + .execute( + "INSERT INTO summaries \ + (request_id, created_at, model, route, provider, engine, caller_endpoint_id, caller_addr, caller_path_type) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(request_id) DO UPDATE SET \ + model = COALESCE(summaries.model, excluded.model), \ + route = COALESCE(summaries.route, excluded.route), \ + provider = COALESCE(summaries.provider, excluded.provider), \ + engine = COALESCE(summaries.engine, excluded.engine), \ + caller_endpoint_id = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_endpoint_id \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_endpoint_id \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_endpoint_id \ + ELSE excluded.caller_endpoint_id END, \ + caller_addr = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_addr \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_addr \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_addr \ + ELSE excluded.caller_addr END, \ + caller_path_type = CASE \ + WHEN summaries.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (summaries.caller_endpoint_id IS NOT NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL) \ + THEN summaries.caller_path_type \ + WHEN (excluded.caller_path_type IN ('remote_quic_http', 'relay') \ + OR (excluded.caller_endpoint_id IS NOT NULL AND excluded.caller_addr IS NULL AND excluded.caller_path_type IS NULL)) \ + AND (summaries.caller_path_type = 'local_http' \ + OR (summaries.caller_endpoint_id IS NULL AND summaries.caller_addr IS NULL AND summaries.caller_path_type IS NULL)) \ + THEN excluded.caller_path_type \ + WHEN summaries.caller_endpoint_id IS NOT NULL OR summaries.caller_addr IS NOT NULL OR summaries.caller_path_type IS NOT NULL \ + THEN summaries.caller_path_type \ + ELSE excluded.caller_path_type END", + rusqlite::params![ + request_id, + occurred_at, + model, + route, + provider, + engine, + caller.endpoint_id, + caller.addr, + caller.path_type + ], + ) + .map(|_| ()) + .map_err(|error| LogStoreError::InsertFailed(error.to_string())) + } +} diff --git a/crates/mesh-llm-log-store/src/schema.rs b/crates/mesh-llm-log-store/src/schema.rs new file mode 100644 index 0000000000..2a8092473e --- /dev/null +++ b/crates/mesh-llm-log-store/src/schema.rs @@ -0,0 +1,159 @@ +use rusqlite::Connection; + +const INITIAL_SCHEMA: &str = r#" +CREATE TABLE summaries ( + request_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active' + CHECK (state IN ('active', 'completed', 'failed', 'rejected', 'cancelled', 'dropped')), + created_at TEXT NOT NULL, terminal_at TEXT, route TEXT, model TEXT, provider TEXT, + engine TEXT, status_code INTEGER, error_msg TEXT, tenant_id TEXT, account_id TEXT, user_id TEXT, + caller_endpoint_id TEXT, caller_addr TEXT, caller_path_type TEXT +); +CREATE INDEX idx_summaries_created ON summaries (created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_state ON summaries (state); +CREATE INDEX idx_summaries_terminal_order +ON summaries (state, COALESCE(terminal_at, created_at), request_id); +CREATE INDEX idx_summaries_route_created ON summaries (route, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_model_created ON summaries (model, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_provider_created ON summaries (provider, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_engine_created ON summaries (engine, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_status_created ON summaries (status_code, created_at DESC, request_id DESC); +CREATE INDEX idx_summaries_state_created ON summaries (state, created_at DESC, request_id DESC); + +CREATE TABLE lifecycle_events ( + event_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}', + event_type TEXT NOT NULL DEFAULT 'unknown', is_terminal INTEGER NOT NULL DEFAULT 0 + CHECK (is_terminal IN (0, 1)), UNIQUE(request_id, event_id) +); +CREATE UNIQUE INDEX idx_terminal_event_one_per_request +ON lifecycle_events (request_id) WHERE is_terminal = 1; +CREATE INDEX idx_lifecycle_events_occurred ON lifecycle_events (occurred_at DESC, event_id DESC); +CREATE INDEX idx_lifecycle_events_request ON lifecycle_events (request_id); +CREATE INDEX idx_lifecycle_events_request_terminal ON lifecycle_events (request_id, is_terminal); +CREATE INDEX idx_lifecycle_events_request_occurred +ON lifecycle_events (request_id, occurred_at ASC, event_id ASC); + +CREATE TABLE artifact_pointers ( + artifact_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, kind TEXT NOT NULL, metadata_json TEXT, media_kind TEXT, + checksum TEXT, bytes INTEGER NOT NULL DEFAULT 0 CHECK (bytes >= 0), + version INTEGER NOT NULL DEFAULT 1 CHECK (version >= 1), + redacted INTEGER NOT NULL DEFAULT 0 CHECK (redacted IN (0, 1)), + truncated INTEGER NOT NULL DEFAULT 0 CHECK (truncated IN (0, 1)), stored_at TEXT, + missing INTEGER NOT NULL DEFAULT 0 CHECK (missing IN (0, 1)), + corrupt INTEGER NOT NULL DEFAULT 0 CHECK (corrupt IN (0, 1)), + unavailable_reason TEXT CHECK (unavailable_reason IS NULL OR unavailable_reason IN ( + 'streaming_response_not_assembled', 'response_body_not_bounded', + 'capture_content_limit_exceeded', 'capture_memory_budget_exceeded', + 'artifact_capture_disabled', 'artifact_capture_failed' + )), UNIQUE(request_id, artifact_id) +); +CREATE INDEX idx_artifact_pointers_occurred ON artifact_pointers (occurred_at DESC, artifact_id DESC); +CREATE INDEX idx_artifact_pointers_request_occurred +ON artifact_pointers (request_id, occurred_at ASC, artifact_id ASC); + +CREATE TABLE proxy_records ( + attempt_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL REFERENCES summaries(request_id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, target TEXT NOT NULL, provider TEXT, engine TEXT, + started_at TEXT, completed_at TEXT, status_code INTEGER, error_msg TEXT, + UNIQUE(request_id, attempt_id) +); +CREATE INDEX idx_proxy_records_occurred ON proxy_records (occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_request_occurred +ON proxy_records (request_id, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_provider_occurred +ON proxy_records (provider, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_engine_occurred +ON proxy_records (engine, occurred_at DESC, attempt_id DESC); +CREATE INDEX idx_proxy_records_status_occurred +ON proxy_records (status_code, occurred_at DESC, attempt_id DESC); + +CREATE TABLE audit_entries ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT CHECK (sequence > 0), entry_id TEXT NOT NULL UNIQUE, + request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, action TEXT NOT NULL, detail_json TEXT, UNIQUE(request_id, entry_id) +); +CREATE INDEX idx_audit_entries_occurred ON audit_entries (occurred_at DESC, entry_id DESC); +CREATE INDEX idx_audit_entries_actor_occurred +ON audit_entries (actor, occurred_at DESC, entry_id DESC); +CREATE INDEX idx_audit_entries_severity_occurred ON audit_entries ( + CASE WHEN json_valid(detail_json) THEN json_extract(detail_json, '$.severity') END, + occurred_at DESC, entry_id DESC +); + +CREATE TABLE webhook_deliveries ( + delivery_id TEXT PRIMARY KEY, request_id TEXT REFERENCES summaries(request_id) ON DELETE SET NULL, + terminal_outcome TEXT NOT NULL + CHECK (terminal_outcome IN ('completed', 'failed', 'rejected', 'cancelled', 'dropped')), + terminal_status_code INTEGER CHECK (terminal_status_code IS NULL OR terminal_status_code BETWEEN 100 AND 599), + occurred_at TEXT NOT NULL, target_url TEXT NOT NULL, + attempt_number INTEGER NOT NULL DEFAULT 0 CHECK (attempt_number BETWEEN 0 AND 20), + status_code INTEGER CHECK (status_code IS NULL OR status_code BETWEEN 100 AND 599), + response_body TEXT, error_msg TEXT, + state TEXT NOT NULL DEFAULT 'succeeded' + CHECK (state IN ('pending', 'in_flight', 'succeeded', 'retry', 'dead_letter', 'manual_retry')), + created_at TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT '', next_attempt_at TEXT, + lease_expires_at TEXT, claim_generation INTEGER NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), + max_attempts INTEGER NOT NULL DEFAULT 1 CHECK (max_attempts BETWEEN 1 AND 20), + last_error_code TEXT CHECK (last_error_code IS NULL OR last_error_code IN + ('timeout', 'transport', 'http_4xx', 'http_5xx', 'configuration')), + UNIQUE(request_id, delivery_id) +); +CREATE INDEX idx_webhook_deliveries_occurred ON webhook_deliveries (occurred_at DESC, delivery_id DESC); +CREATE INDEX idx_webhook_deliveries_eligible +ON webhook_deliveries (state, next_attempt_at, lease_expires_at, created_at, delivery_id); +CREATE INDEX idx_webhook_deliveries_ready +ON webhook_deliveries (COALESCE(next_attempt_at, created_at), created_at, delivery_id) +WHERE state IN ('pending', 'retry', 'manual_retry'); +CREATE INDEX idx_webhook_deliveries_expired_lease +ON webhook_deliveries (lease_expires_at, created_at, delivery_id) WHERE state = 'in_flight'; + +CREATE TABLE cleanup_runs ( + run_id TEXT PRIMARY KEY, occurred_at TEXT NOT NULL, policy_name TEXT NOT NULL, + cutoff_before TEXT NOT NULL, deleted_count INTEGER NOT NULL DEFAULT 0 CHECK (deleted_count >= 0), + duration_ms INTEGER CHECK (duration_ms IS NULL OR duration_ms >= 0) +); +CREATE INDEX idx_cleanup_runs_occurred ON cleanup_runs (occurred_at DESC, run_id DESC); + +CREATE TABLE maintenance_operations ( + operation_id TEXT PRIMARY KEY, action TEXT NOT NULL CHECK (action IN ('cleanup', 'delete_one')), + cutoff_before TEXT NOT NULL, request_limit INTEGER NOT NULL CHECK (request_limit BETWEEN 1 AND 100), + reason TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('previewed', 'completed', 'partial')), + planned_requests INTEGER NOT NULL CHECK (planned_requests >= 0), + planned_events INTEGER NOT NULL CHECK (planned_events >= 0), + planned_artifacts INTEGER NOT NULL CHECK (planned_artifacts >= 0), + planned_proxy_records INTEGER NOT NULL CHECK (planned_proxy_records >= 0), + planned_database_rows INTEGER NOT NULL CHECK (planned_database_rows >= 0), + executed_requests INTEGER NOT NULL DEFAULT 0 CHECK (executed_requests >= 0), + executed_events INTEGER NOT NULL DEFAULT 0 CHECK (executed_events >= 0), + executed_artifacts INTEGER NOT NULL DEFAULT 0 CHECK (executed_artifacts >= 0), + executed_proxy_records INTEGER NOT NULL DEFAULT 0 CHECK (executed_proxy_records >= 0), + executed_database_rows INTEGER NOT NULL DEFAULT 0 CHECK (executed_database_rows >= 0), + has_more INTEGER NOT NULL CHECK (has_more IN (0, 1)), created_at TEXT NOT NULL, + completed_at TEXT, selection_fingerprint TEXT NOT NULL DEFAULT '', + artifact_files_removed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_removed >= 0), + artifact_files_failed INTEGER NOT NULL DEFAULT 0 CHECK (artifact_files_failed >= 0), + artifact_file_failure_class TEXT + CHECK (artifact_file_failure_class IS NULL OR artifact_file_failure_class IN ('io', 'unsafe_path')), + preview_audit_id TEXT, execution_audit_id TEXT, cleanup_filters_json TEXT NOT NULL DEFAULT '{}' +); +CREATE TABLE maintenance_operation_targets ( + operation_id TEXT NOT NULL REFERENCES maintenance_operations(operation_id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), request_id TEXT NOT NULL, + PRIMARY KEY (operation_id, request_id), UNIQUE (operation_id, ordinal) +); +CREATE INDEX idx_maintenance_operation_targets_operation +ON maintenance_operation_targets (operation_id, ordinal); +CREATE TABLE pending_artifact_deletions ( + artifact_id TEXT PRIMARY KEY, request_id TEXT NOT NULL +); +CREATE INDEX idx_pending_artifact_deletions_request +ON pending_artifact_deletions (request_id, artifact_id); +"#; + +pub(super) fn initialize(connection: &Connection) -> Result<(), rusqlite::Error> { + connection.execute_batch(INITIAL_SCHEMA) +} diff --git a/crates/mesh-llm-log-store/src/space_maintenance_tests.rs b/crates/mesh-llm-log-store/src/space_maintenance_tests.rs index a6e66b152d..071d224ccd 100644 --- a/crates/mesh-llm-log-store/src/space_maintenance_tests.rs +++ b/crates/mesh-llm-log-store/src/space_maintenance_tests.rs @@ -50,6 +50,10 @@ fn new_store_reports_zero_reclaimed_pages_when_nothing_can_be_reclaimed() { .expect("new store auto-vacuum mode"); assert_eq!(auto_vacuum, 2); + store + .conn() + .execute_batch("PRAGMA analysis_limit = 400; PRAGMA optimize;") + .expect("prepare optimizer metadata"); let before = page_count(&store); let maintenance = store .maintain_space_after_cleanup() diff --git a/crates/mesh-llm-log-store/src/store.rs b/crates/mesh-llm-log-store/src/store.rs index ba27a14967..a6ff898acd 100644 --- a/crates/mesh-llm-log-store/src/store.rs +++ b/crates/mesh-llm-log-store/src/store.rs @@ -59,7 +59,14 @@ impl LogStore { let conn = Connection::open(&db_path).map_err(|e| { LogStoreError::IoError(std::io::Error::other(format!("sqlite open: {}", e))) })?; - prepare_private_database_files(&db_path)?; + conn.busy_timeout(std::time::Duration::from_secs(30)) + .map_err(LogStoreError::Sqlite)?; + + if let Some((found, supported)) = crate::migrations::incompatible_schema(&conn) + .map_err(|error| LogStoreError::MigrationFailed(error.to_string()))? + { + return Err(LogStoreError::SchemaIncompatible { found, supported }); + } if new_database { // This only takes effect before schema allocation. Existing stores @@ -71,15 +78,9 @@ impl LogStore { let pragmas = " PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; - PRAGMA busy_timeout = 30000; "; conn.execute_batch(pragmas).map_err(LogStoreError::Sqlite)?; - if let Some((found, supported)) = crate::migrations::incompatible_schema(&conn) - .map_err(|error| LogStoreError::MigrationFailed(error.to_string()))? - { - return Err(LogStoreError::SchemaIncompatible { found, supported }); - } crate::migrations::apply_migrations(&conn) .map_err(|e| LogStoreError::MigrationFailed(e.to_string()))?; prepare_private_database_files(&db_path)?; diff --git a/crates/mesh-llm-log-store/tests/public_api_compat.rs b/crates/mesh-llm-log-store/tests/public_api_compat.rs new file mode 100644 index 0000000000..ee9f5598bc --- /dev/null +++ b/crates/mesh-llm-log-store/tests/public_api_compat.rs @@ -0,0 +1,82 @@ +use std::collections::BTreeMap; + +use mesh_llm_log_store::{ + AuditEntryFilters, AuditEntryRow, LogStore, LogStoreError, Page, QueryPage, RequestQuery, + RequestRecord, +}; + +type LegacyAuditPageMethod = fn( + &LogStore, + Option, + Option<&str>, + AuditEntryFilters, +) -> Result, LogStoreError>; +type LegacyAuditSequenceMethod = + fn(&LogStore, u64, usize, AuditEntryFilters) -> Result, LogStoreError>; +type LegacySummaryMetadataMethod = fn( + &LogStore, + &str, + Option<&str>, + Option<&str>, + Option<&str>, + Option<&str>, + &str, +) -> Result<(), LogStoreError>; + +#[test] +fn released_request_record_struct_literal_remains_source_compatible() { + let record = RequestRecord { + request_id: "released-request".to_owned(), + outcome: "completed".to_owned(), + created_at: "2026-08-22T12:00:00.000000000Z".to_owned(), + terminal_at: Some("2026-08-22T12:00:01.000000000Z".to_owned()), + route: Some("chat".to_owned()), + model: Some("released-model".to_owned()), + provider: Some("released-provider".to_owned()), + engine: Some("released-engine".to_owned()), + status_code: Some(200), + }; + + assert_eq!(record.request_id, "released-request"); +} + +#[test] +fn released_audit_entry_row_struct_literal_remains_source_compatible() { + let row = AuditEntryRow { + sequence: 7, + entry_id: "released-audit".to_owned(), + request_id: Some("released-request".to_owned()), + occurred_at: "2026-08-22T12:00:02.000000000Z".to_owned(), + source: "runtime".to_owned(), + code: "runtime_ready".to_owned(), + severity: None, + context_version: Some(1), + subject_kind: Some("runtime".to_owned()), + subject_id: Some("released-host".to_owned()), + operation_id: Some("released-operation".to_owned()), + correlation_request_id: Some("released-correlation".to_owned()), + reason_code: Some("released-reason".to_owned()), + outcome: Some("ready".to_owned()), + duration_ms: Some(11), + numeric_summaries: BTreeMap::from([("models".to_owned(), 2)]), + }; + + assert_eq!(row.entry_id, "released-audit"); +} + +#[test] +fn released_query_and_list_methods_keep_legacy_return_shapes() { + let _: fn(&LogStore, &str) -> Result, LogStoreError> = + LogStore::query_request; + let _: fn(&LogStore, &[String]) -> Result, LogStoreError> = + LogStore::query_requests_by_ids; + let _: fn(&LogStore, &RequestQuery) -> Result, LogStoreError> = + LogStore::query_requests; + let _: LegacyAuditPageMethod = LogStore::list_audit_entries; + let _: LegacyAuditSequenceMethod = LogStore::list_audit_entries_after_sequence; +} + +#[test] +fn released_summary_metadata_method_keeps_legacy_signature() { + let _: LegacySummaryMetadataMethod = LogStore::upsert_summary_metadata; +} diff --git a/crates/mesh-llm-log-store/tests/public_query_projections.rs b/crates/mesh-llm-log-store/tests/public_query_projections.rs new file mode 100644 index 0000000000..783fd99587 --- /dev/null +++ b/crates/mesh-llm-log-store/tests/public_query_projections.rs @@ -0,0 +1,387 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use mesh_llm_log_store::{ + AuditEntryDetail, AuditEntryFilters, AuditEntryRow, AuditEntrySeverity, LogStore, QuerySort, + RealClock, RequestQuery, RequestRecord, RequestRecordWithCaller, +}; + +const CALLER_ENDPOINT_ID: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const CALLER_ADDR: &str = "192.0.2.42:11204"; +const CALLER_PATH_TYPE: &str = "remote_quic_http"; +const AUDIT_REMOTE_ADDR: &str = "198.51.100.27:443"; +const AUDIT_PATH_TYPE: &str = "direct"; +const AUDIT_COMMAND: &str = "mesh-llm runtime guardrails --mode metrics --port 41731"; + +fn open_store() -> (tempfile::TempDir, LogStore) { + let root = tempfile::tempdir().expect("create public projection store"); + let store = LogStore::open(root.path(), Arc::new(RealClock)).expect("open log store"); + (root, store) +} + +fn request_query(limit: usize, cursor: Option) -> RequestQuery { + RequestQuery { + limit, + cursor, + from: None, + to: None, + route: None, + exclude_route: None, + exclude_route_prefix: None, + model: None, + provider: None, + engine: None, + status_code: None, + outcome: None, + sort: QuerySort::Descending, + } +} + +fn insert_request_with_caller( + store: &LogStore, + request_id: &str, + created_at: &str, + caller_endpoint_id: &str, + caller_addr: &str, +) { + store + .upsert_summary_metadata_with_caller( + request_id, + Some("caller-model"), + Some("responses"), + Some("caller-provider"), + Some("caller-engine"), + Some(caller_endpoint_id), + Some(caller_addr), + Some(CALLER_PATH_TYPE), + created_at, + ) + .expect("insert request caller metadata"); +} + +fn assert_released_request(record: &RequestRecord, request_id: &str) { + assert_eq!(record.request_id, request_id); + assert_eq!(record.outcome, "active"); + assert_eq!(record.route.as_deref(), Some("responses")); + assert_eq!(record.model.as_deref(), Some("caller-model")); + assert_eq!(record.provider.as_deref(), Some("caller-provider")); + assert_eq!(record.engine.as_deref(), Some("caller-engine")); +} + +fn assert_caller(detailed: &RequestRecordWithCaller, endpoint_id: &str, caller_addr: &str) { + assert_eq!(detailed.caller_endpoint_id.as_deref(), Some(endpoint_id)); + assert_eq!(detailed.caller_addr.as_deref(), Some(caller_addr)); + assert_eq!(detailed.caller_path_type.as_deref(), Some(CALLER_PATH_TYPE)); +} + +#[test] +fn detailed_request_preserves_caller_while_legacy_query_keeps_released_projection() { + // Given + let (_root, store) = open_store(); + insert_request_with_caller( + &store, + "request-with-caller", + "2026-08-22T12:00:00Z", + CALLER_ENDPOINT_ID, + CALLER_ADDR, + ); + + // When + let detailed = store + .query_request_with_caller("request-with-caller") + .expect("query detailed request") + .expect("detailed request"); + let legacy = store + .query_request("request-with-caller") + .expect("query legacy request") + .expect("legacy request"); + + // Then + assert_released_request(&detailed.request, "request-with-caller"); + assert_caller(&detailed, CALLER_ENDPOINT_ID, CALLER_ADDR); + assert_released_request(&legacy, "request-with-caller"); +} + +#[test] +fn detailed_request_pagination_keeps_legacy_order_and_cursor() { + // Given + let (_root, store) = open_store(); + for (request_id, created_at, endpoint_digit, caller_addr) in [ + ("request-a", "2026-08-22T12:00:01Z", 'a', "192.0.2.10:11204"), + ("request-b", "2026-08-22T12:00:02Z", 'b', "192.0.2.11:11204"), + ("request-c", "2026-08-22T12:00:03Z", 'c', "192.0.2.12:11204"), + ] { + let endpoint_id = endpoint_digit.to_string().repeat(64); + insert_request_with_caller(&store, request_id, created_at, &endpoint_id, caller_addr); + } + + // When + let detailed_first = store + .query_requests_with_caller(&request_query(2, None)) + .expect("query first detailed page"); + let legacy_first = store + .query_requests(&request_query(2, None)) + .expect("query first legacy page"); + let detailed_second = store + .query_requests_with_caller(&request_query(2, detailed_first.next_cursor.clone())) + .expect("query second detailed page"); + + // Then + assert_eq!(detailed_first.next_cursor, legacy_first.next_cursor); + assert_eq!( + detailed_first + .items + .iter() + .map(|item| item.request.request_id.as_str()) + .collect::>(), + ["request-c", "request-b"] + ); + assert_eq!( + legacy_first + .items + .iter() + .map(|item| item.request_id.as_str()) + .collect::>(), + ["request-c", "request-b"] + ); + assert_eq!(detailed_second.items[0].request.request_id, "request-a"); + assert!(detailed_second.next_cursor.is_none()); +} + +#[test] +fn detailed_request_batch_omits_missing_ids_across_chunk_boundary() { + // Given + let (_root, store) = open_store(); + let mut requested_ids = Vec::new(); + for index in 0..=100_u8 { + let request_id = format!("request-{index:03}"); + let endpoint_id = format!("{index:064x}"); + let caller_addr = format!("192.0.2.{}:11204", index + 1); + insert_request_with_caller( + &store, + &request_id, + "2026-08-22T12:00:00Z", + &endpoint_id, + &caller_addr, + ); + requested_ids.push(request_id); + } + requested_ids.push("request-missing".to_owned()); + + // When + let detailed = store + .query_requests_by_ids_with_caller(&requested_ids) + .expect("query detailed request batch"); + let legacy = store + .query_requests_by_ids(&requested_ids) + .expect("query legacy request batch"); + + // Then + assert_eq!(detailed.len(), 101); + assert_eq!(legacy.len(), 101); + let detailed_by_id = detailed + .iter() + .map(|item| (item.request.request_id.as_str(), item)) + .collect::>(); + let legacy_ids = legacy + .iter() + .map(|item| item.request_id.as_str()) + .collect::>(); + assert!(!detailed_by_id.contains_key("request-missing")); + assert!(!legacy_ids.contains("request-missing")); + for index in [99_u8, 100_u8] { + let request_id = format!("request-{index:03}"); + let endpoint_id = format!("{index:064x}"); + let caller_addr = format!("192.0.2.{}:11204", index + 1); + assert_caller( + detailed_by_id + .get(request_id.as_str()) + .expect("detailed request at chunk boundary"), + &endpoint_id, + &caller_addr, + ); + } +} + +fn insert_detailed_audit( + store: &LogStore, + entry_id: &str, + occurred_at: &str, + remote_addr: &str, + command_summary: &str, +) { + let detail = serde_json::json!({ + "severity": "info", + "context_version": 1, + "subject_kind": "mesh_peer", + "subject_id": entry_id, + "remote_addr": remote_addr, + "path_type": AUDIT_PATH_TYPE, + "operation_id": format!("operation-{entry_id}"), + "request_id": format!("correlation-{entry_id}"), + "reason_code": "accepted", + "outcome": "ready", + "command_summary": command_summary, + "duration_ms": 17, + "numeric_summaries": {"peers": 3} + }); + store + .insert_audit_entry( + entry_id, + None, + occurred_at, + "cli", + "command_completed", + Some(&detail.to_string()), + ) + .expect("insert detailed audit entry"); +} + +fn assert_released_audit(row: &AuditEntryRow, entry_id: &str) { + assert_eq!(row.entry_id, entry_id); + assert_eq!(row.source, "cli"); + assert_eq!(row.code, "command_completed"); + assert_eq!(row.severity, Some(AuditEntrySeverity::Info)); + assert_eq!(row.context_version, Some(1)); + assert_eq!(row.subject_kind.as_deref(), Some("mesh_peer")); + assert_eq!(row.subject_id.as_deref(), Some(entry_id)); + assert_eq!(row.reason_code.as_deref(), Some("accepted")); + assert_eq!(row.outcome.as_deref(), Some("ready")); + assert_eq!(row.duration_ms, Some(17)); + assert_eq!(row.numeric_summaries.get("peers"), Some(&3)); +} + +fn assert_audit_detail( + detail: &AuditEntryDetail, + entry_id: &str, + remote_addr: &str, + command_summary: &str, +) { + assert_released_audit(&detail.entry, entry_id); + assert_eq!(detail.remote_addr.as_deref(), Some(remote_addr)); + assert_eq!(detail.path_type.as_deref(), Some(AUDIT_PATH_TYPE)); + assert_eq!(detail.command_summary.as_deref(), Some(command_summary)); +} + +#[test] +fn detailed_audit_preserves_path_remote_and_command_while_legacy_list_stays_released() { + // Given + let (_root, store) = open_store(); + insert_detailed_audit( + &store, + "audit-detailed", + "2026-08-22T12:00:00Z", + AUDIT_REMOTE_ADDR, + AUDIT_COMMAND, + ); + + // When + let detailed = store + .list_audit_entry_details(Some(1), None, AuditEntryFilters::default()) + .expect("list detailed audit entries"); + let legacy = store + .list_audit_entries(Some(1), None, AuditEntryFilters::default()) + .expect("list legacy audit entries"); + + // Then + assert_audit_detail( + &detailed.items[0], + "audit-detailed", + AUDIT_REMOTE_ADDR, + AUDIT_COMMAND, + ); + assert_released_audit(&legacy.items[0], "audit-detailed"); +} + +#[test] +fn detailed_audit_pagination_keeps_legacy_order_and_cursor_boundary() { + // Given + let (_root, store) = open_store(); + for (entry_id, remote_addr, command) in [ + ("audit-a", "198.51.100.10:443", AUDIT_COMMAND), + ("audit-b", "198.51.100.11:443", AUDIT_COMMAND), + ("audit-c", "198.51.100.12:443", AUDIT_COMMAND), + ] { + insert_detailed_audit( + &store, + entry_id, + "2026-08-22T12:00:00Z", + remote_addr, + command, + ); + } + + // When + let detailed_first = store + .list_audit_entry_details(Some(2), None, AuditEntryFilters::default()) + .expect("list first detailed audit page"); + let legacy_first = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .expect("list first legacy audit page"); + let detailed_second = store + .list_audit_entry_details( + Some(2), + detailed_first.next_cursor.as_deref(), + AuditEntryFilters::default(), + ) + .expect("list second detailed audit page"); + + // Then + assert_eq!(detailed_first.next_cursor, legacy_first.next_cursor); + assert_eq!( + detailed_first + .items + .iter() + .map(|item| item.entry.entry_id.as_str()) + .collect::>(), + ["audit-c", "audit-b"] + ); + assert_eq!(detailed_second.items[0].entry.entry_id, "audit-a"); + assert!(detailed_second.next_cursor.is_none()); +} + +#[test] +fn detailed_audit_sequence_reconciliation_resumes_after_exact_boundary() { + // Given + let (root, store) = open_store(); + let writer = LogStore::open(root.path(), Arc::new(RealClock)).expect("open audit writer"); + for (entry_id, remote_addr, command) in [ + ("audit-sequence-a", "203.0.113.10:443", AUDIT_COMMAND), + ("audit-sequence-b", "203.0.113.11:443", AUDIT_COMMAND), + ("audit-sequence-c", "203.0.113.12:443", AUDIT_COMMAND), + ] { + insert_detailed_audit( + &writer, + entry_id, + "2026-08-22T12:00:00Z", + remote_addr, + command, + ); + } + + // When + let first = store + .list_audit_entry_details_after_sequence(0, 2, AuditEntryFilters::default()) + .expect("reconcile first detailed audit batch"); + let boundary = u64::try_from(first[1].entry.sequence).expect("positive audit sequence"); + let second = store + .list_audit_entry_details_after_sequence(boundary, 2, AuditEntryFilters::default()) + .expect("reconcile second detailed audit batch"); + + // Then + assert_eq!( + first + .iter() + .map(|item| item.entry.entry_id.as_str()) + .collect::>(), + ["audit-sequence-a", "audit-sequence-b"] + ); + assert_eq!(second.len(), 1); + assert_audit_detail( + &second[0], + "audit-sequence-c", + "203.0.113.12:443", + AUDIT_COMMAND, + ); + assert!(second[0].entry.sequence > i64::try_from(boundary).expect("sequence fits i64")); +} diff --git a/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs b/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs new file mode 100644 index 0000000000..61d25b7c19 --- /dev/null +++ b/crates/mesh-llm-log-store/tests/raw_audit_scalars.rs @@ -0,0 +1,60 @@ +use std::sync::Arc; + +use mesh_llm_log_store::{AuditEntryFilters, AuditEntrySeverity, LogStore, RealClock}; + +#[test] +fn audit_entry_reads_sanitize_raw_actor_and_action_scalars() { + // Given: rows written outside the typed repository with malformed and oversized scalars. + let temp = tempfile::tempdir().expect("create temporary log store"); + let store = LogStore::open(temp.path(), Arc::new(RealClock)).expect("open log store"); + let home = std::env::var("HOME").expect("HOME should be available to the test"); + let malformed_actor = format!("{home}/runtime\nactor"); + let secret_action = "Bearer raw-sql-action-secret"; + let oversized_actor = "a".repeat(1_100); + let oversized_action = "z".repeat(1_100); + store + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, ?5)", + rusqlite::params![ + "raw-audit-malformed", + "2026-08-22T12:00:01.000000000Z", + malformed_actor, + secret_action, + r#"{"severity":"warning"}"#, + ], + ) + .expect("insert malformed raw audit row"); + store + .conn() + .execute( + "INSERT INTO audit_entries \ + (entry_id, request_id, occurred_at, actor, action, detail_json) \ + VALUES (?1, NULL, ?2, ?3, ?4, NULL)", + rusqlite::params![ + "raw-audit-oversized", + "2026-08-22T12:00:00.000000000Z", + oversized_actor, + oversized_action, + ], + ) + .expect("insert oversized raw audit row"); + + // When: the shared durable read projection maps both rows. + let page = store + .list_audit_entries(Some(2), None, AuditEntryFilters::default()) + .expect("list raw audit rows"); + + // Then: ordering and detail projection survive while both scalar columns are canonicalized. + assert_eq!(page.items[0].entry_id, "raw-audit-malformed"); + assert_eq!(page.items[0].source, "~/runtimeactor"); + assert_eq!(page.items[0].code, "[REDACTED]"); + assert_eq!(page.items[0].severity, Some(AuditEntrySeverity::Warning)); + assert_eq!(page.items[1].entry_id, "raw-audit-oversized"); + assert_eq!(page.items[1].source.chars().count(), 1_024); + assert!(page.items[1].source.ends_with("... [TRUNCATED]")); + assert_eq!(page.items[1].code.chars().count(), 1_024); + assert!(page.items[1].code.ends_with("... [TRUNCATED]")); +} diff --git a/crates/mesh-llm/src/commands/mod.rs b/crates/mesh-llm/src/commands/mod.rs index 096fd8c472..725afd86cb 100644 --- a/crates/mesh-llm/src/commands/mod.rs +++ b/crates/mesh-llm/src/commands/mod.rs @@ -28,7 +28,9 @@ pub async fn dispatch(cli: &Cli) -> Result { _ => mesh_llm_commands::operational_logging::command_family(cmd), }; let command_boundary = - mesh_llm_commands::operational_logging::CommandDispatchBoundary::start_family(family); + mesh_llm_commands::operational_logging::CommandDispatchBoundary::start_with_cli( + cli, family, + ); let result = dispatch_command(cli, cmd).await; command_boundary.finish(&result); result?; diff --git a/crates/mesh-llm/src/lib.rs b/crates/mesh-llm/src/lib.rs index 0a277115d7..56fcd5613e 100644 --- a/crates/mesh-llm/src/lib.rs +++ b/crates/mesh-llm/src/lib.rs @@ -152,7 +152,9 @@ fn install_cli_operational_audit_bridge() { }; let bridge: mesh_llm_commands::operational_logging::CliOperationalAuditBridge = Arc::new( - |family: mesh_llm_events::CliCommandFamily, outcome: mesh_llm_events::CliCommandOutcome| { + |family: mesh_llm_events::CliCommandFamily, + outcome: mesh_llm_events::CliCommandOutcome, + summary: Option| { let Some(state) = mesh_llm_host_runtime::logging_runtime_state() else { return; }; @@ -171,7 +173,8 @@ fn install_cli_operational_audit_bridge() { .with_context( OperationalAuditContext::new() .subject(OperationalAuditSubjectKind::CliCommand, family.as_str()) - .outcome(outcome.as_str()), + .outcome(outcome.as_str()) + .command_summary(summary.as_ref().map_or("", |summary| summary.as_str())), ); let _ = state.write_operational_audit(record); }, diff --git a/docs/LOGGING.md b/docs/LOGGING.md index b2566747fe..6565255c77 100644 --- a/docs/LOGGING.md +++ b/docs/LOGGING.md @@ -5,6 +5,10 @@ records. It is separate from the public mesh, normal runtime status events, and OTLP telemetry. Open the Logs tab in the embedded console to inspect the request ledger, then select a row to open its detail view. +Ledger search accepts either a full endpoint ID or the abbreviated form shown +in the console, such as `9f0c…bb04`, for request caller identities and mesh-peer +audit subjects. + ## What is retained The logging service keeps compact request metadata in the node-local @@ -36,6 +40,62 @@ Artifact content is deliberately conservative: The console renders diagnostic text as text. It does not render log content as HTML, and it never automatically downloads optional artifact content. +### Caller and mesh-peer context + +Request summaries can include `callerEndpointId`, `callerAddr`, and +`callerPathType`. Mesh operational audits can include `subjectKind`, +`subjectId`, `remoteAddr`, and `pathType`, together with bounded outcome, +reason, duration, and numeric-summary fields appropriate to the audit code. +These fields are optional because local HTTP callers have no mesh endpoint ID, +not every boundary has a live connection, and attribution is only recorded when +the relevant caller or peer context was observed. + +The request caller vocabulary is `local_http`, `remote_quic_http`, and +`relay`. Skippy stage connections and streams are transport work without a +top-level request ID, so they never fabricate request-summary rows. Staged QUIC +is represented as authenticated QUIC audit evidence, not as a synthetic request +row. Its authenticated peer and selected direct/relay path appear on the local +`mesh_quic_inbound_accepted` audit entry. + +An authenticated QUIC endpoint identity can be known even when the selected +path was not observed or its value was not recognized. In that case the request +summary keeps `callerEndpointId` and omits both `callerAddr` and +`callerPathType`. It does not infer a direct or relay path. Endpoint-only caller +identity is not a fourth caller path type. + +For a selected direct path, the ledger records `pathType: "direct"` and the +observed socket address when it is available. For a relay path, it records +`pathType: "relay"` and omits the address; it never substitutes a relay server +address for the peer. When the relevant context was not observed, these fields +remain absent rather than fabricating attribution. + +### Exact mesh audit contract + +| Code | Outcome | Reason | Numeric summaries | Duration owner | +| -------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------------------------------------ | +| `gossip_direct_peer_promoted` | `promoted` | none | `direct_peers` | none | +| `gossip_peer_removed` | `removed` | `stale_direct_and_transitive`, `heartbeat_unreachable`, `peer_down_probe_failed`, `closed_connection_no_address`, `reconnect_failed`, `recovered_gossip_failed`, `clean_shutdown`, or `tunnel_open_failed` | `direct_peers` | none | +| `gossip_policy_rejected` | `rejected` | `owner_attestation_required`, `owner_attestation_expired`, `owner_attestation_invalid`, `owner_attestation_node_mismatch`, `owner_revoked`, `owner_attestation_revoked`, `owner_node_revoked`, `owner_attestation_protocol_unsupported`, or `owner_untrusted` | none | none | +| `gossip_incompatible_version_rejected` | `rejected` | `protocol_version_unsupported` | `peer_gen`, `local_gen` | none | +| `mesh_quic_inbound_accepted` | `accepted` or `readmitted` | none | negotiated family `protocol_gen` (`4` for the current Skippy stage protocol) | none | +| `mesh_control_connection_accepted` | `accepted` | none | `protocol_gen` | none | +| `mesh_control_alpn_rejected` | `rejected` | `alpn_unsupported` | none | none | +| `mesh_quic_handler_failed` | `failed` | `capacity` or `internal` | none | QUIC handler boundary, when timed | +| `mesh_control_handler_failed` | `failed` | `capacity` or `internal` | none | control handler boundary, when timed | + +`gossip_incompatible_version_rejected` is emitted when a direct peer is +rejected during admission for an incompatible version. A transitive +announcement below the local version floor is dropped without emitting this +event. + +`mesh_auto_join_succeeded` and `mesh_auto_join_failed` remain generic, +context-free entries. Pre-authentication failures may be sparse because peer +identity is unavailable before authentication completes. Direct paths may +include the observed address. Relay paths use `pathType: "relay"` and omit the +address, never substituting the relay address. Records may omit typed context +when it was not observed. Audit and request records are trusted-local only, with no OTLP export, +gossip advertisement, or peer replication. + ## Local access model The log API is a trusted-local management surface. Every `/api/logs/**` route @@ -44,6 +104,10 @@ headers are present. It is not advertised through mesh gossip and must not be put behind a public reverse proxy. Use the local embedded console rather than sharing its log routes with other users or nodes. +Audit and request records remain node-local. No OTLP exporter or telemetry +survey reads them, and no gossip or replication path sends them to peers. The +bounded export API is also trusted-local and requires the same access checks. + ### Invite tokens are public connection information Mesh invite tokens are intentionally shareable connection descriptors, not @@ -85,6 +149,13 @@ enters reconnecting and then bounded polling mode until a stream connection is available again. A stale or polling indication means the REST ledger remains the authority; it does not mean a request has failed. +SSE `stream_error` uses two stable codes. `invalid_event` means one retained +entry could not be projected safely. `audit_reconcile_failed` means durable +audit reconciliation failed. In the audit stream, clients preserve the `a1:` +cursor, mark audit data stale, hydrate authoritatively through +`GET /api/logs/audit`, then resume or reconnect. This recovery is fail-open, +so it does not fail inference or the original request. + ## Retention and maintenance Logging settings are available in the production Configuration page. The two settings with a @@ -154,18 +225,29 @@ authoritative transition. The following matrix is the current support contract; it is intentionally narrower than every line that may appear in terminal or native debug output. -| Owner | Captured now | Displayed in Logs | -|---|---|---| -| CLI dispatch | Parsed command start and terminal result, bounded parse failure, and normalized command family | Yes, with command family and outcome | -| Host runtime | Startup, ready, shutdown start, and authoritative shutdown completion | Yes, with runtime subject context where available | -| Model lifecycle | Resolve/load start, ready, load failure, unload start, unload completion/failure, and unexpected model exit at existing Rust-owned boundaries | Yes, with model and runtime-instance correlation | -| Configuration and diagnostics | Existing apply and diagnostics outcomes | Yes | -| Discovery, mesh, and local serving | Existing join/discovery, connection, readiness, and availability outcomes already owned by Rust | Yes | -| OpenAI-compatible inference | Admission, route and attempts, stream boundaries, usage, terminal outcome, and optional request/response artifacts | Yes; payload bodies require `redacted_artifacts` and an explicit operator read | -| Logging operations | Queue/persistence failures, cleanup/delete/export/retry operations, audited reads, and local authorization rejection | Yes; routine management records remain filterable | +| Owner | Captured now | Displayed in Logs | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| CLI dispatch | Parsed command start and terminal result, bounded parse failure, and normalized command family | Yes, with command family and outcome | +| Host runtime | Startup, ready, shutdown start, and authoritative shutdown completion | Yes, with runtime subject context where available | +| Model lifecycle | Resolve/load start, ready, load failure, unload start, unload completion/failure, and unexpected model exit at existing Rust-owned boundaries | Yes, with model and runtime-instance correlation | +| Configuration and diagnostics | Existing apply and diagnostics outcomes | Yes | +| Discovery, mesh, and local serving | Existing join/discovery, connection, readiness, and availability outcomes already owned by Rust | Yes | +| OpenAI-compatible inference | Admission, route and attempts, stream boundaries, usage, terminal outcome, and optional request/response artifacts | Yes; payload bodies require `redacted_artifacts` and an explicit operator read | +| Logging operations | Queue/persistence failures, cleanup/delete/export/retry operations, audited reads, and local authorization rejection | Yes; routine management records remain filterable | The request ledger is canonical for inference lifecycle records. Operational logging does not emit duplicate audit rows for those same request events. + +CLI dispatch audits may include the optional `commandSummary` field. It is a +bounded projection of the parsed command, not raw argv: positional values, +paths, URLs, credentials, model and plugin names, identifiers, and reasons are +represented by `[REDACTED]` or omitted. The summary is limited to 32 tokens and +256 characters, rejects control text, and preserves only the fixed command +vocabulary plus explicitly supplied numeric or enum values in their approved +option contexts. Rows without this field remain valid when attribution was not +observed. The same contract is enforced when records +enter host context, durable storage, REST, live SSE, and replay SSE; malformed +internal or persisted summaries are omitted rather than exposed. Streaming response artifacts are assembled only in `redacted_artifacts` mode, bounded by the configured artifact limit while frames arrive, and redacted before persistence. An incomplete stream or an over-limit stream is recorded @@ -174,7 +256,7 @@ as explicitly unavailable rather than as a partial payload. Lower-level native runtime/model phases, stage and topology lifecycle, session and capacity events, tokenization/prefill/decode progress, KV/cache pressure, device degradation, and reducer-owned availability remain deferred to the -future event pipeline in [`.omo/specs/event-system.md`](../.omo/specs/event-system.md). +future event pipeline. Do not change the mesh protocol, Skippy ABI, native callbacks, or low-level runtime ownership solely to add logging hooks before that system is implemented. @@ -183,12 +265,14 @@ runtime ownership solely to add logging hooks before that system is implemented. - **The Logs tab says unsupported.** The connected host predates the local log API or has it disabled. Upgrade or use a host that exposes the service; do not point the console at status/runtime SSE as a workaround. -- **The Logs tab reports a database version mismatch.** Compare the database - and runtime schema versions shown in the warning. Update MeshLLM when the - database is newer, or restore/update to a build that recognizes an older - schema, then restart the node. The incompatible store is left unchanged and - inference remains available; repeated retries and manual database edits are - not recovery steps. +- **The Logs tab reports `logging_schema_incompatible`.** The warning includes + typed `schema_version` and `supported_schema_version` details. Unknown or + incompatible schemas are left unchanged. Logging metadata is unavailable, + but inference remains available. Update or restore a compatible build. If + you intentionally start new history, stop the node and move or back up + `log_store.db`, `log_store.db-wal`, and `log_store.db-shm` together, then + restart. Do not edit `PRAGMA user_version`, retry until it changes, or + automatically migrate, reset, or delete an unknown schema. - **The ledger says reconnecting, polling, gap, or stale.** Check local host availability first. The page will hydrate from the request listing after a replay gap and uses bounded polling only while the dedicated stream is @@ -199,11 +283,10 @@ runtime ownership solely to add logging hooks before that system is implemented. - **A maintenance operation is rejected.** Reopen the operation, request a fresh preview when required, review the scoped count, and provide a valid audit reason. Never bypass the preview by editing local log-store files. -- **Rolling back a host.** Export the permitted, bounded operator view before - changing versions. An older binary may show the unsupported state and must - not be assumed to understand a newer local log store. Follow the release - rollback procedure for the binary and its application state; do not manually - copy or edit log database or artifact files. +- **Rolling back a host.** Restore or update to a compatible build. If the + whole application state is moved, stop the node first and move or back up + `log_store.db`, `log_store.db-wal`, and `log_store.db-shm` together. Do not + edit individual database files or `PRAGMA user_version`. For the wire and UI contracts behind this guide, see the logging section of [the architecture notes](design/DESIGN.md#operator-request-logging) and the diff --git a/scripts/tests/test_logging_api_docs.py b/scripts/tests/test_logging_api_docs.py index 027c10f860..dfa7e8a403 100644 --- a/scripts/tests/test_logging_api_docs.py +++ b/scripts/tests/test_logging_api_docs.py @@ -9,6 +9,7 @@ ROOT = Path(__file__).resolve().parents[2] PAGE = ROOT / "website" / "src" / "docs" / "pages" / "logging-api.md" +OPERATOR_GUIDE = ROOT / "docs" / "LOGGING.md" DOCS_NAV = ROOT / "website" / "src" / "_data" / "docs.js" API_REFERENCE = ROOT / "website" / "src" / "docs" / "pages" / "api-reference.md" @@ -101,11 +102,67 @@ def test_page_covers_recovery_privacy_and_configuration_contracts(self) -> None: with self.subTest(anchor=anchor): self.assertIn(anchor, page) + def test_stream_error_audit_recovery_contract_is_anchored(self) -> None: + for document in (PAGE, OPERATOR_GUIDE): + with self.subTest(document=document): + text = document.read_text(encoding="utf-8") + for anchor in ( + "invalid_event", + "audit_reconcile_failed", + "a1:", + "GET /api/logs/audit", + ): + self.assertIn(anchor, text) + def test_page_is_linked_from_developer_navigation_and_api_reference(self) -> None: expected_url = "/docs/pages/logging-api/" self.assertIn(expected_url, DOCS_NAV.read_text(encoding="utf-8")) self.assertIn(expected_url, API_REFERENCE.read_text(encoding="utf-8")) + def test_incompatible_version_audit_scope_is_explicit_in_authored_docs(self) -> None: + for document in (OPERATOR_GUIDE, PAGE): + with self.subTest(document=document): + text = document.read_text(encoding="utf-8") + normalized = re.sub(r"\s+", " ", text.lower()) + self.assertIn("`gossip_incompatible_version_rejected` is emitted when a direct peer is", text) + self.assertIn("A transitive\nannouncement below the local version floor is dropped without emitting this", text) + + def test_schema_incompatibility_contract_is_typed_and_version_neutral(self) -> None: + for document in (OPERATOR_GUIDE, PAGE): + with self.subTest(document=document): + text = document.read_text(encoding="utf-8") + normalized = re.sub(r"\s+", " ", text.lower()) + for term in ( + "logging_schema_incompatible", + "schema_version", + "supported_schema_version", + "left unchanged", + "inference remains available", + "log_store.db-wal", + "log_store.db-shm", + "PRAGMA user_version", + ): + self.assertIn(term.lower(), normalized) + self.assertRegex(normalized, r"logging metadata (?:is|becomes) unavailable") + self.assertNotRegex(normalized, r"legacy (rows|entries)|previously retained|older schema|newer schema") + + def test_stable_503_table_lists_all_codes(self) -> None: + page = self.page() + row = re.search(r"^\|\s*503\s*\|\s*([^|]+?)\s*\|\s*$", page, flags=re.MULTILINE) + self.assertIsNotNone(row, "the stable 503 error-code table row must exist") + assert row is not None + self.assertEqual( + set(re.findall(r"`([^`]+)`", row.group(1))), + { + "artifact_deletion_unavailable", + "export_timed_out", + "maintenance_cancelled", + "logging_unavailable", + "store_unavailable", + "logging_schema_incompatible", + }, + ) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_logging_module_boundaries.py b/scripts/tests/test_logging_module_boundaries.py index 8898eab6fe..8fe5d091d1 100644 --- a/scripts/tests/test_logging_module_boundaries.py +++ b/scripts/tests/test_logging_module_boundaries.py @@ -42,6 +42,33 @@ def assert_owner_module( f"new owner module exceeds {MAX_NEW_FILE_LINES} lines: {child}", ) + def assert_semantic_owner( + self, + parent: str, + module_name: str, + child: str, + owned_symbols: tuple[str, ...], + ) -> None: + self.assert_owner_module( + parent, + module_name, + child, + forbidden_parent_text=owned_symbols, + ) + child_source = self.source(child) + for symbol in owned_symbols: + self.assertIn(symbol, child_source, f"{symbol!r} missing from {child}") + pure_lines = [ + line + for line in child_source.splitlines() + if line.strip() and not line.lstrip().startswith("//") + ] + self.assertLessEqual( + len(pure_lines), + 250, + f"semantic production owner exceeds 250 pure lines: {child}", + ) + def test_production_logging_responsibilities_have_named_owners(self) -> None: test_extractions = ( ( @@ -102,6 +129,22 @@ def test_production_logging_responsibilities_have_named_owners(self) -> None: "crates/mesh-llm-log-store/src/maintenance/execution.rs", forbidden_parent_text=("impl ArtifactFileStore",), ) + self.assert_semantic_owner( + "crates/mesh-llm-host-runtime/src/logging/service/operational_audit.rs", + "context", + "crates/mesh-llm-host-runtime/src/logging/service/operational_audit/context.rs", + ( + "OPERATIONAL_AUDIT_CONTEXT_VERSION", + "MAX_CONTEXT_VALUE_CHARS", + "MAX_NUMERIC_SUMMARIES", + "pub enum OperationalAuditSubjectKind", + "pub enum OperationalAuditPathType", + "pub struct OperationalAuditContext", + "fn insert_optional_string", + "fn valid_static_code", + "fn bounded_context_value", + ), + ) for parent in ( "crates/mesh-llm-host-runtime/src/logging/webhook_delivery.rs", @@ -120,6 +163,75 @@ def test_production_logging_responsibilities_have_named_owners(self) -> None: f"new production owner remains oversized: {parent}", ) + def test_log_store_repository_responsibilities_have_named_owners(self) -> None: + repositories = "crates/mesh-llm-log-store/src/repositories.rs" + audit = "crates/mesh-llm-log-store/src/repositories/audit.rs" + self.assert_owner_module( + repositories, + "audit", + audit, + forbidden_parent_text=( + "pub struct AuditEntryRow", + "struct StoredAuditDetail", + "fn audit_entry_query_parts", + "fn audit_entry_row", + "pub fn insert_audit_entry", + "pub fn list_audit_entries(", + "pub fn list_audit_entries_after_sequence(", + ), + ) + self.assert_owner_module( + audit, + "detail", + "crates/mesh-llm-log-store/src/repositories/audit/detail.rs", + forbidden_parent_text=( + "struct StoredAuditDetail", + "fn bounded_audit_value", + "fn bounded_command_summary", + "fn bounded_audit_code", + "fn bounded_code", + ), + ) + self.assert_owner_module( + repositories, + "caller_metadata", + "crates/mesh-llm-log-store/src/repositories/caller_metadata.rs", + forbidden_parent_text=("pub fn upsert_summary_metadata(",), + ) + + def test_mesh_connection_responsibilities_have_named_owners(self) -> None: + connections = "crates/mesh-llm-host-runtime/src/mesh/connections.rs" + self.assert_semantic_owner( + connections, + "inbound", + "crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs", + ( + "pub(crate) async fn handle_incoming(", + "pub(crate) async fn handle_control_incoming(", + "pub(crate) async fn accept_mesh_stream(", + "pub(crate) async fn admitted_mesh_stream(", + ), + ) + self.assert_semantic_owner( + connections, + "tunnel", + "crates/mesh-llm-host-runtime/src/mesh/connections/tunnel.rs", + ( + "pub(crate) async fn dispatch_mesh_stream(", + "pub(crate) async fn forward_tunnel_stream(", + "pub(crate) async fn forward_tunnel_http_stream(", + "pub(crate) async fn _dispatch_streams(", + "pub(crate) async fn authenticated_peer_path(", + "pub(crate) async fn remove_connection_if_stable_id(", + ), + ) + self.assert_semantic_owner( + "crates/mesh-llm-host-runtime/src/mesh/connections/inbound.rs", + "stage", + "crates/mesh-llm-host-runtime/src/mesh/connections/inbound/stage.rs", + ("pub(crate) async fn handle_stage_alpn(",), + ) + def test_oversized_characterization_suites_are_split_by_concern(self) -> None: suites = ( ( @@ -159,6 +271,54 @@ def test_oversized_characterization_suites_are_split_by_concern(self) -> None: f"test suite parent remains oversized: {parent}", ) + def test_audit_and_gossip_suites_have_semantic_children(self) -> None: + suites = ( + ( + "crates/mesh-llm-log-store/src/api_acceptance_tests/summary_audit.rs", + ( + ("mod basic;", "summary_audit/basic.rs"), + ("mod sanitization;", "summary_audit/sanitization.rs"), + ("mod query;", "summary_audit/query.rs"), + ), + ), + ( + "crates/mesh-llm-host-runtime/src/mesh/tests/gossip.rs", + ( + ( + 'include!("gossip/merge_and_refresh.rs");', + "gossip/merge_and_refresh.rs", + ), + ('include!("gossip/admission.rs");', "gossip/admission.rs"), + ('include!("gossip/discovery.rs");', "gossip/discovery.rs"), + ), + ), + ) + for parent, children in suites: + parent_path = ROOT / parent + parent_source = parent_path.read_text(encoding="utf-8") + with self.subTest(parent=parent): + self.assertTrue(parent_source.strip(), f"empty test parent: {parent}") + self.assertNotIn("#[test]", parent_source) + self.assertNotIn("#[tokio::test", parent_source) + self.assertLessEqual( + len(parent_source.splitlines()), + MAX_NEW_FILE_LINES, + f"test suite parent remains oversized: {parent}", + ) + for declaration, relative_child in children: + child = str(Path(parent).parent / relative_child) + child_path = ROOT / child + with self.subTest(parent=parent, child=child): + self.assertIn(declaration, parent_source) + self.assertTrue(child_path.is_file(), f"missing test child: {child}") + child_source = child_path.read_text(encoding="utf-8") + self.assertTrue(child_source.strip(), f"empty test child: {child}") + self.assertLessEqual( + len(child_source.splitlines()), + MAX_NEW_FILE_LINES, + f"test suite child remains oversized: {child}", + ) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py index f13658edda..f790edd92b 100644 --- a/scripts/tests/test_static_abi_artifacts.py +++ b/scripts/tests/test_static_abi_artifacts.py @@ -96,6 +96,21 @@ def test_native_sdk_reuse_is_verification_only(self) -> None: self.assertIn("SKIPPY_LLAMA_AUTO_BUILD=0", package_script) self.assertIn("MESH_LLM_AUTO_BUILD_LLAMA=0", package_script) + def test_skippy_ffi_links_mtmd_hash_dependency_after_mtmd(self) -> None: + build_script = (ROOT / "crates" / "skippy-ffi" / "build.rs").read_text( + encoding="utf-8", + ) + + mtmd_link = 'println!("cargo:rustc-link-lib=static=mtmd");' + hash_link = 'println!("cargo:rustc-link-lib=static=vendor-hash");' + self.assertIn('build_dir.join("vendor/hash")', build_script) + self.assertIn('"vendor/hash/libvendor-hash.a"', build_script) + self.assertIn('"vendor/hash/vendor-hash.lib"', build_script) + self.assertLess( + build_script.index(mtmd_link), + build_script.index(hash_link), + ) + def test_dynamic_output_probe_is_pipefail_safe(self) -> None: build_script = (ROOT / "scripts" / "build-llama.sh").read_text( encoding="utf-8", diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index 9b630779a9..16f6e2f11b 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -2775,15 +2775,15 @@ ], "crates/mesh-llm/src/lib.rs": [ { - "line": 286, + "line": 289, "macro_name": "print!" }, { - "line": 338, + "line": 341, "macro_name": "print!" }, { - "line": 339, + "line": 342, "macro_name": "eprintln!" } ], diff --git a/website/src/docs/pages/logging-api.md b/website/src/docs/pages/logging-api.md index 4509d8f2ed..a5f1aaee06 100644 --- a/website/src/docs/pages/logging-api.md +++ b/website/src/docs/pages/logging-api.md @@ -14,6 +14,10 @@ This is an operator API, not an inference API. Applications should continue to use the [OpenAI-compatible API](/docs/pages/openai-compatible-api/) on port `9337` for inference. +In the embedded Logs console, ledger search accepts either a full endpoint ID +or the abbreviated form shown in the interface, such as `9f0c…bb04`, for +request caller identities and mesh-peer audit subjects. + ## Scope and trust boundary Every `/api/logs` and `/api/logs/**` route requires a trusted local caller. A @@ -31,6 +35,10 @@ these checks. If a local reverse proxy is used, it must connect from loopback and preserve a trusted local `Host` and `Origin`. Rejected requests receive the normal typed `403 forbidden` response before the logging store is touched. +The ledger and its bounded export API remain trusted-local. Audit and request +records are not read by OTLP exporters or telemetry surveys, advertised in +gossip, replicated, or otherwise sent to peer nodes. + Examples below assume: ```bash @@ -100,19 +108,36 @@ modify, or reuse a cursor with different filters. Unless noted otherwise, `GET /api/logs/requests` merges the bounded active snapshot with durable history. An active request takes precedence over a durable row with the same ID. Its response contains `requestId`, `outcome`, `createdAt`, optional -`terminalAt`, `route`, `model`, `provider`, `engine`, and `statusCode`, plus a -`source` of `active` or `durable`. +`terminalAt`, `route`, `model`, `provider`, `engine`, `statusCode`, +`callerEndpointId`, `callerAddr`, and `callerPathType`, plus a `source` of +`active` or `durable`. + +Caller fields are observational and optional. The vocabulary is `"local_http"`, +`"remote_quic_http"`, and `"relay"`. A direct remote HTTP path uses +`callerPathType: "remote_quic_http"` and includes the observed peer address +when available. A relay path uses `"relay"` and omits `callerAddr`; a relay +server address is never substituted. Local management requests use +`"local_http"` and have no mesh endpoint ID. Caller fields are absent when +attribution was not observed, rather than being fabricated. Skippy stage streams have no top-level +request ID and therefore never create request-summary rows. Staged QUIC is +represented as authenticated QUIC audit evidence, not as a synthetic request +row. + +Authentication can establish `callerEndpointId` when the selected path was not +observed or its value was not recognized. Such a request omits `callerAddr` and +`callerPathType`; the API does not infer `"remote_quic_http"` or `"relay"`. +Endpoint-only identity is not a fourth caller path type. Supported query parameters: -| Parameter | Values | -|---|---| -| `limit`, `cursor`, `sort` | Keyset pagination controls; request listing defaults to `sort=desc`. | -| `from`, `to` | Inclusive RFC 3339 creation-time bounds. | -| `route`, `model`, `provider`, `engine` | Exact metadata matches, each at most 128 bytes. | -| `status` | HTTP status from 100 through 599. | -| `outcome` | `active`, `completed`, `failed`, `rejected`, `cancelled`, or `dropped`. | -| `source` | `active` or `durable`; omit it to merge both sources. | +| Parameter | Values | +| -------------------------------------- | ----------------------------------------------------------------------- | +| `limit`, `cursor`, `sort` | Keyset pagination controls; request listing defaults to `sort=desc`. | +| `from`, `to` | Inclusive RFC 3339 creation-time bounds. | +| `route`, `model`, `provider`, `engine` | Exact metadata matches, each at most 128 bytes. | +| `status` | HTTP status from 100 through 599. | +| `outcome` | `active`, `completed`, `failed`, `rejected`, `cancelled`, or `dropped`. | +| `source` | `active` or `durable`; omit it to merge both sources. | Unknown or duplicate parameters are rejected rather than ignored. @@ -147,10 +172,55 @@ reduced to scheme, host, and port; credentials, paths, query strings, and fragments are not returned. `GET /api/logs/audit` returns sparse operational audit entries with `entryId`, -`occurredAt`, `source`, `code`, and optional `severity`. It accepts `limit`, -`cursor`, `source`, and `severity`. Sources are `logging_service`, `runtime`, -`mesh`, `cli`, or `logs_api`; severities are `info`, `warning`, or `error`. Audit responses -contain no request ID, detail JSON, or free-form message. +`occurredAt`, `source`, `code`, and optional `severity`, `contextVersion`, +`subjectKind`, `subjectId`, `remoteAddr`, `pathType`, `operationId`, `requestId`, +`reasonCode`, `outcome`, `durationMs`, and `numericSummaries`. It accepts +`limit`, `cursor`, `source`, and `severity`. Sources are `logging_service`, +`runtime`, `mesh`, `cli`, or `logs_api`; severities are `info`, `warning`, or +`error`. CLI audit entries may also contain the optional `commandSummary`. +It is a bounded parsed-command projection, never raw argv: positional values, +paths, URLs, credentials, model and plugin names, identifiers, and reasons are +redacted or omitted. It accepts at most 32 tokens and 256 characters, rejects +control text, and retains only fixed command vocabulary plus explicitly supplied +numeric or enum values in their approved option contexts. Entries may omit it +when attribution was not observed. Audit responses contain no +arbitrary detail JSON or free-form message; malformed summaries are omitted at +every REST and SSE projection boundary. + +For mesh-peer audits, `subjectId` is the authenticated endpoint ID. Direct +paths can include the observed peer `remoteAddr`; relay paths omit it and never +expose the relay server as the peer. `pathType` is absent when no live path was +available, and entries can omit typed context fields when that context was not +observed. Authenticated +Skippy stage connections use `mesh_quic_inbound_accepted` for this peer/path +evidence before stage stream dispatch. + +### Exact mesh audit contract + +| Code | Outcome | Reason | Numeric summaries | Duration owner | +| -------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------------------------------------ | +| `gossip_direct_peer_promoted` | `promoted` | none | `direct_peers` | none | +| `gossip_peer_removed` | `removed` | `stale_direct_and_transitive`, `heartbeat_unreachable`, `peer_down_probe_failed`, `closed_connection_no_address`, `reconnect_failed`, `recovered_gossip_failed`, `clean_shutdown`, or `tunnel_open_failed` | `direct_peers` | none | +| `gossip_policy_rejected` | `rejected` | `owner_attestation_required`, `owner_attestation_expired`, `owner_attestation_invalid`, `owner_attestation_node_mismatch`, `owner_revoked`, `owner_attestation_revoked`, `owner_node_revoked`, `owner_attestation_protocol_unsupported`, or `owner_untrusted` | none | none | +| `gossip_incompatible_version_rejected` | `rejected` | `protocol_version_unsupported` | `peer_gen`, `local_gen` | none | +| `mesh_quic_inbound_accepted` | `accepted` or `readmitted` | none | negotiated family `protocol_gen` (`4` for the current Skippy stage protocol) | none | +| `mesh_control_connection_accepted` | `accepted` | none | `protocol_gen` | none | +| `mesh_control_alpn_rejected` | `rejected` | `alpn_unsupported` | none | none | +| `mesh_quic_handler_failed` | `failed` | `capacity` or `internal` | none | QUIC handler boundary, when timed | +| `mesh_control_handler_failed` | `failed` | `capacity` or `internal` | none | control handler boundary, when timed | + +`gossip_incompatible_version_rejected` is emitted when a direct peer is +rejected during admission for an incompatible version. A transitive +announcement below the local version floor is dropped without emitting this +event. + +`mesh_auto_join_succeeded` and `mesh_auto_join_failed` remain generic, +context-free entries. Pre-authentication failures may be sparse because peer +identity is unavailable before authentication completes. Direct paths may +include the observed address. Relay paths use `pathType: "relay"` and omit +`remoteAddr`, never substituting the relay address. Entries may omit typed +context when it was not observed. Audit and request records remain trusted-local only, with no +OTLP export, gossip advertisement, or peer replication. ## Live SSE @@ -177,6 +247,11 @@ GET /api/logs/events?audit=true&source=runtime&severity=warning Audit mode accepts `source`, `severity`, and its cursor, but not lifecycle channels or filters. +`audit_entry` frames use the same sparse projection as `GET /api/logs/audit`, +including optional `commandSummary`. Live and replay frames enforce the same +redaction, token, control-character, and length contract as durable rows; a +malformed persisted or internal summary is omitted instead of being streamed. + Each accepted connection first receives matching entries retained after its cursor, then live updates. Lifecycle event IDs use the v1 vector form `v1:..`, for example `v1:42.7.3`. Audit event IDs @@ -186,12 +261,12 @@ newer position so reconnects do not move backwards. The stream emits: -| SSE event | Meaning | -|---|---| -| `log_event` | A privacy-safe lifecycle projection. | -| `audit_entry` | A sparse audit projection in audit mode. | -| `replay_gap` | Requested entries have left the bounded replay window. | -| `stream_error` | One retained entry could not be projected safely; its code is `invalid_event`. | +| SSE event | Meaning | +| -------------- | ------------------------------------------------------------------------------ | +| `log_event` | A privacy-safe lifecycle projection. | +| `audit_entry` | A sparse audit projection in audit mode. | +| `replay_gap` | Requested entries have left the bounded replay window. | +| `stream_error` | A retained entry could not be projected safely (`invalid_event`), or durable audit reconciliation failed (`audit_reconcile_failed`). | If a client lags but the relevant entries remain in replay, the session catches up from that buffer. If entries have been evicted, `replay_gap` identifies the @@ -201,6 +276,13 @@ best-effort durable cursor. Recover lifecycle history from history from `GET /api/logs/audit`. Resume SSE from the gap frame's event ID after the REST reconciliation. +For `stream_error`, `invalid_event` means one retained entry could not be +projected safely. `audit_reconcile_failed` means durable audit reconciliation +failed. Clients preserve the `a1:` cursor, mark audit data stale, hydrate it +authoritatively through `GET /api/logs/audit`, then resume or reconnect the +audit stream. Audit recovery is fail-open: it does not fail inference or the +original request. + Each frame is capped at 16 KiB. A connection has a 64-frame handoff queue, receives a keepalive every 15 seconds, and is disconnected when writes remain blocked. Treat disconnect as a reconnect signal and use `Last-Event-ID`; do not @@ -323,7 +405,7 @@ An operator who has a dead-letter delivery ID can request a new bounded attempt cycle with `POST /api/logs/webhooks/{deliveryId}/retry`: ```json -{"reason":"receiver recovered"} +{ "reason": "receiver recovered" } ``` The response outcome is `scheduled` or, for an idempotent repeat, @@ -357,21 +439,26 @@ Across REST, SSE, status, export, audit, and webhook surfaces: REST errors have one stable envelope: ```json -{"error":{"code":"invalid_query","message":"limit must be between 1 and 100"}} +{ + "error": { + "code": "invalid_query", + "message": "limit must be between 1 and 100" + } +} ``` Clients should branch on `error.code`; messages are explanatory, not a parsing contract. -| HTTP status | Stable codes | -|---|---| -| 400 | `invalid_request`, `invalid_query`, `invalid_cursor`, `cursor_expired`, `invalid_id`, `invalid_webhook_delivery_id` | -| 403 | `forbidden`, `artifact_export_forbidden` | -| 404 | `not_found` | -| 405 | `method_not_allowed` | -| 406 | `not_acceptable` | -| 409 | `maintenance_conflict`, `request_active`, `webhook_not_retryable` | -| 503 | `export_timed_out`, `maintenance_cancelled`, `logging_unavailable`, `store_unavailable` | +| HTTP status | Stable codes | +| ----------- | ------------------------------------------------------------------------------------------------------------------- | +| 400 | `invalid_request`, `invalid_query`, `invalid_cursor`, `cursor_expired`, `invalid_id`, `invalid_webhook_delivery_id` | +| 403 | `forbidden`, `artifact_export_forbidden` | +| 404 | `not_found` | +| 405 | `method_not_allowed` | +| 406 | `not_acceptable` | +| 409 | `maintenance_conflict`, `request_active`, `webhook_not_retryable` | +| 503 | `artifact_deletion_unavailable`, `export_timed_out`, `maintenance_cancelled`, `logging_schema_incompatible`, `logging_unavailable`, `store_unavailable` | Methods, request bodies, query keys, duplicate parameters, identifiers, and enum values are validated strictly. A store or audit failure does not cause @@ -411,26 +498,26 @@ dead_letter_retention_secs = 259200 application-state root. It must not be empty, traverse out of its scope, name a protected system/root directory, or be world-writable when it already exists. -| Field | Allowed range | Apply behavior | -|---|---:|---| -| `logging.enabled` | Boolean, default `true` | Restart | -| `logging.application_state_root` | Safe private path | Restart | -| `logging.summary_line_limit` | 1–65,536 | Restart | -| `logging.event_buffer_size` | 50–100,000 | Restart | -| `logging.retention_ttl_secs` | 3,600–7,776,000 seconds | Live | -| `logging.retention_max_rows` | 1–1,000,000 terminal summaries | Restart | -| `logging.replay_capacity` | 1–10,000 events | Live | -| `logging.queue_capacity` | 64–131,072 entries | Restart | -| `logging.artifact.capture_mode` | `metadata_only` or `redacted_artifacts` | Restart | -| `logging.artifact.byte_limit_bytes` | 1 KiB–16 MiB | Restart | -| `logging.artifact.aggregate_limit_bytes` | 512 KiB–500 MiB | Restart | -| `logging.export_limit_bytes` | 64 KiB–100 MiB | Restart | -| `logging.cleanup_cadence_secs` | 300–86,400 seconds | Restart | -| `logging.webhook.enabled` | Boolean, default `false` | Restart | -| `logging.webhook.url` | Valid constrained HTTP(S) URL | Restart | -| `logging.webhook.max_attempts` | 1–20 | Restart | -| `logging.webhook.timeout_secs` | 1–60 seconds | Restart | -| `logging.webhook.dead_letter_retention_secs` | 3,600–1,555,200 seconds | Restart | +| Field | Allowed range | Apply behavior | +| -------------------------------------------- | --------------------------------------: | -------------- | +| `logging.enabled` | Boolean, default `true` | Restart | +| `logging.application_state_root` | Safe private path | Restart | +| `logging.summary_line_limit` | 1–65,536 | Restart | +| `logging.event_buffer_size` | 50–100,000 | Restart | +| `logging.retention_ttl_secs` | 3,600–7,776,000 seconds | Live | +| `logging.retention_max_rows` | 1–1,000,000 terminal summaries | Restart | +| `logging.replay_capacity` | 1–10,000 events | Live | +| `logging.queue_capacity` | 64–131,072 entries | Restart | +| `logging.artifact.capture_mode` | `metadata_only` or `redacted_artifacts` | Restart | +| `logging.artifact.byte_limit_bytes` | 1 KiB–16 MiB | Restart | +| `logging.artifact.aggregate_limit_bytes` | 512 KiB–500 MiB | Restart | +| `logging.export_limit_bytes` | 64 KiB–100 MiB | Restart | +| `logging.cleanup_cadence_secs` | 300–86,400 seconds | Restart | +| `logging.webhook.enabled` | Boolean, default `false` | Restart | +| `logging.webhook.url` | Valid constrained HTTP(S) URL | Restart | +| `logging.webhook.max_attempts` | 1–20 | Restart | +| `logging.webhook.timeout_secs` | 1–60 seconds | Restart | +| `logging.webhook.dead_letter_retention_secs` | 3,600–1,555,200 seconds | Restart | `logging.retention_ttl_secs` and `logging.replay_capacity` are the only logging settings currently applied to a running service. Other changes are validated @@ -448,3 +535,18 @@ must continue to tolerate absent and unknown fields. SSE IDs beginning with `v1:` and `a1:` version only the local replay cursor syntax; they are not Mesh protocol versions. Persist cursors only for reconnect/recovery and handle `replay_gap` instead of assuming replay retention is permanent. + +### Log-store schema and recovery + +The durable log store uses one complete schema version, `1`. The forward +migration registry is intentionally empty. The API reports an unknown or +incompatible store with `logging_schema_incompatible` and typed details in +`schema_version` and `supported_schema_version`. + +Mesh does not automatically migrate, reset, or delete an unknown schema. The +store is left unchanged, logging metadata becomes unavailable, and inference +remains available. The database is not modified while the schema is unknown. +Recover by updating or restoring a compatible build. To +start new history instead, stop Mesh and move or back up `log_store.db`, +`log_store.db-wal`, and `log_store.db-shm` together before restarting. Do not +edit `PRAGMA user_version`.