From 382f61a16395fef30d48303a9b50a30d0808ed20 Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Thu, 15 Oct 2026 21:20:43 +0800 Subject: [PATCH 1/6] feat: update ov privacy commands for agent use --- crates/ov_cli/src/client.rs | 64 +++ crates/ov_cli/src/commands/privacy.rs | 130 ++++++ crates/ov_cli/src/handlers.rs | 66 +++ crates/ov_cli/src/help_ui.rs | 16 +- crates/ov_cli/src/main.rs | 389 ++++++++++++++---- docs/en/api/01-overview.md | 6 + docs/en/api/10-privacy.md | 126 ++++++ docs/en/concepts/13-privacy.md | 8 + docs/zh/api/01-overview.md | 6 + docs/zh/api/10-privacy.md | 130 ++++++ docs/zh/concepts/13-privacy.md | 8 + docs/zh/privacy-review.md | 208 ++++++++++ openviking/privacy/__init__.py | 33 +- openviking/privacy/models.py | 11 + openviking/privacy/reference.py | 143 +++++++ openviking/privacy/sanitizer.py | 214 ++++++++++ openviking/privacy/scan.py | 156 +++++++ openviking/privacy/service.py | 100 ++++- openviking/server/routers/privacy_configs.py | 256 +++++++++++- openviking/service/core.py | 5 +- openviking/service/fs_service.py | 38 +- openviking/service/reindex_executor.py | 7 +- openviking/storage/content_write.py | 48 ++- openviking/storage/context_update_plan.py | 8 +- openviking/storage/viking_fs/__init__.py | 5 + openviking/storage/viking_fs/_ops.py | 15 + openviking/utils/resource_processor.py | 90 +++- openviking/utils/skill_processor.py | 73 +++- tests/server/test_api_privacy_configs.py | 306 ++++++++++++++ tests/server/test_privacy_sanitizer_writes.py | 168 ++++++++ tests/unit/test_privacy_sanitizer.py | 198 +++++++++ 31 files changed, 2927 insertions(+), 104 deletions(-) create mode 100644 docs/zh/privacy-review.md create mode 100644 openviking/privacy/reference.py create mode 100644 openviking/privacy/sanitizer.py create mode 100644 openviking/privacy/scan.py create mode 100644 tests/server/test_privacy_sanitizer_writes.py create mode 100644 tests/unit/test_privacy_sanitizer.py diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs index 2ec9b8c608..dce1549cac 100644 --- a/crates/ov_cli/src/client.rs +++ b/crates/ov_cli/src/client.rs @@ -1740,6 +1740,32 @@ impl HttpClient { self.get(&path, &[]).await } + pub async fn privacy_get_field( + &self, + category: &str, + target_key: &str, + field: &str, + reveal: bool, + ) -> Result { + let path = format!("/api/v1/privacy-configs/{}/{}/field", category, target_key); + let mut params = vec![("field".to_string(), field.to_string())]; + if reveal { + params.push(("reveal".to_string(), "true".to_string())); + } + self.get(&path, ¶ms).await + } + + pub async fn privacy_resolve_selector( + &self, + selector: &str, + ) -> Result { + self.post( + "/api/v1/privacy-configs/resolve", + &serde_json::json!({"selector": selector}), + ) + .await + } + pub async fn privacy_upsert( &self, category: &str, @@ -1789,6 +1815,44 @@ impl HttpClient { self.post(&path, &body).await } + pub async fn privacy_delete( + &self, + category: &str, + target_key: &str, + ) -> Result { + let path = format!("/api/v1/privacy-configs/{}/{}", category, target_key); + self.delete(&path, &[]).await + } + + pub async fn privacy_delete_field( + &self, + category: &str, + target_key: &str, + field: &str, + ) -> Result { + let path = format!("/api/v1/privacy-configs/{}/{}/field", category, target_key); + self.delete(&path, &[("field".to_string(), field.to_string())]) + .await + } + + pub async fn privacy_delete_selector( + &self, + selector: &str, + ) -> Result { + self.delete_with_body( + "/api/v1/privacy-configs/selector", + &serde_json::json!({"selector": selector}), + ) + .await + } + + pub async fn privacy_scan( + &self, + body: &serde_json::Value, + ) -> Result { + self.post("/api/v1/privacy-configs/scan", body).await + } + // ============ Watch Management (RFC #2104) ============ pub async fn list_watches(&self, active_only: bool) -> Result { diff --git a/crates/ov_cli/src/commands/privacy.rs b/crates/ov_cli/src/commands/privacy.rs index a99cf1dbe4..f8d13be542 100644 --- a/crates/ov_cli/src/commands/privacy.rs +++ b/crates/ov_cli/src/commands/privacy.rs @@ -3,6 +3,7 @@ use crate::error::{Error, Result}; use crate::output::{OutputFormat, output_success}; use serde_json::{Map, Value}; use std::collections::{HashMap, HashSet}; +use std::io::{self, IsTerminal, Write}; fn parse_kv_pairs(pairs: &[String]) -> Result> { let mut parsed: HashMap = HashMap::new(); @@ -246,6 +247,47 @@ pub async fn get_current( Ok(()) } +pub async fn reference( + client: &HttpClient, + category: &str, + target_key: &str, + field: &str, + output_format: OutputFormat, + compact: bool, +) -> Result<()> { + let result = client + .privacy_get_field(category, target_key, field, false) + .await?; + output_success(&result, output_format, compact); + Ok(()) +} + +pub async fn resolve( + client: &HttpClient, + selector_or_category: &str, + target_key: Option<&str>, + field: Option<&str>, + output_format: OutputFormat, + compact: bool, +) -> Result<()> { + let result = match (target_key, field) { + (None, None) => client.privacy_resolve_selector(selector_or_category).await?, + (Some(target_key), Some(field)) => { + client + .privacy_get_field(selector_or_category, target_key, field, true) + .await? + } + _ => { + return Err(Error::Client( + "Resolve accepts either one reference/field URI or category target field." + .to_string(), + )); + } + }; + output_success(&result, output_format, compact); + Ok(()) +} + pub async fn list_versions( client: &HttpClient, category: &str, @@ -296,6 +338,94 @@ pub async fn activate( Ok(()) } +pub async fn delete( + client: &HttpClient, + selector_or_category: &str, + target_key: Option<&str>, + field: Option<&str>, + yes: bool, + output_format: OutputFormat, + compact: bool, +) -> Result<()> { + let display = match (target_key, field) { + (None, None) => selector_or_category.to_string(), + (Some(target), None) => format!("{selector_or_category}/{target}"), + (Some(target), Some(field)) => format!("{selector_or_category}/{target}#{field}"), + (None, Some(_)) => unreachable!("Clap cannot supply the third positional alone"), + }; + if !yes && !confirm_delete(&display)? { + output_success( + &serde_json::json!({ + "selector": display, + "deleted": false, + "cancelled": true, + }), + output_format, + compact, + ); + return Ok(()); + } + + let result = match (target_key, field) { + (None, None) => client.privacy_delete_selector(selector_or_category).await?, + (Some(target), None) => client.privacy_delete(selector_or_category, target).await?, + (Some(target), Some(field)) => { + client + .privacy_delete_field(selector_or_category, target, field) + .await? + } + (None, Some(_)) => unreachable!("Clap cannot supply the third positional alone"), + }; + output_success(&result, output_format, compact); + Ok(()) +} + +pub async fn scan( + client: &HttpClient, + scopes: &[String], + target_uri: Option<&str>, + fix: bool, + wait: bool, + node_limit: usize, + output_format: OutputFormat, + compact: bool, +) -> Result<()> { + if target_uri.is_some() && !scopes.is_empty() { + return Err(Error::Client( + "Pass either --target or --scope, not both.".to_string(), + )); + } + let body = serde_json::json!({ + "scopes": scopes, + "target_uri": target_uri.unwrap_or(""), + "fix": fix, + "wait": wait, + "node_limit": node_limit, + }); + let result = client.privacy_scan(&body).await?; + output_success(&result, output_format, compact); + Ok(()) +} + +fn confirm_delete(display: &str) -> Result { + if !io::stdin().is_terminal() || !io::stdout().is_terminal() { + return Err(Error::Client( + "Deleting a privacy target requires confirmation. Pass --yes to skip the prompt." + .to_string(), + )); + } + + print!( + "Delete Privacy selection {}? [y/N]: ", + display + ); + io::stdout().flush()?; + let mut answer = String::new(); + io::stdin().read_line(&mut answer)?; + let answer = answer.trim().to_ascii_lowercase(); + Ok(answer == "y" || answer == "yes") +} + pub async fn upsert( client: &HttpClient, category: &str, diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs index 73e1202f36..a079d0b106 100644 --- a/crates/ov_cli/src/handlers.rs +++ b/crates/ov_cli/src/handlers.rs @@ -820,6 +820,36 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> ) .await } + PrivacyCommands::Reference { + category, + target_key, + field, + } => { + commands::privacy::reference( + &client, + &category, + &target_key, + &field, + ctx.output_format, + ctx.compact, + ) + .await + } + PrivacyCommands::Resolve { + selector_or_category, + target_key, + field, + } => { + commands::privacy::resolve( + &client, + &selector_or_category, + target_key.as_deref(), + field.as_deref(), + ctx.output_format, + ctx.compact, + ) + .await + } PrivacyCommands::Upsert { category, target_key, @@ -886,6 +916,42 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> ) .await } + PrivacyCommands::Delete { + selector_or_category, + target_key, + field, + yes, + } => { + commands::privacy::delete( + &client, + &selector_or_category, + target_key.as_deref(), + field.as_deref(), + yes, + ctx.output_format, + ctx.compact, + ) + .await + } + PrivacyCommands::Scan { + scopes, + target, + fix, + wait, + node_limit, + } => { + commands::privacy::scan( + &client, + &scopes, + target.as_deref(), + fix, + wait, + node_limit, + ctx.output_format, + ctx.compact, + ) + .await + } } } diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs index 493bafd283..ddeeaa0f73 100644 --- a/crates/ov_cli/src/help_ui.rs +++ b/crates/ov_cli/src/help_ui.rs @@ -772,7 +772,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ }, CommandHelpSpec { path: &["privacy"], - purpose: "Manage privacy config categories, targets, versions, and activation.", + purpose: "Manage privacy targets, field references, values, versions, and cleanup.", examples: &[ HelpItem { label: "ov privacy categories", @@ -782,6 +782,18 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ label: "ov privacy get ", description: "Show active values for one target.", }, + HelpItem { + label: "ov privacy reference ", + description: "Show the owner-bound reference without revealing the value.", + }, + HelpItem { + label: "ov privacy resolve ", + description: "Explicitly reveal the referenced active field value.", + }, + HelpItem { + label: "ov privacy scan --scope resources,skills,memories", + description: "Dry-run scan for registered Privacy values in stored content.", + }, ], next_steps: &[HelpItem { label: "ov privacy --help", @@ -3173,7 +3185,7 @@ mod tests { ), ( "privacy", - "Manage privacy config categories, targets, versions, and activation.", + "Manage privacy targets, field references, values, versions, and cleanup.", ), ("admin", "Manage accounts, users, roles, and API keys."), ( diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs index 4ccc201ce5..65a616f309 100644 --- a/crates/ov_cli/src/main.rs +++ b/crates/ov_cli/src/main.rs @@ -1162,7 +1162,7 @@ enum Commands { /// [Data] Privacy config management commands Privacy { #[command(subcommand)] - action: PrivacyCommands, + action: Box, }, /// [Data] Export context as .ovpack Export { @@ -1958,6 +1958,30 @@ enum PrivacyCommands { #[arg(value_name = "target-key")] target_key: String, }, + /// Show an owner-bound reference for one active field without revealing its value + Reference { + /// Privacy config category + #[arg(value_name = "category")] + category: String, + /// Privacy config target key + #[arg(value_name = "target-key")] + target_key: String, + /// Field in the current values object + #[arg(value_name = "field")] + field: String, + }, + /// Reveal one active field value for immediate use + Resolve { + /// Owner-bound reference, field URI, or privacy category + #[arg(value_name = "reference-or-uri-or-category")] + selector_or_category: String, + /// Privacy config target key (category form only) + #[arg(value_name = "target-key")] + target_key: Option, + /// Field in current values (category form only) + #[arg(value_name = "field")] + field: Option, + }, /// Upsert privacy config values Upsert { /// Privacy config category @@ -2019,6 +2043,39 @@ enum PrivacyCommands { #[arg(value_name = "version")] version: i32, }, + /// Delete a target or make one active field unavailable + Delete { + /// Privacy reference, URI, or config category + #[arg(value_name = "reference-or-uri-or-category")] + selector_or_category: String, + /// Privacy config target key (category form only) + #[arg(value_name = "target-key")] + target_key: Option, + /// Field to remove from active values (optional) + #[arg(value_name = "field")] + field: Option, + /// Skip the confirmation prompt + #[arg(long)] + yes: bool, + }, + /// Scan persisted content for registered Privacy values + Scan { + /// Comma-separated scopes: resources,skills,memories + #[arg(long = "scope", value_delimiter = ',', value_name = "scope")] + scopes: Vec, + /// Scan one exact viking:// file or directory instead of default scopes + #[arg(long, conflicts_with = "scopes", value_name = "uri")] + target: Option, + /// Rewrite unambiguous matches and refresh derived indexes + #[arg(long)] + fix: bool, + /// Wait for refresh/reindex work after fixing + #[arg(long)] + wait: bool, + /// Maximum files/nodes enumerated per root + #[arg(long, default_value_t = 10000)] + node_limit: usize, + }, } #[derive(Subcommand)] @@ -2804,7 +2861,17 @@ fn is_session_subcommand(token: &str) -> bool { fn is_privacy_subcommand(token: &str) -> bool { matches!( token, - "categories" | "list" | "get" | "upsert" | "versions" | "version" | "activate" + "categories" + | "list" + | "get" + | "reference" + | "resolve" + | "upsert" + | "versions" + | "version" + | "activate" + | "delete" + | "scan" ) } @@ -3595,7 +3662,7 @@ async fn main() { Commands::Observer { action } => handlers::handle_observer(action, ctx).await, Commands::Session { action } => handlers::handle_session(action, ctx).await, Commands::Admin { action } => handlers::handle_admin(action, ctx).await, - Commands::Privacy { action } => handlers::handle_privacy(action, ctx).await, + Commands::Privacy { action } => handlers::handle_privacy(*action, ctx).await, Commands::Ls { uri, simple, @@ -4652,10 +4719,14 @@ mod tests { &["ov", "privacy", "categories"], &["ov", "privacy", "list"], &["ov", "privacy", "get"], + &["ov", "privacy", "reference"], + &["ov", "privacy", "resolve"], &["ov", "privacy", "upsert"], &["ov", "privacy", "versions"], &["ov", "privacy", "version"], &["ov", "privacy", "activate"], + &["ov", "privacy", "delete"], + &["ov", "privacy", "scan"], &["ov", "observer", "queue"], &["ov", "observer", "vikingdb"], &["ov", "observer", "models"], @@ -5971,95 +6042,257 @@ mod tests { ); } + fn on_cli_parser_stack(test: impl FnOnce() + Send + 'static) { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(test) + .expect("spawn CLI parser test") + .join() + .expect("CLI parser test thread"); + } + #[test] fn cli_parses_privacy_upsert_with_key_dynamic_flag() { - let cli = Cli::parse_from(preprocess_privacy_args(vec![ - OsString::from("ov"), - OsString::from("privacy"), - OsString::from("upsert"), - OsString::from("skill"), - OsString::from("demo"), - OsString::from("--key-api_key"), - OsString::from("secret-v2"), - ])); + on_cli_parser_stack(|| { + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("upsert"), + OsString::from("skill"), + OsString::from("demo"), + OsString::from("--key-api_key"), + OsString::from("secret-v2"), + ])); - match cli.command { - Commands::Privacy { action } => match action { - PrivacyCommands::Upsert { - category, - target_key, - key, - .. - } => { - assert_eq!(category, "skill"); - assert_eq!(target_key, "demo"); - assert_eq!(key, vec!["api_key=secret-v2"]); + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Upsert { + category, + target_key, + key, + .. + } => { + assert_eq!(category, "skill"); + assert_eq!(target_key, "demo"); + assert_eq!(key, vec!["api_key=secret-v2"]); + } + _ => panic!("expected privacy upsert"), + }, + _ => panic!("expected privacy command"), + } + }); + } + + #[test] + fn cli_parses_privacy_reference_resolve_and_delete() { + on_cli_parser_stack(|| { + for (subcommand, expected_field) in [("reference", "api_key"), ("resolve", "token")] { + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from(subcommand), + OsString::from("credential"), + OsString::from("github-prod"), + OsString::from(expected_field), + ])); + + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Reference { + category, + target_key, + field, + } => { + assert_eq!(subcommand, "reference"); + assert_eq!(category, "credential"); + assert_eq!(target_key, "github-prod"); + assert_eq!(field, expected_field); + } + PrivacyCommands::Resolve { + selector_or_category, + target_key, + field, + } => { + assert_eq!(subcommand, "resolve"); + assert_eq!(selector_or_category, "credential"); + assert_eq!(target_key.as_deref(), Some("github-prod")); + assert_eq!(field.as_deref(), Some(expected_field)); + } + _ => panic!("expected privacy field command"), + }, + _ => panic!("expected privacy command"), } - _ => panic!("expected privacy upsert"), - }, - _ => panic!("expected privacy command"), - } + } + + let scan = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("scan"), + OsString::from("--scope"), + OsString::from("resources,memories"), + OsString::from("--fix"), + OsString::from("--wait"), + ])); + match scan.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Scan { + scopes, + target, + fix, + wait, + node_limit, + } => { + assert_eq!(scopes, vec!["resources", "memories"]); + assert!(target.is_none()); + assert!(fix && wait); + assert_eq!(node_limit, 10000); + } + _ => panic!("expected privacy scan"), + }, + _ => panic!("expected privacy command"), + } + + let reference = + "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}"; + let resolved = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("resolve"), + OsString::from(reference), + ])); + match resolved.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Resolve { + selector_or_category, + target_key, + field, + } => { + assert_eq!(selector_or_category, reference); + assert!(target_key.is_none() && field.is_none()); + } + _ => panic!("expected privacy resolve"), + }, + _ => panic!("expected privacy command"), + } + + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("delete"), + OsString::from("credential"), + OsString::from("github-prod"), + OsString::from("--yes"), + ])); + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Delete { + selector_or_category, + target_key, + field, + yes, + } => { + assert_eq!(selector_or_category, "credential"); + assert_eq!(target_key.as_deref(), Some("github-prod")); + assert!(field.is_none()); + assert!(yes); + } + _ => panic!("expected privacy delete"), + }, + _ => panic!("expected privacy command"), + } + + let field_delete = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("delete"), + OsString::from("credential"), + OsString::from("github-prod"), + OsString::from("api_key"), + OsString::from("--yes"), + ])); + match field_delete.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Delete { + selector_or_category, + target_key, + field, + yes, + } => { + assert_eq!(selector_or_category, "credential"); + assert_eq!(target_key.as_deref(), Some("github-prod")); + assert_eq!(field.as_deref(), Some("api_key")); + assert!(yes); + } + _ => panic!("expected privacy field delete"), + }, + _ => panic!("expected privacy command"), + } + }); } #[test] fn cli_parses_privacy_shortcut_as_get() { - let cli = Cli::parse_from(preprocess_privacy_args(vec![ - OsString::from("ov"), - OsString::from("privacy"), - OsString::from("skill"), - OsString::from("demo"), - ])); + on_cli_parser_stack(|| { + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("skill"), + OsString::from("demo"), + ])); - match cli.command { - Commands::Privacy { action } => match action { - PrivacyCommands::Get { - category, - target_key, - } => { - assert_eq!(category, "skill"); - assert_eq!(target_key, "demo"); - } - _ => panic!("expected privacy get"), - }, - _ => panic!("expected privacy command"), - } + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Get { + category, + target_key, + } => { + assert_eq!(category, "skill"); + assert_eq!(target_key, "demo"); + } + _ => panic!("expected privacy get"), + }, + _ => panic!("expected privacy command"), + } + }); } #[test] fn cli_parses_privacy_shortcut_after_compact_without_value() { - let cli = Cli::try_parse_from(preprocess_cli_args(vec![ - OsString::from("ov"), - OsString::from("--compact"), - OsString::from("privacy"), - OsString::from("skill"), - OsString::from("demo"), - ])) - .expect("--compact without a value should not consume the privacy command"); + on_cli_parser_stack(|| { + let cli = Cli::try_parse_from(preprocess_cli_args(vec![ + OsString::from("ov"), + OsString::from("--compact"), + OsString::from("privacy"), + OsString::from("skill"), + OsString::from("demo"), + ])) + .expect("--compact without a value should not consume the privacy command"); - assert!(cli.compact); - match cli.command { - Commands::Privacy { action } => match action { - PrivacyCommands::Get { - category, - target_key, - } => { - assert_eq!(category, "skill"); - assert_eq!(target_key, "demo"); - } - _ => panic!("expected privacy get"), - }, - _ => panic!("expected privacy command"), - } + assert!(cli.compact); + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Get { + category, + target_key, + } => { + assert_eq!(category, "skill"); + assert_eq!(target_key, "demo"); + } + _ => panic!("expected privacy get"), + }, + _ => panic!("expected privacy command"), + } - let explicit_false = Cli::try_parse_from(preprocess_cli_args(vec![ - OsString::from("ov"), - OsString::from("-c"), - OsString::from("false"), - OsString::from("privacy"), - OsString::from("skill"), - OsString::from("demo"), - ])) - .expect("-c false should keep its explicit bool value"); - assert!(!explicit_false.compact); + let explicit_false = Cli::try_parse_from(preprocess_cli_args(vec![ + OsString::from("ov"), + OsString::from("-c"), + OsString::from("false"), + OsString::from("privacy"), + OsString::from("skill"), + OsString::from("demo"), + ])) + .expect("-c false should keep its explicit bool value"); + assert!(!explicit_false.compact); + }); } } diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md index b9707b347a..b2c6211c08 100644 --- a/docs/en/api/01-overview.md +++ b/docs/en/api/01-overview.md @@ -521,12 +521,18 @@ This catalog follows the routes actually mounted by the server. Each group headi | PUT | `/api/v1/admin/accounts/{account_id}/groups/{group_id}/members/{user_id}` | Add a group member | | DELETE | `/api/v1/admin/accounts/{account_id}/groups/{group_id}/members/{user_id}` | Remove a group member | | GET | `/api/v1/privacy-configs` | List privacy configuration categories | +| POST | `/api/v1/privacy-configs/scan` | Scan or repair registered values in content | | GET | `/api/v1/privacy-configs/{category}` | List category targets | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | Get the active configuration | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | Get a field reference or explicitly reveal its value | +| POST | `/api/v1/privacy-configs/resolve` | Explicitly reveal a field by reference or URI | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | List configuration versions | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | Get one version | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | Write and activate a new version | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | Activate a version | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | Delete a target and every version | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | Remove the active field | +| DELETE | `/api/v1/privacy-configs/selector` | Delete a target or field by URI/reference | ### [OpenViking Gateway Management](25-gateway.md) diff --git a/docs/en/api/10-privacy.md b/docs/en/api/10-privacy.md index 5338c44331..dd8b82b4f6 100644 --- a/docs/en/api/10-privacy.md +++ b/docs/en/api/10-privacy.md @@ -13,9 +13,15 @@ Activating a version changes only the values stored in OpenViking. It does not r | Method | Path | Description | |--------|------|-------------| | GET | `/api/v1/privacy-configs` | List privacy config categories | +| POST | `/api/v1/privacy-configs/scan` | Scan or repair registered values in stored content | | GET | `/api/v1/privacy-configs/{category}` | List targets under a category | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | Get active config (`meta + current`) | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | Get a field reference; optionally reveal its value | +| POST | `/api/v1/privacy-configs/resolve` | Explicitly reveal a field by reference or URI | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | Upsert and activate a new/current version | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | Delete a target and every version | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | Remove a field from the active config | +| DELETE | `/api/v1/privacy-configs/selector` | Delete by target URI or field reference/URI | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | List version numbers | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | Get a specific version snapshot | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | Activate a specific version | @@ -31,6 +37,28 @@ Activating a version changes only the values stored in OpenViking. It does not r --- +## Scan Stored Content + +Explicitly scan persisted text. The default is a dry run and never returns the matched value. With `fix=true`, unambiguous matches are replaced and affected content is refreshed. + +```text +POST /api/v1/privacy-configs/scan +``` + +```json +{ + "scopes": ["resources", "skills", "memories"], + "target_uri": "", + "fix": false, + "wait": false, + "node_limit": 10000 +} +``` + +Use either `scopes` or `target_uri`. When one value maps to multiple Privacy fields, the result reports `ambiguous` and never chooses a reference arbitrarily. + +--- + ## Data Structures ### current (active version snapshot) @@ -168,6 +196,47 @@ curl -X GET "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-sea --- +### get_privacy_field() + +Get one field from the active version. By default the endpoint returns an owner-bound reference and configuration status without returning the value. Add `reveal=true` to explicitly read that one field for immediate use. + +```text +GET /api/v1/privacy-configs/{category}/{target_key}/field?field={field}&reveal=false +``` + +```json +{ + "status": "ok", + "result": { + "owner_user_id": "alice", + "category": "credential", + "target_key": "github-prod", + "field": "api_key", + "version": 1, + "configured": true, + "reference": "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}" + } +} +``` + +With `reveal=true`, the response also contains that field's `value`. Sibling fields are never returned. A missing target or field returns `NOT_FOUND`. + +An owner-bound reference or field URI can instead be sent to the selector endpoint: + +```text +POST /api/v1/privacy-configs/resolve +``` + +```json +{"selector":"{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}"} +``` + +Field URIs may use `viking://~/privacy/...#/values/...` or an explicit `viking://user/{user_id}/privacy/...#/values/...`. An explicit owner that differs from the current identity returns `PERMISSION_DENIED`. + +References generated, stored, and returned by OpenViking always use the whitespace-free canonical form. The parser accepts spaces or tabs immediately after `{{` and before `}}`, such as `{{ ov_privacy:alice:... }}`, then normalizes the result to `{{ov_privacy:alice:...}}`. Whitespace is not accepted inside the owner, URI, or path tokens. + +--- + ### upsert_privacy_config() Write a new version and set it as active. @@ -230,6 +299,43 @@ curl -X POST "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-se --- +### delete_privacy_config() + +Delete a target's current value, metadata, and complete version history. Repeated deletion is safe: a missing target returns `deleted: false`. + +```text +DELETE /api/v1/privacy-configs/{category}/{target_key} +``` + +```json +{ + "status": "ok", + "result": { + "category": "credential", + "target_key": "github-prod", + "deleted": true + } +} +``` + +Field deletion affects only the active configuration and does not expose a version parameter. If fields remain, it publishes a new active snapshot without that field and keeps historical snapshots. Removing the final field deletes the complete target, including current values, metadata, and all history, and returns `target_deleted: true`. + +```text +DELETE /api/v1/privacy-configs/{category}/{target_key}/field?field={field} +``` + +Target URIs, field URIs, and owner-bound references can also be deleted through: + +```text +DELETE /api/v1/privacy-configs/selector +``` + +```json +{"selector":"viking://~/privacy/credential/github-prod#/values/api_key"} +``` + +--- + ### list_privacy_versions() List all version numbers for a target. @@ -349,6 +455,14 @@ ov privacy list skill ov privacy get skill byted-viking-search-knowledgebase ov privacy skill byted-viking-search-knowledgebase +# Get the owner-bound reference without returning the value +ov privacy reference credential github-prod api_key + +# Explicitly reveal one field in the active version +ov privacy resolve credential github-prod api_key +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' +ov privacy resolve 'viking://~/privacy/credential/github-prod#/values/api_key' + # Upsert with full JSON snapshot ov privacy upsert skill byted-viking-search-knowledgebase \ --values-json '{"api_key":"secret-2","base_url":"https://example.com"}' @@ -364,6 +478,18 @@ ov privacy upsert skill byted-viking-search-knowledgebase \ ov privacy versions skill byted-viking-search-knowledgebase ov privacy version skill byted-viking-search-knowledgebase 2 ov privacy activate skill byted-viking-search-knowledgebase 2 + +# Delete the target and every version; non-interactive use requires --yes +ov privacy delete credential github-prod --yes +ov privacy delete 'viking://~/privacy/credential/github-prod' --yes + +# Remove the active field; historical snapshots remain +ov privacy delete credential github-prod api_key --yes +ov privacy delete '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' --yes + +# Stored-content scans are dry-run unless --fix is passed +ov privacy scan --scope resources,skills,memories +ov privacy scan --target viking://~/resources/project-a --fix --wait ``` --- diff --git a/docs/en/concepts/13-privacy.md b/docs/en/concepts/13-privacy.md index c423178163..e4b846668c 100644 --- a/docs/en/concepts/13-privacy.md +++ b/docs/en/concepts/13-privacy.md @@ -128,14 +128,22 @@ Current matching is suffix-based: `/skills/{name}/SKILL.md`, so it supports user - Content plane: `read` automatically restores placeholders - Separation of concerns: content files hold placeholders; privacy service holds sensitive values and versions +PrivacySanitizer does not infer sensitivity from field names. String values in the active Privacy catalog participate in automatic content scanning and reference replacement only when they contain at least 6 Unicode characters. Shorter values can still be stored, referenced, and explicitly resolved, but are excluded from global substring replacement to reduce false positives in ordinary text. The matcher catalog is cached per user and process-local revision; local changes invalidate it immediately, and a 3600-second TTL bounds staleness after changes made by another worker or node. + Common commands: ```bash ov privacy categories ov privacy list skill ov privacy skill search-web +ov privacy reference skill search-web api_key +ov privacy resolve skill search-web api_key +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/skill/search-web#/values/api_key}}' ov privacy upsert skill search-web --values-json '{"api_key":"..."}' ov privacy activate skill search-web 1 +ov privacy delete skill search-web api_key --yes +ov privacy delete skill search-web --yes +ov privacy scan --scope resources,skills,memories ov read viking://~/skills/search-web/SKILL.md ``` diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md index 00e917525c..9b564c0b24 100644 --- a/docs/zh/api/01-overview.md +++ b/docs/zh/api/01-overview.md @@ -516,12 +516,18 @@ JSON 输出 - 错误： | PUT | `/api/v1/admin/accounts/{account_id}/groups/{group_id}/members/{user_id}` | 添加用户组成员 | | DELETE | `/api/v1/admin/accounts/{account_id}/groups/{group_id}/members/{user_id}` | 移除用户组成员 | | GET | `/api/v1/privacy-configs` | 列出隐私配置分类 | +| POST | `/api/v1/privacy-configs/scan` | 扫描或修复正文中的登记值 | | GET | `/api/v1/privacy-configs/{category}` | 列出分类目标 | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | 获取生效配置 | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | 获取单字段引用或显式读取值 | +| POST | `/api/v1/privacy-configs/resolve` | 按字段 reference/URI 显式读取值 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | 列出配置版本 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | 获取指定版本 | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | 写入并激活新版本 | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | 激活指定版本 | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | 删除目标及全部版本 | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | 删除当前字段 | +| DELETE | `/api/v1/privacy-configs/selector` | 按 URI/reference 删除目标或字段 | ### [OpenViking 网关管理](25-gateway.md) diff --git a/docs/zh/api/10-privacy.md b/docs/zh/api/10-privacy.md index 649c5a4b80..1a84719543 100644 --- a/docs/zh/api/10-privacy.md +++ b/docs/zh/api/10-privacy.md @@ -22,9 +22,15 @@ | 方法 | 路径 | 说明 | |------|------|------| | GET | `/api/v1/privacy-configs` | 列出隐私配置分类 | +| POST | `/api/v1/privacy-configs/scan` | 扫描或修复已存储正文中的登记值 | | GET | `/api/v1/privacy-configs/{category}` | 列出分类下目标 | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | 获取当前生效配置（meta + current） | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | 获取单字段引用；显式选择时返回值 | +| POST | `/api/v1/privacy-configs/resolve` | 按字段 reference/URI 显式读取值 | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | 写入新版本并激活 | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | 删除目标及全部版本 | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | 删除当前生效配置中的字段 | +| DELETE | `/api/v1/privacy-configs/selector` | 按 target URI 或字段 reference/URI 删除 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | 列出版本号 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | 获取指定版本详情 | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | 激活指定版本 | @@ -72,6 +78,28 @@ ## API 参考 +### scan_privacy_content() + +显式扫描已经持久化的文本内容。默认 dry-run，不返回命中的原值；`fix=true` 时替换无歧义命中并刷新相应内容。 + +```text +POST /api/v1/privacy-configs/scan +``` + +```json +{ + "scopes": ["resources", "skills", "memories"], + "target_uri": "", + "fix": false, + "wait": false, + "node_limit": 10000 +} +``` + +`scopes` 与 `target_uri` 二选一。重复值对应多个 Privacy 字段时报告 `ambiguous`，不会任意选取引用。 + +--- + ### list_privacy_categories() 列出当前用户下已有隐私配置的分类。 @@ -168,6 +196,51 @@ curl -X GET "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-sea --- +### get_privacy_field() + +获取当前生效版本中的单个字段。默认只返回带 owner 的引用和配置状态，不返回实际值；`reveal=true` 是供立即使用场景显式读取单字段的接口。 + +**HTTP API** + +```text +GET /api/v1/privacy-configs/{category}/{target_key}/field?field={field}&reveal=false +``` + +默认响应： + +```json +{ + "status": "ok", + "result": { + "owner_user_id": "alice", + "category": "credential", + "target_key": "github-prod", + "field": "api_key", + "version": 1, + "configured": true, + "reference": "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}" + } +} +``` + +传入 `reveal=true` 时，响应额外包含该字段的 `value`。接口不会返回同一 target 的其他字段。target 或 field 不存在时返回 `NOT_FOUND`。 + +也可以将 owner-bound reference 或字段 URI 传给统一解析接口： + +```text +POST /api/v1/privacy-configs/resolve +``` + +```json +{"selector":"{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}"} +``` + +字段 URI 支持 `viking://~/privacy/...#/values/...` 和显式 `viking://user/{user_id}/privacy/...#/values/...`。显式 owner 与当前身份不一致时返回 `PERMISSION_DENIED`。 + +OpenViking 生成、保存和返回的 canonical reference 始终不带空格。解析器兼容 `{{` 后和 `}}` 前的空格或 Tab，例如 `{{ ov_privacy:alice:... }}`，解析后仍规范化为 `{{ov_privacy:alice:...}}`；owner、URI 和路径字段内部不允许插入空白。 + +--- + ### upsert_privacy_config() 写入新版本并将其设为当前生效版本。 @@ -230,6 +303,43 @@ curl -X POST "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-se --- +### delete_privacy_config() + +删除一个 target 的当前值、元信息和完整版本历史。重复删除是安全的：目标不存在时返回 `deleted: false`。 + +```text +DELETE /api/v1/privacy-configs/{category}/{target_key} +``` + +```json +{ + "status": "ok", + "result": { + "category": "credential", + "target_key": "github-prod", + "deleted": true + } +} +``` + +字段删除只影响当前生效配置，对外不要求指定版本。删除后仍有字段时，实现会发布一个移除该字段的新 active 快照，已有历史版本继续保留；删除最后一个字段时会直接删除整个 target，包括当前值、元信息和全部历史，并返回 `target_deleted: true`。 + +```text +DELETE /api/v1/privacy-configs/{category}/{target_key}/field?field={field} +``` + +target URI、字段 URI 和 owner-bound reference 也可以通过以下接口删除： + +```text +DELETE /api/v1/privacy-configs/selector +``` + +```json +{"selector":"viking://~/privacy/credential/github-prod#/values/api_key"} +``` + +--- + ### list_privacy_versions() 列出 target 的所有版本号。 @@ -349,6 +459,14 @@ ov privacy list skill ov privacy get skill byted-viking-search-knowledgebase ov privacy skill byted-viking-search-knowledgebase +# 默认只取得 owner 绑定引用，不返回实际值 +ov privacy reference credential github-prod api_key + +# 显式读取当前生效版本中的一个字段 +ov privacy resolve credential github-prod api_key +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' +ov privacy resolve 'viking://~/privacy/credential/github-prod#/values/api_key' + # 更新（整包 JSON） ov privacy upsert skill byted-viking-search-knowledgebase \ --values-json '{"api_key":"secret-2","base_url":"https://example.com"}' @@ -364,6 +482,18 @@ ov privacy upsert skill byted-viking-search-knowledgebase \ ov privacy versions skill byted-viking-search-knowledgebase ov privacy version skill byted-viking-search-knowledgebase 2 ov privacy activate skill byted-viking-search-knowledgebase 2 + +# 删除目标及全部版本；非交互环境必须显式 --yes +ov privacy delete credential github-prod --yes +ov privacy delete 'viking://~/privacy/credential/github-prod' --yes + +# 删除当前字段；历史快照仍保留 +ov privacy delete credential github-prod api_key --yes +ov privacy delete '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' --yes + +# 存量内容扫描默认为 dry-run；--fix 才会改写 +ov privacy scan --scope resources,skills,memories +ov privacy scan --target viking://~/resources/project-a --fix --wait ``` --- diff --git a/docs/zh/concepts/13-privacy.md b/docs/zh/concepts/13-privacy.md index bdcf9948d5..6c5ebf8044 100644 --- a/docs/zh/concepts/13-privacy.md +++ b/docs/zh/concepts/13-privacy.md @@ -128,14 +128,22 @@ fs.read(uri) - 内容面：通过 `read` 自动恢复占位符 - 两者解耦：内容文件负责占位符，隐私服务负责敏感值与版本 +PrivacySanitizer 不根据字段名猜测敏感性。当前 active Privacy 中长度不少于 6 个 Unicode 字符的字符串值会参与正文自动扫描与引用替换；更短的值仍可保存、引用和显式读取，但不会参与全文替换，以降低普通正文误命中。匹配目录按用户和本进程 revision 缓存，本进程修改会立即失效，另外设有 3600 秒缓存 TTL 以覆盖多 worker/多节点修改。 + 常用命令： ```bash ov privacy categories ov privacy list skill ov privacy skill search-web +ov privacy reference skill search-web api_key +ov privacy resolve skill search-web api_key +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/skill/search-web#/values/api_key}}' ov privacy upsert skill search-web --values-json '{"api_key":"..."}' ov privacy activate skill search-web 1 +ov privacy delete skill search-web api_key --yes +ov privacy delete skill search-web --yes +ov privacy scan --scope resources,skills,memories ov read viking://~/skills/search-web/SKILL.md ``` diff --git a/docs/zh/privacy-review.md b/docs/zh/privacy-review.md new file mode 100644 index 0000000000..bb4f09fb41 --- /dev/null +++ b/docs/zh/privacy-review.md @@ -0,0 +1,208 @@ +# OpenViking Privacy 审查简报 + +审查日期：2026-10-15；代码基线：`9baf83f0`。依据：[Privacy API](api/10-privacy.md)、[Privacy 概念](concepts/13-privacy.md)，并核对当前实现、Agent 接入、测试和 CI。Agent 审查重点为仓库内 MCP、SDK/CLI、Codex/通用 hook 共享采集代码与 OpenClaw 使用指导。本文是审查结果及建议方案，不涉及功能修改。 + +## 结论 + +**Privacy 已具备用户范围的敏感配置分离、版本保存和读取还原能力，但当前不能保证 Agent 接入后会主动、妥善地用它保存外部系统账号密钥，也不能依据现有测试通过就认定敏感信息不会泄漏。** + +主要原因是：Agent 缺少直接的 Privacy 管理工具与完整操作指导；自动保护集中于技能正文，依赖模型识别及有限的文本替换；还原后的真实值可以进入模型上下文及会话采集；版本写入的故障一致性存在可复现问题。现有测试能验证若干功能和并发行为，对这些安全边界的保障不足。 + +两份参考文档已经如实说明“抽取不能保证全面”“附件不一定处理”“读取返回真实值”“不等于加密”。这些限制应成为 Agent 接入和测试设计的约束，不能只靠文档提示来落实。 + +## 1. 当前机制及适用边界 + +| 能力 | 当前实现与含义 | +|---|---| +| 配置组织 | 按调用者的 `viking://user/{user_id}/privacy/{category}/{target_key}` 保存。API 的 `category` 不限于 `skill`，所以可以人工保存任意系统的账号、密码、密钥；自动提取和还原使用 `category=skill`、技能名作为目标。 | +| 版本管理 | `current.json`、`history/version_001.json` 等保存完整 `values`；`.meta.json` 保存当前/最新版本及作者等。相同值不新增版本；不同值完整替换并激活；回滚后再次写入按最新版本递增。 | +| 技能写入 | `SkillProcessor` 在准备阶段调用账户配置的 VLM，识别正文中的配置，再对成功匹配的片段占位符化，保存 Privacy 后写入技能。更新与训练产生技能的相关路径也使用隐私处理。 | +| 技能读取 | `FSService.read` 识别 `/skills/{name}/SKILL.md`，按**当前调用者**的配置还原占位符。Privacy API、CLI 和技能详情也能返回真实值。共享技能的密钥仍按用户保存；同一用户的同名技能共用同一个 Privacy 目标。 | +| 权限与加密 | Privacy 复用请求身份和 VikingFS 命名空间/ACL，具有用户空间隔离；ROOT 可跨用户访问。它没有独立的“仅执行可用、不可读取”密钥权限。存储加密是另一项配置，默认关闭。 | +| 生命周期 | `DELETE /api/v1/skills/{name}` / `ov skills remove` 删除相应 Privacy；通用 `forget` / `ov rm` 删除技能目录会留下配置。API 未提供通用 Privacy 删除或单个历史版本销毁端点。激活版本不会轮换或撤销第三方密钥。 | + +实现依据：[配置服务](../../openviking/privacy/service.py)、[技能处理](../../openviking/utils/skill_processor.py)、[读取服务](../../openviking/service/fs_service.py)、[访问判断](../../openviking/core/namespace.py)、[存储加密](concepts/10-encryption.md)。 + +## 2. 建议落地方案 + +本阶段先解决一个确定性场景：当 `resources`、`skills`、`memories` 的待写入正文中出现**当前写入用户已登记在 Privacy 中的密钥**时，在落盘及派生处理之前强制识别并替换为统一引用。Privacy 新增后的存量反向扫描不默认自动运行，而由显式 CLI 命令触发。 + +当前工作分支已经完成第一轮落地：`ov privacy reference` 默认只返回 owner 绑定引用；`ov privacy resolve` 可以通过三段参数、owner-bound reference 或字段 URI 显式读取单个字段；`ov privacy delete` 可以删除整个 target，也可以让当前 active 配置中的单个字段不可用：仍有其他字段时内部发布新快照并保留历史，删除最后一个字段时删除整个 target 及其历史；PrivacySanitizer 不按字段名猜测，对当前 active Privacy 中字符数不少于 6 的字符串值实施自动保护；普通文本写入、Skill 写入、Resource 导入、直接 VikingFS 文本写入和 `read_visible` 已接入；`ov privacy scan` 默认 dry-run，`--fix` 显式修复存量正文。会话采集、Gateway 内部还原以及所有外部导出/备份路径仍需后续验收，不能把本轮接入理解为全链路已经闭环。 + +### 2.1 统一引用格式 + +建议采用： + +```text +{{ov_privacy:{owner_user_id}:viking://~/privacy/{category}/{target_key}#/values/{field}}} +``` + +例如用户 `user_id_1` 向公共技能区写入包含 GitHub 密钥的内容，替换结果为： + +```text +Authorization: Bearer {{ov_privacy:user_id_1:viking://~/privacy/credential/github-prod#/values/api_key}} +``` + +语义约定： + +- `owner_user_id` 是 Privacy 配置所有者，也是本次自动替换的溯源身份；本阶段不增加引用协议版本，未来出现不兼容语法时再加入。 +- 引用内部的 `~` **在 Privacy 引用解析器内**相对于 `owner_user_id` 解析，而不是相对于当前阅读者解析；普通 `viking://~` 在其他 API 中仍保持“当前调用者”语义。 +- `category + target_key` 定位配置目标，`#/values/{field}` 定位单个字段；不能只引用 target，否则可能一次暴露同一目标下的多个值。 +- `{owner_user_id}`、`category`、`target_key` 和 `field` 必须走统一的标识符校验与转义，不能通过字符串拼接后直接访问存储。 +- 引用始终解析当前 active version，不绑定 `current.json` 等物理文件。密钥轮换后正文引用无需变化。 +- 自动 Sanitizer 只生成所有者绑定引用，不生成调用者绑定引用。这使 `/agent/skills` 等公共区域中的内容可追溯，同时阻止其他用户将引用误解析为自己的同名配置。 +- canonical reference 始终采用无空格形式。解析器仅兼容 `{{` 后、`}}` 前的空格或 Tab，解析后立即规范化为无空格形式；owner、URI、category、target 和 field 内部不接受任意空白。 + +当其他用户读取公共内容时，应保留引用并返回结构化状态，而不是尝试跨用户读取： + +```text +[Privacy Config Notice] +owner: user_id_1 +status: forbidden +action: ask the owner or an administrator for access or configuration guidance +``` + +这项设计会向有权读取公共内容的用户显示配置所有者 ID，是为了支持溯源和协作；如部署不希望暴露真实 user ID，后续可以用可审计的 `owner_ref` 替代，但不能移除服务端真实 owner 绑定。 + +### 2.2 高效 PrivacySanitizer + +PrivacySanitizer 应成为统一底层能力，而不是 Skill 专用的 LLM 后处理。建议工作方式如下： + +1. 按 `(account_id, user_id, privacy_revision)` 缓存当前生效的敏感值目录。每次该用户 upsert/activate/delete Privacy 后只失效对应缓存；缓存同时使用 3600 秒 TTL，多 worker/多节点上的外部修改最多一小时后被本进程重新加载。 +2. Privacy 是显式保护边界，不再通过 `_SECRET_FIELD_NAMES` 或字段名模式猜测敏感性；当前 active Privacy 中字符数不少于 6 的字符串值进入匹配目录，包括用户自定义字段名。更短的值仍可保存和显式 resolve，但不会参与正文全文替换，以避免 `cn`、`1` 等造成大量误命中。 +3. 在受控内存中构建多模式匹配器（如 Aho-Corasick），扫描复杂度保持为 `O(正文长度 + 命中数)`；重叠命中选择最长值，同一个值对应多个 Privacy 引用时标记为歧义并拒绝自动替换。持久缓存和日志只保存 HMAC/引用元数据，不记录原值。 +4. 对当前用户发起的 user 空间写入生效；写入 `/agent/skills` 等公共区域时仍使用**写入者**的 Privacy 目录，并在引用中写入该用户 ID。 +5. 先匹配完整原值，再按明确白名单处理 URL 编码、JSON 转义、PEM 换行标准化等变体。多个 Privacy 字段对应同一值、过短密钥造成高歧义、或无法可靠生成引用时，应拒绝/隔离写入并返回候选引用，不能任意选择。 +6. Privacy 自身存储路径不参与扫描。审计只记录正文 URI、owner、引用、位置和匹配规则，不记录命中的秘密。 + +必须覆盖以下写入和派生入口： + +- 普通 `write`、`edit`、batch write； +- Skill 的 frontmatter、正文、description、metadata 和文本附件； +- Resource 解析后的正文及文本派生物； +- Memory 最终序列化正文和可进入上下文的字段； +- 写入 `.abstract.md`、`.overview.md`、关键词索引、向量 payload 或发送摘要模型之前。 + +建议采用“两道门”：第一道在各类内容进入持久化/摘要/索引之前替换；第二道在 `read`、search inline content 和 context assembler 输出前再次扫描，拦截绕过统一写入入口或尚未完成手工修复的存量内容。第二道只做当前响应脱敏，不在读取路径中隐式改写存储。 + +### 2.3 显式存量扫描与修复 + +Privacy 新增、轮换后暂不默认扫描全部存量，以避免不可预期的大范围 IO 和内容修改。提供显式 CLI： + +```bash +# 默认只报告命中，不修改 +ov privacy scan --scope resources,skills,memories +ov privacy scan --target viking://~/resources/project-a + +# 用户确认后替换正文，并刷新摘要和索引 +ov privacy scan --scope resources,skills,memories --fix --wait +``` + +建议约束： + +- 普通用户只能扫描其可读且可写的内容，并只使用自己的 Privacy 目录；管理员代用户扫描必须显式指定 owner 并留下审计。 +- `scan` 默认 dry-run，只返回 URI、字段引用、命中数量及是否可自动修复，不返回原值。 +- `--fix` 在持有内容锁时重新扫描，采用 expected revision/CAS 防止覆盖并发修改；修复后删除旧索引记录，重新生成摘要和索引。 +- 历史快照、导出、外部备份和已发送日志不承诺由该命令清理，需单独的保留与销毁策略。 + +### 2.4 上下文表示与 Agent 还原 + +上下文正文保持统一引用，不在组装阶段自动展开，并可以附加不含密钥的状态： + +```xml + +``` + +本阶段允许 Agent 自己发起还原，但应是**显式、字段级、受权限控制**的操作： + +1. Agent 识别引用并提取 owner、category、target 和 field，不自行拼接或改写引用。 +2. 当当前身份有权读取 owner 的 Privacy 时，调用字段级 resolve/read 接口获取当前 active value；接口只返回被引用字段，不返回整个 target。 +3. Agent 将值用于当前外部系统调用；工具输入、工具输出、错误、会话采集和后续写入仍需经过 PrivacySanitizer，避免还原值再次进入 resources、skills 或 memories。 +4. 当前用户无权读取时，解析器返回 `forbidden + owner_user_id`。Agent 应说明需联系该 owner/管理员，不能回退到猜测、搜索或跨用户读取。 +5. 缺少 target/field 时返回 `missing`；Agent 可以引导 owner 补充 Privacy，不能把占位符当作真实参数调用。 + +OpenViking Gateway 可以提供更安全的增强路径：Gateway 在转发外部请求前于内部解析引用、注入密钥，并在响应返回 Agent 前再次脱敏，使明文不进入模型上下文。普通 OpenViking 接入仍允许 Agent 显式还原，两种方式共用同一引用、权限校验、审计和 Sanitizer，不应维护两套占位符协议。 + +## 3. 主要发现与建议 + +优先级：P0 表示应在承诺“自动安全保存生产凭据”前解决；P1 表示下一阶段补齐；P2 表示完善运维和产品语义。下表分别标明执行复现与代码审查证据。 + +| 优先级 | 发现及影响 | 建议与证据 | +|---|---|---| +| P0 | **Agent 不能自然发现完整的 Privacy 使用方法。** MCP 没有 Privacy 写入、查询、激活工具；Python SDK 也没有对应方法。插件指导包含“不要回显凭据”“不要把凭据放进技能”，但没有教如何将用户交付的账号密钥转存、引用、使用及清理。仅有 HTTP/CLI 能力不能证明 Agent 会选对保存路径。 | 增加明确的 `set/list/delete/resolve-field` 等工具与接入技能；普通管理响应默认只返回 owner、键名、引用和配置版本，仅显式字段还原返回该字段值。给出账号密钥处理示例及工具不可用时的分支。依据：[MCP 注册](../../openviking/server/mcp_endpoint.py)、[插件记忆指导](../../examples/codex-memory-plugin/skills/openviking-memory/SKILL.md)、[插件技能指导](../../examples/codex-memory-plugin/skills/openviking-skills/SKILL.md)、`sdk/python`。这是接入完备性判断，尚未通过真实 Agent 行为评测证明其选择结果。 | +| P0 | **模型识别成功也不等于清除成功。** 仅替换引号包裹的完整值，或 `:` / `=` 后直到行尾的完整裸值。Bearer、自然语言、带行尾注释等会保留原值；同一密钥在赋值和说明文字同时出现时，会一边保存 Privacy、一边留下明文。 | 对已经登记的密钥改用确定性的 PrivacySanitizer 全文匹配和残留校验，不再依赖字段格式；有歧义或未处理秘密时拒绝/隔离写入。对尚未登记的新秘密仍需检测或模型辅助，但不能把“模型未识别”解释为安全。已用虚构密钥执行复现，见下一节；依据：[替换器](../../openviking/privacy/skill_placeholder.py)。 | +| P0 | **无法区分“没有秘密”和“提取没有成功”。** 非 JSON、错误结构、匹配失败可得到空 `values` 并保留原文；模型调用异常会传播，但正常返回无效内容没有严格失败状态。提示词要求输出缺值配置项，代码却过滤空值并只返回实际替换成功的项，所需配置清单会丢失。 | 定义严格输出 schema、提取状态、已识别/已替换/未解决三类结果；失败时阻止敏感内容持久化。保留缺值字段及引用，供 Agent 请求配置。已复现无效响应和空值；依据：[提取器](../../openviking/privacy/skill_extractor.py)、[提取提示词](../../openviking/prompts/templates/skill/privacy_extraction.yaml)。 | +| P0 | **保护未贯穿持久化链路。** 只替换 `skill_dict.content`；description、metadata 和辅助文件不受同等处理，且附件会随技能写入并排入递归语义处理。普通 `write`、资源导入、`remember`、会话消息没有自动转存 Privacy 的机制。原始技能正文在清理前已经发给配置的 VLM。 | 覆盖所有技能文本、元数据和附件；在首次保存及外发模型前执行秘密检测。资源/记忆/会话应使用统一敏感数据策略，以引用替代密钥，并支持无需向模型提交原值的确定性保存接口。依据：[SkillLoader](../../openviking/core/skill_loader.py)、[技能写入与附件](../../openviking/utils/skill_processor.py)、[MCP remember](../../openviking/server/mcp_endpoint.py)、[会话保存](../../openviking/session/session.py)。附件/派生数据扩散是代码链路风险，本次未运行真实模型泄漏评测。 | +| P0 | **密钥还原后可能再次被保存。** `read` 会把实际值注入上下文；未引用配置也被完整追加到 Notice。采集过滤只改文本，工具输入/输出保持原样，即使配置文本脱敏规则也无效。工具结果采集默认因 harness 而异，DSH 默认开启，不能笼统认为所有插件都默认采集工具结果。 | 普通接入允许 Agent 经字段级、权限受控接口显式还原，但 Notice 只能列 owner/键名/状态，还原值不能随整个 target 返回；工具输入、输出、错误、会话采集和再次写入统一经过 Sanitizer。Gateway 可在内部解析和注入，避免明文进入模型。已复现 Notice 回显及文本过滤生效而工具载荷不变；依据：[还原器](../../openviking/privacy/skill_restore.py)、[采集过滤](../../examples/codex-memory-plugin/scripts/shared/capture-utils.mjs)、[默认配置](../../examples/codex-memory-plugin/scripts/shared/config-schema.mjs)。 | +| P0 | **版本写入缺少多文件事务保障。** `upsert` 依次写 history、current、meta，读操作不持同一锁。meta 写入失败后 current 可以已切到新版本，但 meta 仍为旧版本；下一次写入会复用版本号并覆盖历史。锁防并发写入，不能解决部分写入和读取中间态。 | 引入原子版本发布/提交指针及故障恢复，版本号不能只依赖可能过时的 meta。验证 history、current、meta 的一致性和历史不可覆盖；对 activate、取消、重启也做故障注入。已用真实配置服务加内存存储故障注入复现，见下一节；依据：[配置服务](../../openviking/privacy/service.py)。 | +| P1 | **保留占位符的技能更新可能删掉仍需要的配置。** `PUT /skills/{name}` 对空提取结果执行 `delete_if_empty=True`。传入已占位符化内容、模型返回缺值字段时，仍有配置引用却会走删除分支。现有测试只覆盖“内容确实不再需要配置时删除”。 | 检查现存占位符与字段引用，区分未提供值、显式删除和提取失败；保留仍被引用的值。已在实际 prepare/apply 方法组合中复现删除调用；完整 HTTP 情景应补回归测试。依据：[技能更新](../../openviking/server/routers/skills.py)、[apply_skill_privacy](../../openviking/utils/skill_processor.py)、[现有删除测试](../../tests/server/test_api_skills.py)。 | +| P1 | **核心 Privacy 回归没有明确接入现有 CI。** `tests/server` 中相关测试可本地运行，但工作流中的主 API 集成任务运行 `tests/api_test`，该目录无 Privacy 专项测试；Lite 任务仅运行 cuVS 指定测试。当前配置中未发现直接运行这些 Privacy 核心测试的 job，也未配置 Privacy 覆盖率门槛。 | 增加 PR 必跑的 Privacy 回归任务，将本简报列出的安全不变量列为门禁；涉及提示词或 Agent 技能的 Markdown/YAML 修改也要触发。依据：[API CI](../../.github/workflows/api_test.yml)、[PR CI](../../.github/workflows/pr.yml)、[Lite CI](../../.github/workflows/_test_lite.yml)、[pytest 配置](../../pyproject.toml)。本次审查的是仓库配置，未核验远端 branch protection 和 CI 历史运行。 | +| P2 | **版本记录不等于完整安全审计或删除。** `get_current/get_version` 不更新 `last_accessed_at`，也未在 Privacy 服务记录逐次读取审计；旧密钥长期存在历史快照。全文 CLI 输出、命令参数、本地 JSON 文件和备份都是额外副本；存储快照枚举默认没有排除 Privacy。 | 增加读取/使用审计、历史保留/销毁策略、默认脱敏和输入 stdin 等方式；备份与加密策略纳入生命周期。依据：[配置服务](../../openviking/privacy/service.py)、[CLI](../../crates/ov_cli/src/commands/privacy.rs)、[快照枚举](../../crates/ragfs/src/git/enumerate.rs)。是否实际备份取决于部署，本次未核验外部备份。 | +| P2 | **部分更新和目标命名容易造成误操作。** HTTP upsert 整包替换；CLI `--key-*` 是先读再合并再写，两个客户端并行更新不同键仍可能丢更新。`category=skill` 的同名目标没有独立的服务/账号/环境维度。 | 增加原子 PATCH 或 expected_version/CAS；明确命名及绑定规则。人工保存非技能凭据时按服务、环境、账号组织目标，并告知它们不会自动在技能读取中还原。依据：[Privacy API](api/10-privacy.md)、[CLI 合并写入](../../crates/ov_cli/src/commands/privacy.rs)、[技能还原](../../openviking/service/fs_service.py)。并行 CLI 丢更新是读改写语义分析，本次未执行双客户端复现。 | + +## 4. 本次验证与现有测试的证明范围 + +本地使用 macOS / Python 3.12。所有复现只使用虚构密钥，未调用真实模型或外部账号系统。 + +现有测试执行命令： + +```bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin \ + -q -o addopts='' --tb=short \ + tests/server/test_privacy_config_service.py \ + tests/server/test_api_privacy_configs.py \ + tests/server/test_skill_privacy_lock.py \ + tests/unit/test_skill_privacy_cancellation.py \ + tests/session/test_commit_skill_inference.py + +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin \ + -q -o addopts='' --tb=short \ + tests/server/test_api_skills.py tests/server/test_skill_update_lock.py \ + -k 'privacy or rollback' +``` + +第一组 **35 passed**；第二组 **6 passed、15 deselected**，共 **41 个选定的相关测试通过**。上述命令使用显式插件并关闭自动插件加载。运行未开启覆盖率，因此不声称已经获得 Privacy 覆盖率统计。 + +| 测试层 | 已有保障 | 仍缺的关键证明 | +|---|---|---| +| 配置服务/API | 版本递增、同值去重、激活、增加键、404；部分底层错误传播 | 多文件故障一致性、损坏数据恢复、重启、读取中间态、原子部分更新 | +| 抽取/还原 | 固定模型返回值、引用恢复、缺值与额外配置 Notice、引号/换行/CRLF/正则字符处理 | 真实模型识别质量；Bearer、自然语言、注释、重复残留、私钥、提示注入、元数据/附件 | +| 更新与并发 | 8 次并发写版本测试使用内存存储锁；另有服务级更新回滚、等待写入和取消释放锁测试 | 多进程真实存储竞争、进程崩溃、带占位符更新、历史快照不被覆盖；不能把单进程测试扩大为分布式保证 | +| 身份与隔离 | 当前用户路径写入和通用命名空间策略；API 测试主要使用 DevAuth/ROOT | Privacy 专项的无密钥/普通用户/跨用户/跨账户读写历史拒绝测试，以及共享技能的每用户绑定 | +| Agent 与整条链路 | 插件有采集和文本过滤测试；训练产出技能有隐私处理调用测试 | Agent 是否选对 Privacy、是否避免回显；原值是否进入 messages/history/记忆/摘要/工具日志/导出；真实模型与完整 Agent 的行为评测 | + +**额外执行复现（调用当前实现，非正式回归测试）：** + +- 已给提取器正确的虚构值：`api_key=FAKE_SECRET` 能替换；`Authorization: Bearer FAKE_SECRET`、`密码是 FAKE_SECRET`、`api_key=FAKE_SECRET # prod` 保留明文。赋值后再加 `Use FAKE_SECRET here`，Privacy 成功提取但正文仍有原值。 +- VLM 返回 `I cannot process this request.` 或 `{"values":{"api_key":""}}`，提取结果为空且原正文不变；无明确失败状态。 +- `restore_skill_content` 给不含引用的正文配上 `unused_key=FAKE_SECRET`，Notice 包含完整虚构密钥。 +- `shapeCaptureParts` 配置替换规则后，文本中的虚构密钥变为 `[redacted]`，同一消息的 `tool_output` 中原值保留。 +- 对 `UserPrivacyConfigService` 的 `.meta.json` 写入注入异常：先写 v1，再写 v2 失败后得到 `meta.active_version=1`、`current.version=2`；随后写另一值仍分配 v2，并覆盖原来的 `history/version_002.json`。 +- 对包含 `{{ov_privacy:skill:audit-skill:api_key}}` 的内容调用 `prepare_skill_privacy`，VLM 返回空字段后，再以 `delete_if_empty=True` 调用 `apply_skill_privacy`：内容仍有引用，配置删除方法被调用。 + +## 5. 推荐的 Agent 使用流程 + +**当前可以通过 HTTP/CLI 人工管理秘密，但尚无经过验证的“接入即可自动学会”流程。** `remember`、普通 `write`、资源导入都不应被当作外部账号密钥的保存入口；也不应为了利用自动抽取而先把真实密钥写进技能正文。 + +建议把如下流程落实为工具协议和 Agent 接入技能，而不是只添加一句“注意隐私”： + +1. 接收到账号密码、AK/SK、token、私钥等时，先识别所属用户、服务、环境和账号，选定 Privacy 目标。持久化技能只写字段声明或引用。 +2. 使用专用工具保存值，工具返回带 owner 的统一引用、键名和版本号；普通回复与记忆只记“已配置”及用途。现有 HTTP 写入会回显值，接入包装器应处理响应及采集内容。 +3. 使用时，普通接入允许 Agent 在当前身份权限内通过字段级接口显式还原；Gateway 接入优先在内部解析并注入。两条路径都必须在工具输出、错误、会话采集和再次写入时重新脱敏。 +4. owner 不同、值缺失、提取失败或权限不足时返回 `forbidden/missing` 和可联系的 owner；不把未解析引用当成密钥，不通过搜索绕过权限。外部凭据轮换后更新 Privacy；明确回滚不能恢复已被第三方撤销的密钥。 +5. 更新技能保留仍被引用的配置；移除技能/账号时清理 current、history 及保留策略涵盖的副本。用元信息验证成功，不通过打印密钥确认。 + +## 6. 建议的验收顺序 + +1. **先实现高效写入 Sanitizer 和输出防线。** 覆盖正文、frontmatter、附件、普通写入、资源、memory、摘要、索引和采集的虚构秘密样本；验证 user 写入及 `/agent/skills` 写入均生成 owner 绑定引用，失败必须阻止发布或转入明确状态。 +2. **补引用解析、身份与 Agent/Gateway 回归。** 验证 owner 本人字段级还原、其他用户得到 forbidden 与 owner 提示、缺值处理、轮换后解析 active value；分别验证 Agent 显式还原和 Gateway 内部注入不会把值再次写回内容或会话。 +3. **提供显式存量扫描。** 验证 `ov privacy scan` 默认只报告，`--fix` 使用内容 revision、刷新摘要和索引，且不越过用户权限；暂不把 Privacy upsert 后全量反向扫描设为默认行为。 +4. **补事务回归并接入 PR CI。** 对 history/current/meta 每一步注入写失败、取消及重启；检查版本一致、历史不覆盖，并把 Sanitizer、引用解析、身份隔离和 CLI 修复测试设为门禁。覆盖率辅助发现未测分支，不能替代安全断言。 +5. **再验证真实 Agent 行为。** 用虚构外部账号构建完整任务：保存凭据、读取公共技能、owner/非 owner 还原、轮换、缺配置和修复；检查模型回复、工具调用和会话中是否出现非预期明文，并记录模型与提示词版本。 + +完成这些验收前，适合将 Privacy 定位为**有用的敏感配置基础设施**；生产凭据的安全性仍需要接入层、执行层和存储生命周期共同落实。 diff --git a/openviking/privacy/__init__.py b/openviking/privacy/__init__.py index e01fb95f79..0d7dc6b3d0 100644 --- a/openviking/privacy/__init__.py +++ b/openviking/privacy/__init__.py @@ -2,7 +2,26 @@ # SPDX-License-Identifier: AGPL-3.0 """Privacy config utilities for OpenViking.""" -from openviking.privacy.models import UserPrivacyConfigMeta, UserPrivacyConfigVersion +from openviking.privacy.models import ( + UserPrivacyConfigMeta, + UserPrivacyConfigVersion, + UserPrivacyFieldDeleteResult, +) +from openviking.privacy.reference import ( + PrivacyFieldReference, + PrivacyTargetReference, + build_privacy_field_reference, + parse_privacy_field_reference, + parse_privacy_field_selector, + parse_privacy_target_selector, +) +from openviking.privacy.sanitizer import ( + DEFAULT_MATCHER_CACHE_TTL_SECONDS, + MIN_PROTECTED_VALUE_CHARS, + PrivacyReplacement, + PrivacySanitizationResult, + PrivacySanitizer, +) from openviking.privacy.service import UserPrivacyConfigService from openviking.privacy.skill_extractor import ( SkillPrivacyExtractionResult, @@ -19,7 +38,19 @@ __all__ = [ "UserPrivacyConfigMeta", "UserPrivacyConfigVersion", + "UserPrivacyFieldDeleteResult", "UserPrivacyConfigService", + "PrivacyFieldReference", + "PrivacyTargetReference", + "build_privacy_field_reference", + "parse_privacy_field_reference", + "parse_privacy_field_selector", + "parse_privacy_target_selector", + "PrivacyReplacement", + "MIN_PROTECTED_VALUE_CHARS", + "DEFAULT_MATCHER_CACHE_TTL_SECONDS", + "PrivacySanitizationResult", + "PrivacySanitizer", "SkillPrivacyExtractionResult", "extract_skill_privacy_values", "SkillPrivacyPlaceholderizationResult", diff --git a/openviking/privacy/models.py b/openviking/privacy/models.py index 8fe7e3090b..f680c3af56 100644 --- a/openviking/privacy/models.py +++ b/openviking/privacy/models.py @@ -82,3 +82,14 @@ def from_dict(cls, data: Dict[str, Any]) -> "UserPrivacyConfigVersion": created_by=data.get("created_by", ""), change_reason=data.get("change_reason", ""), ) + + +@dataclass(frozen=True) +class UserPrivacyFieldDeleteResult: + """Outcome of removing a field from the active Privacy target.""" + + category: str + target_key: str + field: str + target_deleted: bool + active_version: int | None = None diff --git a/openviking/privacy/reference.py b/openviking/privacy/reference.py new file mode 100644 index 0000000000..d27c5e1adb --- /dev/null +++ b/openviking/privacy/reference.py @@ -0,0 +1,143 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Canonical owner-bound references to individual privacy fields.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from urllib.parse import quote, unquote + +from openviking.core.identifiers import validate_user_id + +_REFERENCE_RE = re.compile( + r"^\{\{[ \t]*ov_privacy:(?P[^\s:{}]+):viking://~/privacy/" + r"(?P[^/\s]+)/(?P[^#\s]+)#/values/" + r"(?P[^}\s]+)[ \t]*\}\}$" +) +_FIELD_URI_RE = re.compile( + r"^viking://(?P~|user/(?P[^/]+))/privacy/" + r"(?P[^/]+)/(?P[^#]+)#/values/(?P.+)$" +) +_TARGET_URI_RE = re.compile( + r"^viking://(?P~|user/(?P[^/]+))/privacy/" + r"(?P[^/]+)/(?P[^#]+)$" +) + + +@dataclass(frozen=True) +class PrivacyFieldReference: + owner_user_id: str + category: str + target_key: str + field: str + + def render(self) -> str: + return build_privacy_field_reference( + self.owner_user_id, + self.category, + self.target_key, + self.field, + ) + + +@dataclass(frozen=True) +class PrivacyTargetReference: + owner_user_id: str + category: str + target_key: str + + +def build_privacy_field_reference( + owner_user_id: str, + category: str, + target_key: str, + field: str, +) -> str: + """Build the canonical reference without exposing the referenced value.""" + validation_error = validate_user_id(owner_user_id) + if validation_error: + raise ValueError(validation_error) + parts = { + "category": category, + "target_key": target_key, + "field": field, + } + for name, value in parts.items(): + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + + return ( + f"{{{{ov_privacy:{owner_user_id}:viking://~/privacy/" + f"{quote(category, safe='')}/{quote(target_key, safe='')}" + f"#/values/{quote(field, safe='')}}}}}" + ) + + +def parse_privacy_field_reference(value: str) -> PrivacyFieldReference | None: + """Parse a reference, tolerating only whitespace next to the outer braces. + + Rendering remains canonical and whitespace-free. Whitespace inside owner, + URI, target, or field tokens is invalid and must be percent-encoded when it + belongs to a path segment. + """ + match = _REFERENCE_RE.fullmatch(value) + if match is None: + return None + owner = match.group("owner") + if validate_user_id(owner): + return None + decoded = { + name: unquote(match.group(name)) + for name in ("category", "target", "field") + } + if any(not item for item in decoded.values()): + return None + return PrivacyFieldReference( + owner_user_id=owner, + category=decoded["category"], + target_key=decoded["target"], + field=decoded["field"], + ) + + +def parse_privacy_field_selector( + value: str, + *, + current_user_id: str, +) -> PrivacyFieldReference | None: + """Parse either an owner-bound reference or a field-level Privacy URI.""" + reference = parse_privacy_field_reference(value) + if reference is not None: + return reference + match = _FIELD_URI_RE.fullmatch(value) + if match is None: + return None + owner = current_user_id if match.group("scope") == "~" else match.group("owner") + if not owner or validate_user_id(owner): + return None + return PrivacyFieldReference( + owner_user_id=owner, + category=unquote(match.group("category")), + target_key=unquote(match.group("target")), + field=unquote(match.group("field")), + ) + + +def parse_privacy_target_selector( + value: str, + *, + current_user_id: str, +) -> PrivacyTargetReference | None: + """Parse a target-level current-user or explicit-user Privacy URI.""" + match = _TARGET_URI_RE.fullmatch(value) + if match is None: + return None + owner = current_user_id if match.group("scope") == "~" else match.group("owner") + if not owner or validate_user_id(owner): + return None + return PrivacyTargetReference( + owner_user_id=owner, + category=unquote(match.group("category")), + target_key=unquote(match.group("target")), + ) diff --git a/openviking/privacy/sanitizer.py b/openviking/privacy/sanitizer.py new file mode 100644 index 0000000000..2bb640388b --- /dev/null +++ b/openviking/privacy/sanitizer.py @@ -0,0 +1,214 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Deterministic replacement of registered secrets with owner-bound references.""" + +from __future__ import annotations + +import asyncio +import time +from collections import deque +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable + +from openviking.privacy.reference import build_privacy_field_reference +from openviking.server.identity import RequestContext + +if TYPE_CHECKING: + from openviking.privacy.service import UserPrivacyConfigService + +_AMBIGUOUS_PLACEHOLDER = "{{ov_privacy:ambiguous}}" +MIN_PROTECTED_VALUE_CHARS = 6 +DEFAULT_MATCHER_CACHE_TTL_SECONDS = 3600.0 + + +@dataclass(frozen=True) +class PrivacyReplacement: + start: int + end: int + reference: str + + +@dataclass(frozen=True) +class PrivacySanitizationResult: + content: str + replacements: tuple[PrivacyReplacement, ...] = () + ambiguous_references: tuple[tuple[str, ...], ...] = () + + @property + def changed(self) -> bool: + return bool(self.replacements) + + +@dataclass +class _AutomatonNode: + transitions: dict[str, int] = field(default_factory=dict) + failure: int = 0 + outputs: list[str] = field(default_factory=list) + + +class _SecretMatcher: + """Small pure-Python Aho-Corasick matcher built once per catalog revision.""" + + def __init__(self, references_by_secret: dict[str, str | tuple[str, ...]]): + self._references_by_secret = references_by_secret + self._nodes = [_AutomatonNode()] + for secret in references_by_secret: + state = 0 + for char in secret: + state = self._nodes[state].transitions.setdefault(char, len(self._nodes)) + if state == len(self._nodes): + self._nodes.append(_AutomatonNode()) + self._nodes[state].outputs.append(secret) + self._build_failures() + + def _build_failures(self) -> None: + pending: deque[int] = deque() + for state in self._nodes[0].transitions.values(): + pending.append(state) + while pending: + state = pending.popleft() + for char, next_state in self._nodes[state].transitions.items(): + pending.append(next_state) + fallback = self._nodes[state].failure + while fallback and char not in self._nodes[fallback].transitions: + fallback = self._nodes[fallback].failure + self._nodes[next_state].failure = self._nodes[fallback].transitions.get(char, 0) + self._nodes[next_state].outputs.extend( + self._nodes[self._nodes[next_state].failure].outputs + ) + + def replacements( + self, content: str + ) -> tuple[list[PrivacyReplacement], list[tuple[str, ...]]]: + matches: list[tuple[int, int, str]] = [] + state = 0 + for index, char in enumerate(content): + while state and char not in self._nodes[state].transitions: + state = self._nodes[state].failure + state = self._nodes[state].transitions.get(char, 0) + for secret in self._nodes[state].outputs: + matches.append((index + 1 - len(secret), index + 1, secret)) + + # Prefer the longest match at the same position and discard overlaps. + matches.sort(key=lambda item: (item[0], -(item[1] - item[0]), item[1])) + selected: list[PrivacyReplacement] = [] + ambiguous: list[tuple[str, ...]] = [] + cursor = 0 + for start, end, secret in matches: + if start < cursor: + continue + target = self._references_by_secret[secret] + if isinstance(target, tuple): + ambiguous.append(target) + selected.append(PrivacyReplacement(start, end, _AMBIGUOUS_PLACEHOLDER)) + else: + selected.append(PrivacyReplacement(start, end, target)) + cursor = end + return selected, ambiguous + + +class PrivacySanitizer: + """Sanitize text with the active sensitive values owned by the current user.""" + + def __init__( + self, + privacy: "UserPrivacyConfigService", + *, + cache_ttl_seconds: float = DEFAULT_MATCHER_CACHE_TTL_SECONDS, + monotonic_clock: Callable[[], float] = time.monotonic, + ): + if cache_ttl_seconds <= 0: + raise ValueError("cache_ttl_seconds must be positive") + self._privacy = privacy + self._cache_ttl_seconds = float(cache_ttl_seconds) + self._monotonic_clock = monotonic_clock + self._cache: dict[tuple[str, str, int], tuple[float, _SecretMatcher]] = {} + self._cache_lock = asyncio.Lock() + + async def sanitize( + self, + content: str, + *, + ctx: RequestContext, + uri: str = "", + ) -> PrivacySanitizationResult: + if not content or self._is_privacy_storage_uri(uri, ctx): + return PrivacySanitizationResult(content=content) + matcher = await self._matcher(ctx) + replacements, ambiguous = matcher.replacements(content) + if not replacements: + return PrivacySanitizationResult( + content=content, + ambiguous_references=tuple(ambiguous), + ) + parts: list[str] = [] + cursor = 0 + for replacement in replacements: + parts.append(content[cursor : replacement.start]) + parts.append(replacement.reference) + cursor = replacement.end + parts.append(content[cursor:]) + return PrivacySanitizationResult( + content="".join(parts), + replacements=tuple(replacements), + ambiguous_references=tuple(ambiguous), + ) + + async def _matcher(self, ctx: RequestContext) -> _SecretMatcher: + revision = self._privacy.catalog_revision(ctx) + cache_key = (ctx.account_id, ctx.user.user_id, revision) + now = self._monotonic_clock() + cached = self._cache.get(cache_key) + if cached is not None and now - cached[0] < self._cache_ttl_seconds: + return cached[1] + async with self._cache_lock: + now = self._monotonic_clock() + cached = self._cache.get(cache_key) + if cached is not None and now - cached[0] < self._cache_ttl_seconds: + return cached[1] + matcher = await self._build_matcher(ctx) + owner_key = cache_key[:2] + self._cache = { + key: item for key, item in self._cache.items() if key[:2] != owner_key + } + self._cache[cache_key] = (now, matcher) + return matcher + + async def _build_matcher(self, ctx: RequestContext) -> _SecretMatcher: + references: dict[str, list[str]] = {} + for category in await self._privacy.list_categories(ctx): + for target_key in await self._privacy.list_targets(ctx, category): + current = await self._privacy.get_current(ctx, category, target_key) + if current is None: + continue + for field_name, value in current.values.items(): + # Privacy is the explicit protection boundary. Do not infer + # sensitivity from field names. Very short values are not + # safe for global substring replacement because ordinary + # prose would produce excessive false positives. + if ( + not isinstance(value, str) + or len(value) < MIN_PROTECTED_VALUE_CHARS + ): + continue + reference = build_privacy_field_reference( + ctx.user.user_id, + category, + target_key, + field_name, + ) + references.setdefault(value, []).append(reference) + + targets: dict[str, str | tuple[str, ...]] = {} + for secret, candidates in references.items(): + unique = tuple(sorted(set(candidates))) + if len(unique) == 1: + targets[secret] = unique[0] + else: + targets[secret] = unique + return _SecretMatcher(targets) + + @staticmethod + def _is_privacy_storage_uri(uri: str, ctx: RequestContext) -> bool: + root = f"viking://user/{ctx.user.user_id}/privacy" + return uri.rstrip("/") == root or uri.startswith(f"{root}/") diff --git a/openviking/privacy/scan.py b/openviking/privacy/scan.py new file mode 100644 index 0000000000..ebf5cf321c --- /dev/null +++ b/openviking/privacy/scan.py @@ -0,0 +1,156 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Explicit scan and repair workflow for already persisted Privacy values.""" + +from __future__ import annotations + +from typing import Any, Iterable + +from openviking.core.namespace import canonical_user_root, classify_uri, resolve_request_uri +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.server.identity import RequestContext +from openviking_cli.exceptions import InvalidArgumentError, NotFoundError + +_SCOPES = frozenset({"resources", "skills", "memories"}) +_SKIP_NAMES = frozenset( + { + ".abstract.md", + ".overview.md", + ".relations.json", + ".source.json", + ".image_mappings.json", + ".artifact_manifest.json", + } +) + + +def default_privacy_scan_roots(ctx: RequestContext, scopes: Iterable[str]) -> list[str]: + user_root = canonical_user_root(ctx) + roots: list[str] = [] + for scope in scopes: + if scope not in _SCOPES: + raise InvalidArgumentError( + f"unsupported privacy scan scope: {scope}", + details={"scope": scope, "allowed": sorted(_SCOPES)}, + ) + if scope == "resources": + roots.extend([f"{user_root}/resources", "viking://resources"]) + elif scope == "skills": + roots.extend([f"{user_root}/skills", "viking://agent/skills"]) + else: + roots.append(f"{user_root}/memories") + return list(dict.fromkeys(roots)) + + +async def scan_privacy_content( + *, + service: Any, + ctx: RequestContext, + sanitizer: PrivacySanitizer, + scopes: list[str], + target_uri: str = "", + fix: bool = False, + wait: bool = False, + node_limit: int = 10000, +) -> dict[str, Any]: + """Scan readable text and optionally rewrite unambiguous matches.""" + if target_uri and scopes: + raise InvalidArgumentError("pass either target_uri or scopes, not both") + roots = ( + [resolve_request_uri(target_uri, ctx)] + if target_uri + else default_privacy_scan_roots(ctx, scopes or sorted(_SCOPES)) + ) + files: list[str] = [] + truncated = False + for root in roots: + try: + stat = await service.viking_fs.stat(root, ctx=ctx, skip_count=True) + except (NotFoundError, FileNotFoundError): + continue + if stat.get("isDir", stat.get("is_dir", False)): + entries = await service.viking_fs.tree( + root, + output="original", + show_all_hidden=True, + node_limit=node_limit, + level_limit=None, + ctx=ctx, + ) + if len(entries) >= node_limit: + truncated = True + files.extend( + str(entry.get("uri") or "") + for entry in entries + if not entry.get("isDir") and entry.get("uri") + ) + else: + files.append(root) + + matches: list[dict[str, Any]] = [] + repaired = 0 + reindex_roots: set[str] = set() + for uri in dict.fromkeys(files): + if uri.rstrip("/").rsplit("/", 1)[-1] in _SKIP_NAMES: + continue + try: + raw = await service.viking_fs.read_file_bytes(uri, ctx=ctx) + content = raw.decode("utf-8") + except UnicodeDecodeError: + continue + except (NotFoundError, FileNotFoundError): + continue + result = await sanitizer.sanitize(content, ctx=ctx, uri=uri) + if not result.replacements and not result.ambiguous_references: + continue + item = { + "uri": uri, + "occurrences": len(result.replacements), + "references": sorted({replacement.reference for replacement in result.replacements}), + "ambiguous": [list(candidates) for candidates in result.ambiguous_references], + "fixable": not result.ambiguous_references, + "fixed": False, + } + if fix and item["fixable"]: + classification = classify_uri(uri) + if classification.is_skill: + path = service.viking_fs._uri_to_path(uri, ctx=ctx) + lease = await service.viking_fs._async_agfs.pathlock_acquire_exact(path) + try: + await service.viking_fs.write_file( + uri, result.content, ctx=ctx, lease_ref=lease + ) + finally: + await service.viking_fs._async_agfs.pathlock_release(lease) + parts = uri.split("/skills/", 1) + skill_name = parts[1].split("/", 1)[0] if len(parts) == 2 else "" + if skill_name: + reindex_roots.add(f"{parts[0]}/skills/{skill_name}") + else: + await service.fs.write(uri, result.content, ctx=ctx, wait=wait) + item["fixed"] = True + repaired += 1 + matches.append(item) + + reindex_tasks: list[dict[str, Any]] = [] + if fix: + for root in sorted(reindex_roots): + reindex_tasks.append( + await service.reindex( + uri=root, + mode="semantic_and_vectors", + wait=wait, + force=True, + ctx=ctx, + ) + ) + return { + "roots": roots, + "scanned_files": len(dict.fromkeys(files)), + "matched_files": len(matches), + "repaired_files": repaired, + "dry_run": not fix, + "truncated": truncated, + "matches": matches, + "reindex": reindex_tasks, + } diff --git a/openviking/privacy/service.py b/openviking/privacy/service.py index 1bbef6c20f..88e7326728 100644 --- a/openviking/privacy/service.py +++ b/openviking/privacy/service.py @@ -16,7 +16,11 @@ parse_version_filename, version_uri, ) -from openviking.privacy.models import UserPrivacyConfigMeta, UserPrivacyConfigVersion +from openviking.privacy.models import ( + UserPrivacyConfigMeta, + UserPrivacyConfigVersion, + UserPrivacyFieldDeleteResult, +) from openviking.server.identity import RequestContext from openviking.storage.viking_fs import VikingFS from openviking.utils.time_utils import get_current_timestamp @@ -28,6 +32,15 @@ class UserPrivacyConfigService: def __init__(self, viking_fs: VikingFS): self._viking_fs = viking_fs + self._catalog_revisions: dict[tuple[str, str], int] = {} + + def catalog_revision(self, ctx: RequestContext) -> int: + """Return the process-local revision of the caller's active value catalog.""" + return self._catalog_revisions.get((ctx.account_id, ctx.user.user_id), 0) + + def _bump_catalog_revision(self, ctx: RequestContext) -> None: + key = (ctx.account_id, ctx.user.user_id) + self._catalog_revisions[key] = self._catalog_revisions.get(key, 0) + 1 def _user_root(self, ctx: RequestContext) -> str: return canonical_user_root(ctx) @@ -63,7 +76,11 @@ async def delete( async with self._config_lock( ctx, category, target_key, owner_lease_ref=owner_lease_ref ) as lease: - if not await self._viking_fs.exists(root_uri, ctx=ctx): + # Acquiring a path lock may materialize the target directory. Treat + # only a persisted current snapshot as an existing privacy target, + # so repeated deletion is idempotent and an empty lock root is not + # reported as deleted configuration. + if not await self.exists(ctx, category, target_key): return False if owner_lease_ref is not None: # An enclosing Skill update still needs this root's lock for @@ -71,6 +88,7 @@ async def delete( await self._clear_contents(ctx, category, target_key, lease) else: await self._viking_fs.rm(root_uri, recursive=True, ctx=ctx, lease_ref=lease) + self._bump_catalog_revision(ctx) return True async def _clear_contents( @@ -160,7 +178,7 @@ async def _save_meta( async def get_current( self, ctx: RequestContext, category: str, target_key: str - ) -> Optional[UserPrivacyConfigVersion]: + ) -> Optional[UserPrivacyFieldDeleteResult]: try: content = await self._viking_fs.read_file( current_uri(self._user_space(ctx), category, target_key), ctx=ctx @@ -283,6 +301,7 @@ async def upsert( if labels is not None: meta.labels = dict(labels) await self._save_meta(ctx, category, target_key, meta, lease) + self._bump_catalog_revision(ctx) return version async def activate_version( @@ -313,4 +332,79 @@ async def activate_version( meta.updated_by = updated_by meta.last_accessed_at = get_current_timestamp() await self._save_meta(ctx, category, target_key, meta, lease) + self._bump_catalog_revision(ctx) return snapshot + + async def delete_field( + self, + ctx: RequestContext, + category: str, + target_key: str, + field: str, + updated_by: str = "", + ) -> Optional[UserPrivacyConfigVersion]: + """Remove one field from the active values by publishing a new snapshot. + + Historical snapshots remain intact while the target still has fields. + Deleting the final field removes the complete target and its history. + """ + async with self._config_lock(ctx, category, target_key) as lease: + meta = await self.get_meta(ctx, category, target_key) + current = await self.get_current(ctx, category, target_key) + if meta is None or current is None or field not in current.values: + return None + + values = dict(current.values) + values.pop(field) + if not values: + await self._viking_fs.rm( + self.get_config_root(ctx, category, target_key), + recursive=True, + ctx=ctx, + lease_ref=lease, + ) + self._bump_catalog_revision(ctx) + return UserPrivacyFieldDeleteResult( + category=category, + target_key=target_key, + field=field, + target_deleted=True, + ) + + now = get_current_timestamp() + version_number = meta.latest_version + 1 + snapshot = UserPrivacyConfigVersion( + version=version_number, + category=category, + target_key=target_key, + values=values, + created_at=now, + created_by=updated_by, + change_reason=f"delete field: {field}", + ) + serialized = json.dumps(snapshot.to_dict(), ensure_ascii=False) + await self._viking_fs.write_file( + version_uri(self._user_space(ctx), category, target_key, version_number), + serialized, + ctx=ctx, + lease_ref=lease, + ) + await self._viking_fs.write_file( + current_uri(self._user_space(ctx), category, target_key), + serialized, + ctx=ctx, + lease_ref=lease, + ) + meta.active_version = version_number + meta.latest_version = version_number + meta.updated_by = updated_by + meta.last_accessed_at = now + await self._save_meta(ctx, category, target_key, meta, lease) + self._bump_catalog_revision(ctx) + return UserPrivacyFieldDeleteResult( + category=category, + target_key=target_key, + field=field, + target_deleted=False, + active_version=snapshot.version, + ) diff --git a/openviking/server/routers/privacy_configs.py b/openviking/server/routers/privacy_configs.py index dda361186d..c962adef85 100644 --- a/openviking/server/routers/privacy_configs.py +++ b/openviking/server/routers/privacy_configs.py @@ -4,14 +4,21 @@ from typing import Any, Dict, Optional -from fastapi import APIRouter, Depends, Path -from pydantic import BaseModel, ConfigDict +from fastapi import APIRouter, Depends, Path, Query +from pydantic import BaseModel, ConfigDict, Field +from openviking.privacy.reference import ( + build_privacy_field_reference, + parse_privacy_field_selector, + parse_privacy_target_selector, +) +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.privacy.scan import scan_privacy_content from openviking.server.auth import get_request_context -from openviking_cli.exceptions import NotFoundError from openviking.server.dependencies import get_service from openviking.server.identity import RequestContext from openviking.server.models import Response +from openviking_cli.exceptions import InvalidArgumentError, NotFoundError, PermissionDeniedError router = APIRouter(prefix="/api/v1/privacy-configs", tags=["privacy-configs"]) @@ -40,6 +47,30 @@ class ActivatePrivacyConfigVersionRequest(BaseModel): version: int +class ScanPrivacyContentRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + scopes: list[str] = Field(default_factory=list) + target_uri: str = "" + fix: bool = False + wait: bool = False + node_limit: int = 10000 + + +class PrivacySelectorRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + selector: str + + +def _require_selector_owner(owner_user_id: str, ctx: RequestContext) -> None: + if owner_user_id != ctx.user.user_id: + raise PermissionDeniedError( + f"Privacy reference belongs to user {owner_user_id}", + resource=owner_user_id, + ) + + @router.get("") async def list_privacy_categories( _ctx: RequestContext = Depends(get_request_context), @@ -50,6 +81,126 @@ async def list_privacy_categories( return Response(status="ok", result=result) +@router.post("/scan") +async def scan_privacy_configs( + request: ScanPrivacyContentRequest, + _ctx: RequestContext = Depends(get_request_context), +): + service = get_service() + privacy = service.privacy_configs + if privacy is None: + return Response(status="ok", result={"matches": [], "matched_files": 0}) + result = await scan_privacy_content( + service=service, + ctx=_ctx, + sanitizer=PrivacySanitizer(privacy), + scopes=request.scopes, + target_uri=request.target_uri, + fix=request.fix, + wait=request.wait, + node_limit=max(1, min(request.node_limit, 100000)), + ) + return Response(status="ok", result=result) + + +@router.post("/resolve") +async def resolve_privacy_reference( + request: PrivacySelectorRequest, + _ctx: RequestContext = Depends(get_request_context), +): + reference = parse_privacy_field_selector( + request.selector, + current_user_id=_ctx.user.user_id, + ) + if reference is None: + raise InvalidArgumentError( + "selector must be an owner-bound Privacy reference or a field-level Privacy URI" + ) + _require_selector_owner(reference.owner_user_id, _ctx) + service = get_service() + privacy = service.privacy_configs + await _require_privacy_target(privacy, _ctx, reference.category, reference.target_key) + current = await privacy.get_current(_ctx, reference.category, reference.target_key) + if current is None or reference.field not in current.values: + raise NotFoundError(request.selector, "privacy config field") + value = current.values[reference.field] + return Response( + status="ok", + result={ + "owner_user_id": reference.owner_user_id, + "category": reference.category, + "target_key": reference.target_key, + "field": reference.field, + "version": current.version, + "configured": value is not None and str(value) != "", + "reference": reference.render(), + "value": value, + }, + ) + + +@router.delete("/selector") +async def delete_privacy_selector( + request: PrivacySelectorRequest, + _ctx: RequestContext = Depends(get_request_context), +): + service = get_service() + privacy = service.privacy_configs + field_reference = parse_privacy_field_selector( + request.selector, + current_user_id=_ctx.user.user_id, + ) + if field_reference is not None: + _require_selector_owner(field_reference.owner_user_id, _ctx) + outcome = await privacy.delete_field( + _ctx, + field_reference.category, + field_reference.target_key, + field_reference.field, + updated_by=_ctx.user.user_id, + ) + if outcome is None: + raise NotFoundError(request.selector, "privacy config field") + return Response( + status="ok", + result={ + "owner_user_id": field_reference.owner_user_id, + "category": field_reference.category, + "target_key": field_reference.target_key, + "field": field_reference.field, + "deleted": True, + "scope": "field", + "target_deleted": outcome.target_deleted, + "active_version": outcome.active_version, + }, + ) + + target_reference = parse_privacy_target_selector( + request.selector, + current_user_id=_ctx.user.user_id, + ) + if target_reference is None: + raise InvalidArgumentError( + "selector must be a Privacy target URI, field URI, or owner-bound reference" + ) + _require_selector_owner(target_reference.owner_user_id, _ctx) + deleted = await privacy.delete( + _ctx, + target_reference.category, + target_reference.target_key, + ) + return Response( + status="ok", + result={ + "owner_user_id": target_reference.owner_user_id, + "category": target_reference.category, + "target_key": target_reference.target_key, + "deleted": deleted, + "scope": "target", + }, + ) + + @router.get("/{category}") async def list_privacy_targets( category: str = Path(..., description="Privacy config category"), @@ -81,6 +232,83 @@ async def get_privacy_current( ) +@router.get("/{category}/{target_key}/field") +async def get_privacy_field( + category: str = Path(..., description="Privacy config category"), + target_key: str = Path(..., description="Privacy config target key"), + field: str = Query(..., min_length=1, description="Field in current values"), + reveal: bool = Query(False, description="Include the active field value"), + _ctx: RequestContext = Depends(get_request_context), +): + """Return one active field and its owner-bound reference. + + The safe default omits the value. Callers must opt in with ``reveal=true`` + when the plaintext is required for immediate use. + """ + service = get_service() + privacy = service.privacy_configs + await _require_privacy_target(privacy, _ctx, category, target_key) + current = await privacy.get_current(_ctx, category, target_key) + if current is None or field not in current.values: + raise NotFoundError( + f"{category}/{target_key}/fields/{field}", + "privacy config field", + ) + + value = current.values[field] + result = { + "owner_user_id": _ctx.user.user_id, + "category": category, + "target_key": target_key, + "field": field, + "version": current.version, + "configured": value is not None and str(value) != "", + "reference": build_privacy_field_reference( + _ctx.user.user_id, + category, + target_key, + field, + ), + } + if reveal: + result["value"] = value + return Response(status="ok", result=result) + + +@router.delete("/{category}/{target_key}/field") +async def delete_privacy_field( + category: str = Path(..., description="Privacy config category"), + target_key: str = Path(..., description="Privacy config target key"), + field: str = Query(..., min_length=1, description="Field to remove from active values"), + _ctx: RequestContext = Depends(get_request_context), +): + service = get_service() + privacy = service.privacy_configs + await _require_privacy_target(privacy, _ctx, category, target_key) + outcome = await privacy.delete_field( + _ctx, + category, + target_key, + field, + updated_by=_ctx.user.user_id, + ) + if outcome is None: + raise NotFoundError(f"{category}/{target_key}/fields/{field}", "privacy config field") + return Response( + status="ok", + result={ + "owner_user_id": _ctx.user.user_id, + "category": category, + "target_key": target_key, + "field": field, + "deleted": True, + "scope": "field", + "target_deleted": outcome.target_deleted, + "active_version": outcome.active_version, + }, + ) + + @router.get("/{category}/{target_key}/versions") async def list_privacy_versions( category: str = Path(..., description="Privacy config category"), @@ -133,6 +361,28 @@ async def upsert_privacy_config( return Response(status="ok", result=None if result is None else result.to_dict()) +@router.delete("/{category}/{target_key}") +async def delete_privacy_config( + category: str = Path(..., description="Privacy config category"), + target_key: str = Path(..., description="Privacy config target key"), + _ctx: RequestContext = Depends(get_request_context), +): + """Delete a target's current value, metadata, and complete version history.""" + service = get_service() + privacy = service.privacy_configs + deleted = False + if privacy is not None: + deleted = await privacy.delete(_ctx, category, target_key) + return Response( + status="ok", + result={ + "category": category, + "target_key": target_key, + "deleted": deleted, + }, + ) + + @router.post("/{category}/{target_key}/activate") async def activate_privacy_version( request: ActivatePrivacyConfigVersionRequest, diff --git a/openviking/service/core.py b/openviking/service/core.py index 593c4c2308..4b0f868fd3 100644 --- a/openviking/service/core.py +++ b/openviking/service/core.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional from openviking.core.directories import DirectoryInitializer -from openviking.privacy import UserPrivacyConfigService +from openviking.privacy import PrivacySanitizer, UserPrivacyConfigService from openviking.resource.uri_mutation_coordinator import UriMutationCoordinator from openviking.resource.watch_scheduler import WatchScheduler from openviking.server.identity import RequestContext, Role @@ -519,6 +519,9 @@ async def initialize(self) -> None: user_count, ) self._privacy_config_service = UserPrivacyConfigService(self._viking_fs) + privacy_sanitizer = PrivacySanitizer(self._privacy_config_service) + self._viking_fs.set_privacy_sanitizer(privacy_sanitizer) + self._resource_processor.set_privacy_sanitizer(privacy_sanitizer) # Initialize processors self._skill_processor = SkillProcessor( diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py index 5b1910751d..6a644cc748 100644 --- a/openviking/service/fs_service.py +++ b/openviking/service/fs_service.py @@ -20,6 +20,7 @@ uri_parts, ) from openviking.privacy import ( + PrivacySanitizer, UserPrivacyConfigService, get_skill_name_from_uri, restore_skill_content, @@ -105,6 +106,9 @@ def __init__( self._viking_fs = viking_fs self._vikingdb = vikingdb self._privacy_config_service = privacy_config_service + self._privacy_sanitizer = ( + PrivacySanitizer(privacy_config_service) if privacy_config_service is not None else None + ) self._resource_memory_link_service = resource_memory_link_service self._watch_scheduler = watch_scheduler self._uri_mutation_coordinator = uri_mutation_coordinator or UriMutationCoordinator() @@ -122,6 +126,9 @@ def set_dependencies( self._viking_fs = viking_fs self._vikingdb = vikingdb self._privacy_config_service = privacy_config_service + self._privacy_sanitizer = ( + PrivacySanitizer(privacy_config_service) if privacy_config_service is not None else None + ) self._resource_memory_link_service = resource_memory_link_service self._watch_scheduler = watch_scheduler if uri_mutation_coordinator is not None: @@ -1090,10 +1097,18 @@ async def read_visible( # Resolve id to URI before classify_uri / visible_content which expect # a real viking:// URI. resolved_uri = await self._resolve_uri(uri, ctx) - if not classify_uri(resolved_uri).is_memory: - return await self.read(resolved_uri, ctx=ctx, offset=offset, limit=limit) content = await self.read(resolved_uri, ctx=ctx) - return visible_content(content, uri=resolved_uri, offset=offset, limit=limit) + if classify_uri(resolved_uri).is_memory: + content = visible_content(content, uri=resolved_uri) + if self._privacy_sanitizer is not None: + content = ( + await self._privacy_sanitizer.sanitize(content, ctx=ctx, uri=resolved_uri) + ).content + if offset == 0 and limit == -1: + return content + lines = content.splitlines(keepends=True) + sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit] + return "".join(sliced) async def abstract(self, uri: str, ctx: RequestContext) -> str: """Read L0 abstract (.abstract.md).""" @@ -1210,7 +1225,11 @@ async def write( ) -> Dict[str, Any]: """Create or update a file and refresh semantics/vectors.""" viking_fs = self._ensure_initialized() - coordinator = ContentWriteCoordinator(viking_fs=viking_fs, vikingdb=self._vikingdb) + coordinator = ContentWriteCoordinator( + viking_fs=viking_fs, + vikingdb=self._vikingdb, + privacy_sanitizer=self._privacy_sanitizer, + ) return await coordinator.write( uri=uri, content=content, @@ -1235,7 +1254,11 @@ async def batch_write( ) -> Dict[str, Any]: """Apply multiple file writes and aggregate downstream refresh.""" viking_fs = self._ensure_initialized() - coordinator = ContentWriteCoordinator(viking_fs=viking_fs, vikingdb=self._vikingdb) + coordinator = ContentWriteCoordinator( + viking_fs=viking_fs, + vikingdb=self._vikingdb, + privacy_sanitizer=self._privacy_sanitizer, + ) return await coordinator.batch_write( root_uri=root_uri, operations=operations, @@ -1255,7 +1278,10 @@ async def set_tags( ) -> Dict[str, Any]: """Set explicit retrieval tags for a file or directory semantic nodes.""" viking_fs = self._ensure_initialized() - coordinator = ContentWriteCoordinator(viking_fs=viking_fs) + coordinator = ContentWriteCoordinator( + viking_fs=viking_fs, + privacy_sanitizer=self._privacy_sanitizer, + ) return await coordinator.set_tags( uri=uri, tags=tags, diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py index bd8131a675..4c5e104382 100644 --- a/openviking/service/reindex_executor.py +++ b/openviking/service/reindex_executor.py @@ -701,7 +701,12 @@ async def _reindex_rfv( target_uri=uri, processing_mode=mode, ingest_options=run.ingest_options, - force=run.force, + # A Skill package has exactly one capability document plus its L0/L1 + # sidecars. ``vectors_only`` is expected to rebuild all three even + # when their md5 values are unchanged (the public API regression + # contract relies on this), while other RFV targets retain the + # caller-controlled force behavior. + force=run.force or (context_type == "skill" and mode == "vectors_only"), scalar_target_levels=frozenset({0, 1, 2}), ) snapshot = await build_rfv_snapshot( diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py index 2c474fce85..1427c5d42f 100644 --- a/openviking/storage/content_write.py +++ b/openviking/storage/content_write.py @@ -19,6 +19,7 @@ relative_uri_path, uri_parts, ) +from openviking.privacy.sanitizer import PrivacySanitizer from openviking.resource.processing_mode import ( DEFAULT_PROCESSING_MODE, ProcessingMode, @@ -192,9 +193,52 @@ def statuses(self, *, wait: bool) -> tuple[str, str]: class ContentWriteCoordinator: """Write a file (create or modify) and trigger downstream maintenance.""" - def __init__(self, viking_fs: VikingFS, vikingdb: Any = None): + def __init__( + self, + viking_fs: VikingFS, + vikingdb: Any = None, + privacy_sanitizer: PrivacySanitizer | None = None, + ): self._viking_fs = viking_fs self._vikingdb = vikingdb + self._privacy_sanitizer = privacy_sanitizer + + async def _sanitize_text( + self, + content: str, + *, + uri: str, + ctx: RequestContext, + ) -> str: + if self._privacy_sanitizer is None: + return content + result = await self._privacy_sanitizer.sanitize(content, ctx=ctx, uri=uri) + if result.ambiguous_references: + raise InvalidArgumentError( + "Content contains a Privacy value mapped to multiple fields; " + "remove the duplicate mapping or insert the intended reference explicitly", + details={ + "uri": uri, + "candidate_references": [ + list(candidates) for candidates in result.ambiguous_references + ], + }, + ) + return result.content + + async def _sanitize_batch_operations( + self, + operations: list[dict[str, Any]], + *, + ctx: RequestContext, + ) -> None: + if self._privacy_sanitizer is None: + return + for operation in operations: + content = operation.get("content") + uri = operation.get("uri") + if isinstance(content, str) and isinstance(uri, str): + operation["content"] = await self._sanitize_text(content, uri=uri, ctx=ctx) async def write( self, @@ -216,6 +260,7 @@ async def write( ) self._ensure_content_write_policy(normalized_uri) await self._viking_fs._ensure_access(normalized_uri, ctx, action=AclAction.WRITE) + content = await self._sanitize_text(content, uri=normalized_uri, ctx=ctx) ingest_options = IngestOptions.from_search_tags(tags, mode=tag_mode) if acl is not None: ingest_options = replace( @@ -296,6 +341,7 @@ async def batch_write( normalized_operations = await self._normalize_batch_operations( normalized_root, operations, ctx=ctx ) + await self._sanitize_batch_operations(normalized_operations, ctx=ctx) target_paths = [ self._viking_fs._uri_to_path(operation["uri"], ctx=ctx) diff --git a/openviking/storage/context_update_plan.py b/openviking/storage/context_update_plan.py index 776ffff59f..127196df40 100644 --- a/openviking/storage/context_update_plan.py +++ b/openviking/storage/context_update_plan.py @@ -596,7 +596,13 @@ def _rfv_file_summary( file_name = file_uri.rsplit("/", 1)[-1] else: parent_uri = _uri(snapshot.request.target_uri, _parent(relative_path)) - overview = snapshot.source_contents.get((parent_uri, 1)) + # RFV keeps both the parsed sidecar body and its raw frontmatter-bearing + # source. ``parse_overview_file_summaries`` needs the latter; parsing the + # already-stripped body again misinterprets a skill body that begins with + # its own YAML frontmatter as the sidecar envelope. + overview = snapshot.source_raw_contents.get((parent_uri, 1)) + if overview is None: + overview = snapshot.source_contents.get((parent_uri, 1)) if isinstance(overview, (str, bytes)): summaries = overview_summaries.get(parent_uri) if summaries is None: diff --git a/openviking/storage/viking_fs/__init__.py b/openviking/storage/viking_fs/__init__.py index 4af01d28ed..83944e387a 100644 --- a/openviking/storage/viking_fs/__init__.py +++ b/openviking/storage/viking_fs/__init__.py @@ -162,11 +162,16 @@ def __init__( ) self._background_tasks: set = set() self._deletion_guard: Optional[Callable[[str, str], bool]] = None + self._privacy_sanitizer: Optional[Any] = None def set_vlm_resolver(self, resolver: "VLMResolver") -> None: """Set the VLM resolver supplied by the owning service.""" self._vlm_resolver = resolver + def set_privacy_sanitizer(self, sanitizer: Optional[Any]) -> None: + """Install the account service's text sanitizer for direct writes.""" + self._privacy_sanitizer = sanitizer + VikingFS.__module__ = __name__ diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py index 214829b218..9d0e3fd523 100644 --- a/openviking/storage/viking_fs/_ops.py +++ b/openviking/storage/viking_fs/_ops.py @@ -1978,6 +1978,21 @@ async def write_file( await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref) if isinstance(content, str): + sanitizer = getattr(self, "_privacy_sanitizer", None) + if sanitizer is not None and ctx is not None: + result = await sanitizer.sanitize(content, ctx=ctx, uri=uri) + if result.ambiguous_references: + raise InvalidArgumentError( + "Content contains a Privacy value mapped to multiple fields; " + "insert the intended owner-bound reference explicitly", + details={ + "uri": uri, + "candidate_references": [ + list(candidates) for candidates in result.ambiguous_references + ], + }, + ) + content = result.content content = content.encode("utf-8") await self._async_agfs.write( diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py index 2b3512d4bd..679f2eeead 100644 --- a/openviking/utils/resource_processor.py +++ b/openviking/utils/resource_processor.py @@ -20,6 +20,7 @@ from openviking.parse.image_rewrite import rewrite_image_uris from openviking.parse.mode import ParseMode, normalize_parse_mode from openviking.parse.tree_builder import TreeBuilder +from openviking.privacy.sanitizer import PrivacySanitizer from openviking.resource.processing_mode import ( DEFAULT_PROCESSING_MODE, VECTORS_ONLY, @@ -115,6 +116,83 @@ def __init__( self._vlm_processor = None self._media_processor = None self._summarizer = None + self._privacy_sanitizer: PrivacySanitizer | None = None + + def set_privacy_sanitizer(self, sanitizer: PrivacySanitizer | None) -> None: + self._privacy_sanitizer = sanitizer + + async def _sanitize_parse_artifact( + self, + *, + store: Any, + artifact_ref: Any, + doc_rel: str, + target_root_uri: str, + ctx: RequestContext, + ) -> None: + """Rewrite text in a temporary parse artifact before planning or persistence.""" + if self._privacy_sanitizer is None: + return + from openviking.parse.output import read_artifact_manifest, write_artifact_manifest + from openviking.utils.content_hash import content_md5 + from openviking_cli.exceptions import InvalidArgumentError + + base = (doc_rel or "").strip("/") + prefix = f"{base}/" if base else "" + manifest = await read_artifact_manifest(store, artifact_ref) + changed = False + + async def sanitize_file(artifact_path: str, relative: str) -> None: + nonlocal changed + data = await store.read_bytes(artifact_ref, artifact_path) + try: + content = data.decode("utf-8") + except UnicodeDecodeError: + return + uri = ( + target_root_uri.rstrip("/") + if not relative + else f"{target_root_uri.rstrip('/')}/{relative}" + ) + result = await self._privacy_sanitizer.sanitize(content, ctx=ctx, uri=uri) + if result.ambiguous_references: + raise InvalidArgumentError( + "Resource content contains a Privacy value mapped to multiple fields; " + "insert the intended owner-bound reference explicitly", + details={ + "uri": uri, + "candidate_references": [ + list(candidates) for candidates in result.ambiguous_references + ], + }, + ) + if result.content == content: + return + final_bytes = result.content.encode("utf-8") + await store.write_bytes(artifact_ref, artifact_path, final_bytes) + manifest[artifact_path] = content_md5(final_bytes) + changed = True + + async def walk(directory: str) -> None: + for entry in await store.list(artifact_ref, directory): + if entry.is_dir: + await walk(entry.rel_path) + continue + if entry.name.startswith("."): + continue + relative = ( + entry.rel_path[len(prefix) :] + if prefix and entry.rel_path.startswith(prefix) + else entry.rel_path + ) + await sanitize_file(entry.rel_path, relative) + + if getattr(artifact_ref, "root_type", "dir") == "file": + await sanitize_file(base, "") + else: + await walk(base) + if changed: + await write_artifact_manifest(store, artifact_ref, manifest) async def github_token_for( self, @@ -338,6 +416,14 @@ async def _commit_directory_artifact_with_plan( lease_ref=lease_ref, ) telemetry = get_current_telemetry() + await self._sanitize_parse_artifact( + store=output_store, + artifact_ref=artifact_ref, + doc_rel=doc_rel, + target_root_uri=root_uri, + ctx=ctx, + ) + commit_store = _DocRelStore(output_store, doc_rel) artifact_backend = str(getattr(artifact_ref, "backend", "unknown")) update_plan_started_at = time.perf_counter() try: @@ -397,7 +483,7 @@ async def _commit_directory_artifact_with_plan( with telemetry.measure("resource.update_plan.diff_and_compile"): diff, context_plan = await build_context_update_plan_from_snapshot( snapshot=rnfv, - store=_DocRelStore(output_store, doc_rel), + store=commit_store, artifact_ref=artifact_ref, target=target, vikingdb=self.vikingdb, @@ -439,7 +525,7 @@ async def _commit_directory_artifact_with_plan( with telemetry.measure("resource.content_commit"): await execute_content_tree_actions( context_plan.content_tree_actions, - store=_DocRelStore(output_store, doc_rel), + store=commit_store, artifact_ref=artifact_ref, target=target, ) diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.py index d7bd4eb03e..4932ff68de 100644 --- a/openviking/utils/skill_processor.py +++ b/openviking/utils/skill_processor.py @@ -22,6 +22,7 @@ from openviking.core.namespace import canonical_user_root from openviking.core.skill_loader import SkillLoader from openviking.privacy import ( + PrivacySanitizer, UserPrivacyConfigService, extract_skill_privacy_values, ) @@ -112,6 +113,11 @@ def __init__( ) self.vikingdb = vikingdb self._privacy_config_service = privacy_config_service + self._privacy_sanitizer = ( + PrivacySanitizer(privacy_config_service) + if isinstance(privacy_config_service, UserPrivacyConfigService) + else None + ) self._vlm_resolver = vlm_resolver async def process_skill( @@ -485,6 +491,19 @@ async def prepare_skill_privacy( return skill_dict, {} content = skill_dict.get("content", "") + sanitized_by_catalog = await self._sanitize_registered_privacy( + content, + ctx=ctx, + uri=( + f"{canonical_user_root(ctx)}/skills/" + f"{skill_dict.get('name', '')}/SKILL.md" + ), + ) + prepared_skill = skill_dict + if sanitized_by_catalog != content: + prepared_skill = deepcopy(skill_dict) + prepared_skill["content"] = sanitized_by_catalog + content = sanitized_by_catalog extraction_kwargs = { "skill_name": skill_dict.get("name", ""), "skill_description": skill_dict.get("description", ""), @@ -497,12 +516,35 @@ async def prepare_skill_privacy( extraction_kwargs["vlm"] = await self._vlm_resolver.get_vlm(ctx.account_id) extraction_result = await extract_skill_privacy_values(**extraction_kwargs) if not extraction_result.values: - return skill_dict, {} + return prepared_skill, {} - sanitized = deepcopy(skill_dict) + sanitized = deepcopy(prepared_skill) sanitized["content"] = extraction_result.sanitized_content return sanitized, extraction_result.values + async def _sanitize_registered_privacy( + self, + content: str, + *, + ctx: RequestContext, + uri: str, + ) -> str: + if self._privacy_sanitizer is None or not content: + return content + result = await self._privacy_sanitizer.sanitize(content, ctx=ctx, uri=uri) + if result.ambiguous_references: + raise InvalidArgumentError( + "Skill content contains a Privacy value mapped to multiple fields; " + "insert the intended owner-bound reference explicitly", + details={ + "uri": uri, + "candidate_references": [ + list(candidates) for candidates in result.ambiguous_references + ], + }, + ) + return result.content + async def apply_skill_privacy( self, skill_dict: Dict[str, Any], @@ -586,10 +628,26 @@ async def _write_skill_content( """Write main skill content to VikingFS.""" from openviking.storage.abstract_overview import write_abstract_overview - await viking_fs.write_file( - f"{skill_dir_uri}/SKILL.md", + skill_md_uri = f"{skill_dir_uri}/SKILL.md" + serialized = await self._sanitize_registered_privacy( SkillLoader.to_skill_md(skill_dict), ctx=ctx, + uri=skill_md_uri, + ) + abstract = await self._sanitize_registered_privacy( + abstract, + ctx=ctx, + uri=f"{skill_dir_uri}/.abstract.md", + ) + overview = await self._sanitize_registered_privacy( + overview, + ctx=ctx, + uri=f"{skill_dir_uri}/.overview.md", + ) + await viking_fs.write_file( + skill_md_uri, + serialized, + ctx=ctx, lease_ref=lease_ref, ) await write_abstract_overview( @@ -629,9 +687,14 @@ async def _write_auxiliary_files( is_text = False if is_text: + text_content = await self._sanitize_registered_privacy( + file_bytes.decode("utf-8"), + ctx=ctx, + uri=aux_uri, + ) await viking_fs.write_file( aux_uri, - file_bytes.decode("utf-8"), + text_content, ctx=ctx, lease_ref=lease_ref, ) diff --git a/tests/server/test_api_privacy_configs.py b/tests/server/test_api_privacy_configs.py index 9eb818566a..ce34c6b4dc 100644 --- a/tests/server/test_api_privacy_configs.py +++ b/tests/server/test_api_privacy_configs.py @@ -3,6 +3,10 @@ import httpx +from openviking.privacy.reference import build_privacy_field_reference +from openviking.server.identity import RequestContext, Role +from openviking_cli.session.user_id import UserIdentifier + async def test_privacy_config_api_roundtrip(client: httpx.AsyncClient): upsert = await client.post( @@ -120,3 +124,305 @@ async def test_privacy_config_target_not_found(client: httpx.AsyncClient): activate_body = activate.json() assert activate_body["status"] == "error" assert activate_body["error"]["code"] == "NOT_FOUND" + + +async def test_privacy_field_reference_is_safe_by_default_and_reveal_is_explicit( + client: httpx.AsyncClient, +): + created = await client.post( + "/api/v1/privacy-configs/credential/github-prod", + json={"values": {"api/key": "secret-value", "region": "cn"}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + + reference_response = await client.get( + "/api/v1/privacy-configs/credential/github-prod/field", + params={"field": "api/key"}, + ) + assert reference_response.status_code == 200, reference_response.text + reference = reference_response.json()["result"] + assert reference == { + "owner_user_id": owner, + "category": "credential", + "target_key": "github-prod", + "field": "api/key", + "version": 1, + "configured": True, + "reference": ( + f"{{{{ov_privacy:{owner}:viking://~/privacy/credential/" + "github-prod#/values/api%2Fkey}}" + ), + } + assert "value" not in reference + + resolved_response = await client.get( + "/api/v1/privacy-configs/credential/github-prod/field", + params={"field": "api/key", "reveal": True}, + ) + assert resolved_response.status_code == 200, resolved_response.text + resolved = resolved_response.json()["result"] + assert resolved["reference"] == reference["reference"] + assert resolved["value"] == "secret-value" + + missing_response = await client.get( + "/api/v1/privacy-configs/credential/github-prod/field", + params={"field": "missing"}, + ) + assert missing_response.status_code == 404 + assert missing_response.json()["error"]["code"] == "NOT_FOUND" + + +async def test_privacy_config_delete_is_idempotent_and_removes_history( + client: httpx.AsyncClient, +): + for value in ("secret-v1", "secret-v2"): + created = await client.post( + "/api/v1/privacy-configs/credential/delete-me", + json={"values": {"api_key": value}}, + ) + assert created.status_code == 200, created.text + + deleted_response = await client.delete( + "/api/v1/privacy-configs/credential/delete-me" + ) + assert deleted_response.status_code == 200, deleted_response.text + assert deleted_response.json()["result"] == { + "category": "credential", + "target_key": "delete-me", + "deleted": True, + } + + get_response = await client.get("/api/v1/privacy-configs/credential/delete-me") + assert get_response.status_code == 404 + + deleted_again = await client.delete( + "/api/v1/privacy-configs/credential/delete-me" + ) + assert deleted_again.status_code == 200, deleted_again.text + assert deleted_again.json()["result"]["deleted"] is False + + +async def test_privacy_scan_is_dry_run_by_default_and_fix_rewrites_content( + client: httpx.AsyncClient, + service, +): + secret = "legacy-value-to-repair" + created = await client.post( + "/api/v1/privacy-configs/credential/legacy-service", + json={"values": {"custom_field": secret}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + uri = f"viking://user/{owner}/legacy-privacy-content.md" + ctx = RequestContext( + user=UserIdentifier(service.user.account_id, owner), + role=Role.USER, + ) + raw_path = service.viking_fs._uri_to_path(uri, ctx=ctx) + await service.viking_fs._async_agfs.write( + raw_path, + f"before {secret} after".encode("utf-8"), + ) + + dry_run = await client.post( + "/api/v1/privacy-configs/scan", + json={"target_uri": uri}, + ) + assert dry_run.status_code == 200, dry_run.text + report = dry_run.json()["result"] + assert report["dry_run"] is True + assert report["matched_files"] == 1 + assert report["repaired_files"] == 0 + assert report["matches"][0]["uri"] == uri + assert secret not in str(report) + assert secret in await service.viking_fs.read_file(uri, ctx=ctx) + + fixed = await client.post( + "/api/v1/privacy-configs/scan", + json={"target_uri": uri, "fix": True}, + ) + assert fixed.status_code == 200, fixed.text + fixed_report = fixed.json()["result"] + assert fixed_report["repaired_files"] == 1 + stored = await service.viking_fs.read_file(uri, ctx=ctx) + assert secret not in stored + assert build_privacy_field_reference( + owner, "credential", "legacy-service", "custom_field" + ) in stored + + +async def test_privacy_scan_fix_repairs_skill_and_reindexes_package( + client: httpx.AsyncClient, + service, +): + secret = "legacy-skill-registered-value" + created = await client.post( + "/api/v1/privacy-configs/credential/legacy-skill-service", + json={"values": {"custom": secret}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + skill_md = """--- +name: privacy-scan-skill +description: Privacy scan fixture +--- + +# Instructions + +Use no credentials yet. +""" + added = await client.post( + "/api/v1/skills", + json={"data": skill_md, "wait": True}, + ) + assert added.status_code == 200, added.text + skill_uri = f"{added.json()['result']['root_uri']}/SKILL.md" + ctx = RequestContext( + user=UserIdentifier(service.user.account_id, owner), + role=Role.USER, + ) + raw_path = service.viking_fs._uri_to_path(skill_uri, ctx=ctx) + await service.viking_fs._async_agfs.write( + raw_path, + skill_md.replace("Use no credentials yet.", f"credential={secret}").encode("utf-8"), + ) + + fixed = await client.post( + "/api/v1/privacy-configs/scan", + json={"target_uri": skill_uri, "fix": True, "wait": True}, + ) + assert fixed.status_code == 200, fixed.text + report = fixed.json()["result"] + assert report["repaired_files"] == 1 + assert report["reindex"] + stored = await service.viking_fs.read_file(skill_uri, ctx=ctx) + assert secret not in stored + assert build_privacy_field_reference( + owner, + "credential", + "legacy-skill-service", + "custom", + ) in stored + + +async def test_privacy_resolve_accepts_reference_and_field_uri_and_checks_owner( + client: httpx.AsyncClient, +): + created = await client.post( + "/api/v1/privacy-configs/credential/selector-service", + json={"values": {"api/key": "selector-secret"}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + reference = build_privacy_field_reference( + owner, "credential", "selector-service", "api/key" + ) + + for selector in ( + reference, + reference.replace("{{ov_privacy:", "{{ ov_privacy:").replace("}}", " }}"), + "viking://~/privacy/credential/selector-service#/values/api%2Fkey", + f"viking://user/{owner}/privacy/credential/selector-service#/values/api%2Fkey", + ): + resolved = await client.post( + "/api/v1/privacy-configs/resolve", + json={"selector": selector}, + ) + assert resolved.status_code == 200, resolved.text + result = resolved.json()["result"] + assert result["value"] == "selector-secret" + assert result["reference"] == reference + + spaced_reference = reference.replace("{{ov_privacy:", "{{ ov_privacy:").replace( + "}}", " }}" + ) + normalized_delete = await client.request( + "DELETE", + "/api/v1/privacy-configs/selector", + json={"selector": spaced_reference.replace("api%2Fkey", "missing")}, + ) + assert normalized_delete.status_code == 404 + + foreign = reference.replace(f"ov_privacy:{owner}:", "ov_privacy:other-user:") + denied = await client.post( + "/api/v1/privacy-configs/resolve", + json={"selector": foreign}, + ) + assert denied.status_code == 403 + assert denied.json()["error"]["code"] == "PERMISSION_DENIED" + + +async def test_privacy_field_delete_publishes_new_active_snapshot_and_keeps_history( + client: httpx.AsyncClient, +): + created = await client.post( + "/api/v1/privacy-configs/credential/delete-field-service", + json={"values": {"api_key": "secret", "region": "cn"}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + + deleted = await client.delete( + "/api/v1/privacy-configs/credential/delete-field-service/field", + params={"field": "api_key"}, + ) + assert deleted.status_code == 200, deleted.text + result = deleted.json()["result"] + assert result["scope"] == "field" + assert result["target_deleted"] is False + assert result["active_version"] == 2 + + current = ( + await client.get("/api/v1/privacy-configs/credential/delete-field-service") + ).json()["result"]["current"] + assert current["values"] == {"region": "cn"} + old = ( + await client.get( + "/api/v1/privacy-configs/credential/delete-field-service/versions/1" + ) + ).json()["result"] + assert old["values"]["api_key"] == "secret" + + region_reference = build_privacy_field_reference( + owner, "credential", "delete-field-service", "region" + ) + deleted_last = await client.request( + "DELETE", + "/api/v1/privacy-configs/selector", + json={"selector": region_reference}, + ) + assert deleted_last.status_code == 200, deleted_last.text + last_result = deleted_last.json()["result"] + assert last_result["target_deleted"] is True + assert last_result["active_version"] is None + missing_target = await client.get( + "/api/v1/privacy-configs/credential/delete-field-service" + ) + assert missing_target.status_code == 404 + missing_history = await client.get( + "/api/v1/privacy-configs/credential/delete-field-service/versions/1" + ) + assert missing_history.status_code == 404 + + missing = await client.request( + "DELETE", + "/api/v1/privacy-configs/selector", + json={"selector": region_reference}, + ) + assert missing.status_code == 404 + + +async def test_privacy_delete_selector_accepts_target_uri(client: httpx.AsyncClient): + created = await client.post( + "/api/v1/privacy-configs/credential/delete-target-uri", + json={"values": {"key": "value"}}, + ) + assert created.status_code == 200, created.text + deleted = await client.request( + "DELETE", + "/api/v1/privacy-configs/selector", + json={"selector": "viking://~/privacy/credential/delete-target-uri"}, + ) + assert deleted.status_code == 200, deleted.text + assert deleted.json()["result"]["scope"] == "target" diff --git a/tests/server/test_privacy_sanitizer_writes.py b/tests/server/test_privacy_sanitizer_writes.py new file mode 100644 index 0000000000..2df4c9528d --- /dev/null +++ b/tests/server/test_privacy_sanitizer_writes.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +import pytest + +from openviking.privacy.reference import build_privacy_field_reference +from openviking.server.identity import RequestContext, Role +from openviking_cli.exceptions import InvalidArgumentError +from openviking_cli.session.user_id import UserIdentifier + + +async def test_user_and_public_content_writes_replace_registered_privacy_values(service): + ctx = RequestContext(user=service.user, role=Role.USER) + secret = "registered-custom-value" + await service.privacy_configs.upsert( + ctx, + "custom", + "external-system", + {"not_a_conventional_secret_name": secret}, + updated_by=ctx.user.user_id, + ) + reference = build_privacy_field_reference( + ctx.user.user_id, + "custom", + "external-system", + "not_a_conventional_secret_name", + ) + + private_uri = f"viking://user/{ctx.user.user_id}/privacy-sanitized.md" + public_uri = "viking://resources/privacy-sanitized.md" + for uri in (private_uri, public_uri): + await service.fs.write(uri, f"credential={secret}", ctx=ctx, wait=False) + stored = await service.viking_fs.read_file(uri, ctx=ctx) + assert secret not in stored + assert stored == f"credential={reference}" + + +async def test_content_write_rejects_registered_value_with_ambiguous_references(service): + ctx = RequestContext(user=service.user, role=Role.USER) + duplicate = "same-registered-value" + await service.privacy_configs.upsert( + ctx, + "credential", + "first", + {"key": duplicate}, + updated_by=ctx.user.user_id, + ) + await service.privacy_configs.upsert( + ctx, + "credential", + "second", + {"key": duplicate}, + updated_by=ctx.user.user_id, + ) + + with pytest.raises(InvalidArgumentError, match="mapped to multiple fields") as exc_info: + await service.fs.write( + f"viking://user/{ctx.user.user_id}/ambiguous.md", + duplicate, + ctx=ctx, + wait=False, + ) + details = exc_info.value.details + assert details is not None + assert len(details["candidate_references"][0]) == 2 + assert duplicate not in str(details) + + # Legacy content may predate write-time sanitization. The visible read path + # must fail closed without choosing either field or returning the value. + legacy_uri = f"viking://user/{ctx.user.user_id}/legacy-ambiguous.md" + legacy_path = service.viking_fs._uri_to_path(legacy_uri, ctx=ctx) + await service.viking_fs._async_agfs.write(legacy_path, duplicate.encode("utf-8")) + assert await service.fs.read_visible(legacy_uri, ctx=ctx) == "{{ov_privacy:ambiguous}}" + + +async def test_sanitizer_uses_only_the_writing_users_privacy_catalog(service): + owner = RequestContext(user=service.user, role=Role.USER) + other = RequestContext( + user=UserIdentifier(owner.account_id, "other-writer"), + role=Role.USER, + ) + owner_secret = "owner-only-registered-value" + await service.initialize_user_directories(other) + await service.privacy_configs.upsert( + owner, + "credential", + "owner-service", + {"key": owner_secret}, + updated_by=owner.user.user_id, + ) + + uri = "viking://resources/other-writer-content.md" + await service.fs.write(uri, owner_secret, ctx=other, wait=False) + assert await service.viking_fs.read_file(uri, ctx=other) == owner_secret + + +async def test_visible_read_sanitizes_legacy_content_that_bypassed_content_write(service): + ctx = RequestContext(user=service.user, role=Role.USER) + secret = "legacy-registered-value" + await service.privacy_configs.upsert( + ctx, + "credential", + "legacy-service", + {"key": secret}, + updated_by=ctx.user.user_id, + ) + uri = f"viking://user/{ctx.user.user_id}/legacy-content.md" + await service.viking_fs.write_file(uri, f"before {secret} after", ctx=ctx) + + visible = await service.fs.read_visible(uri, ctx=ctx) + assert secret not in visible + assert visible == ( + "before " + + build_privacy_field_reference( + ctx.user.user_id, + "credential", + "legacy-service", + "key", + ) + + " after" + ) + + +async def test_resource_import_sanitizes_parse_artifact_before_persistence(service, tmp_path): + ctx = RequestContext(user=service.user, role=Role.USER) + secret = "resource-import-registered-value" + await service.privacy_configs.upsert( + ctx, + "credential", + "import-service", + {"custom": secret}, + updated_by=ctx.user.user_id, + ) + source = tmp_path / "privacy-resource.md" + source.write_text(f"# Imported\n\ncredential={secret}\n", encoding="utf-8") + + result = await service.resources.add_resource( + path=str(source), + ctx=ctx, + wait=True, + ) + assert result["status"] == "success" + root_uri = result["root_uri"] + stat = await service.viking_fs.stat(root_uri, ctx=ctx, skip_count=True) + uris = [root_uri] + if stat.get("isDir"): + uris = [ + entry["uri"] + for entry in await service.viking_fs.tree( + root_uri, + output="original", + show_all_hidden=True, + node_limit=None, + level_limit=None, + ctx=ctx, + ) + if not entry.get("isDir") and not entry["name"].startswith(".") + ] + stored = "\n".join( + [await service.viking_fs.read_file(uri, ctx=ctx) for uri in uris] + ) + assert secret not in stored + assert build_privacy_field_reference( + ctx.user.user_id, + "credential", + "import-service", + "custom", + ) in stored diff --git a/tests/unit/test_privacy_sanitizer.py b/tests/unit/test_privacy_sanitizer.py new file mode 100644 index 0000000000..1bef435ac8 --- /dev/null +++ b/tests/unit/test_privacy_sanitizer.py @@ -0,0 +1,198 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +from types import SimpleNamespace + +from openviking.privacy.reference import ( + PrivacyFieldReference, + build_privacy_field_reference, + parse_privacy_field_reference, + parse_privacy_field_selector, + parse_privacy_target_selector, +) +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.server.identity import RequestContext, Role +from openviking_cli.session.user_id import UserIdentifier + + +class _PrivacyCatalog: + def __init__(self, configs=None): + self.configs = configs or {} + self.revision = 0 + self.build_reads = 0 + + def catalog_revision(self, _ctx): + return self.revision + + async def list_categories(self, _ctx): + self.build_reads += 1 + return sorted(self.configs) + + async def list_targets(self, _ctx, category): + return sorted(self.configs.get(category, {})) + + async def get_current(self, _ctx, category, target_key): + values = self.configs.get(category, {}).get(target_key) + return None if values is None else SimpleNamespace(values=values) + + +def _ctx(user_id="alice"): + return RequestContext(user=UserIdentifier("account-a", user_id), role=Role.USER) + + +def test_privacy_reference_roundtrip_encodes_individual_segments(): + rendered = build_privacy_field_reference("alice", "custom config", "prod/key", "api/key") + assert rendered == ( + "{{ov_privacy:alice:viking://~/privacy/custom%20config/" + "prod%2Fkey#/values/api%2Fkey}}" + ) + assert parse_privacy_field_reference(rendered) == PrivacyFieldReference( + owner_user_id="alice", + category="custom config", + target_key="prod/key", + field="api/key", + ) + spaced = rendered.replace("{{ov_privacy:", "{{ ov_privacy:").replace("}}", " \t}}") + parsed_spaced = parse_privacy_field_reference(spaced) + assert parsed_spaced is not None + assert parsed_spaced.render() == rendered + assert parse_privacy_field_reference(f"prefix {rendered}") is None + assert parse_privacy_field_reference(rendered.replace(":alice:", ": alice:")) is None + assert parse_privacy_field_reference(rendered.replace("privacy/", "privacy/ ")) is None + assert parse_privacy_field_reference(rendered.replace("{{", "{{\n")) is None + assert parse_privacy_field_selector( + "viking://~/privacy/custom%20config/prod%2Fkey#/values/api%2Fkey", + current_user_id="alice", + ) == PrivacyFieldReference( + owner_user_id="alice", + category="custom config", + target_key="prod/key", + field="api/key", + ) + target = parse_privacy_target_selector( + "viking://user/alice/privacy/custom%20config/prod%2Fkey", + current_user_id="other", + ) + assert target is not None + assert target.owner_user_id == "alice" + assert target.category == "custom config" + assert target.target_key == "prod/key" + + +async def test_sanitizer_protects_registered_strings_at_least_six_chars_without_field_guessing(): + catalog = _PrivacyCatalog( + { + "custom": { + "service": { + "five_chars": "abcde", + "six_chars": "abcdef", + "base_url": "https://private.example", + "empty": "", + "numeric": 42, + } + } + } + ) + sanitizer = PrivacySanitizer(catalog) + + result = await sanitizer.sanitize( + "short=abcde token=abcdef endpoint=https://private.example empty= numeric=42", + ctx=_ctx(), + uri="viking://user/alice/memories/preferences/item.md", + ) + + assert result.changed + assert "short=abcde" in result.content + assert "token={{ov_privacy:alice:viking://~/privacy/custom/service#/values/six_chars}}" in ( + result.content + ) + assert ( + "endpoint={{ov_privacy:alice:viking://~/privacy/custom/service#/values/base_url}}" + in result.content + ) + assert "numeric=42" in result.content + + +async def test_sanitizer_prefers_longest_overlap_and_does_not_replace_ambiguous_values(): + catalog = _PrivacyCatalog( + { + "credential": { + "short": {"value": "abc"}, + "long": {"value": "abcdef"}, + "duplicate-a": {"value": "same-secret"}, + "duplicate-b": {"other": "same-secret"}, + } + } + ) + sanitizer = PrivacySanitizer(catalog) + + result = await sanitizer.sanitize("abcdef same-secret", ctx=_ctx()) + + assert result.content.startswith( + "{{ov_privacy:alice:viking://~/privacy/credential/long#/values/value}}" + ) + assert result.content.endswith(" {{ov_privacy:ambiguous}}") + assert len(result.replacements) == 2 + assert len(result.ambiguous_references) == 1 + assert len(result.ambiguous_references[0]) == 2 + + +async def test_sanitizer_cache_rebuilds_only_after_catalog_revision_changes(): + catalog = _PrivacyCatalog({"credential": {"service": {"key": "first1"}}}) + sanitizer = PrivacySanitizer(catalog) + + first = await sanitizer.sanitize("first1", ctx=_ctx()) + second = await sanitizer.sanitize("first1", ctx=_ctx()) + assert first.changed and second.changed + assert catalog.build_reads == 1 + + catalog.configs["credential"]["service"]["key"] = "second2" + catalog.revision += 1 + rebuilt = await sanitizer.sanitize("first1 second2", ctx=_ctx()) + assert rebuilt.content.startswith("first1 ") + assert rebuilt.content.endswith( + "{{ov_privacy:alice:viking://~/privacy/credential/service#/values/key}}" + ) + assert catalog.build_reads == 2 + + +async def test_sanitizer_cache_expires_after_ttl_even_without_local_revision_change(): + catalog = _PrivacyCatalog({"credential": {"service": {"key": "first1"}}}) + now = [100.0] + sanitizer = PrivacySanitizer( + catalog, + cache_ttl_seconds=3600, + monotonic_clock=lambda: now[0], + ) + + initial = await sanitizer.sanitize("first1", ctx=_ctx()) + assert initial.changed + assert catalog.build_reads == 1 + + # Simulate a change made by another worker: the local revision does not move. + catalog.configs["credential"]["service"]["key"] = "second2" + now[0] += 3599 + stale = await sanitizer.sanitize("first1 second2", ctx=_ctx()) + assert stale.content.endswith(" second2") + assert catalog.build_reads == 1 + + now[0] += 1 + refreshed = await sanitizer.sanitize("first1 second2", ctx=_ctx()) + assert refreshed.content.startswith("first1 ") + assert refreshed.content.endswith( + "{{ov_privacy:alice:viking://~/privacy/credential/service#/values/key}}" + ) + assert catalog.build_reads == 2 + + +async def test_sanitizer_never_rewrites_privacy_storage_itself(): + catalog = _PrivacyCatalog({"credential": {"service": {"key": "secret"}}}) + sanitizer = PrivacySanitizer(catalog) + result = await sanitizer.sanitize( + "secret", + ctx=_ctx(), + uri="viking://user/alice/privacy/credential/service/current.json", + ) + assert result.content == "secret" + assert not result.changed + assert catalog.build_reads == 0 From 414dbf2eb7a2e76647e99de9b5b4aa00404ffc81 Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Fri, 16 Oct 2026 13:50:57 +0800 Subject: [PATCH 2/6] feat(privacy): add native sanitization and field workflows Co-authored-by: TRAE CLI --- Cargo.lock | 1 + crates/ov_cli/src/client.rs | 19 ++ crates/ov_cli/src/commands/privacy.rs | 156 ++++++++---- crates/ov_cli/src/handlers.rs | 6 +- crates/ov_cli/src/help_ui.rs | 12 +- crates/ov_cli/src/main.rs | 138 +++++++---- crates/ragfs-python/src/lib.rs | 8 +- crates/ragfs-python/src/privacy.rs | 54 +++++ crates/ragfs/Cargo.toml | 5 + crates/ragfs/benches/privacy_matcher.rs | 31 +++ crates/ragfs/src/lib.rs | 3 +- crates/ragfs/src/privacy_matcher.rs | 213 +++++++++++++++++ docs/en/api/01-overview.md | 1 + docs/en/api/10-privacy.md | 82 ++++++- docs/en/concepts/13-privacy.md | 33 ++- docs/zh/api/01-overview.md | 1 + docs/zh/api/10-privacy.md | 81 ++++++- docs/zh/concepts/13-privacy.md | 33 ++- openviking/privacy/helpers.py | 4 + openviking/privacy/reference.py | 15 +- openviking/privacy/sanitizer.py | 116 ++------- openviking/privacy/service.py | 144 +++++++---- openviking/privacy/validation.py | 236 +++++++++++++++++++ openviking/pyagfs/__init__.py | 28 ++- openviking/server/routers/privacy_configs.py | 79 ++++++- tests/server/test_api_privacy_configs.py | 105 +++++++++ tests/server/test_privacy_config_service.py | 48 +++- tests/unit/test_privacy_sanitizer.py | 28 ++- tests/unit/test_privacy_validation.py | 71 ++++++ 29 files changed, 1448 insertions(+), 303 deletions(-) create mode 100644 crates/ragfs-python/src/privacy.rs create mode 100644 crates/ragfs/benches/privacy_matcher.rs create mode 100644 crates/ragfs/src/privacy_matcher.rs create mode 100644 openviking/privacy/validation.py create mode 100644 tests/unit/test_privacy_validation.py diff --git a/Cargo.lock b/Cargo.lock index 626c9324f3..299f45c035 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4051,6 +4051,7 @@ name = "ragfs" version = "0.1.0" dependencies = [ "aes-gcm", + "aho-corasick", "anyhow", "async-trait", "aws-config", diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs index dce1549cac..cb2078f9d0 100644 --- a/crates/ov_cli/src/client.rs +++ b/crates/ov_cli/src/client.rs @@ -1755,6 +1755,15 @@ impl HttpClient { self.get(&path, ¶ms).await } + pub async fn privacy_list_fields( + &self, + category: &str, + target_key: &str, + ) -> Result { + let path = format!("/api/v1/privacy-configs/{}/{}/fields", category, target_key); + self.get(&path, &[]).await + } + pub async fn privacy_resolve_selector( &self, selector: &str, @@ -1776,6 +1785,16 @@ impl HttpClient { self.post(&path, body).await } + pub async fn privacy_patch( + &self, + category: &str, + target_key: &str, + body: &serde_json::Value, + ) -> Result { + let path = format!("/api/v1/privacy-configs/{}/{}", category, target_key); + self.patch(&path, body, &[]).await + } + pub async fn privacy_list_versions( &self, category: &str, diff --git a/crates/ov_cli/src/commands/privacy.rs b/crates/ov_cli/src/commands/privacy.rs index f8d13be542..1ad9637df4 100644 --- a/crates/ov_cli/src/commands/privacy.rs +++ b/crates/ov_cli/src/commands/privacy.rs @@ -5,20 +5,20 @@ use serde_json::{Map, Value}; use std::collections::{HashMap, HashSet}; use std::io::{self, IsTerminal, Write}; -fn parse_kv_pairs(pairs: &[String]) -> Result> { +fn parse_field_pairs(pairs: &[String]) -> Result> { let mut parsed: HashMap = HashMap::new(); for pair in pairs { - let (key, value) = pair.split_once('=').ok_or_else(|| { + let (field, value) = pair.split_once('=').ok_or_else(|| { Error::Client(format!( - "Invalid --key format: {} (expected key=value)", + "Invalid --field format: {} (expected field=value)", pair )) })?; - let key = key.trim(); - if key.is_empty() { - return Err(Error::Client("--key key cannot be empty".to_string())); + let field = field.trim(); + if field.is_empty() { + return Err(Error::Client("--field name cannot be empty".to_string())); } - parsed.insert(key.to_string(), value.to_string()); + parsed.insert(field.to_string(), value.to_string()); } Ok(parsed) } @@ -56,7 +56,7 @@ fn parse_values_json( fn to_kv_row(key: &str, value: &Value) -> Value { serde_json::json!({ - "key": key, + "field": key, "value": match value { Value::String(s) => Value::String(s.clone()), _ => Value::String(value.to_string()), @@ -107,8 +107,8 @@ fn extract_kv_cells(rows: &[Value]) -> Vec<(String, String)> { let Some(obj) = row.as_object() else { continue; }; - let key = obj - .get("key") + let field = obj + .get("field") .map(|v| match v { Value::String(s) => s.clone(), _ => v.to_string(), @@ -121,23 +121,23 @@ fn extract_kv_cells(rows: &[Value]) -> Vec<(String, String)> { _ => v.to_string(), }) .unwrap_or_default(); - cells.push((key, value)); + cells.push((field, value)); } cells } fn render_kv_ascii_table(rows: &[Value]) -> String { let cells = extract_kv_cells(rows); - let mut key_width = string_width("key"); + let mut field_width = string_width("field"); let mut value_width = string_width("value"); for (k, v) in &cells { - key_width = key_width.max(string_width(k)); + field_width = field_width.max(string_width(k)); value_width = value_width.max(string_width(v)); } let border = format!( "+{}+{}+", - "-".repeat(key_width + 2), + "-".repeat(field_width + 2), "-".repeat(value_width + 2) ); @@ -146,7 +146,7 @@ fn render_kv_ascii_table(rows: &[Value]) -> String { out.push('\n'); out.push_str(&format!( "| {} | {} |\n", - pad_right("key", key_width), + pad_right("field", field_width), pad_right("value", value_width) )); out.push_str(&border); @@ -155,7 +155,7 @@ fn render_kv_ascii_table(rows: &[Value]) -> String { for (k, v) in cells { out.push_str(&format!( "| {} | {} |\n", - pad_right(&k, key_width), + pad_right(&k, field_width), pad_right(&v, value_width) )); } @@ -251,12 +251,13 @@ pub async fn reference( client: &HttpClient, category: &str, target_key: &str, - field: &str, + field: Option<&str>, output_format: OutputFormat, compact: bool, ) -> Result<()> { + let field = infer_field(client, "reference", category, target_key, field).await?; let result = client - .privacy_get_field(category, target_key, field, false) + .privacy_get_field(category, target_key, &field, false) .await?; output_success(&result, output_format, compact); Ok(()) @@ -272,14 +273,16 @@ pub async fn resolve( ) -> Result<()> { let result = match (target_key, field) { (None, None) => client.privacy_resolve_selector(selector_or_category).await?, - (Some(target_key), Some(field)) => { + (Some(target_key), field) => { + let field = + infer_field(client, "resolve", selector_or_category, target_key, field).await?; client - .privacy_get_field(selector_or_category, target_key, field, true) + .privacy_get_field(selector_or_category, target_key, &field, true) .await? } - _ => { + (None, Some(_)) => { return Err(Error::Client( - "Resolve accepts either one reference/field URI or category target field." + "Resolve accepts either one reference/field URI or category target [field]." .to_string(), )); } @@ -288,6 +291,50 @@ pub async fn resolve( Ok(()) } +async fn infer_field( + client: &HttpClient, + command: &str, + category: &str, + target_key: &str, + field: Option<&str>, +) -> Result { + if let Some(field) = field { + return Ok(field.to_string()); + } + + let result = client.privacy_list_fields(category, target_key).await?; + let entries = result + .get("fields") + .and_then(Value::as_array) + .ok_or_else(|| Error::Parse("Expected fields array in Privacy response".to_string()))?; + let fields = entries + .iter() + .filter_map(|entry| entry.get("field").and_then(Value::as_str)) + .map(ToString::to_string) + .collect::>(); + select_inferred_field(command, category, target_key, fields) +} + +fn select_inferred_field( + command: &str, + category: &str, + target_key: &str, + mut fields: Vec, +) -> Result { + fields.sort(); + + match fields.as_slice() { + [field] => Ok(field.clone()), + [] => Err(Error::Client(format!( + "Privacy target {category}/{target_key} has no active fields." + ))), + _ => Err(Error::Client(format!( + "Privacy target {category}/{target_key} has multiple active fields: {}. Specify one, for example: ov privacy {command} {category} {target_key} ", + fields.join(", ") + ))), + } +} + pub async fn list_versions( client: &HttpClient, category: &str, @@ -432,7 +479,7 @@ pub async fn upsert( target_key: &str, values_json: Option<&str>, values_file: Option<&str>, - key_pairs: &[String], + field_pairs: &[String], change_reason: &str, labels_json: Option<&str>, output_format: OutputFormat, @@ -441,40 +488,21 @@ pub async fn upsert( let parsed_values = parse_values_json(values_json, values_file)?; let mut final_values: Map = parsed_values.clone().unwrap_or_default(); - let key_overrides = parse_kv_pairs(key_pairs)?; + let field_overrides = parse_field_pairs(field_pairs)?; let mut requested_keys: HashSet = HashSet::new(); if let Some(values_from_json) = parsed_values.as_ref() { requested_keys.extend(values_from_json.keys().cloned()); } - requested_keys.extend(key_overrides.keys().cloned()); + requested_keys.extend(field_overrides.keys().cloned()); - if !key_overrides.is_empty() { - let current = client.privacy_get_current(category, target_key).await?; - let mut base_values = current - .get("current") - .and_then(|v| v.get("values")) - .and_then(Value::as_object) - .ok_or_else(|| { - Error::Parse( - "privacy get current response missing current.values object".to_string(), - ) - })? - .clone(); - - if let Some(values_from_json) = parsed_values { - for (key, value) in values_from_json { - base_values.insert(key, value); - } - } - for (key, value) in key_overrides { - base_values.insert(key, Value::String(value)); - } - final_values = base_values; + let partial_update = !field_overrides.is_empty(); + for (field, value) in field_overrides { + final_values.insert(field, Value::String(value)); } if final_values.is_empty() { return Err(Error::Client( - "No values provided; use --values-json/--values-file or --key-{key} " + "No values provided; use --values-json/--values-file, --field , or --field- " .to_string(), )); } @@ -500,9 +528,11 @@ pub async fn upsert( payload["labels"] = labels; } - let result = client - .privacy_upsert(category, target_key, &payload) - .await?; + let result = if partial_update { + client.privacy_patch(category, target_key, &payload).await? + } else { + client.privacy_upsert(category, target_key, &payload).await? + }; if matches!(output_format, OutputFormat::Table) { let values = result .get("values") @@ -531,6 +561,30 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn inferred_field_requires_an_unambiguous_target() { + let one = vec!["pat".to_string()]; + assert_eq!( + select_inferred_field("reference", "credential", "github", one) + .expect("one field should be inferred"), + "pat" + ); + + let multiple = vec!["region".to_string(), "pat".to_string()]; + let error = select_inferred_field("resolve", "credential", "github", multiple) + .expect_err("multiple fields must not be guessed") + .to_string(); + assert!(error.contains("multiple active fields: pat, region")); + assert!(error.contains("ov privacy resolve credential github ")); + } + + #[test] + fn privacy_table_uses_field_header() { + let rendered = render_kv_ascii_table(&[json!({"field": "pat", "value": "secret"})]); + assert!(rendered.contains("| field | value |")); + assert!(!rendered.contains("| key ")); + } + #[test] fn version_like_table_output_keeps_profile_section() { let result = json!({ diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs index a079d0b106..64e88d0c72 100644 --- a/crates/ov_cli/src/handlers.rs +++ b/crates/ov_cli/src/handlers.rs @@ -829,7 +829,7 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> &client, &category, &target_key, - &field, + field.as_deref(), ctx.output_format, ctx.compact, ) @@ -855,7 +855,7 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> target_key, values_json, values_file, - key, + field, change_reason, labels_json, } => { @@ -865,7 +865,7 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> &target_key, values_json.as_deref(), values_file.as_deref(), - &key, + &field, &change_reason, labels_json.as_deref(), ctx.output_format, diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs index ddeeaa0f73..6ff36207e9 100644 --- a/crates/ov_cli/src/help_ui.rs +++ b/crates/ov_cli/src/help_ui.rs @@ -783,12 +783,16 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ description: "Show active values for one target.", }, HelpItem { - label: "ov privacy reference ", - description: "Show the owner-bound reference without revealing the value.", + label: "ov privacy reference [field]", + description: "Show a reference; infer the field only when exactly one exists.", }, HelpItem { - label: "ov privacy resolve ", - description: "Explicitly reveal the referenced active field value.", + label: "ov privacy resolve [field]", + description: "Reveal a field; infer it only when exactly one exists.", + }, + HelpItem { + label: "ov privacy upsert --field- ", + description: "Update one field; --key and --key- remain compatible.", }, HelpItem { label: "ov privacy scan --scope resources,skills,memories", diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs index 65a616f309..943ff84f80 100644 --- a/crates/ov_cli/src/main.rs +++ b/crates/ov_cli/src/main.rs @@ -1968,7 +1968,7 @@ enum PrivacyCommands { target_key: String, /// Field in the current values object #[arg(value_name = "field")] - field: String, + field: Option, }, /// Reveal one active field value for immediate use Resolve { @@ -2000,9 +2000,9 @@ enum PrivacyCommands { value_name = "path" )] values_file: Option, - /// Existing key updates in key=value format (repeatable) - #[arg(long = "key", value_name = "key=value")] - key: Vec, + /// Field updates in field=value format (repeatable; --key is a compatibility alias) + #[arg(long = "field", alias = "key", value_name = "field=value")] + field: Vec, /// Change reason #[arg(long, default_value = "", value_name = "text")] change_reason: String, @@ -2897,7 +2897,7 @@ fn preprocess_privacy_get_shortcut(args: Vec) -> Vec { out } -fn preprocess_privacy_upsert_key_flags(args: Vec) -> Vec { +fn preprocess_privacy_upsert_field_flags(args: Vec) -> Vec { let Some(cmd_idx) = find_command_index(&args) else { return args; }; @@ -2923,8 +2923,11 @@ fn preprocess_privacy_upsert_key_flags(args: Vec) -> Vec { continue; } - if i > cmd_idx + 1 && arg_lossy.starts_with("--key-") { - let suffix = &arg_lossy[6..]; + let dynamic_field = arg_lossy + .strip_prefix("--field-") + .or_else(|| arg_lossy.strip_prefix("--key-")); + if i > cmd_idx + 1 && dynamic_field.is_some() { + let suffix = dynamic_field.expect("checked above"); if suffix.is_empty() { converted.push(args[i].clone()); i += 1; @@ -2932,7 +2935,7 @@ fn preprocess_privacy_upsert_key_flags(args: Vec) -> Vec { } if let Some((key, value)) = suffix.split_once('=') { - converted.push(OsString::from("--key")); + converted.push(OsString::from("--field")); converted.push(OsString::from(format!("{}={}", key, value))); i += 1; continue; @@ -2940,7 +2943,7 @@ fn preprocess_privacy_upsert_key_flags(args: Vec) -> Vec { if i + 1 < args.len() { let next_val = args[i + 1].to_string_lossy(); - converted.push(OsString::from("--key")); + converted.push(OsString::from("--field")); converted.push(OsString::from(format!("{}={}", suffix, next_val))); i += 2; continue; @@ -3013,7 +3016,7 @@ fn pre_parse_output_options(args: &[OsString]) -> (OutputFormat, bool) { fn preprocess_privacy_args(args: Vec) -> Vec { let args = preprocess_privacy_get_shortcut(args); - preprocess_privacy_upsert_key_flags(args) + preprocess_privacy_upsert_field_flags(args) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -6011,35 +6014,37 @@ mod tests { } #[test] - fn preprocess_key_dynamic_flag_to_static_form() { - let args = vec![ - OsString::from("ov"), - OsString::from("privacy"), - OsString::from("upsert"), - OsString::from("skill"), - OsString::from("demo"), - OsString::from("--key-api_key"), - OsString::from("secret-v1"), - ]; + fn preprocess_field_dynamic_flags_to_static_form() { + for dynamic_flag in ["--field-api_key", "--key-api_key"] { + let args = vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("upsert"), + OsString::from("skill"), + OsString::from("demo"), + OsString::from(dynamic_flag), + OsString::from("secret-v1"), + ]; - let converted = preprocess_privacy_args(args); - let converted_strs: Vec = converted - .into_iter() - .map(|s| s.to_string_lossy().to_string()) - .collect(); + let converted = preprocess_privacy_args(args); + let converted_strs: Vec = converted + .into_iter() + .map(|s| s.to_string_lossy().to_string()) + .collect(); - assert_eq!( - converted_strs, - vec![ - "ov", - "privacy", - "upsert", - "skill", - "demo", - "--key", - "api_key=secret-v1", - ] - ); + assert_eq!( + converted_strs, + vec![ + "ov", + "privacy", + "upsert", + "skill", + "demo", + "--field", + "api_key=secret-v1", + ] + ); + } } fn on_cli_parser_stack(test: impl FnOnce() + Send + 'static) { @@ -6052,7 +6057,7 @@ mod tests { } #[test] - fn cli_parses_privacy_upsert_with_key_dynamic_flag() { + fn cli_parses_privacy_upsert_with_field_dynamic_flag() { on_cli_parser_stack(|| { let cli = Cli::parse_from(preprocess_privacy_args(vec![ OsString::from("ov"), @@ -6060,7 +6065,7 @@ mod tests { OsString::from("upsert"), OsString::from("skill"), OsString::from("demo"), - OsString::from("--key-api_key"), + OsString::from("--field-api_key"), OsString::from("secret-v2"), ])); @@ -6069,17 +6074,38 @@ mod tests { PrivacyCommands::Upsert { category, target_key, - key, + field, .. } => { assert_eq!(category, "skill"); assert_eq!(target_key, "demo"); - assert_eq!(key, vec!["api_key=secret-v2"]); + assert_eq!(field, vec!["api_key=secret-v2"]); } _ => panic!("expected privacy upsert"), }, _ => panic!("expected privacy command"), } + + for flag in ["--field", "--key"] { + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from("upsert"), + OsString::from("skill"), + OsString::from("demo"), + OsString::from(flag), + OsString::from("api_key=secret-v3"), + ])); + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Upsert { field, .. } => { + assert_eq!(field, vec!["api_key=secret-v3"]); + } + _ => panic!("expected privacy upsert"), + }, + _ => panic!("expected privacy command"), + } + } }); } @@ -6106,7 +6132,7 @@ mod tests { assert_eq!(subcommand, "reference"); assert_eq!(category, "credential"); assert_eq!(target_key, "github-prod"); - assert_eq!(field, expected_field); + assert_eq!(field.as_deref(), Some(expected_field)); } PrivacyCommands::Resolve { selector_or_category, @@ -6124,6 +6150,34 @@ mod tests { } } + for subcommand in ["reference", "resolve"] { + let cli = Cli::parse_from(preprocess_privacy_args(vec![ + OsString::from("ov"), + OsString::from("privacy"), + OsString::from(subcommand), + OsString::from("credential"), + OsString::from("github-prod"), + ])); + + match cli.command { + Commands::Privacy { action } => match *action { + PrivacyCommands::Reference { field, .. } => { + assert_eq!(subcommand, "reference"); + assert!(field.is_none()); + } + PrivacyCommands::Resolve { + target_key, field, .. + } => { + assert_eq!(subcommand, "resolve"); + assert_eq!(target_key.as_deref(), Some("github-prod")); + assert!(field.is_none()); + } + _ => panic!("expected privacy field command"), + }, + _ => panic!("expected privacy command"), + } + } + let scan = Cli::parse_from(preprocess_privacy_args(vec![ OsString::from("ov"), OsString::from("privacy"), diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs index de9219683c..badde19e44 100644 --- a/crates/ragfs-python/src/lib.rs +++ b/crates/ragfs-python/src/lib.rs @@ -919,6 +919,7 @@ fn cache_policy_from_config(config: &RagfsCacheConfig) -> CachePolicy { } mod git; +mod privacy; fn py_detach_blocking(py: Python<'_>, f: F) -> T where @@ -3035,6 +3036,7 @@ impl RAGFSBindingClient { #[pymodule] fn ragfs_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(reopen_tracing_file, m)?)?; // Import the Python business exception so native code raises the same type @@ -3469,7 +3471,8 @@ mod tests { ) .unwrap(); - let cache_config = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap(); + let cache_config = + cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap(); let stack_config = cache_config.stack_config().unwrap(); assert!(cache_config.runtime_enabled); @@ -3668,7 +3671,8 @@ mod tests { ) .unwrap(); - let error = cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err(); + let error = + cache_config_from_ov_conf(path.to_str().unwrap(), "test-service").unwrap_err(); assert!(error.contains(field)); fs::remove_file(path).unwrap(); diff --git a/crates/ragfs-python/src/privacy.rs b/crates/ragfs-python/src/privacy.rs new file mode 100644 index 0000000000..9d5566b414 --- /dev/null +++ b/crates/ragfs-python/src/privacy.rs @@ -0,0 +1,54 @@ +//! Thin PyO3 wrapper around RAGFS's compiled Privacy matcher. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use ragfs::privacy_matcher::PrivacyMatcher; +use std::collections::HashMap; +use std::sync::Arc; + +/// Compiled multi-pattern matcher reused across Python sanitizer calls. +#[pyclass(name = "PrivacyMatcher", module = "ragfs_python")] +pub(crate) struct PyPrivacyMatcher { + inner: Arc, +} + +#[pymethods] +impl PyPrivacyMatcher { + /// Build one immutable matcher from `value -> candidate references`. + #[new] + #[pyo3(signature = (references_by_value, ambiguous_placeholder="{{ov_privacy:ambiguous}}"))] + fn new( + py: Python<'_>, + references_by_value: HashMap>, + ambiguous_placeholder: &str, + ) -> PyResult { + let entries = references_by_value.into_iter().collect(); + let ambiguous_placeholder = ambiguous_placeholder.to_string(); + let matcher = py + .detach(move || PrivacyMatcher::new(entries, ambiguous_placeholder)) + .map_err(PyValueError::new_err)?; + Ok(Self { + inner: Arc::new(matcher), + }) + } + + /// Return `(content, replacements, ambiguous_references)`. + /// + /// Replacement positions are Unicode character offsets, matching Python + /// string slicing rather than UTF-8 byte offsets. The GIL is released while + /// RAGFS scans and rewrites the input. + fn sanitize( + &self, + py: Python<'_>, + content: String, + ) -> (String, Vec<(usize, usize, String)>, Vec>) { + let matcher = self.inner.clone(); + let result = py.detach(move || matcher.sanitize(&content)); + let replacements = result + .replacements + .into_iter() + .map(|replacement| (replacement.start, replacement.end, replacement.reference)) + .collect(); + (result.content, replacements, result.ambiguous_references) + } +} diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml index b941490a7c..393643c553 100644 --- a/crates/ragfs/Cargo.toml +++ b/crates/ragfs/Cargo.toml @@ -33,6 +33,10 @@ name = "dynamic_provider" path = "tests/dynamic_provider.rs" required-features = ["cache"] +[[bench]] +name = "privacy_matcher" +harness = false + [dependencies] # Async runtime tokio = { version = "1.38", features = ["full"] } @@ -62,6 +66,7 @@ filelocks = "0.1.0" # Data structures radix_trie = "0.2" +aho-corasick = "1.1" xxhash-rust = { version = "0.8", features = ["xxh3"] } # Error handling diff --git a/crates/ragfs/benches/privacy_matcher.rs b/crates/ragfs/benches/privacy_matcher.rs new file mode 100644 index 0000000000..7b919ed7ce --- /dev/null +++ b/crates/ragfs/benches/privacy_matcher.rs @@ -0,0 +1,31 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use ragfs::privacy_matcher::PrivacyMatcher; + +fn benchmark_privacy_matcher(c: &mut Criterion) { + let mut group = c.benchmark_group("privacy_matcher"); + for pattern_count in [100usize, 1_000] { + let entries: Vec<_> = (0..pattern_count) + .map(|index| { + ( + format!("mock-secret-{index:06}-value"), + vec![format!("{{{{privacy-ref-{index}}}}}")], + ) + }) + .collect(); + let matcher = PrivacyMatcher::new(entries, "{{ov_privacy:ambiguous}}").unwrap(); + let mut content = "ordinary context without a registered value\n".repeat(20_000); + content.push_str("credential=mock-secret-000042-value\n"); + group.throughput(Throughput::Bytes(content.len() as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(pattern_count), + &content, + |bench, input| { + bench.iter(|| matcher.sanitize(black_box(input))); + }, + ); + } + group.finish(); +} + +criterion_group!(benches, benchmark_privacy_matcher); +criterion_main!(benches); diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rs index 0b81357be9..54ff3f5286 100644 --- a/crates/ragfs/src/lib.rs +++ b/crates/ragfs/src/lib.rs @@ -35,11 +35,12 @@ pub mod cache; pub mod cache_runtime; pub mod core; pub mod crypto; +pub mod git; pub mod lock; pub mod metrics; pub mod multibackend; -pub mod git; pub mod plugins; +pub mod privacy_matcher; pub mod shape; // Re-export core types for convenience diff --git a/crates/ragfs/src/privacy_matcher.rs b/crates/ragfs/src/privacy_matcher.rs new file mode 100644 index 0000000000..590f1ca6b2 --- /dev/null +++ b/crates/ragfs/src/privacy_matcher.rs @@ -0,0 +1,213 @@ +//! High-throughput replacement of registered sensitive values in UTF-8 text. +//! +//! This module deliberately knows nothing about OpenViking users, Privacy +//! paths, or authorization. Callers provide opaque patterns and replacement +//! references; RAGFS owns only automaton construction and text rewriting. + +use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; +use std::collections::HashSet; + +/// Replacement emitted for one non-overlapping leftmost-longest match. +#[derive(Clone, PartialEq, Eq)] +pub struct PrivacyReplacement { + /// Match start as a Unicode character offset. + pub start: usize, + /// Match end as an exclusive Unicode character offset. + pub end: usize, + /// Opaque reference or ambiguity placeholder inserted into the output. + pub reference: String, +} + +/// Sanitized text and metadata for its replacements. +#[derive(Clone, PartialEq, Eq)] +pub struct PrivacySanitizeResult { + /// Rewritten UTF-8 text. + pub content: String, + /// Non-overlapping replacements in source order. + pub replacements: Vec, + /// Reference candidates for ambiguous values that occurred in the input. + pub ambiguous_references: Vec>, +} + +#[derive(Clone)] +enum PatternTarget { + Reference(String), + Ambiguous(Vec), +} + +/// Immutable Aho-Corasick matcher for one Privacy catalog revision. +pub struct PrivacyMatcher { + matcher: Option, + targets: Vec, + ambiguous_placeholder: String, +} + +impl PrivacyMatcher { + /// Build a matcher from `(sensitive value, candidate references)` entries. + /// + /// Candidate references are sorted and deduplicated. One candidate produces + /// a normal replacement; multiple candidates produce the supplied ambiguity + /// placeholder and are reported in the result. Empty patterns and entries + /// without references are rejected. + pub fn new( + entries: Vec<(String, Vec)>, + ambiguous_placeholder: impl Into, + ) -> Result { + let mut patterns = Vec::with_capacity(entries.len()); + let mut targets = Vec::with_capacity(entries.len()); + let mut seen_patterns = HashSet::with_capacity(entries.len()); + + for (pattern, mut references) in entries { + if pattern.is_empty() { + return Err("privacy match pattern must not be empty".to_string()); + } + if !seen_patterns.insert(pattern.clone()) { + return Err("privacy match patterns must be unique".to_string()); + } + references.sort_unstable(); + references.dedup(); + if references.is_empty() { + return Err("privacy match pattern requires at least one reference".to_string()); + } + patterns.push(pattern); + targets.push(if references.len() == 1 { + PatternTarget::Reference(references.pop().expect("one reference remains")) + } else { + PatternTarget::Ambiguous(references) + }); + } + + let matcher = if patterns.is_empty() { + None + } else { + Some( + AhoCorasickBuilder::new() + .match_kind(MatchKind::LeftmostLongest) + .build(patterns) + .map_err(|err| format!("failed to build Privacy matcher: {err}"))?, + ) + }; + Ok(Self { + matcher, + targets, + ambiguous_placeholder: ambiguous_placeholder.into(), + }) + } + + /// Rewrite all non-overlapping leftmost-longest matches. + pub fn sanitize(&self, content: &str) -> PrivacySanitizeResult { + let Some(matcher) = &self.matcher else { + return PrivacySanitizeResult { + content: content.to_string(), + replacements: Vec::new(), + ambiguous_references: Vec::new(), + }; + }; + + let matches: Vec<_> = matcher.find_iter(content).collect(); + if matches.is_empty() { + return PrivacySanitizeResult { + content: content.to_string(), + replacements: Vec::new(), + ambiguous_references: Vec::new(), + }; + } + + let extra_capacity = matches.len().saturating_mul(32); + let mut rewritten = String::with_capacity(content.len().saturating_add(extra_capacity)); + let mut replacements = Vec::with_capacity(matches.len()); + let mut ambiguous_references = Vec::new(); + let mut source_byte_cursor = 0; + let mut source_char_cursor = 0; + + for matched in matches { + let start = matched.start(); + let end = matched.end(); + rewritten.push_str(&content[source_byte_cursor..start]); + source_char_cursor += content[source_byte_cursor..start].chars().count(); + let char_start = source_char_cursor; + source_char_cursor += content[start..end].chars().count(); + let char_end = source_char_cursor; + + let target = &self.targets[matched.pattern().as_usize()]; + let reference = match target { + PatternTarget::Reference(reference) => reference.clone(), + PatternTarget::Ambiguous(candidates) => { + ambiguous_references.push(candidates.clone()); + self.ambiguous_placeholder.clone() + } + }; + rewritten.push_str(&reference); + replacements.push(PrivacyReplacement { + start: char_start, + end: char_end, + reference, + }); + source_byte_cursor = end; + } + rewritten.push_str(&content[source_byte_cursor..]); + + PrivacySanitizeResult { + content: rewritten, + replacements, + ambiguous_references, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matcher(entries: Vec<(&str, Vec<&str>)>) -> PrivacyMatcher { + PrivacyMatcher::new( + entries + .into_iter() + .map(|(value, refs)| { + ( + value.to_string(), + refs.into_iter().map(str::to_string).collect(), + ) + }) + .collect(), + "{{ov_privacy:ambiguous}}", + ) + .unwrap() + } + + #[test] + fn uses_leftmost_longest_and_unicode_character_offsets() { + let matcher = matcher(vec![ + ("秘密", vec!["short"]), + ("秘密值", vec!["long"]), + ("token", vec!["token-ref"]), + ]); + let result = matcher.sanitize("前缀秘密值 token"); + assert_eq!(result.content, "前缀long token-ref"); + assert_eq!(result.replacements.len(), 2); + assert_eq!( + (result.replacements[0].start, result.replacements[0].end), + (2, 5) + ); + assert_eq!( + (result.replacements[1].start, result.replacements[1].end), + (6, 11) + ); + } + + #[test] + fn reports_ambiguity_without_exposing_pattern() { + let matcher = matcher(vec![("secret", vec!["ref-b", "ref-a", "ref-a"])]); + let result = matcher.sanitize("x secret y"); + assert_eq!(result.content, "x {{ov_privacy:ambiguous}} y"); + assert_eq!(result.ambiguous_references, vec![vec!["ref-a", "ref-b"]]); + } + + #[test] + fn empty_catalog_is_a_noop() { + let matcher = PrivacyMatcher::new(Vec::new(), "ambiguous").unwrap(); + let result = matcher.sanitize("unchanged"); + assert_eq!(result.content, "unchanged"); + assert!(result.replacements.is_empty()); + } +} diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md index fbe4b7a161..bd47dc1bc9 100644 --- a/docs/en/api/01-overview.md +++ b/docs/en/api/01-overview.md @@ -528,6 +528,7 @@ This catalog follows the routes actually mounted by the server. Each group headi | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | List configuration versions | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | Get one version | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | Write and activate a new version | +| PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | Atomically merge fields; create if missing | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | Activate a version | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | Delete a target and every version | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | Remove the active field | diff --git a/docs/en/api/10-privacy.md b/docs/en/api/10-privacy.md index dd8b82b4f6..7cf39619d0 100644 --- a/docs/en/api/10-privacy.md +++ b/docs/en/api/10-privacy.md @@ -16,9 +16,11 @@ Activating a version changes only the values stored in OpenViking. It does not r | POST | `/api/v1/privacy-configs/scan` | Scan or repair registered values in stored content | | GET | `/api/v1/privacy-configs/{category}` | List targets under a category | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | Get active config (`meta + current`) | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | List safe active-field metadata without values | | GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | Get a field reference; optionally reveal its value | | POST | `/api/v1/privacy-configs/resolve` | Explicitly reveal a field by reference or URI | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | Upsert and activate a new/current version | +| PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | Atomically merge fields; create a missing target | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | Delete a target and every version | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | Remove a field from the active config | | DELETE | `/api/v1/privacy-configs/selector` | Delete by target URI or field reference/URI | @@ -96,6 +98,23 @@ Use either `scopes` or `target_uri`. When one value maps to multiple Privacy fie } ``` +### Input format and capacity limits + +The server applies the same validation to HTTP, CLI, Embedded, and internal writes: + +| Item | Limit | +|------|------| +| `category` / `target_key` | Non-empty, at most 255 ASCII characters; only letters, digits, `_`, `-`, `.`, and `@`; `.` and `..` are forbidden and at most one `@` is allowed | +| Top-level values fields | At most 128 | +| Field name | Non-empty, at most 255 UTF-8 bytes, no control characters | +| Individual string | At most 64 KiB UTF-8 | +| Serialized values | At most 1 MiB | +| JSON nesting | At most 16 levels; at most 4096 items in an individual object or array | +| labels | At most 64 entries and 64 KiB serialized | +| `change_reason` | At most 1024 characters | + +Violations return `INVALID_ARGUMENT`. Error `details` identify the field, reason, actual size, and limit without including the sensitive value itself. + --- ## API Reference @@ -196,6 +215,35 @@ curl -X GET "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-sea --- +### list_privacy_fields() + +List active field names, configuration status, and owner-bound references without returning plaintext values. The CLI uses this endpoint when `reference` or the category form of `resolve` omits the field. It infers the field only when exactly one exists; multiple fields are listed to the user and require an explicit choice. + +```text +GET /api/v1/privacy-configs/{category}/{target_key}/fields +``` + +```json +{ + "status": "ok", + "result": { + "owner_user_id": "alice", + "category": "credential", + "target_key": "github-prod", + "version": 1, + "fields": [ + { + "field": "pat", + "configured": true, + "reference": "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/pat}}" + } + ] + } +} +``` + +--- + ### get_privacy_field() Get one field from the active version. By default the endpoint returns an owner-bound reference and configuration status without returning the value. Add `reveal=true` to explicitly read that one field for immediate use. @@ -299,6 +347,29 @@ curl -X POST "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-se --- +### patch_privacy_config() + +Merge `values` into the current configuration while holding the server-side target lock. A missing target starts from an empty object and creates version 1. An existing target keeps untouched fields and creates a new version. Read, merge, and write happen under the same lock, so concurrent updates to distinct fields do not suffer from client-side read-modify-write loss. + +```text +PATCH /api/v1/privacy-configs/{category}/{target_key} +``` + +```json +{ + "values": {"pat": "github_pat_..."}, + "change_reason": "configure GitHub PAT" +} +``` + +Canonical CLI `--field` and `--field-*` flags use this endpoint without reading current values in the CLI. `--key` and `--key-*` remain backward-compatible aliases: + +```bash +ov privacy upsert credential github --field-pat 'github_pat_...' +``` + +--- + ### delete_privacy_config() Delete a target's current value, metadata, and complete version history. Repeated deletion is safe: a missing target returns `deleted: false`. @@ -455,10 +526,13 @@ ov privacy list skill ov privacy get skill byted-viking-search-knowledgebase ov privacy skill byted-viking-search-knowledgebase -# Get the owner-bound reference without returning the value +# Get the owner-bound reference without returning the value. +# The field may be omitted only when the target has exactly one field. +ov privacy reference credential github-prod ov privacy reference credential github-prod api_key -# Explicitly reveal one field in the active version +# Explicitly reveal one field in the active version (same inference rule) +ov privacy resolve credential github-prod ov privacy resolve credential github-prod api_key ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' ov privacy resolve 'viking://~/privacy/credential/github-prod#/values/api_key' @@ -470,9 +544,9 @@ ov privacy upsert skill byted-viking-search-knowledgebase \ # Read values from a local JSON file instead of command arguments ov privacy upsert skill byted-viking-search-knowledgebase --values-file ./privacy-values.json -# Partial key update (CLI merges with current first) +# Update one field; creates a missing target, otherwise merges atomically on the server ov privacy upsert skill byted-viking-search-knowledgebase \ - --key-api_key secret-3 + --field-api_key secret-3 # Version query and activation ov privacy versions skill byted-viking-search-knowledgebase diff --git a/docs/en/concepts/13-privacy.md b/docs/en/concepts/13-privacy.md index e4b846668c..8bd059808f 100644 --- a/docs/en/concepts/13-privacy.md +++ b/docs/en/concepts/13-privacy.md @@ -128,23 +128,30 @@ Current matching is suffix-based: `/skills/{name}/SKILL.md`, so it supports user - Content plane: `read` automatically restores placeholders - Separation of concerns: content files hold placeholders; privacy service holds sensitive values and versions -PrivacySanitizer does not infer sensitivity from field names. String values in the active Privacy catalog participate in automatic content scanning and reference replacement only when they contain at least 6 Unicode characters. Shorter values can still be stored, referenced, and explicitly resolved, but are excluded from global substring replacement to reduce false positives in ordinary text. The matcher catalog is cached per user and process-local revision; local changes invalidate it immediately, and a 3600-second TTL bounds staleness after changes made by another worker or node. +PrivacySanitizer does not infer sensitivity from field names. String values in the active Privacy catalog participate in automatic content scanning and reference replacement only when they contain at least 6 Unicode characters. Shorter values can still be stored, referenced, and explicitly resolved, but are excluded from global substring replacement to reduce false positives in ordinary text. Python owns catalog loading, reference generation, and cache policy; matcher construction, leftmost-longest matching, and text rewriting run in RAGFS's Rust Aho-Corasick implementation through a reusable PyO3 matcher that releases the GIL while scanning. The matcher catalog is cached per user and process-local revision; local changes invalidate it immediately, and a 3600-second TTL bounds staleness after changes made by another worker or node. -Common commands: +Using a GitHub Personal Access Token (PAT) as an example: ```bash -ov privacy categories -ov privacy list skill -ov privacy skill search-web -ov privacy reference skill search-web api_key -ov privacy resolve skill search-web api_key -ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/skill/search-web#/values/api_key}}' -ov privacy upsert skill search-web --values-json '{"api_key":"..."}' -ov privacy activate skill search-web 1 -ov privacy delete skill search-web api_key --yes -ov privacy delete skill search-web --yes +# Create or rotate the PAT; credential/github is the target and pat is the field +ov privacy upsert credential github --field-pat 'github_pat_...' + +# Omit the field when pat is the target's only field +ov privacy reference credential github +ov privacy resolve credential github + +# Equivalent explicit-field forms +ov privacy reference credential github pat +ov privacy resolve credential github pat +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github#/values/pat}}' + +# Inspect/activate history, or remove pat (the final field also removes the target) +ov privacy versions credential github +ov privacy activate credential github 1 +ov privacy delete credential github pat --yes + +# Scan persisted content; the default only reports, while --fix replaces plaintext PATs ov privacy scan --scope resources,skills,memories -ov read viking://~/skills/search-web/SKILL.md ``` --- diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md index b9a5348894..e5598a6b8b 100644 --- a/docs/zh/api/01-overview.md +++ b/docs/zh/api/01-overview.md @@ -523,6 +523,7 @@ JSON 输出 - 错误： | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | 列出配置版本 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | 获取指定版本 | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | 写入并激活新版本 | +| PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | 原子合并字段；不存在时创建 | | POST | `/api/v1/privacy-configs/{category}/{target_key}/activate` | 激活指定版本 | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | 删除目标及全部版本 | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | 删除当前字段 | diff --git a/docs/zh/api/10-privacy.md b/docs/zh/api/10-privacy.md index 1a84719543..9b4f254d4a 100644 --- a/docs/zh/api/10-privacy.md +++ b/docs/zh/api/10-privacy.md @@ -25,9 +25,11 @@ | POST | `/api/v1/privacy-configs/scan` | 扫描或修复已存储正文中的登记值 | | GET | `/api/v1/privacy-configs/{category}` | 列出分类下目标 | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | 获取当前生效配置（meta + current） | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | 列出不含明文值的安全字段元数据 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | 获取单字段引用；显式选择时返回值 | | POST | `/api/v1/privacy-configs/resolve` | 按字段 reference/URI 显式读取值 | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | 写入新版本并激活 | +| PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | 原子合并字段；目标不存在时创建 | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | 删除目标及全部版本 | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | 删除当前生效配置中的字段 | | DELETE | `/api/v1/privacy-configs/selector` | 按 target URI 或字段 reference/URI 删除 | @@ -74,6 +76,23 @@ } ``` +### 输入格式与容量限制 + +服务端对 HTTP、CLI、Embedded 和内部写入执行同一套验证： + +| 项目 | 限制 | +|------|------| +| `category` / `target_key` | 非空、最多 255 个 ASCII 字符；仅允许字母、数字、`_`、`-`、`.`、`@`；禁止 `.`、`..`，`@` 最多一个 | +| values 顶层字段 | 最多 128 个 | +| 字段名 | 非空、最多 255 UTF-8 字节、禁止控制字符 | +| 单个字符串值 | 最多 64 KiB UTF-8 | +| values 序列化总量 | 最多 1 MiB | +| JSON 嵌套 | 最多 16 层；单个对象或数组最多 4096 项 | +| labels | 最多 64 项，序列化总量最多 64 KiB | +| `change_reason` | 最多 1024 个字符 | + +违反限制时返回 `INVALID_ARGUMENT`，`details` 包含字段、原因、实际大小和允许上限。错误信息不会包含敏感值正文。 + --- ## API 参考 @@ -196,6 +215,35 @@ curl -X GET "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-sea --- +### list_privacy_fields() + +列出当前生效字段的名称、配置状态和 owner-bound reference，不返回任何明文值。CLI 在 `reference` 或 `resolve` 的 category 形式省略字段时使用该接口：仅当 target 恰好有一个字段时自动推断；存在多个字段时列出可选项并要求显式指定。 + +```text +GET /api/v1/privacy-configs/{category}/{target_key}/fields +``` + +```json +{ + "status": "ok", + "result": { + "owner_user_id": "alice", + "category": "credential", + "target_key": "github-prod", + "version": 1, + "fields": [ + { + "field": "pat", + "configured": true, + "reference": "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/pat}}" + } + ] + } +} +``` + +--- + ### get_privacy_field() 获取当前生效版本中的单个字段。默认只返回带 owner 的引用和配置状态，不返回实际值；`reveal=true` 是供立即使用场景显式读取单字段的接口。 @@ -303,6 +351,29 @@ curl -X POST "http://localhost:1933/api/v1/privacy-configs/skill/byted-viking-se --- +### patch_privacy_config() + +在服务端 target 锁内将 `values` 合并到当前配置。target 不存在时以空对象为基础创建版本 1；target 已存在时保留未涉及的字段并创建新版本。读取、合并和写入在同一把锁内完成，并发更新不同字段不会因客户端 read-modify-write 丢失。 + +```text +PATCH /api/v1/privacy-configs/{category}/{target_key} +``` + +```json +{ + "values": {"pat": "github_pat_..."}, + "change_reason": "configure GitHub PAT" +} +``` + +CLI 规范参数 `--field` 与 `--field-*` 使用该接口，不在 CLI 中预读 current；`--key` 与 `--key-*` 继续作为兼容别名： + +```bash +ov privacy upsert credential github --field-pat 'github_pat_...' +``` + +--- + ### delete_privacy_config() 删除一个 target 的当前值、元信息和完整版本历史。重复删除是安全的：目标不存在时返回 `deleted: false`。 @@ -459,10 +530,12 @@ ov privacy list skill ov privacy get skill byted-viking-search-knowledgebase ov privacy skill byted-viking-search-knowledgebase -# 默认只取得 owner 绑定引用，不返回实际值 +# 默认只取得 owner 绑定引用，不返回实际值；仅有一个字段时可省略 field +ov privacy reference credential github-prod ov privacy reference credential github-prod api_key -# 显式读取当前生效版本中的一个字段 +# 显式读取当前生效版本中的一个字段，省略规则相同 +ov privacy resolve credential github-prod ov privacy resolve credential github-prod api_key ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' ov privacy resolve 'viking://~/privacy/credential/github-prod#/values/api_key' @@ -474,9 +547,9 @@ ov privacy upsert skill byted-viking-search-knowledgebase \ # 从本地 JSON 文件读取值，避免把值直接写入命令参数 ov privacy upsert skill byted-viking-search-knowledgebase --values-file ./privacy-values.json -# 仅更新部分 key（先读取 current 再合并） +# 更新单个 field；target 不存在时直接创建，存在时在服务端原子合并 ov privacy upsert skill byted-viking-search-knowledgebase \ - --key-api_key secret-3 + --field-api_key secret-3 # 版本查询与切换 ov privacy versions skill byted-viking-search-knowledgebase diff --git a/docs/zh/concepts/13-privacy.md b/docs/zh/concepts/13-privacy.md index 6c5ebf8044..616b06e1ce 100644 --- a/docs/zh/concepts/13-privacy.md +++ b/docs/zh/concepts/13-privacy.md @@ -128,23 +128,30 @@ fs.read(uri) - 内容面：通过 `read` 自动恢复占位符 - 两者解耦：内容文件负责占位符，隐私服务负责敏感值与版本 -PrivacySanitizer 不根据字段名猜测敏感性。当前 active Privacy 中长度不少于 6 个 Unicode 字符的字符串值会参与正文自动扫描与引用替换；更短的值仍可保存、引用和显式读取，但不会参与全文替换，以降低普通正文误命中。匹配目录按用户和本进程 revision 缓存，本进程修改会立即失效，另外设有 3600 秒缓存 TTL 以覆盖多 worker/多节点修改。 +PrivacySanitizer 不根据字段名猜测敏感性。当前 active Privacy 中长度不少于 6 个 Unicode 字符的字符串值会参与正文自动扫描与引用替换；更短的值仍可保存、引用和显式读取，但不会参与全文替换，以降低普通正文误命中。Python 负责目录、引用和缓存策略，匹配器构建、最长匹配与正文重写由 `ragfs` 的 Rust Aho-Corasick 实现完成，并通过 PyO3 复用不可变 matcher、扫描时释放 GIL。匹配目录按用户和本进程 revision 缓存，本进程修改会立即失效，另外设有 3600 秒缓存 TTL 以覆盖多 worker/多节点修改。 -常用命令： +以保存 GitHub Personal Access Token（PAT）为例： ```bash -ov privacy categories -ov privacy list skill -ov privacy skill search-web -ov privacy reference skill search-web api_key -ov privacy resolve skill search-web api_key -ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/skill/search-web#/values/api_key}}' -ov privacy upsert skill search-web --values-json '{"api_key":"..."}' -ov privacy activate skill search-web 1 -ov privacy delete skill search-web api_key --yes -ov privacy delete skill search-web --yes +# 首次写入或轮换 GitHub PAT；credential/github 是 target，pat 是 field +ov privacy upsert credential github --field-pat 'github_pat_...' + +# target 只有 pat 一个字段时可以省略 field +ov privacy reference credential github +ov privacy resolve credential github + +# 显式指定 field 的等价形式 +ov privacy reference credential github pat +ov privacy resolve credential github pat +ov privacy resolve '{{ov_privacy:alice:viking://~/privacy/credential/github#/values/pat}}' + +# 查看/切换历史版本，或删除 pat 字段（最后一个字段删除后 target 也会删除） +ov privacy versions credential github +ov privacy activate credential github 1 +ov privacy delete credential github pat --yes + +# 扫描已经持久化的正文；默认仅报告，--fix 才会替换明文 PAT ov privacy scan --scope resources,skills,memories -ov read viking://~/skills/search-web/SKILL.md ``` --- diff --git a/openviking/privacy/helpers.py b/openviking/privacy/helpers.py index 957ea70d48..ee77b2ac14 100644 --- a/openviking/privacy/helpers.py +++ b/openviking/privacy/helpers.py @@ -6,10 +6,14 @@ import re from typing import Any +from openviking.privacy.validation import validate_privacy_identifier + _VERSION_RE = re.compile(r"^version_(\d+)\.json$") def config_root_uri(user_space: str, category: str, target_key: str) -> str: + validate_privacy_identifier(category, "category") + validate_privacy_identifier(target_key, "target_key") return f"viking://user/{user_space}/privacy/{category}/{target_key}" diff --git a/openviking/privacy/reference.py b/openviking/privacy/reference.py index d27c5e1adb..8fdf139c1b 100644 --- a/openviking/privacy/reference.py +++ b/openviking/privacy/reference.py @@ -9,6 +9,10 @@ from urllib.parse import quote, unquote from openviking.core.identifiers import validate_user_id +from openviking.privacy.validation import ( + validate_privacy_field_name, + validate_privacy_identifier, +) _REFERENCE_RE = re.compile( r"^\{\{[ \t]*ov_privacy:(?P[^\s:{}]+):viking://~/privacy/" @@ -58,14 +62,9 @@ def build_privacy_field_reference( validation_error = validate_user_id(owner_user_id) if validation_error: raise ValueError(validation_error) - parts = { - "category": category, - "target_key": target_key, - "field": field, - } - for name, value in parts.items(): - if not isinstance(value, str) or not value: - raise ValueError(f"{name} must be a non-empty string") + validate_privacy_identifier(category, "category") + validate_privacy_identifier(target_key, "target_key") + validate_privacy_field_name(field, location="field") return ( f"{{{{ov_privacy:{owner_user_id}:viking://~/privacy/" diff --git a/openviking/privacy/sanitizer.py b/openviking/privacy/sanitizer.py index 2bb640388b..5756ae80b7 100644 --- a/openviking/privacy/sanitizer.py +++ b/openviking/privacy/sanitizer.py @@ -6,11 +6,11 @@ import asyncio import time -from collections import deque -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable from openviking.privacy.reference import build_privacy_field_reference +from openviking.pyagfs import get_privacy_matcher_class from openviking.server.identity import RequestContext if TYPE_CHECKING: @@ -39,74 +39,6 @@ def changed(self) -> bool: return bool(self.replacements) -@dataclass -class _AutomatonNode: - transitions: dict[str, int] = field(default_factory=dict) - failure: int = 0 - outputs: list[str] = field(default_factory=list) - - -class _SecretMatcher: - """Small pure-Python Aho-Corasick matcher built once per catalog revision.""" - - def __init__(self, references_by_secret: dict[str, str | tuple[str, ...]]): - self._references_by_secret = references_by_secret - self._nodes = [_AutomatonNode()] - for secret in references_by_secret: - state = 0 - for char in secret: - state = self._nodes[state].transitions.setdefault(char, len(self._nodes)) - if state == len(self._nodes): - self._nodes.append(_AutomatonNode()) - self._nodes[state].outputs.append(secret) - self._build_failures() - - def _build_failures(self) -> None: - pending: deque[int] = deque() - for state in self._nodes[0].transitions.values(): - pending.append(state) - while pending: - state = pending.popleft() - for char, next_state in self._nodes[state].transitions.items(): - pending.append(next_state) - fallback = self._nodes[state].failure - while fallback and char not in self._nodes[fallback].transitions: - fallback = self._nodes[fallback].failure - self._nodes[next_state].failure = self._nodes[fallback].transitions.get(char, 0) - self._nodes[next_state].outputs.extend( - self._nodes[self._nodes[next_state].failure].outputs - ) - - def replacements( - self, content: str - ) -> tuple[list[PrivacyReplacement], list[tuple[str, ...]]]: - matches: list[tuple[int, int, str]] = [] - state = 0 - for index, char in enumerate(content): - while state and char not in self._nodes[state].transitions: - state = self._nodes[state].failure - state = self._nodes[state].transitions.get(char, 0) - for secret in self._nodes[state].outputs: - matches.append((index + 1 - len(secret), index + 1, secret)) - - # Prefer the longest match at the same position and discard overlaps. - matches.sort(key=lambda item: (item[0], -(item[1] - item[0]), item[1])) - selected: list[PrivacyReplacement] = [] - ambiguous: list[tuple[str, ...]] = [] - cursor = 0 - for start, end, secret in matches: - if start < cursor: - continue - target = self._references_by_secret[secret] - if isinstance(target, tuple): - ambiguous.append(target) - selected.append(PrivacyReplacement(start, end, _AMBIGUOUS_PLACEHOLDER)) - else: - selected.append(PrivacyReplacement(start, end, target)) - cursor = end - return selected, ambiguous - - class PrivacySanitizer: """Sanitize text with the active sensitive values owned by the current user.""" @@ -122,7 +54,9 @@ def __init__( self._privacy = privacy self._cache_ttl_seconds = float(cache_ttl_seconds) self._monotonic_clock = monotonic_clock - self._cache: dict[tuple[str, str, int], tuple[float, _SecretMatcher]] = {} + matcher_class: Any = get_privacy_matcher_class() + self._matcher_class = matcher_class + self._cache: dict[tuple[str, str, int], tuple[float, Any]] = {} self._cache_lock = asyncio.Lock() async def sanitize( @@ -135,26 +69,18 @@ async def sanitize( if not content or self._is_privacy_storage_uri(uri, ctx): return PrivacySanitizationResult(content=content) matcher = await self._matcher(ctx) - replacements, ambiguous = matcher.replacements(content) - if not replacements: - return PrivacySanitizationResult( - content=content, - ambiguous_references=tuple(ambiguous), - ) - parts: list[str] = [] - cursor = 0 - for replacement in replacements: - parts.append(content[cursor : replacement.start]) - parts.append(replacement.reference) - cursor = replacement.end - parts.append(content[cursor:]) + rewritten, raw_replacements, raw_ambiguous = matcher.sanitize(content) + replacements = tuple( + PrivacyReplacement(start=start, end=end, reference=reference) + for start, end, reference in raw_replacements + ) return PrivacySanitizationResult( - content="".join(parts), - replacements=tuple(replacements), - ambiguous_references=tuple(ambiguous), + content=rewritten, + replacements=replacements, + ambiguous_references=tuple(tuple(items) for items in raw_ambiguous), ) - async def _matcher(self, ctx: RequestContext) -> _SecretMatcher: + async def _matcher(self, ctx: RequestContext) -> Any: revision = self._privacy.catalog_revision(ctx) cache_key = (ctx.account_id, ctx.user.user_id, revision) now = self._monotonic_clock() @@ -174,7 +100,7 @@ async def _matcher(self, ctx: RequestContext) -> _SecretMatcher: self._cache[cache_key] = (now, matcher) return matcher - async def _build_matcher(self, ctx: RequestContext) -> _SecretMatcher: + async def _build_matcher(self, ctx: RequestContext) -> Any: references: dict[str, list[str]] = {} for category in await self._privacy.list_categories(ctx): for target_key in await self._privacy.list_targets(ctx, category): @@ -199,14 +125,10 @@ async def _build_matcher(self, ctx: RequestContext) -> _SecretMatcher: ) references.setdefault(value, []).append(reference) - targets: dict[str, str | tuple[str, ...]] = {} + targets: dict[str, list[str]] = {} for secret, candidates in references.items(): - unique = tuple(sorted(set(candidates))) - if len(unique) == 1: - targets[secret] = unique[0] - else: - targets[secret] = unique - return _SecretMatcher(targets) + targets[secret] = sorted(set(candidates)) + return self._matcher_class(targets, _AMBIGUOUS_PLACEHOLDER) @staticmethod def _is_privacy_storage_uri(uri: str, ctx: RequestContext) -> bool: diff --git a/openviking/privacy/service.py b/openviking/privacy/service.py index 88e7326728..70651a6490 100644 --- a/openviking/privacy/service.py +++ b/openviking/privacy/service.py @@ -21,6 +21,12 @@ UserPrivacyConfigVersion, UserPrivacyFieldDeleteResult, ) +from openviking.privacy.validation import ( + validate_privacy_field_name, + validate_privacy_identifier, + validate_privacy_metadata, + validate_privacy_values, +) from openviking.server.identity import RequestContext from openviking.storage.viking_fs import VikingFS from openviking.utils.time_utils import get_current_timestamp @@ -207,6 +213,7 @@ async def list_categories(self, ctx: RequestContext) -> list[str]: return sorted(entry["name"] for entry in entries if entry.get("name")) async def list_targets(self, ctx: RequestContext, category: str) -> list[str]: + validate_privacy_identifier(category, "category") uri = f"{self._user_root(ctx)}/privacy/{category}" try: entries = await self._viking_fs.ls(uri, ctx=ctx) @@ -243,66 +250,118 @@ async def upsert( async with self._config_lock( ctx, category, target_key, owner_lease_ref=owner_lease_ref ) as lease: - await self._ensure_root(ctx, category, target_key, lease) - now = get_current_timestamp() - meta = await self.get_meta(ctx, category, target_key) - current = await self.get_current(ctx, category, target_key) - - if current and canonicalize_values(current.values) == canonicalize_values(values): - if meta is None: - meta = UserPrivacyConfigMeta( - category=category, - target_key=target_key, - active_version=current.version, - latest_version=current.version, - created_at=now, - ) - meta.updated_by = updated_by - meta.last_accessed_at = now - if labels is not None: - meta.labels = dict(labels) - await self._save_meta(ctx, category, target_key, meta, lease) - return current - - version_number = 1 if meta is None else meta.latest_version + 1 - version = UserPrivacyConfigVersion( - version=version_number, + return await self._upsert_locked( + ctx=ctx, category=category, target_key=target_key, - values=dict(values), - created_at=now, - created_by=updated_by, + values=values, + updated_by=updated_by, change_reason=change_reason, - ) - - await self._viking_fs.write_file( - version_uri(self._user_space(ctx), category, target_key, version_number), - json.dumps(version.to_dict(), ensure_ascii=False), - ctx=ctx, + labels=labels, lease_ref=lease, ) - await self._viking_fs.write_file( - current_uri(self._user_space(ctx), category, target_key), - json.dumps(version.to_dict(), ensure_ascii=False), + + async def patch_values( + self, + ctx: RequestContext, + category: str, + target_key: str, + updates: dict[str, Any], + updated_by: str = "", + change_reason: str = "", + labels: Optional[dict[str, Any]] = None, + ) -> UserPrivacyConfigVersion: + """Atomically merge fields into current values, creating the target if absent.""" + async with self._config_lock(ctx, category, target_key) as lease: + current = await self.get_current(ctx, category, target_key) + values = dict(current.values) if current is not None else {} + values.update(updates) + return await self._upsert_locked( ctx=ctx, + category=category, + target_key=target_key, + values=values, + updated_by=updated_by, + change_reason=change_reason, + labels=labels, lease_ref=lease, ) + async def _upsert_locked( + self, + *, + ctx: RequestContext, + category: str, + target_key: str, + values: dict[str, Any], + updated_by: str, + change_reason: str, + labels: Optional[dict[str, Any]], + lease_ref: Optional[dict[str, Any]], + ) -> UserPrivacyConfigVersion: + """Persist a complete snapshot while the caller holds the target lock.""" + validate_privacy_values(values) + validate_privacy_metadata(change_reason=change_reason, labels=labels) + await self._ensure_root(ctx, category, target_key, lease_ref) + now = get_current_timestamp() + meta = await self.get_meta(ctx, category, target_key) + current = await self.get_current(ctx, category, target_key) + + if current and canonicalize_values(current.values) == canonicalize_values(values): if meta is None: meta = UserPrivacyConfigMeta( category=category, target_key=target_key, + active_version=current.version, + latest_version=current.version, created_at=now, ) - meta.active_version = version_number - meta.latest_version = version_number meta.updated_by = updated_by meta.last_accessed_at = now if labels is not None: meta.labels = dict(labels) - await self._save_meta(ctx, category, target_key, meta, lease) - self._bump_catalog_revision(ctx) - return version + await self._save_meta(ctx, category, target_key, meta, lease_ref) + return current + + version_number = 1 if meta is None else meta.latest_version + 1 + version = UserPrivacyConfigVersion( + version=version_number, + category=category, + target_key=target_key, + values=dict(values), + created_at=now, + created_by=updated_by, + change_reason=change_reason, + ) + + await self._viking_fs.write_file( + version_uri(self._user_space(ctx), category, target_key, version_number), + json.dumps(version.to_dict(), ensure_ascii=False), + ctx=ctx, + lease_ref=lease_ref, + ) + await self._viking_fs.write_file( + current_uri(self._user_space(ctx), category, target_key), + json.dumps(version.to_dict(), ensure_ascii=False), + ctx=ctx, + lease_ref=lease_ref, + ) + + if meta is None: + meta = UserPrivacyConfigMeta( + category=category, + target_key=target_key, + created_at=now, + ) + meta.active_version = version_number + meta.latest_version = version_number + meta.updated_by = updated_by + meta.last_accessed_at = now + if labels is not None: + meta.labels = dict(labels) + await self._save_meta(ctx, category, target_key, meta, lease_ref) + self._bump_catalog_revision(ctx) + return version async def activate_version( self, @@ -342,12 +401,13 @@ async def delete_field( target_key: str, field: str, updated_by: str = "", - ) -> Optional[UserPrivacyConfigVersion]: + ) -> Optional[UserPrivacyFieldDeleteResult]: """Remove one field from the active values by publishing a new snapshot. Historical snapshots remain intact while the target still has fields. Deleting the final field removes the complete target and its history. """ + validate_privacy_field_name(field, location="field") async with self._config_lock(ctx, category, target_key) as lease: meta = await self.get_meta(ctx, category, target_key) current = await self.get_current(ctx, category, target_key) diff --git a/openviking/privacy/validation.py b/openviking/privacy/validation.py new file mode 100644 index 0000000000..650e826ae0 --- /dev/null +++ b/openviking/privacy/validation.py @@ -0,0 +1,236 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Validation and resource limits for Privacy identifiers and payloads.""" + +from __future__ import annotations + +import json +import math +import re +from typing import Any, Mapping, Sequence + +from openviking_cli.exceptions import InvalidArgumentError + +PRIVACY_IDENTIFIER_MAX_CHARS = 255 +PRIVACY_FIELD_NAME_MAX_BYTES = 255 +PRIVACY_MAX_FIELDS = 128 +PRIVACY_MAX_STRING_BYTES = 64 * 1024 +PRIVACY_VALUES_MAX_BYTES = 1024 * 1024 +PRIVACY_VALUES_MAX_DEPTH = 16 +PRIVACY_MAX_COLLECTION_ITEMS = 4096 +PRIVACY_MAX_LABELS = 64 +PRIVACY_LABELS_MAX_BYTES = 64 * 1024 +PRIVACY_CHANGE_REASON_MAX_CHARS = 1024 + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_.@-]+$") + + +def validate_privacy_identifier(value: str, field: str) -> str: + """Validate one URI path segment used by the Privacy storage layout.""" + reason = None + if not isinstance(value, str): + reason = "must be a string" + elif not value: + reason = "must not be empty" + elif value != value.strip(): + reason = "must not contain leading or trailing whitespace" + elif len(value) > PRIVACY_IDENTIFIER_MAX_CHARS: + reason = f"must not exceed {PRIVACY_IDENTIFIER_MAX_CHARS} characters" + elif value in {".", ".."}: + reason = "must not be '.' or '..'" + elif not value.isascii() or _IDENTIFIER_RE.fullmatch(value) is None: + reason = "may contain only ASCII letters, digits, '_', '-', '.', and '@'" + elif value.count("@") > 1: + reason = "must contain at most one '@'" + if reason is not None: + raise InvalidArgumentError( + f"Invalid Privacy {field}: {reason}.", + details={ + "field": field, + "reason": reason, + "allowed_pattern": "[A-Za-z0-9_.@-]+", + "max_characters": PRIVACY_IDENTIFIER_MAX_CHARS, + }, + ) + return value + + +def validate_privacy_field_name(value: str, *, location: str = "values") -> str: + """Validate a JSON object key without restricting user-defined field names.""" + if not isinstance(value, str) or not value: + reason = "must be a non-empty string" + elif len(value.encode("utf-8")) > PRIVACY_FIELD_NAME_MAX_BYTES: + reason = f"must not exceed {PRIVACY_FIELD_NAME_MAX_BYTES} UTF-8 bytes" + elif any(ord(char) < 0x20 or ord(char) == 0x7F for char in value): + reason = "must not contain control characters" + else: + return value + raise InvalidArgumentError( + f"Invalid Privacy field name in {location}: {reason}.", + details={ + "field": location, + "reason": reason, + "max_bytes": PRIVACY_FIELD_NAME_MAX_BYTES, + }, + ) + + +def _validate_json_tree(value: Any, *, location: str, depth: int) -> None: + if depth > PRIVACY_VALUES_MAX_DEPTH: + raise InvalidArgumentError( + f"Privacy {location} exceeds the maximum JSON nesting depth of " + f"{PRIVACY_VALUES_MAX_DEPTH}.", + details={ + "field": location, + "reason": "maximum nesting depth exceeded", + "max_depth": PRIVACY_VALUES_MAX_DEPTH, + }, + ) + if isinstance(value, str): + size = len(value.encode("utf-8")) + if size > PRIVACY_MAX_STRING_BYTES: + raise InvalidArgumentError( + f"Privacy string at {location} exceeds the maximum size of " + f"{PRIVACY_MAX_STRING_BYTES} UTF-8 bytes.", + details={ + "field": location, + "reason": "string value too large", + "actual_bytes": size, + "max_bytes": PRIVACY_MAX_STRING_BYTES, + }, + ) + return + if isinstance(value, Mapping): + if len(value) > PRIVACY_MAX_COLLECTION_ITEMS: + raise InvalidArgumentError( + f"Privacy object at {location} contains {len(value)} entries; at most " + f"{PRIVACY_MAX_COLLECTION_ITEMS} are allowed.", + details={ + "field": location, + "reason": "collection too large", + "actual_items": len(value), + "max_items": PRIVACY_MAX_COLLECTION_ITEMS, + }, + ) + for key, item in value.items(): + validate_privacy_field_name(key, location=location) + _validate_json_tree(item, location=f"{location}.{key}", depth=depth + 1) + return + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + if len(value) > PRIVACY_MAX_COLLECTION_ITEMS: + raise InvalidArgumentError( + f"Privacy array at {location} contains {len(value)} items; at most " + f"{PRIVACY_MAX_COLLECTION_ITEMS} are allowed.", + details={ + "field": location, + "reason": "collection too large", + "actual_items": len(value), + "max_items": PRIVACY_MAX_COLLECTION_ITEMS, + }, + ) + for index, item in enumerate(value): + _validate_json_tree(item, location=f"{location}[{index}]", depth=depth + 1) + return + if isinstance(value, float) and not math.isfinite(value): + raise InvalidArgumentError( + f"Privacy value at {location} must be a finite JSON number.", + details={"field": location, "reason": "non-finite number"}, + ) + if value is not None and not isinstance(value, (bool, int, float)): + raise InvalidArgumentError( + f"Privacy value at {location} must be JSON-compatible.", + details={"field": location, "reason": "unsupported value type"}, + ) + + +def _serialized_size(value: Mapping[str, Any]) -> int: + return len( + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + ) + + +def validate_privacy_values(values: Mapping[str, Any]) -> None: + """Validate the complete active values snapshot before persistence.""" + if not isinstance(values, Mapping): + raise InvalidArgumentError( + "Privacy values must be a JSON object.", + details={"field": "values", "reason": "expected object"}, + ) + if len(values) > PRIVACY_MAX_FIELDS: + raise InvalidArgumentError( + f"Privacy values contain {len(values)} fields; at most {PRIVACY_MAX_FIELDS} are allowed.", + details={ + "field": "values", + "reason": "too many fields", + "actual_fields": len(values), + "max_fields": PRIVACY_MAX_FIELDS, + }, + ) + _validate_json_tree(values, location="values", depth=1) + size = _serialized_size(values) + if size > PRIVACY_VALUES_MAX_BYTES: + raise InvalidArgumentError( + f"Privacy values serialize to {size} bytes; the maximum is " + f"{PRIVACY_VALUES_MAX_BYTES} bytes.", + details={ + "field": "values", + "reason": "serialized payload too large", + "actual_bytes": size, + "max_bytes": PRIVACY_VALUES_MAX_BYTES, + }, + ) + + +def validate_privacy_metadata( + *, + change_reason: str, + labels: Mapping[str, Any] | None, +) -> None: + """Validate optional snapshot metadata without exposing values in errors.""" + if not isinstance(change_reason, str): + raise InvalidArgumentError( + "Privacy change_reason must be a string.", + details={"field": "change_reason", "reason": "expected string"}, + ) + if len(change_reason) > PRIVACY_CHANGE_REASON_MAX_CHARS: + raise InvalidArgumentError( + f"Privacy change_reason must not exceed {PRIVACY_CHANGE_REASON_MAX_CHARS} characters.", + details={ + "field": "change_reason", + "reason": "value too long", + "actual_characters": len(change_reason), + "max_characters": PRIVACY_CHANGE_REASON_MAX_CHARS, + }, + ) + if labels is None: + return + if not isinstance(labels, Mapping): + raise InvalidArgumentError( + "Privacy labels must be a JSON object.", + details={"field": "labels", "reason": "expected object"}, + ) + if len(labels) > PRIVACY_MAX_LABELS: + raise InvalidArgumentError( + f"Privacy labels contain {len(labels)} entries; at most {PRIVACY_MAX_LABELS} are allowed.", + details={ + "field": "labels", + "reason": "too many labels", + "actual_fields": len(labels), + "max_fields": PRIVACY_MAX_LABELS, + }, + ) + _validate_json_tree(labels, location="labels", depth=1) + size = _serialized_size(labels) + if size > PRIVACY_LABELS_MAX_BYTES: + raise InvalidArgumentError( + f"Privacy labels serialize to {size} bytes; the maximum is " + f"{PRIVACY_LABELS_MAX_BYTES} bytes.", + details={ + "field": "labels", + "reason": "serialized payload too large", + "actual_bytes": size, + "max_bytes": PRIVACY_LABELS_MAX_BYTES, + }, + ) diff --git a/openviking/pyagfs/__init__.py b/openviking/pyagfs/__init__.py index ba3905975b..fd900cc2f9 100644 --- a/openviking/pyagfs/__init__.py +++ b/openviking/pyagfs/__init__.py @@ -98,17 +98,16 @@ def _find_ragfs_so(): return None -def _load_rust_binding(): - """Attempt to load the Rust (PyO3) binding client. +def _load_rust_module(): + """Load the complete Rust PyO3 module from the installed or vendored artifact. Prefers the pip-installed ``ragfs_python`` package (e.g. from maturin develop), then falls back to the vendored native extension in openviking/lib/. """ - # Prefer pip-installed version (handles @rpath correctly) try: - from ragfs_python import RAGFSBindingClient as _Rust + import ragfs_python as rust_module - return _Rust, None + return rust_module except ImportError: pass @@ -121,13 +120,19 @@ def _load_rust_binding(): raise ImportError(f"Unable to load Rust binding spec from {so_path}") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) - return mod.RAGFSBindingClient, None + return mod except Exception: pass raise ImportError("Rust binding not available") +def _load_rust_binding(): + """Attempt to load the Rust (PyO3) binding client.""" + module = _load_rust_module() + return module.RAGFSBindingClient, None + + def get_binding_client(): """Get the RAGFS binding client class. @@ -142,6 +147,17 @@ def get_binding_client(): raise ImportError("ragfs_python native library is not available: " + str(exc)) from exc +def get_privacy_matcher_class(): + """Return the native RAGFS PrivacyMatcher class.""" + try: + module = _load_rust_module() + return module.PrivacyMatcher + except (ImportError, AttributeError) as exc: + raise ImportError( + "ragfs_python PrivacyMatcher is not available; rebuild the native extension" + ) from exc + + def reopen_rust_tracing_file() -> None: """Reopen the Rust tracing log file after Python rotates the active file.""" try: diff --git a/openviking/server/routers/privacy_configs.py b/openviking/server/routers/privacy_configs.py index c962adef85..0747e04cb0 100644 --- a/openviking/server/routers/privacy_configs.py +++ b/openviking/server/routers/privacy_configs.py @@ -14,6 +14,7 @@ ) from openviking.privacy.sanitizer import PrivacySanitizer from openviking.privacy.scan import scan_privacy_content +from openviking.privacy.validation import validate_privacy_field_name from openviking.server.auth import get_request_context from openviking.server.dependencies import get_service from openviking.server.identity import RequestContext @@ -41,6 +42,14 @@ class UpsertPrivacyConfigRequest(BaseModel): labels: Optional[Dict[str, Any]] = None +class PatchPrivacyConfigRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + values: Dict[str, Any] + change_reason: str = "" + labels: Optional[Dict[str, Any]] = None + + class ActivatePrivacyConfigVersionRequest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -60,7 +69,7 @@ class ScanPrivacyContentRequest(BaseModel): class PrivacySelectorRequest(BaseModel): model_config = ConfigDict(extra="forbid") - selector: str + selector: str = Field(..., min_length=1, max_length=2048) def _require_selector_owner(owner_user_id: str, ctx: RequestContext) -> None: @@ -117,6 +126,7 @@ async def resolve_privacy_reference( "selector must be an owner-bound Privacy reference or a field-level Privacy URI" ) _require_selector_owner(reference.owner_user_id, _ctx) + validate_privacy_field_name(reference.field, location="field") service = get_service() privacy = service.privacy_configs await _require_privacy_target(privacy, _ctx, reference.category, reference.target_key) @@ -232,6 +242,48 @@ async def get_privacy_current( ) +@router.get("/{category}/{target_key}/fields") +async def list_privacy_fields( + category: str = Path(..., description="Privacy config category"), + target_key: str = Path(..., description="Privacy config target key"), + _ctx: RequestContext = Depends(get_request_context), +): + """List active field metadata without returning plaintext values.""" + service = get_service() + privacy = service.privacy_configs + await _require_privacy_target(privacy, _ctx, category, target_key) + current = await privacy.get_current(_ctx, category, target_key) + if current is None: + raise NotFoundError(f"{category}/{target_key}", "privacy config") + + fields = [] + for field in sorted(current.values): + value = current.values[field] + fields.append( + { + "field": field, + "configured": value is not None and str(value) != "", + "reference": build_privacy_field_reference( + _ctx.user.user_id, + category, + target_key, + field, + ), + } + ) + + return Response( + status="ok", + result={ + "owner_user_id": _ctx.user.user_id, + "category": category, + "target_key": target_key, + "version": current.version, + "fields": fields, + }, + ) + + @router.get("/{category}/{target_key}/field") async def get_privacy_field( category: str = Path(..., description="Privacy config category"), @@ -282,6 +334,7 @@ async def delete_privacy_field( field: str = Query(..., min_length=1, description="Field to remove from active values"), _ctx: RequestContext = Depends(get_request_context), ): + validate_privacy_field_name(field, location="field") service = get_service() privacy = service.privacy_configs await _require_privacy_target(privacy, _ctx, category, target_key) @@ -361,6 +414,30 @@ async def upsert_privacy_config( return Response(status="ok", result=None if result is None else result.to_dict()) +@router.patch("/{category}/{target_key}") +async def patch_privacy_config( + request: PatchPrivacyConfigRequest, + category: str = Path(..., description="Privacy config category"), + target_key: str = Path(..., description="Privacy config target key"), + _ctx: RequestContext = Depends(get_request_context), +): + """Atomically merge fields into current values, creating the target if absent.""" + service = get_service() + privacy = service.privacy_configs + result = None + if privacy is not None: + result = await privacy.patch_values( + ctx=_ctx, + category=category, + target_key=target_key, + updates=request.values, + updated_by=_ctx.user.user_id, + change_reason=request.change_reason, + labels=request.labels, + ) + return Response(status="ok", result=None if result is None else result.to_dict()) + + @router.delete("/{category}/{target_key}") async def delete_privacy_config( category: str = Path(..., description="Privacy config category"), diff --git a/tests/server/test_api_privacy_configs.py b/tests/server/test_api_privacy_configs.py index ce34c6b4dc..74789c61b5 100644 --- a/tests/server/test_api_privacy_configs.py +++ b/tests/server/test_api_privacy_configs.py @@ -76,6 +76,62 @@ async def test_privacy_config_update_allows_new_keys(client: httpx.AsyncClient): assert body["result"]["values"]["new_key"] == "x" +async def test_privacy_config_patch_creates_target_and_atomically_merges_fields( + client: httpx.AsyncClient, +): + created = await client.patch( + "/api/v1/privacy-configs/credential/github", + json={"values": {"pat": "github_pat_mock_value"}, "change_reason": "initial"}, + ) + assert created.status_code == 200, created.text + assert created.json()["result"]["version"] == 1 + assert created.json()["result"]["values"] == {"pat": "github_pat_mock_value"} + + updated = await client.patch( + "/api/v1/privacy-configs/credential/github", + json={"values": {"username": "alice"}, "change_reason": "add username"}, + ) + assert updated.status_code == 200, updated.text + assert updated.json()["result"]["version"] == 2 + assert updated.json()["result"]["values"] == { + "pat": "github_pat_mock_value", + "username": "alice", + } + + +async def test_privacy_api_rejects_invalid_target_with_structured_format_error( + client: httpx.AsyncClient, +): + response = await client.patch( + "/api/v1/privacy-configs/credential/two@@signs", + json={"values": {"pat": "github_pat_mock_value"}}, + ) + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["code"] == "INVALID_ARGUMENT" + assert error["message"] == "Invalid Privacy target_key: must contain at most one '@'." + assert error["details"] == { + "field": "target_key", + "reason": "must contain at most one '@'", + "allowed_pattern": "[A-Za-z0-9_.@-]+", + "max_characters": 255, + } + + +async def test_privacy_api_rejects_oversized_values_without_echoing_secret( + client: httpx.AsyncClient, +): + secret = "sensitive-marker-" + "x" * (64 * 1024) + response = await client.post( + "/api/v1/privacy-configs/credential/oversized", + json={"values": {"pat": secret}}, + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "INVALID_ARGUMENT" + assert response.json()["error"]["details"]["reason"] == "string value too large" + assert secret not in response.text + + async def test_privacy_config_get_version_not_found(client: httpx.AsyncClient): create = await client.post( "/api/v1/privacy-configs/skill/demo-skill-version-not-found", @@ -173,6 +229,55 @@ async def test_privacy_field_reference_is_safe_by_default_and_reveal_is_explicit assert missing_response.json()["error"]["code"] == "NOT_FOUND" +async def test_privacy_field_list_returns_metadata_without_values( + client: httpx.AsyncClient, +): + created = await client.post( + "/api/v1/privacy-configs/credential/github-fields", + json={"values": {"region": "", "pat": "github-secret-value"}}, + ) + assert created.status_code == 200, created.text + owner = created.json()["result"]["created_by"] + + response = await client.get( + "/api/v1/privacy-configs/credential/github-fields/fields" + ) + assert response.status_code == 200, response.text + result = response.json()["result"] + assert result == { + "owner_user_id": owner, + "category": "credential", + "target_key": "github-fields", + "version": 1, + "fields": [ + { + "field": "pat", + "configured": True, + "reference": ( + f"{{{{ov_privacy:{owner}:viking://~/privacy/credential/" + "github-fields#/values/pat}}" + ), + }, + { + "field": "region", + "configured": False, + "reference": ( + f"{{{{ov_privacy:{owner}:viking://~/privacy/credential/" + "github-fields#/values/region}}" + ), + }, + ], + } + assert "github-secret-value" not in response.text + assert all("value" not in field for field in result["fields"]) + + missing = await client.get( + "/api/v1/privacy-configs/credential/missing-fields/fields" + ) + assert missing.status_code == 404 + assert missing.json()["error"]["code"] == "NOT_FOUND" + + async def test_privacy_config_delete_is_idempotent_and_removes_history( client: httpx.AsyncClient, ): diff --git a/tests/server/test_privacy_config_service.py b/tests/server/test_privacy_config_service.py index 66a1c646d5..596239649b 100644 --- a/tests/server/test_privacy_config_service.py +++ b/tests/server/test_privacy_config_service.py @@ -9,9 +9,10 @@ from openviking.privacy.service import UserPrivacyConfigService from openviking.privacy.skill_extractor import extract_skill_privacy_values from openviking.privacy.skill_placeholder import placeholderize_skill_content_with_blocks +from openviking.privacy.validation import PRIVACY_MAX_FIELDS from openviking.server.identity import RequestContext, Role from openviking.storage.viking_fs import VikingFS -from openviking_cli.exceptions import NotFoundError +from openviking_cli.exceptions import InvalidArgumentError, NotFoundError from openviking_cli.session.user_id import UserIdentifier @@ -128,6 +129,51 @@ async def test_privacy_config_concurrent_updates_keep_every_version(): assert await privacy.list_versions(ctx, "skill", "concurrent-skill") == list(range(1, 9)) +@pytest.mark.asyncio +async def test_privacy_config_concurrent_patches_do_not_lose_distinct_fields(): + ctx = RequestContext(user=UserIdentifier.the_default_user("privacy_patch"), role=Role.ROOT) + privacy = UserPrivacyConfigService(_MemoryPrivacyStorage()) + results = await asyncio.gather( + *( + privacy.patch_values( + ctx=ctx, + category="credential", + target_key="github", + updates={f"key_{index}": f"value-{index}"}, + updated_by=ctx.user.user_id, + ) + for index in range(8) + ) + ) + + assert sorted(result.version for result in results) == list(range(1, 9)) + current = await privacy.get_current(ctx, "credential", "github") + assert current is not None + assert current.values == {f"key_{index}": f"value-{index}" for index in range(8)} + + +@pytest.mark.asyncio +async def test_privacy_config_patch_validates_final_snapshot_and_preserves_current_on_failure(): + ctx = RequestContext(user=UserIdentifier.the_default_user("privacy_limit"), role=Role.ROOT) + privacy = UserPrivacyConfigService(_MemoryPrivacyStorage()) + original = {f"key_{index}": index for index in range(PRIVACY_MAX_FIELDS)} + await privacy.upsert(ctx, "credential", "full", original, updated_by=ctx.user.user_id) + + with pytest.raises(InvalidArgumentError, match="at most 128"): + await privacy.patch_values( + ctx, + "credential", + "full", + {"one_more": "value"}, + updated_by=ctx.user.user_id, + ) + + current = await privacy.get_current(ctx, "credential", "full") + assert current is not None + assert current.version == 1 + assert current.values == original + + @pytest.mark.asyncio async def test_privacy_config_exists_propagates_storage_errors(): class FailingStorage: diff --git a/tests/unit/test_privacy_sanitizer.py b/tests/unit/test_privacy_sanitizer.py index 1bef435ac8..9eb05566c8 100644 --- a/tests/unit/test_privacy_sanitizer.py +++ b/tests/unit/test_privacy_sanitizer.py @@ -40,16 +40,22 @@ def _ctx(user_id="alice"): return RequestContext(user=UserIdentifier("account-a", user_id), role=Role.USER) +def test_privacy_sanitizer_uses_native_ragfs_matcher(): + sanitizer = PrivacySanitizer(_PrivacyCatalog()) + assert sanitizer._matcher_class.__module__ == "ragfs_python" + assert sanitizer._matcher_class.__name__ == "PrivacyMatcher" + + def test_privacy_reference_roundtrip_encodes_individual_segments(): - rendered = build_privacy_field_reference("alice", "custom config", "prod/key", "api/key") + rendered = build_privacy_field_reference("alice", "custom-config", "prod-key", "api/key") assert rendered == ( - "{{ov_privacy:alice:viking://~/privacy/custom%20config/" - "prod%2Fkey#/values/api%2Fkey}}" + "{{ov_privacy:alice:viking://~/privacy/custom-config/" + "prod-key#/values/api%2Fkey}}" ) assert parse_privacy_field_reference(rendered) == PrivacyFieldReference( owner_user_id="alice", - category="custom config", - target_key="prod/key", + category="custom-config", + target_key="prod-key", field="api/key", ) spaced = rendered.replace("{{ov_privacy:", "{{ ov_privacy:").replace("}}", " \t}}") @@ -61,22 +67,22 @@ def test_privacy_reference_roundtrip_encodes_individual_segments(): assert parse_privacy_field_reference(rendered.replace("privacy/", "privacy/ ")) is None assert parse_privacy_field_reference(rendered.replace("{{", "{{\n")) is None assert parse_privacy_field_selector( - "viking://~/privacy/custom%20config/prod%2Fkey#/values/api%2Fkey", + "viking://~/privacy/custom-config/prod-key#/values/api%2Fkey", current_user_id="alice", ) == PrivacyFieldReference( owner_user_id="alice", - category="custom config", - target_key="prod/key", + category="custom-config", + target_key="prod-key", field="api/key", ) target = parse_privacy_target_selector( - "viking://user/alice/privacy/custom%20config/prod%2Fkey", + "viking://user/alice/privacy/custom-config/prod-key", current_user_id="other", ) assert target is not None assert target.owner_user_id == "alice" - assert target.category == "custom config" - assert target.target_key == "prod/key" + assert target.category == "custom-config" + assert target.target_key == "prod-key" async def test_sanitizer_protects_registered_strings_at_least_six_chars_without_field_guessing(): diff --git a/tests/unit/test_privacy_validation.py b/tests/unit/test_privacy_validation.py new file mode 100644 index 0000000000..5ef4b743a1 --- /dev/null +++ b/tests/unit/test_privacy_validation.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +import pytest + +from openviking.privacy.validation import ( + PRIVACY_CHANGE_REASON_MAX_CHARS, + PRIVACY_FIELD_NAME_MAX_BYTES, + PRIVACY_MAX_FIELDS, + PRIVACY_MAX_STRING_BYTES, + PRIVACY_VALUES_MAX_BYTES, + PRIVACY_VALUES_MAX_DEPTH, + validate_privacy_identifier, + validate_privacy_metadata, + validate_privacy_values, +) +from openviking_cli.exceptions import InvalidArgumentError + + +@pytest.mark.parametrize( + ("value", "reason"), + [ + ("", "must not be empty"), + ("..", "must not be '.' or '..'"), + (" bad", "leading or trailing whitespace"), + ("bad/path", "may contain only ASCII"), + ("bad%2Fpath", "may contain only ASCII"), + ("two@@signs", "at most one '@'"), + ], +) +def test_privacy_identifier_errors_are_specific(value, reason): + with pytest.raises(InvalidArgumentError, match=reason) as exc_info: + validate_privacy_identifier(value, "target_key") + assert exc_info.value.details["field"] == "target_key" + assert exc_info.value.details["allowed_pattern"] == "[A-Za-z0-9_.@-]+" + + +def test_privacy_values_enforce_field_count_name_and_string_limits_without_echoing_value(): + with pytest.raises(InvalidArgumentError, match="at most 128"): + validate_privacy_values({f"key_{index}": index for index in range(PRIVACY_MAX_FIELDS + 1)}) + + with pytest.raises(InvalidArgumentError, match="field name"): + validate_privacy_values({"x" * (PRIVACY_FIELD_NAME_MAX_BYTES + 1): "value"}) + + secret = "s" * (PRIVACY_MAX_STRING_BYTES + 1) + with pytest.raises(InvalidArgumentError, match="maximum size") as exc_info: + validate_privacy_values({"pat": secret}) + assert secret not in str(exc_info.value) + assert exc_info.value.details["actual_bytes"] == PRIVACY_MAX_STRING_BYTES + 1 + + +def test_privacy_values_enforce_total_size_and_nesting_depth(): + chunk = "x" * PRIVACY_MAX_STRING_BYTES + with pytest.raises(InvalidArgumentError, match="serialize") as exc_info: + validate_privacy_values({f"key_{index}": chunk for index in range(17)}) + assert exc_info.value.details["actual_bytes"] > PRIVACY_VALUES_MAX_BYTES + + nested = "value" + for _ in range(PRIVACY_VALUES_MAX_DEPTH): + nested = {"child": nested} + with pytest.raises(InvalidArgumentError, match="nesting depth"): + validate_privacy_values({"root": nested}) + + +def test_privacy_metadata_limits_change_reason(): + with pytest.raises(InvalidArgumentError, match="change_reason") as exc_info: + validate_privacy_metadata( + change_reason="x" * (PRIVACY_CHANGE_REASON_MAX_CHARS + 1), + labels=None, + ) + assert exc_info.value.details["max_characters"] == PRIVACY_CHANGE_REASON_MAX_CHARS From cc6e84262c7621fb0ce9ffed7907475606d02efb Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Fri, 16 Oct 2026 14:15:10 +0800 Subject: [PATCH 3/6] feat: update ov privacy commands for agent use --- crates/ov_cli/src/commands/privacy.rs | 15 +- crates/ov_cli/src/handlers.rs | 8 +- crates/ov_cli/src/help_ui.rs | 4 +- crates/ov_cli/src/main.rs | 51 +-- crates/ragfs-python/src/privacy.rs | 23 ++ crates/ragfs/src/privacy_matcher.rs | 85 +++-- docs/en/api/10-privacy.md | 16 +- docs/en/concepts/13-privacy.md | 4 +- docs/zh/api/10-privacy.md | 16 +- docs/zh/concepts/13-privacy.md | 4 +- docs/zh/privacy-review.md | 208 ------------ openviking/privacy/sanitizer.py | 16 +- openviking/privacy/scan.py | 335 ++++++++++++++----- openviking/server/routers/privacy_configs.py | 14 +- openviking/storage/viking_fs/_ops.py | 19 +- tests/server/test_api_privacy_configs.py | 23 +- tests/unit/test_privacy_sanitizer.py | 15 + tests/unit/test_privacy_scan.py | 108 ++++++ 18 files changed, 572 insertions(+), 392 deletions(-) delete mode 100644 docs/zh/privacy-review.md create mode 100644 tests/unit/test_privacy_scan.py diff --git a/crates/ov_cli/src/commands/privacy.rs b/crates/ov_cli/src/commands/privacy.rs index 1ad9637df4..9214cc88b9 100644 --- a/crates/ov_cli/src/commands/privacy.rs +++ b/crates/ov_cli/src/commands/privacy.rs @@ -429,24 +429,15 @@ pub async fn delete( pub async fn scan( client: &HttpClient, - scopes: &[String], - target_uri: Option<&str>, + uri: &str, fix: bool, - wait: bool, - node_limit: usize, + node_limit: u16, output_format: OutputFormat, compact: bool, ) -> Result<()> { - if target_uri.is_some() && !scopes.is_empty() { - return Err(Error::Client( - "Pass either --target or --scope, not both.".to_string(), - )); - } let body = serde_json::json!({ - "scopes": scopes, - "target_uri": target_uri.unwrap_or(""), + "uri": uri, "fix": fix, - "wait": wait, "node_limit": node_limit, }); let result = client.privacy_scan(&body).await?; diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs index 64e88d0c72..f589fecb14 100644 --- a/crates/ov_cli/src/handlers.rs +++ b/crates/ov_cli/src/handlers.rs @@ -934,18 +934,14 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()> .await } PrivacyCommands::Scan { - scopes, - target, + uri, fix, - wait, node_limit, } => { commands::privacy::scan( &client, - &scopes, - target.as_deref(), + &uri, fix, - wait, node_limit, ctx.output_format, ctx.compact, diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs index 6ff36207e9..8479ad1162 100644 --- a/crates/ov_cli/src/help_ui.rs +++ b/crates/ov_cli/src/help_ui.rs @@ -795,8 +795,8 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ description: "Update one field; --key and --key- remain compatible.", }, HelpItem { - label: "ov privacy scan --scope resources,skills,memories", - description: "Dry-run scan for registered Privacy values in stored content.", + label: "ov privacy scan --uri ", + description: "Dry-run one explicit URI under strict server resource limits.", }, ], next_steps: &[HelpItem { diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs index 943ff84f80..007cd5a818 100644 --- a/crates/ov_cli/src/main.rs +++ b/crates/ov_cli/src/main.rs @@ -2060,21 +2060,19 @@ enum PrivacyCommands { }, /// Scan persisted content for registered Privacy values Scan { - /// Comma-separated scopes: resources,skills,memories - #[arg(long = "scope", value_delimiter = ',', value_name = "scope")] - scopes: Vec, - /// Scan one exact viking:// file or directory instead of default scopes - #[arg(long, conflicts_with = "scopes", value_name = "uri")] - target: Option, + /// Exact viking:// file or directory to scan + #[arg(long, value_name = "uri")] + uri: String, /// Rewrite unambiguous matches and refresh derived indexes #[arg(long)] fix: bool, - /// Wait for refresh/reindex work after fixing - #[arg(long)] - wait: bool, /// Maximum files/nodes enumerated per root - #[arg(long, default_value_t = 10000)] - node_limit: usize, + #[arg( + long, + default_value_t = 1000, + value_parser = clap::value_parser!(u16).range(1..=1000) + )] + node_limit: u16, }, } @@ -6182,30 +6180,41 @@ mod tests { OsString::from("ov"), OsString::from("privacy"), OsString::from("scan"), - OsString::from("--scope"), - OsString::from("resources,memories"), + OsString::from("--uri"), + OsString::from("viking://~/resources/project-a"), OsString::from("--fix"), - OsString::from("--wait"), ])); match scan.command { Commands::Privacy { action } => match *action { PrivacyCommands::Scan { - scopes, - target, + uri, fix, - wait, node_limit, } => { - assert_eq!(scopes, vec!["resources", "memories"]); - assert!(target.is_none()); - assert!(fix && wait); - assert_eq!(node_limit, 10000); + assert_eq!(uri, "viking://~/resources/project-a"); + assert!(fix); + assert_eq!(node_limit, 1000); } _ => panic!("expected privacy scan"), }, _ => panic!("expected privacy command"), } + for invalid_args in [ + vec!["ov", "privacy", "scan"], + vec![ + "ov", + "privacy", + "scan", + "--uri", + "viking://~/resources/project-a", + "--node-limit", + "1001", + ], + ] { + assert!(Cli::try_parse_from(invalid_args).is_err()); + } + let reference = "{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}"; let resolved = Cli::parse_from(preprocess_privacy_args(vec![ diff --git a/crates/ragfs-python/src/privacy.rs b/crates/ragfs-python/src/privacy.rs index 9d5566b414..b86e1606a4 100644 --- a/crates/ragfs-python/src/privacy.rs +++ b/crates/ragfs-python/src/privacy.rs @@ -51,4 +51,27 @@ impl PyPrivacyMatcher { .collect(); (result.content, replacements, result.ambiguous_references) } + + /// Return the regular result plus whether the replacement cap was exceeded. + fn sanitize_limited( + &self, + py: Python<'_>, + content: String, + max_replacements: usize, + ) -> (String, Vec<(usize, usize, String)>, Vec>, bool) { + let matcher = self.inner.clone(); + let (result, limit_exceeded) = + py.detach(move || matcher.sanitize_limited(&content, max_replacements)); + let replacements = result + .replacements + .into_iter() + .map(|replacement| (replacement.start, replacement.end, replacement.reference)) + .collect(); + ( + result.content, + replacements, + result.ambiguous_references, + limit_exceeded, + ) + } } diff --git a/crates/ragfs/src/privacy_matcher.rs b/crates/ragfs/src/privacy_matcher.rs index 590f1ca6b2..42654326c4 100644 --- a/crates/ragfs/src/privacy_matcher.rs +++ b/crates/ragfs/src/privacy_matcher.rs @@ -96,31 +96,47 @@ impl PrivacyMatcher { /// Rewrite all non-overlapping leftmost-longest matches. pub fn sanitize(&self, content: &str) -> PrivacySanitizeResult { + self.sanitize_limited(content, usize::MAX).0 + } + + /// Rewrite matches while bounding replacement metadata allocation. + /// + /// When the limit is exceeded, the original content is returned with empty + /// metadata and the boolean is set. Callers can then report the file as too + /// dense without retaining a potentially attacker-controlled match vector. + pub fn sanitize_limited( + &self, + content: &str, + max_replacements: usize, + ) -> (PrivacySanitizeResult, bool) { let Some(matcher) = &self.matcher else { - return PrivacySanitizeResult { - content: content.to_string(), - replacements: Vec::new(), - ambiguous_references: Vec::new(), - }; + return ( + PrivacySanitizeResult { + content: content.to_string(), + replacements: Vec::new(), + ambiguous_references: Vec::new(), + }, + false, + ); }; - let matches: Vec<_> = matcher.find_iter(content).collect(); - if matches.is_empty() { - return PrivacySanitizeResult { - content: content.to_string(), - replacements: Vec::new(), - ambiguous_references: Vec::new(), - }; - } - - let extra_capacity = matches.len().saturating_mul(32); - let mut rewritten = String::with_capacity(content.len().saturating_add(extra_capacity)); - let mut replacements = Vec::with_capacity(matches.len()); + let mut rewritten = String::with_capacity(content.len()); + let mut replacements = Vec::new(); let mut ambiguous_references = Vec::new(); let mut source_byte_cursor = 0; let mut source_char_cursor = 0; - for matched in matches { + for matched in matcher.find_iter(content) { + if replacements.len() >= max_replacements { + return ( + PrivacySanitizeResult { + content: content.to_string(), + replacements: Vec::new(), + ambiguous_references: Vec::new(), + }, + true, + ); + } let start = matched.start(); let end = matched.end(); rewritten.push_str(&content[source_byte_cursor..start]); @@ -145,13 +161,20 @@ impl PrivacyMatcher { }); source_byte_cursor = end; } - rewritten.push_str(&content[source_byte_cursor..]); - - PrivacySanitizeResult { - content: rewritten, - replacements, - ambiguous_references, + if replacements.is_empty() { + rewritten.push_str(content); + } else { + rewritten.push_str(&content[source_byte_cursor..]); } + + ( + PrivacySanitizeResult { + content: rewritten, + replacements, + ambiguous_references, + }, + false, + ) } } @@ -210,4 +233,18 @@ mod tests { assert_eq!(result.content, "unchanged"); assert!(result.replacements.is_empty()); } + + #[test] + fn limited_scan_stops_without_returning_partial_rewrites() { + let matcher = matcher(vec![("secret", vec!["ref"])]); + let (result, limit_exceeded) = matcher.sanitize_limited("secret secret secret", 2); + assert!(limit_exceeded); + assert_eq!(result.content, "secret secret secret"); + assert!(result.replacements.is_empty()); + + let (result, limit_exceeded) = matcher.sanitize_limited("secret secret", 2); + assert!(!limit_exceeded); + assert_eq!(result.content, "ref ref"); + assert_eq!(result.replacements.len(), 2); + } } diff --git a/docs/en/api/10-privacy.md b/docs/en/api/10-privacy.md index 7cf39619d0..37f51e10db 100644 --- a/docs/en/api/10-privacy.md +++ b/docs/en/api/10-privacy.md @@ -41,7 +41,7 @@ Activating a version changes only the values stored in OpenViking. It does not r ## Scan Stored Content -Explicitly scan persisted text. The default is a dry run and never returns the matched value. With `fix=true`, unambiguous matches are replaced and affected content is refreshed. +Explicitly scan one persisted file or directory. The default is a dry run and never returns the matched value. With `fix=true`, unambiguous matches are replaced and affected content is refreshed asynchronously. ```text POST /api/v1/privacy-configs/scan @@ -49,15 +49,13 @@ POST /api/v1/privacy-configs/scan ```json { - "scopes": ["resources", "skills", "memories"], - "target_uri": "", + "uri": "viking://~/resources/project-a", "fix": false, - "wait": false, - "node_limit": 10000 + "node_limit": 1000 } ``` -Use either `scopes` or `target_uri`. When one value maps to multiple Privacy fields, the result reports `ambiguous` and never chooses a reference arbitrarily. +`uri` is required. Broad scope-based scans are intentionally unsupported. `node_limit` must be between 1 and 1000. The server also enforces one scan per process, 20 file reads per second, 1 MiB per file, 16 MiB total input, 1,000 replacements per file, at most 32 reference details per matched file, at most 100 repaired files, and a cooperative 60-second scan budget. A second concurrent request receives `RESOURCE_EXHAUSTED`. Repair never waits for semantic/vector refresh work. Responses report skipped files, truncation, the stop reason, and the effective limits. When one value maps to multiple Privacy fields, the result reports `ambiguous` and never chooses a reference arbitrarily. --- @@ -561,9 +559,9 @@ ov privacy delete 'viking://~/privacy/credential/github-prod' --yes ov privacy delete credential github-prod api_key --yes ov privacy delete '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' --yes -# Stored-content scans are dry-run unless --fix is passed -ov privacy scan --scope resources,skills,memories -ov privacy scan --target viking://~/resources/project-a --fix --wait +# Stored-content scans require one URI and are dry-run unless --fix is passed +ov privacy scan --uri viking://~/resources/project-a +ov privacy scan --uri viking://~/resources/project-a --fix --node-limit 500 ``` --- diff --git a/docs/en/concepts/13-privacy.md b/docs/en/concepts/13-privacy.md index 8bd059808f..de62b3b971 100644 --- a/docs/en/concepts/13-privacy.md +++ b/docs/en/concepts/13-privacy.md @@ -150,8 +150,8 @@ ov privacy versions credential github ov privacy activate credential github 1 ov privacy delete credential github pat --yes -# Scan persisted content; the default only reports, while --fix replaces plaintext PATs -ov privacy scan --scope resources,skills,memories +# Scan one explicit URI; the default only reports, while --fix replaces plaintext PATs +ov privacy scan --uri viking://~/resources/project-a ``` --- diff --git a/docs/zh/api/10-privacy.md b/docs/zh/api/10-privacy.md index 9b4f254d4a..39cdf391c9 100644 --- a/docs/zh/api/10-privacy.md +++ b/docs/zh/api/10-privacy.md @@ -99,7 +99,7 @@ ### scan_privacy_content() -显式扫描已经持久化的文本内容。默认 dry-run，不返回命中的原值；`fix=true` 时替换无歧义命中并刷新相应内容。 +显式扫描一个已经持久化的文件或目录。默认 dry-run，不返回命中的原值；`fix=true` 时替换无歧义命中，并异步刷新相应内容。 ```text POST /api/v1/privacy-configs/scan @@ -107,15 +107,13 @@ POST /api/v1/privacy-configs/scan ```json { - "scopes": ["resources", "skills", "memories"], - "target_uri": "", + "uri": "viking://~/resources/project-a", "fix": false, - "wait": false, - "node_limit": 10000 + "node_limit": 1000 } ``` -`scopes` 与 `target_uri` 二选一。重复值对应多个 Privacy 字段时报告 `ambiguous`，不会任意选取引用。 +`uri` 必填，服务端刻意不支持按 scope 发起全域扫描。`node_limit` 只能在 1–1000 之间。服务端还强制限制为每个进程同时一个扫描、每秒最多读取 20 个文件、单文件最多 1 MiB、总输入最多 16 MiB、单文件最多记录 1000 个替换、每个命中文件最多返回 32 条引用详情、单次最多修复 100 个文件，并使用协作式 60 秒扫描预算；第二个并发请求返回 `RESOURCE_EXHAUSTED`。修复不会等待语义和向量刷新完成。响应会返回跳过文件计数、截断状态、停止原因和实际生效的限制。重复值对应多个 Privacy 字段时报告 `ambiguous`，不会任意选取引用。 --- @@ -564,9 +562,9 @@ ov privacy delete 'viking://~/privacy/credential/github-prod' --yes ov privacy delete credential github-prod api_key --yes ov privacy delete '{{ov_privacy:alice:viking://~/privacy/credential/github-prod#/values/api_key}}' --yes -# 存量内容扫描默认为 dry-run；--fix 才会改写 -ov privacy scan --scope resources,skills,memories -ov privacy scan --target viking://~/resources/project-a --fix --wait +# 存量内容扫描必须指定一个 URI，默认为 dry-run；--fix 才会改写 +ov privacy scan --uri viking://~/resources/project-a +ov privacy scan --uri viking://~/resources/project-a --fix --node-limit 500 ``` --- diff --git a/docs/zh/concepts/13-privacy.md b/docs/zh/concepts/13-privacy.md index 616b06e1ce..c5924ee2c9 100644 --- a/docs/zh/concepts/13-privacy.md +++ b/docs/zh/concepts/13-privacy.md @@ -150,8 +150,8 @@ ov privacy versions credential github ov privacy activate credential github 1 ov privacy delete credential github pat --yes -# 扫描已经持久化的正文；默认仅报告，--fix 才会替换明文 PAT -ov privacy scan --scope resources,skills,memories +# 扫描一个明确的 URI；默认仅报告，--fix 才会替换明文 PAT +ov privacy scan --uri viking://~/resources/project-a ``` --- diff --git a/docs/zh/privacy-review.md b/docs/zh/privacy-review.md deleted file mode 100644 index bb4f09fb41..0000000000 --- a/docs/zh/privacy-review.md +++ /dev/null @@ -1,208 +0,0 @@ -# OpenViking Privacy 审查简报 - -审查日期：2026-10-15；代码基线：`9baf83f0`。依据：[Privacy API](api/10-privacy.md)、[Privacy 概念](concepts/13-privacy.md)，并核对当前实现、Agent 接入、测试和 CI。Agent 审查重点为仓库内 MCP、SDK/CLI、Codex/通用 hook 共享采集代码与 OpenClaw 使用指导。本文是审查结果及建议方案，不涉及功能修改。 - -## 结论 - -**Privacy 已具备用户范围的敏感配置分离、版本保存和读取还原能力，但当前不能保证 Agent 接入后会主动、妥善地用它保存外部系统账号密钥，也不能依据现有测试通过就认定敏感信息不会泄漏。** - -主要原因是：Agent 缺少直接的 Privacy 管理工具与完整操作指导；自动保护集中于技能正文，依赖模型识别及有限的文本替换；还原后的真实值可以进入模型上下文及会话采集；版本写入的故障一致性存在可复现问题。现有测试能验证若干功能和并发行为，对这些安全边界的保障不足。 - -两份参考文档已经如实说明“抽取不能保证全面”“附件不一定处理”“读取返回真实值”“不等于加密”。这些限制应成为 Agent 接入和测试设计的约束，不能只靠文档提示来落实。 - -## 1. 当前机制及适用边界 - -| 能力 | 当前实现与含义 | -|---|---| -| 配置组织 | 按调用者的 `viking://user/{user_id}/privacy/{category}/{target_key}` 保存。API 的 `category` 不限于 `skill`，所以可以人工保存任意系统的账号、密码、密钥；自动提取和还原使用 `category=skill`、技能名作为目标。 | -| 版本管理 | `current.json`、`history/version_001.json` 等保存完整 `values`；`.meta.json` 保存当前/最新版本及作者等。相同值不新增版本；不同值完整替换并激活；回滚后再次写入按最新版本递增。 | -| 技能写入 | `SkillProcessor` 在准备阶段调用账户配置的 VLM，识别正文中的配置，再对成功匹配的片段占位符化，保存 Privacy 后写入技能。更新与训练产生技能的相关路径也使用隐私处理。 | -| 技能读取 | `FSService.read` 识别 `/skills/{name}/SKILL.md`，按**当前调用者**的配置还原占位符。Privacy API、CLI 和技能详情也能返回真实值。共享技能的密钥仍按用户保存；同一用户的同名技能共用同一个 Privacy 目标。 | -| 权限与加密 | Privacy 复用请求身份和 VikingFS 命名空间/ACL，具有用户空间隔离；ROOT 可跨用户访问。它没有独立的“仅执行可用、不可读取”密钥权限。存储加密是另一项配置，默认关闭。 | -| 生命周期 | `DELETE /api/v1/skills/{name}` / `ov skills remove` 删除相应 Privacy；通用 `forget` / `ov rm` 删除技能目录会留下配置。API 未提供通用 Privacy 删除或单个历史版本销毁端点。激活版本不会轮换或撤销第三方密钥。 | - -实现依据：[配置服务](../../openviking/privacy/service.py)、[技能处理](../../openviking/utils/skill_processor.py)、[读取服务](../../openviking/service/fs_service.py)、[访问判断](../../openviking/core/namespace.py)、[存储加密](concepts/10-encryption.md)。 - -## 2. 建议落地方案 - -本阶段先解决一个确定性场景：当 `resources`、`skills`、`memories` 的待写入正文中出现**当前写入用户已登记在 Privacy 中的密钥**时，在落盘及派生处理之前强制识别并替换为统一引用。Privacy 新增后的存量反向扫描不默认自动运行，而由显式 CLI 命令触发。 - -当前工作分支已经完成第一轮落地：`ov privacy reference` 默认只返回 owner 绑定引用；`ov privacy resolve` 可以通过三段参数、owner-bound reference 或字段 URI 显式读取单个字段；`ov privacy delete` 可以删除整个 target，也可以让当前 active 配置中的单个字段不可用：仍有其他字段时内部发布新快照并保留历史，删除最后一个字段时删除整个 target 及其历史；PrivacySanitizer 不按字段名猜测，对当前 active Privacy 中字符数不少于 6 的字符串值实施自动保护；普通文本写入、Skill 写入、Resource 导入、直接 VikingFS 文本写入和 `read_visible` 已接入；`ov privacy scan` 默认 dry-run，`--fix` 显式修复存量正文。会话采集、Gateway 内部还原以及所有外部导出/备份路径仍需后续验收，不能把本轮接入理解为全链路已经闭环。 - -### 2.1 统一引用格式 - -建议采用： - -```text -{{ov_privacy:{owner_user_id}:viking://~/privacy/{category}/{target_key}#/values/{field}}} -``` - -例如用户 `user_id_1` 向公共技能区写入包含 GitHub 密钥的内容，替换结果为： - -```text -Authorization: Bearer {{ov_privacy:user_id_1:viking://~/privacy/credential/github-prod#/values/api_key}} -``` - -语义约定： - -- `owner_user_id` 是 Privacy 配置所有者，也是本次自动替换的溯源身份；本阶段不增加引用协议版本，未来出现不兼容语法时再加入。 -- 引用内部的 `~` **在 Privacy 引用解析器内**相对于 `owner_user_id` 解析，而不是相对于当前阅读者解析；普通 `viking://~` 在其他 API 中仍保持“当前调用者”语义。 -- `category + target_key` 定位配置目标，`#/values/{field}` 定位单个字段；不能只引用 target，否则可能一次暴露同一目标下的多个值。 -- `{owner_user_id}`、`category`、`target_key` 和 `field` 必须走统一的标识符校验与转义，不能通过字符串拼接后直接访问存储。 -- 引用始终解析当前 active version，不绑定 `current.json` 等物理文件。密钥轮换后正文引用无需变化。 -- 自动 Sanitizer 只生成所有者绑定引用，不生成调用者绑定引用。这使 `/agent/skills` 等公共区域中的内容可追溯，同时阻止其他用户将引用误解析为自己的同名配置。 -- canonical reference 始终采用无空格形式。解析器仅兼容 `{{` 后、`}}` 前的空格或 Tab，解析后立即规范化为无空格形式；owner、URI、category、target 和 field 内部不接受任意空白。 - -当其他用户读取公共内容时，应保留引用并返回结构化状态，而不是尝试跨用户读取： - -```text -[Privacy Config Notice] -owner: user_id_1 -status: forbidden -action: ask the owner or an administrator for access or configuration guidance -``` - -这项设计会向有权读取公共内容的用户显示配置所有者 ID，是为了支持溯源和协作；如部署不希望暴露真实 user ID，后续可以用可审计的 `owner_ref` 替代，但不能移除服务端真实 owner 绑定。 - -### 2.2 高效 PrivacySanitizer - -PrivacySanitizer 应成为统一底层能力，而不是 Skill 专用的 LLM 后处理。建议工作方式如下： - -1. 按 `(account_id, user_id, privacy_revision)` 缓存当前生效的敏感值目录。每次该用户 upsert/activate/delete Privacy 后只失效对应缓存；缓存同时使用 3600 秒 TTL，多 worker/多节点上的外部修改最多一小时后被本进程重新加载。 -2. Privacy 是显式保护边界，不再通过 `_SECRET_FIELD_NAMES` 或字段名模式猜测敏感性；当前 active Privacy 中字符数不少于 6 的字符串值进入匹配目录，包括用户自定义字段名。更短的值仍可保存和显式 resolve，但不会参与正文全文替换，以避免 `cn`、`1` 等造成大量误命中。 -3. 在受控内存中构建多模式匹配器（如 Aho-Corasick），扫描复杂度保持为 `O(正文长度 + 命中数)`；重叠命中选择最长值，同一个值对应多个 Privacy 引用时标记为歧义并拒绝自动替换。持久缓存和日志只保存 HMAC/引用元数据，不记录原值。 -4. 对当前用户发起的 user 空间写入生效；写入 `/agent/skills` 等公共区域时仍使用**写入者**的 Privacy 目录，并在引用中写入该用户 ID。 -5. 先匹配完整原值，再按明确白名单处理 URL 编码、JSON 转义、PEM 换行标准化等变体。多个 Privacy 字段对应同一值、过短密钥造成高歧义、或无法可靠生成引用时，应拒绝/隔离写入并返回候选引用，不能任意选择。 -6. Privacy 自身存储路径不参与扫描。审计只记录正文 URI、owner、引用、位置和匹配规则，不记录命中的秘密。 - -必须覆盖以下写入和派生入口： - -- 普通 `write`、`edit`、batch write； -- Skill 的 frontmatter、正文、description、metadata 和文本附件； -- Resource 解析后的正文及文本派生物； -- Memory 最终序列化正文和可进入上下文的字段； -- 写入 `.abstract.md`、`.overview.md`、关键词索引、向量 payload 或发送摘要模型之前。 - -建议采用“两道门”：第一道在各类内容进入持久化/摘要/索引之前替换；第二道在 `read`、search inline content 和 context assembler 输出前再次扫描，拦截绕过统一写入入口或尚未完成手工修复的存量内容。第二道只做当前响应脱敏，不在读取路径中隐式改写存储。 - -### 2.3 显式存量扫描与修复 - -Privacy 新增、轮换后暂不默认扫描全部存量，以避免不可预期的大范围 IO 和内容修改。提供显式 CLI： - -```bash -# 默认只报告命中，不修改 -ov privacy scan --scope resources,skills,memories -ov privacy scan --target viking://~/resources/project-a - -# 用户确认后替换正文，并刷新摘要和索引 -ov privacy scan --scope resources,skills,memories --fix --wait -``` - -建议约束： - -- 普通用户只能扫描其可读且可写的内容，并只使用自己的 Privacy 目录；管理员代用户扫描必须显式指定 owner 并留下审计。 -- `scan` 默认 dry-run，只返回 URI、字段引用、命中数量及是否可自动修复，不返回原值。 -- `--fix` 在持有内容锁时重新扫描，采用 expected revision/CAS 防止覆盖并发修改；修复后删除旧索引记录，重新生成摘要和索引。 -- 历史快照、导出、外部备份和已发送日志不承诺由该命令清理，需单独的保留与销毁策略。 - -### 2.4 上下文表示与 Agent 还原 - -上下文正文保持统一引用，不在组装阶段自动展开，并可以附加不含密钥的状态： - -```xml - -``` - -本阶段允许 Agent 自己发起还原，但应是**显式、字段级、受权限控制**的操作： - -1. Agent 识别引用并提取 owner、category、target 和 field，不自行拼接或改写引用。 -2. 当当前身份有权读取 owner 的 Privacy 时，调用字段级 resolve/read 接口获取当前 active value；接口只返回被引用字段，不返回整个 target。 -3. Agent 将值用于当前外部系统调用；工具输入、工具输出、错误、会话采集和后续写入仍需经过 PrivacySanitizer，避免还原值再次进入 resources、skills 或 memories。 -4. 当前用户无权读取时，解析器返回 `forbidden + owner_user_id`。Agent 应说明需联系该 owner/管理员，不能回退到猜测、搜索或跨用户读取。 -5. 缺少 target/field 时返回 `missing`；Agent 可以引导 owner 补充 Privacy，不能把占位符当作真实参数调用。 - -OpenViking Gateway 可以提供更安全的增强路径：Gateway 在转发外部请求前于内部解析引用、注入密钥，并在响应返回 Agent 前再次脱敏，使明文不进入模型上下文。普通 OpenViking 接入仍允许 Agent 显式还原，两种方式共用同一引用、权限校验、审计和 Sanitizer，不应维护两套占位符协议。 - -## 3. 主要发现与建议 - -优先级：P0 表示应在承诺“自动安全保存生产凭据”前解决；P1 表示下一阶段补齐；P2 表示完善运维和产品语义。下表分别标明执行复现与代码审查证据。 - -| 优先级 | 发现及影响 | 建议与证据 | -|---|---|---| -| P0 | **Agent 不能自然发现完整的 Privacy 使用方法。** MCP 没有 Privacy 写入、查询、激活工具；Python SDK 也没有对应方法。插件指导包含“不要回显凭据”“不要把凭据放进技能”，但没有教如何将用户交付的账号密钥转存、引用、使用及清理。仅有 HTTP/CLI 能力不能证明 Agent 会选对保存路径。 | 增加明确的 `set/list/delete/resolve-field` 等工具与接入技能；普通管理响应默认只返回 owner、键名、引用和配置版本，仅显式字段还原返回该字段值。给出账号密钥处理示例及工具不可用时的分支。依据：[MCP 注册](../../openviking/server/mcp_endpoint.py)、[插件记忆指导](../../examples/codex-memory-plugin/skills/openviking-memory/SKILL.md)、[插件技能指导](../../examples/codex-memory-plugin/skills/openviking-skills/SKILL.md)、`sdk/python`。这是接入完备性判断，尚未通过真实 Agent 行为评测证明其选择结果。 | -| P0 | **模型识别成功也不等于清除成功。** 仅替换引号包裹的完整值，或 `:` / `=` 后直到行尾的完整裸值。Bearer、自然语言、带行尾注释等会保留原值；同一密钥在赋值和说明文字同时出现时，会一边保存 Privacy、一边留下明文。 | 对已经登记的密钥改用确定性的 PrivacySanitizer 全文匹配和残留校验，不再依赖字段格式；有歧义或未处理秘密时拒绝/隔离写入。对尚未登记的新秘密仍需检测或模型辅助，但不能把“模型未识别”解释为安全。已用虚构密钥执行复现，见下一节；依据：[替换器](../../openviking/privacy/skill_placeholder.py)。 | -| P0 | **无法区分“没有秘密”和“提取没有成功”。** 非 JSON、错误结构、匹配失败可得到空 `values` 并保留原文；模型调用异常会传播，但正常返回无效内容没有严格失败状态。提示词要求输出缺值配置项，代码却过滤空值并只返回实际替换成功的项，所需配置清单会丢失。 | 定义严格输出 schema、提取状态、已识别/已替换/未解决三类结果；失败时阻止敏感内容持久化。保留缺值字段及引用，供 Agent 请求配置。已复现无效响应和空值；依据：[提取器](../../openviking/privacy/skill_extractor.py)、[提取提示词](../../openviking/prompts/templates/skill/privacy_extraction.yaml)。 | -| P0 | **保护未贯穿持久化链路。** 只替换 `skill_dict.content`；description、metadata 和辅助文件不受同等处理，且附件会随技能写入并排入递归语义处理。普通 `write`、资源导入、`remember`、会话消息没有自动转存 Privacy 的机制。原始技能正文在清理前已经发给配置的 VLM。 | 覆盖所有技能文本、元数据和附件；在首次保存及外发模型前执行秘密检测。资源/记忆/会话应使用统一敏感数据策略，以引用替代密钥，并支持无需向模型提交原值的确定性保存接口。依据：[SkillLoader](../../openviking/core/skill_loader.py)、[技能写入与附件](../../openviking/utils/skill_processor.py)、[MCP remember](../../openviking/server/mcp_endpoint.py)、[会话保存](../../openviking/session/session.py)。附件/派生数据扩散是代码链路风险，本次未运行真实模型泄漏评测。 | -| P0 | **密钥还原后可能再次被保存。** `read` 会把实际值注入上下文；未引用配置也被完整追加到 Notice。采集过滤只改文本，工具输入/输出保持原样，即使配置文本脱敏规则也无效。工具结果采集默认因 harness 而异，DSH 默认开启，不能笼统认为所有插件都默认采集工具结果。 | 普通接入允许 Agent 经字段级、权限受控接口显式还原，但 Notice 只能列 owner/键名/状态，还原值不能随整个 target 返回；工具输入、输出、错误、会话采集和再次写入统一经过 Sanitizer。Gateway 可在内部解析和注入，避免明文进入模型。已复现 Notice 回显及文本过滤生效而工具载荷不变；依据：[还原器](../../openviking/privacy/skill_restore.py)、[采集过滤](../../examples/codex-memory-plugin/scripts/shared/capture-utils.mjs)、[默认配置](../../examples/codex-memory-plugin/scripts/shared/config-schema.mjs)。 | -| P0 | **版本写入缺少多文件事务保障。** `upsert` 依次写 history、current、meta，读操作不持同一锁。meta 写入失败后 current 可以已切到新版本，但 meta 仍为旧版本；下一次写入会复用版本号并覆盖历史。锁防并发写入，不能解决部分写入和读取中间态。 | 引入原子版本发布/提交指针及故障恢复，版本号不能只依赖可能过时的 meta。验证 history、current、meta 的一致性和历史不可覆盖；对 activate、取消、重启也做故障注入。已用真实配置服务加内存存储故障注入复现，见下一节；依据：[配置服务](../../openviking/privacy/service.py)。 | -| P1 | **保留占位符的技能更新可能删掉仍需要的配置。** `PUT /skills/{name}` 对空提取结果执行 `delete_if_empty=True`。传入已占位符化内容、模型返回缺值字段时，仍有配置引用却会走删除分支。现有测试只覆盖“内容确实不再需要配置时删除”。 | 检查现存占位符与字段引用，区分未提供值、显式删除和提取失败；保留仍被引用的值。已在实际 prepare/apply 方法组合中复现删除调用；完整 HTTP 情景应补回归测试。依据：[技能更新](../../openviking/server/routers/skills.py)、[apply_skill_privacy](../../openviking/utils/skill_processor.py)、[现有删除测试](../../tests/server/test_api_skills.py)。 | -| P1 | **核心 Privacy 回归没有明确接入现有 CI。** `tests/server` 中相关测试可本地运行，但工作流中的主 API 集成任务运行 `tests/api_test`，该目录无 Privacy 专项测试；Lite 任务仅运行 cuVS 指定测试。当前配置中未发现直接运行这些 Privacy 核心测试的 job，也未配置 Privacy 覆盖率门槛。 | 增加 PR 必跑的 Privacy 回归任务，将本简报列出的安全不变量列为门禁；涉及提示词或 Agent 技能的 Markdown/YAML 修改也要触发。依据：[API CI](../../.github/workflows/api_test.yml)、[PR CI](../../.github/workflows/pr.yml)、[Lite CI](../../.github/workflows/_test_lite.yml)、[pytest 配置](../../pyproject.toml)。本次审查的是仓库配置，未核验远端 branch protection 和 CI 历史运行。 | -| P2 | **版本记录不等于完整安全审计或删除。** `get_current/get_version` 不更新 `last_accessed_at`，也未在 Privacy 服务记录逐次读取审计；旧密钥长期存在历史快照。全文 CLI 输出、命令参数、本地 JSON 文件和备份都是额外副本；存储快照枚举默认没有排除 Privacy。 | 增加读取/使用审计、历史保留/销毁策略、默认脱敏和输入 stdin 等方式；备份与加密策略纳入生命周期。依据：[配置服务](../../openviking/privacy/service.py)、[CLI](../../crates/ov_cli/src/commands/privacy.rs)、[快照枚举](../../crates/ragfs/src/git/enumerate.rs)。是否实际备份取决于部署，本次未核验外部备份。 | -| P2 | **部分更新和目标命名容易造成误操作。** HTTP upsert 整包替换；CLI `--key-*` 是先读再合并再写，两个客户端并行更新不同键仍可能丢更新。`category=skill` 的同名目标没有独立的服务/账号/环境维度。 | 增加原子 PATCH 或 expected_version/CAS；明确命名及绑定规则。人工保存非技能凭据时按服务、环境、账号组织目标，并告知它们不会自动在技能读取中还原。依据：[Privacy API](api/10-privacy.md)、[CLI 合并写入](../../crates/ov_cli/src/commands/privacy.rs)、[技能还原](../../openviking/service/fs_service.py)。并行 CLI 丢更新是读改写语义分析，本次未执行双客户端复现。 | - -## 4. 本次验证与现有测试的证明范围 - -本地使用 macOS / Python 3.12。所有复现只使用虚构密钥，未调用真实模型或外部账号系统。 - -现有测试执行命令： - -```bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin \ - -q -o addopts='' --tb=short \ - tests/server/test_privacy_config_service.py \ - tests/server/test_api_privacy_configs.py \ - tests/server/test_skill_privacy_lock.py \ - tests/unit/test_skill_privacy_cancellation.py \ - tests/session/test_commit_skill_inference.py - -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin \ - -q -o addopts='' --tb=short \ - tests/server/test_api_skills.py tests/server/test_skill_update_lock.py \ - -k 'privacy or rollback' -``` - -第一组 **35 passed**；第二组 **6 passed、15 deselected**，共 **41 个选定的相关测试通过**。上述命令使用显式插件并关闭自动插件加载。运行未开启覆盖率，因此不声称已经获得 Privacy 覆盖率统计。 - -| 测试层 | 已有保障 | 仍缺的关键证明 | -|---|---|---| -| 配置服务/API | 版本递增、同值去重、激活、增加键、404；部分底层错误传播 | 多文件故障一致性、损坏数据恢复、重启、读取中间态、原子部分更新 | -| 抽取/还原 | 固定模型返回值、引用恢复、缺值与额外配置 Notice、引号/换行/CRLF/正则字符处理 | 真实模型识别质量；Bearer、自然语言、注释、重复残留、私钥、提示注入、元数据/附件 | -| 更新与并发 | 8 次并发写版本测试使用内存存储锁；另有服务级更新回滚、等待写入和取消释放锁测试 | 多进程真实存储竞争、进程崩溃、带占位符更新、历史快照不被覆盖；不能把单进程测试扩大为分布式保证 | -| 身份与隔离 | 当前用户路径写入和通用命名空间策略；API 测试主要使用 DevAuth/ROOT | Privacy 专项的无密钥/普通用户/跨用户/跨账户读写历史拒绝测试，以及共享技能的每用户绑定 | -| Agent 与整条链路 | 插件有采集和文本过滤测试；训练产出技能有隐私处理调用测试 | Agent 是否选对 Privacy、是否避免回显；原值是否进入 messages/history/记忆/摘要/工具日志/导出；真实模型与完整 Agent 的行为评测 | - -**额外执行复现（调用当前实现，非正式回归测试）：** - -- 已给提取器正确的虚构值：`api_key=FAKE_SECRET` 能替换；`Authorization: Bearer FAKE_SECRET`、`密码是 FAKE_SECRET`、`api_key=FAKE_SECRET # prod` 保留明文。赋值后再加 `Use FAKE_SECRET here`，Privacy 成功提取但正文仍有原值。 -- VLM 返回 `I cannot process this request.` 或 `{"values":{"api_key":""}}`，提取结果为空且原正文不变；无明确失败状态。 -- `restore_skill_content` 给不含引用的正文配上 `unused_key=FAKE_SECRET`，Notice 包含完整虚构密钥。 -- `shapeCaptureParts` 配置替换规则后，文本中的虚构密钥变为 `[redacted]`，同一消息的 `tool_output` 中原值保留。 -- 对 `UserPrivacyConfigService` 的 `.meta.json` 写入注入异常：先写 v1，再写 v2 失败后得到 `meta.active_version=1`、`current.version=2`；随后写另一值仍分配 v2，并覆盖原来的 `history/version_002.json`。 -- 对包含 `{{ov_privacy:skill:audit-skill:api_key}}` 的内容调用 `prepare_skill_privacy`，VLM 返回空字段后，再以 `delete_if_empty=True` 调用 `apply_skill_privacy`：内容仍有引用，配置删除方法被调用。 - -## 5. 推荐的 Agent 使用流程 - -**当前可以通过 HTTP/CLI 人工管理秘密，但尚无经过验证的“接入即可自动学会”流程。** `remember`、普通 `write`、资源导入都不应被当作外部账号密钥的保存入口；也不应为了利用自动抽取而先把真实密钥写进技能正文。 - -建议把如下流程落实为工具协议和 Agent 接入技能，而不是只添加一句“注意隐私”： - -1. 接收到账号密码、AK/SK、token、私钥等时，先识别所属用户、服务、环境和账号，选定 Privacy 目标。持久化技能只写字段声明或引用。 -2. 使用专用工具保存值，工具返回带 owner 的统一引用、键名和版本号；普通回复与记忆只记“已配置”及用途。现有 HTTP 写入会回显值，接入包装器应处理响应及采集内容。 -3. 使用时，普通接入允许 Agent 在当前身份权限内通过字段级接口显式还原；Gateway 接入优先在内部解析并注入。两条路径都必须在工具输出、错误、会话采集和再次写入时重新脱敏。 -4. owner 不同、值缺失、提取失败或权限不足时返回 `forbidden/missing` 和可联系的 owner；不把未解析引用当成密钥，不通过搜索绕过权限。外部凭据轮换后更新 Privacy；明确回滚不能恢复已被第三方撤销的密钥。 -5. 更新技能保留仍被引用的配置；移除技能/账号时清理 current、history 及保留策略涵盖的副本。用元信息验证成功，不通过打印密钥确认。 - -## 6. 建议的验收顺序 - -1. **先实现高效写入 Sanitizer 和输出防线。** 覆盖正文、frontmatter、附件、普通写入、资源、memory、摘要、索引和采集的虚构秘密样本；验证 user 写入及 `/agent/skills` 写入均生成 owner 绑定引用，失败必须阻止发布或转入明确状态。 -2. **补引用解析、身份与 Agent/Gateway 回归。** 验证 owner 本人字段级还原、其他用户得到 forbidden 与 owner 提示、缺值处理、轮换后解析 active value；分别验证 Agent 显式还原和 Gateway 内部注入不会把值再次写回内容或会话。 -3. **提供显式存量扫描。** 验证 `ov privacy scan` 默认只报告，`--fix` 使用内容 revision、刷新摘要和索引，且不越过用户权限；暂不把 Privacy upsert 后全量反向扫描设为默认行为。 -4. **补事务回归并接入 PR CI。** 对 history/current/meta 每一步注入写失败、取消及重启；检查版本一致、历史不覆盖，并把 Sanitizer、引用解析、身份隔离和 CLI 修复测试设为门禁。覆盖率辅助发现未测分支，不能替代安全断言。 -5. **再验证真实 Agent 行为。** 用虚构外部账号构建完整任务：保存凭据、读取公共技能、owner/非 owner 还原、轮换、缺配置和修复；检查模型回复、工具调用和会话中是否出现非预期明文，并记录模型与提示词版本。 - -完成这些验收前，适合将 Privacy 定位为**有用的敏感配置基础设施**；生产凭据的安全性仍需要接入层、执行层和存储生命周期共同落实。 diff --git a/openviking/privacy/sanitizer.py b/openviking/privacy/sanitizer.py index 5756ae80b7..f0efc83efc 100644 --- a/openviking/privacy/sanitizer.py +++ b/openviking/privacy/sanitizer.py @@ -33,6 +33,7 @@ class PrivacySanitizationResult: content: str replacements: tuple[PrivacyReplacement, ...] = () ambiguous_references: tuple[tuple[str, ...], ...] = () + match_limit_exceeded: bool = False @property def changed(self) -> bool: @@ -65,11 +66,23 @@ async def sanitize( *, ctx: RequestContext, uri: str = "", + max_replacements: int | None = None, ) -> PrivacySanitizationResult: if not content or self._is_privacy_storage_uri(uri, ctx): return PrivacySanitizationResult(content=content) matcher = await self._matcher(ctx) - rewritten, raw_replacements, raw_ambiguous = matcher.sanitize(content) + match_limit_exceeded = False + if max_replacements is None: + rewritten, raw_replacements, raw_ambiguous = matcher.sanitize(content) + else: + if max_replacements < 0: + raise ValueError("max_replacements must be non-negative") + ( + rewritten, + raw_replacements, + raw_ambiguous, + match_limit_exceeded, + ) = matcher.sanitize_limited(content, max_replacements) replacements = tuple( PrivacyReplacement(start=start, end=end, reference=reference) for start, end, reference in raw_replacements @@ -78,6 +91,7 @@ async def sanitize( content=rewritten, replacements=replacements, ambiguous_references=tuple(tuple(items) for items in raw_ambiguous), + match_limit_exceeded=match_limit_exceeded, ) async def _matcher(self, ctx: RequestContext) -> Any: diff --git a/openviking/privacy/scan.py b/openviking/privacy/scan.py index ebf5cf321c..f27cd628c7 100644 --- a/openviking/privacy/scan.py +++ b/openviking/privacy/scan.py @@ -4,14 +4,29 @@ from __future__ import annotations -from typing import Any, Iterable +import asyncio +import threading +import time +from typing import Any -from openviking.core.namespace import canonical_user_root, classify_uri, resolve_request_uri +from openviking.core.namespace import classify_uri, resolve_request_uri from openviking.privacy.sanitizer import PrivacySanitizer from openviking.server.identity import RequestContext -from openviking_cli.exceptions import InvalidArgumentError, NotFoundError +from openviking_cli.exceptions import ( + InvalidArgumentError, + NotFoundError, + ResourceExhaustedError, +) + +PRIVACY_SCAN_MAX_NODES = 1000 +PRIVACY_SCAN_MAX_FILE_BYTES = 1024 * 1024 +PRIVACY_SCAN_MAX_TOTAL_BYTES = 16 * 1024 * 1024 +PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE = 1000 +PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE = 32 +PRIVACY_SCAN_MAX_REPAIRED_FILES = 100 +PRIVACY_SCAN_MAX_DURATION_SECONDS = 60.0 +PRIVACY_SCAN_FILES_PER_SECOND = 20.0 -_SCOPES = frozenset({"resources", "skills", "memories"}) _SKIP_NAMES = frozenset( { ".abstract.md", @@ -22,24 +37,30 @@ ".artifact_manifest.json", } ) +_SCAN_STATE_GUARD = threading.Lock() +_SCAN_ACTIVE = False -def default_privacy_scan_roots(ctx: RequestContext, scopes: Iterable[str]) -> list[str]: - user_root = canonical_user_root(ctx) - roots: list[str] = [] - for scope in scopes: - if scope not in _SCOPES: - raise InvalidArgumentError( - f"unsupported privacy scan scope: {scope}", - details={"scope": scope, "allowed": sorted(_SCOPES)}, +def _acquire_scan_slot() -> None: + global _SCAN_ACTIVE + with _SCAN_STATE_GUARD: + if _SCAN_ACTIVE: + raise ResourceExhaustedError( + "Another Privacy scan is already running in this server process", + details={"max_concurrent_scans": 1}, ) - if scope == "resources": - roots.extend([f"{user_root}/resources", "viking://resources"]) - elif scope == "skills": - roots.extend([f"{user_root}/skills", "viking://agent/skills"]) - else: - roots.append(f"{user_root}/memories") - return list(dict.fromkeys(roots)) + _SCAN_ACTIVE = True + + +def _release_scan_slot() -> None: + global _SCAN_ACTIVE + with _SCAN_STATE_GUARD: + _SCAN_ACTIVE = False + + +def _entry_size(entry: dict[str, Any]) -> int | None: + size = entry.get("size") + return size if isinstance(size, int) and size >= 0 else None async def scan_privacy_content( @@ -47,89 +68,221 @@ async def scan_privacy_content( service: Any, ctx: RequestContext, sanitizer: PrivacySanitizer, - scopes: list[str], - target_uri: str = "", + uri: str, fix: bool = False, - wait: bool = False, - node_limit: int = 10000, + node_limit: int = PRIVACY_SCAN_MAX_NODES, +) -> dict[str, Any]: + """Scan one explicit URI within strict process, IO, and repair budgets.""" + if not uri: + raise InvalidArgumentError("privacy scan requires one explicit uri") + if not 1 <= node_limit <= PRIVACY_SCAN_MAX_NODES: + raise InvalidArgumentError( + f"privacy scan node_limit must be between 1 and {PRIVACY_SCAN_MAX_NODES}", + details={"node_limit": node_limit, "max_node_limit": PRIVACY_SCAN_MAX_NODES}, + ) + + _acquire_scan_slot() + try: + return await _scan_privacy_content( + service=service, + ctx=ctx, + sanitizer=sanitizer, + uri=uri, + fix=fix, + node_limit=node_limit, + ) + finally: + _release_scan_slot() + + +async def _scan_privacy_content( + *, + service: Any, + ctx: RequestContext, + sanitizer: PrivacySanitizer, + uri: str, + fix: bool, + node_limit: int, ) -> dict[str, Any]: - """Scan readable text and optionally rewrite unambiguous matches.""" - if target_uri and scopes: - raise InvalidArgumentError("pass either target_uri or scopes, not both") - roots = ( - [resolve_request_uri(target_uri, ctx)] - if target_uri - else default_privacy_scan_roots(ctx, scopes or sorted(_SCOPES)) - ) - files: list[str] = [] + root = resolve_request_uri(uri, ctx) + started_at = time.monotonic() + files: list[dict[str, Any]] = [] truncated = False - for root in roots: - try: - stat = await service.viking_fs.stat(root, ctx=ctx, skip_count=True) - except (NotFoundError, FileNotFoundError): - continue - if stat.get("isDir", stat.get("is_dir", False)): - entries = await service.viking_fs.tree( - root, - output="original", - show_all_hidden=True, - node_limit=node_limit, - level_limit=None, - ctx=ctx, - ) - if len(entries) >= node_limit: - truncated = True - files.extend( - str(entry.get("uri") or "") - for entry in entries - if not entry.get("isDir") and entry.get("uri") - ) - else: - files.append(root) + try: + stat = await service.viking_fs.stat(root, ctx=ctx, skip_count=True) + except (NotFoundError, FileNotFoundError): + raise NotFoundError(root, "privacy scan target") from None + if stat.get("isDir", stat.get("is_dir", False)): + entries = await service.viking_fs.tree( + root, + output="original", + show_all_hidden=True, + node_limit=node_limit, + level_limit=None, + ctx=ctx, + ) + # VikingFS does not expose a has_more bit on this internal tree call. + # Treat an exactly full page conservatively as truncated. + truncated = len(entries) >= node_limit + files.extend( + entry + for entry in entries + if not entry.get("isDir") and entry.get("uri") + ) + else: + files.append({"uri": root, "size": stat.get("size")}) + + unique_files: dict[str, dict[str, Any]] = {} + for entry in files: + file_uri = str(entry.get("uri") or "") + if file_uri: + unique_files.setdefault(file_uri, entry) matches: list[dict[str, Any]] = [] repaired = 0 reindex_roots: set[str] = set() - for uri in dict.fromkeys(files): - if uri.rstrip("/").rsplit("/", 1)[-1] in _SKIP_NAMES: + scanned_files = 0 + scanned_bytes = 0 + skipped_files: dict[str, int] = {} + stop_reason: str | None = None + fix_truncated = False + last_read_at = 0.0 + + def skip(reason: str) -> None: + skipped_files[reason] = skipped_files.get(reason, 0) + 1 + + for file_uri, entry in unique_files.items(): + elapsed = time.monotonic() - started_at + if elapsed >= PRIVACY_SCAN_MAX_DURATION_SECONDS: + truncated = True + stop_reason = "duration_limit" + break + if file_uri.rstrip("/").rsplit("/", 1)[-1] in _SKIP_NAMES: + skip("internal_metadata") + continue + + size = _entry_size(entry) + if size is None: + try: + file_stat = await service.viking_fs.stat(file_uri, ctx=ctx, skip_count=True) + size = _entry_size(file_stat) + except (NotFoundError, FileNotFoundError): + skip("not_found") + continue + if size is not None and size > PRIVACY_SCAN_MAX_FILE_BYTES: + skip("file_too_large") + continue + if size is not None and scanned_bytes + size > PRIVACY_SCAN_MAX_TOTAL_BYTES: + truncated = True + stop_reason = "total_bytes_limit" + break + + if last_read_at: + min_interval = 1.0 / PRIVACY_SCAN_FILES_PER_SECOND + delay = min_interval - (time.monotonic() - last_read_at) + if delay > 0: + remaining = PRIVACY_SCAN_MAX_DURATION_SECONDS - ( + time.monotonic() - started_at + ) + if remaining <= delay: + truncated = True + stop_reason = "duration_limit" + break + await asyncio.sleep(delay) + try: + raw = await service.viking_fs.read_file_bytes( + file_uri, + ctx=ctx, + max_bytes=PRIVACY_SCAN_MAX_FILE_BYTES, + ) + last_read_at = time.monotonic() + except ResourceExhaustedError: + skip("file_too_large") continue + except (NotFoundError, FileNotFoundError): + skip("not_found") + continue + + if scanned_bytes + len(raw) > PRIVACY_SCAN_MAX_TOTAL_BYTES: + truncated = True + stop_reason = "total_bytes_limit" + break + scanned_files += 1 + scanned_bytes += len(raw) try: - raw = await service.viking_fs.read_file_bytes(uri, ctx=ctx) content = raw.decode("utf-8") except UnicodeDecodeError: + skip("non_utf8") continue - except (NotFoundError, FileNotFoundError): + result = await sanitizer.sanitize( + content, + ctx=ctx, + uri=file_uri, + max_replacements=PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE, + ) + if result.match_limit_exceeded: + skip("match_limit_exceeded") + matches.append( + { + "uri": file_uri, + "occurrences": PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE + 1, + "occurrences_truncated": True, + "references": [], + "ambiguous": [], + "fixable": False, + "fixed": False, + "reason": "match_limit_exceeded", + } + ) continue - result = await sanitizer.sanitize(content, ctx=ctx, uri=uri) if not result.replacements and not result.ambiguous_references: continue + all_references = sorted( + {replacement.reference for replacement in result.replacements} + ) + all_ambiguous = sorted(set(result.ambiguous_references)) + references_truncated = len(all_references) > PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE + ambiguous_truncated = len(all_ambiguous) > PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE item = { - "uri": uri, + "uri": file_uri, "occurrences": len(result.replacements), - "references": sorted({replacement.reference for replacement in result.replacements}), - "ambiguous": [list(candidates) for candidates in result.ambiguous_references], + "references": all_references[:PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE], + "references_truncated": references_truncated, + "ambiguous": [ + list(candidates[:PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE]) + for candidates in all_ambiguous[:PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE] + ], + "ambiguous_truncated": ambiguous_truncated + or any( + len(candidates) > PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE + for candidates in all_ambiguous + ), "fixable": not result.ambiguous_references, "fixed": False, } if fix and item["fixable"]: - classification = classify_uri(uri) - if classification.is_skill: - path = service.viking_fs._uri_to_path(uri, ctx=ctx) - lease = await service.viking_fs._async_agfs.pathlock_acquire_exact(path) - try: - await service.viking_fs.write_file( - uri, result.content, ctx=ctx, lease_ref=lease - ) - finally: - await service.viking_fs._async_agfs.pathlock_release(lease) - parts = uri.split("/skills/", 1) - skill_name = parts[1].split("/", 1)[0] if len(parts) == 2 else "" - if skill_name: - reindex_roots.add(f"{parts[0]}/skills/{skill_name}") + if repaired >= PRIVACY_SCAN_MAX_REPAIRED_FILES: + item["reason"] = "repair_limit_exceeded" + fix_truncated = True else: - await service.fs.write(uri, result.content, ctx=ctx, wait=wait) - item["fixed"] = True - repaired += 1 + classification = classify_uri(file_uri) + if classification.is_skill: + path = service.viking_fs._uri_to_path(file_uri, ctx=ctx) + lease = await service.viking_fs._async_agfs.pathlock_acquire_exact(path) + try: + await service.viking_fs.write_file( + file_uri, result.content, ctx=ctx, lease_ref=lease + ) + finally: + await service.viking_fs._async_agfs.pathlock_release(lease) + parts = file_uri.split("/skills/", 1) + skill_name = parts[1].split("/", 1)[0] if len(parts) == 2 else "" + if skill_name: + reindex_roots.add(f"{parts[0]}/skills/{skill_name}") + else: + await service.fs.write(file_uri, result.content, ctx=ctx, wait=False) + item["fixed"] = True + repaired += 1 matches.append(item) reindex_tasks: list[dict[str, Any]] = [] @@ -139,18 +292,34 @@ async def scan_privacy_content( await service.reindex( uri=root, mode="semantic_and_vectors", - wait=wait, + wait=False, force=True, ctx=ctx, ) ) return { - "roots": roots, - "scanned_files": len(dict.fromkeys(files)), + "uri": root, + "enumerated_files": len(unique_files), + "scanned_files": scanned_files, + "scanned_bytes": scanned_bytes, "matched_files": len(matches), "repaired_files": repaired, "dry_run": not fix, "truncated": truncated, + "fix_truncated": fix_truncated, + "stop_reason": stop_reason, + "skipped_files": skipped_files, + "limits": { + "max_nodes": PRIVACY_SCAN_MAX_NODES, + "max_file_bytes": PRIVACY_SCAN_MAX_FILE_BYTES, + "max_total_bytes": PRIVACY_SCAN_MAX_TOTAL_BYTES, + "max_replacements_per_file": PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE, + "max_reference_details_per_file": PRIVACY_SCAN_MAX_REFERENCE_DETAILS_PER_FILE, + "max_repaired_files": PRIVACY_SCAN_MAX_REPAIRED_FILES, + "max_duration_seconds": PRIVACY_SCAN_MAX_DURATION_SECONDS, + "files_per_second": PRIVACY_SCAN_FILES_PER_SECOND, + "max_concurrent_scans_per_process": 1, + }, "matches": matches, "reindex": reindex_tasks, } diff --git a/openviking/server/routers/privacy_configs.py b/openviking/server/routers/privacy_configs.py index 0747e04cb0..27316e4584 100644 --- a/openviking/server/routers/privacy_configs.py +++ b/openviking/server/routers/privacy_configs.py @@ -13,7 +13,7 @@ parse_privacy_target_selector, ) from openviking.privacy.sanitizer import PrivacySanitizer -from openviking.privacy.scan import scan_privacy_content +from openviking.privacy.scan import PRIVACY_SCAN_MAX_NODES, scan_privacy_content from openviking.privacy.validation import validate_privacy_field_name from openviking.server.auth import get_request_context from openviking.server.dependencies import get_service @@ -59,11 +59,9 @@ class ActivatePrivacyConfigVersionRequest(BaseModel): class ScanPrivacyContentRequest(BaseModel): model_config = ConfigDict(extra="forbid") - scopes: list[str] = Field(default_factory=list) - target_uri: str = "" + uri: str = Field(..., min_length=1, max_length=2048) fix: bool = False - wait: bool = False - node_limit: int = 10000 + node_limit: int = Field(default=PRIVACY_SCAN_MAX_NODES, ge=1, le=PRIVACY_SCAN_MAX_NODES) class PrivacySelectorRequest(BaseModel): @@ -103,11 +101,9 @@ async def scan_privacy_configs( service=service, ctx=_ctx, sanitizer=PrivacySanitizer(privacy), - scopes=request.scopes, - target_uri=request.target_uri, + uri=request.uri, fix=request.fix, - wait=request.wait, - node_limit=max(1, min(request.node_limit, 100000)), + node_limit=request.node_limit, ) return Response(status="ok", result=result) diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py index 9d0e3fd523..d97325a51f 100644 --- a/openviking/storage/viking_fs/_ops.py +++ b/openviking/storage/viking_fs/_ops.py @@ -43,6 +43,7 @@ InvalidArgumentError, NotFoundError, PermissionDeniedError, + ResourceExhaustedError, ) from openviking_cli.utils.uri import VikingURI @@ -2070,8 +2071,11 @@ async def read_file_bytes( self, uri: str, ctx: Optional[RequestContext] = None, + max_bytes: Optional[int] = None, ) -> bytes: """Read single binary file. Accepts a Viking URI or a 32-char hex vector record id.""" + if max_bytes is not None and max_bytes < 0: + raise InvalidArgumentError("max_bytes must be non-negative") real_ctx = self._ctx_or_default(ctx) uri = await self.resolve_uri(uri, real_ctx) await self._ensure_access(uri, ctx) @@ -2095,8 +2099,21 @@ async def read_file_bytes( f"Cannot read directory as file: {uri}", details={"resource": uri, "expected": "file", "actual": "directory"}, ) + if max_bytes is not None: + size = stat.get("size") if isinstance(stat, dict) else None + if isinstance(size, int) and size > max_bytes: + raise ResourceExhaustedError( + "File exceeds the Privacy scan size limit", + details={"uri": uri, "size_bytes": size, "max_bytes": max_bytes}, + ) try: - raw = self._handle_agfs_read(await self._async_agfs.read(path)) + read_size = -1 if max_bytes is None else max_bytes + 1 + raw = self._handle_agfs_read(await self._async_agfs.read(path, size=read_size)) + if max_bytes is not None and len(raw) > max_bytes: + raise ResourceExhaustedError( + "File exceeds the Privacy scan size limit", + details={"uri": uri, "size_bytes": len(raw), "max_bytes": max_bytes}, + ) return raw except Exception as exc: if is_not_found_error(exc): diff --git a/tests/server/test_api_privacy_configs.py b/tests/server/test_api_privacy_configs.py index 74789c61b5..664a686513 100644 --- a/tests/server/test_api_privacy_configs.py +++ b/tests/server/test_api_privacy_configs.py @@ -332,7 +332,7 @@ async def test_privacy_scan_is_dry_run_by_default_and_fix_rewrites_content( dry_run = await client.post( "/api/v1/privacy-configs/scan", - json={"target_uri": uri}, + json={"uri": uri}, ) assert dry_run.status_code == 200, dry_run.text report = dry_run.json()["result"] @@ -340,12 +340,15 @@ async def test_privacy_scan_is_dry_run_by_default_and_fix_rewrites_content( assert report["matched_files"] == 1 assert report["repaired_files"] == 0 assert report["matches"][0]["uri"] == uri + assert report["limits"]["max_nodes"] == 1000 + assert report["limits"]["max_concurrent_scans_per_process"] == 1 + assert report["scanned_bytes"] > 0 assert secret not in str(report) assert secret in await service.viking_fs.read_file(uri, ctx=ctx) fixed = await client.post( "/api/v1/privacy-configs/scan", - json={"target_uri": uri, "fix": True}, + json={"uri": uri, "fix": True}, ) assert fixed.status_code == 200, fixed.text fixed_report = fixed.json()["result"] @@ -395,7 +398,7 @@ async def test_privacy_scan_fix_repairs_skill_and_reindexes_package( fixed = await client.post( "/api/v1/privacy-configs/scan", - json={"target_uri": skill_uri, "fix": True, "wait": True}, + json={"uri": skill_uri, "fix": True}, ) assert fixed.status_code == 200, fixed.text report = fixed.json()["result"] @@ -411,6 +414,20 @@ async def test_privacy_scan_fix_repairs_skill_and_reindexes_package( ) in stored +async def test_privacy_scan_requires_explicit_uri_and_caps_tree_at_1000_nodes( + client: httpx.AsyncClient, +): + for body in ( + {}, + {"scopes": ["resources"]}, + {"uri": "viking://~/resources", "node_limit": 1001}, + {"uri": "viking://~/resources", "wait": True}, + ): + response = await client.post("/api/v1/privacy-configs/scan", json=body) + assert response.status_code == 400, response.text + assert response.json()["error"]["code"] == "INVALID_ARGUMENT" + + async def test_privacy_resolve_accepts_reference_and_field_uri_and_checks_owner( client: httpx.AsyncClient, ): diff --git a/tests/unit/test_privacy_sanitizer.py b/tests/unit/test_privacy_sanitizer.py index 9eb05566c8..fca6f335ac 100644 --- a/tests/unit/test_privacy_sanitizer.py +++ b/tests/unit/test_privacy_sanitizer.py @@ -143,6 +143,21 @@ async def test_sanitizer_prefers_longest_overlap_and_does_not_replace_ambiguous_ assert len(result.ambiguous_references[0]) == 2 +async def test_sanitizer_bounds_dense_match_metadata_for_scans(): + catalog = _PrivacyCatalog({"credential": {"service": {"key": "secret"}}}) + sanitizer = PrivacySanitizer(catalog) + + result = await sanitizer.sanitize( + "secret secret secret", + ctx=_ctx(), + max_replacements=2, + ) + + assert result.match_limit_exceeded + assert result.content == "secret secret secret" + assert not result.replacements + + async def test_sanitizer_cache_rebuilds_only_after_catalog_revision_changes(): catalog = _PrivacyCatalog({"credential": {"service": {"key": "first1"}}}) sanitizer = PrivacySanitizer(catalog) diff --git a/tests/unit/test_privacy_scan.py b/tests/unit/test_privacy_scan.py new file mode 100644 index 0000000000..ebf0cee981 --- /dev/null +++ b/tests/unit/test_privacy_scan.py @@ -0,0 +1,108 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +from types import SimpleNamespace + +import pytest + +from openviking.privacy import scan as privacy_scan +from openviking.privacy.sanitizer import PrivacyReplacement, PrivacySanitizationResult +from openviking.server.identity import RequestContext, Role +from openviking_cli.exceptions import InvalidArgumentError, ResourceExhaustedError +from openviking_cli.session.user_id import UserIdentifier + + +def _ctx() -> RequestContext: + return RequestContext(user=UserIdentifier("account-a", "alice"), role=Role.USER) + + +class _VikingFS: + def __init__(self): + self.read_limits: list[int | None] = [] + + async def stat(self, uri, *, ctx, skip_count): # noqa: ANN001 + del ctx, skip_count + if uri == "viking://resources/demo": + return {"isDir": True, "size": 0} + raise AssertionError(f"unexpected stat: {uri}") + + async def tree(self, uri, **kwargs): # noqa: ANN001 + assert uri == "viking://resources/demo" + assert kwargs["node_limit"] == privacy_scan.PRIVACY_SCAN_MAX_NODES + return [ + { + "uri": "viking://resources/demo/large.md", + "size": privacy_scan.PRIVACY_SCAN_MAX_FILE_BYTES + 1, + "isDir": False, + }, + { + "uri": "viking://resources/demo/content.md", + "size": 13, + "isDir": False, + }, + { + "uri": "viking://resources/demo/.abstract.md", + "size": 10, + "isDir": False, + }, + ] + + async def read_file_bytes(self, uri, *, ctx, max_bytes): # noqa: ANN001 + del ctx + assert uri == "viking://resources/demo/content.md" + self.read_limits.append(max_bytes) + return b"prefix secret" + + +class _Sanitizer: + async def sanitize(self, content, *, ctx, uri, max_replacements): # noqa: ANN001 + del ctx + assert content == "prefix secret" + assert uri == "viking://resources/demo/content.md" + assert max_replacements == privacy_scan.PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE + return PrivacySanitizationResult( + content="prefix reference", + replacements=(PrivacyReplacement(7, 13, "reference"),), + ) + + +async def test_privacy_scan_applies_file_and_node_budgets_before_reading(): + viking_fs = _VikingFS() + service = SimpleNamespace(viking_fs=viking_fs) + + result = await privacy_scan.scan_privacy_content( + service=service, + ctx=_ctx(), + sanitizer=_Sanitizer(), + uri="viking://resources/demo", + ) + + assert result["enumerated_files"] == 3 + assert result["scanned_files"] == 1 + assert result["matched_files"] == 1 + assert result["skipped_files"] == {"file_too_large": 1, "internal_metadata": 1} + assert viking_fs.read_limits == [privacy_scan.PRIVACY_SCAN_MAX_FILE_BYTES] + + +async def test_privacy_scan_rejects_invalid_budget_and_concurrent_run(): + service = SimpleNamespace(viking_fs=_VikingFS()) + with pytest.raises(InvalidArgumentError, match="node_limit"): + await privacy_scan.scan_privacy_content( + service=service, + ctx=_ctx(), + sanitizer=_Sanitizer(), + uri="viking://resources/demo", + node_limit=privacy_scan.PRIVACY_SCAN_MAX_NODES + 1, + ) + + privacy_scan._SCAN_ACTIVE = True + try: + with pytest.raises(ResourceExhaustedError, match="already running"): + await privacy_scan.scan_privacy_content( + service=service, + ctx=_ctx(), + sanitizer=_Sanitizer(), + uri="viking://resources/demo", + ) + finally: + privacy_scan._SCAN_ACTIVE = False From a586d34ce9ff0f4faa0dddae5aa3ac6bb35e4625 Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Fri, 16 Oct 2026 14:39:03 +0800 Subject: [PATCH 4/6] feat: update ov privacy commands for agent use --- crates/ragfs/Cargo.toml | 4 - crates/ragfs/benches/privacy_matcher.rs | 31 ----- docs/en/api/01-overview.md | 1 + docs/en/api/05-sessions.md | 2 +- docs/en/api/10-privacy.md | 4 +- docs/zh/api/01-overview.md | 1 + docs/zh/api/05-sessions.md | 2 +- docs/zh/api/10-privacy.md | 4 +- openviking/privacy/__init__.py | 33 +---- openviking/privacy/sanitizer.py | 63 +++++++-- openviking/privacy/scan.py | 5 +- openviking/privacy/service.py | 2 +- openviking/server/routers/privacy_configs.py | 2 +- openviking/service/core.py | 17 ++- openviking/service/fs_service.py | 33 +++-- openviking/storage/viking_fs/_ops.py | 18 ++- openviking/utils/resource_processor.py | 15 ++- openviking/utils/skill_processor.py | 33 +++-- tests/server/test_privacy_sanitizer_writes.py | 126 +++++++++++++++++- tests/unit/test_privacy_sanitizer.py | 16 +++ tests/unit/test_privacy_scan.py | 11 +- 21 files changed, 303 insertions(+), 120 deletions(-) delete mode 100644 crates/ragfs/benches/privacy_matcher.rs diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml index 393643c553..c39fb87d7b 100644 --- a/crates/ragfs/Cargo.toml +++ b/crates/ragfs/Cargo.toml @@ -33,10 +33,6 @@ name = "dynamic_provider" path = "tests/dynamic_provider.rs" required-features = ["cache"] -[[bench]] -name = "privacy_matcher" -harness = false - [dependencies] # Async runtime tokio = { version = "1.38", features = ["full"] } diff --git a/crates/ragfs/benches/privacy_matcher.rs b/crates/ragfs/benches/privacy_matcher.rs deleted file mode 100644 index 7b919ed7ce..0000000000 --- a/crates/ragfs/benches/privacy_matcher.rs +++ /dev/null @@ -1,31 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use ragfs::privacy_matcher::PrivacyMatcher; - -fn benchmark_privacy_matcher(c: &mut Criterion) { - let mut group = c.benchmark_group("privacy_matcher"); - for pattern_count in [100usize, 1_000] { - let entries: Vec<_> = (0..pattern_count) - .map(|index| { - ( - format!("mock-secret-{index:06}-value"), - vec![format!("{{{{privacy-ref-{index}}}}}")], - ) - }) - .collect(); - let matcher = PrivacyMatcher::new(entries, "{{ov_privacy:ambiguous}}").unwrap(); - let mut content = "ordinary context without a registered value\n".repeat(20_000); - content.push_str("credential=mock-secret-000042-value\n"); - group.throughput(Throughput::Bytes(content.len() as u64)); - group.bench_with_input( - BenchmarkId::from_parameter(pattern_count), - &content, - |bench, input| { - bench.iter(|| matcher.sanitize(black_box(input))); - }, - ); - } - group.finish(); -} - -criterion_group!(benches, benchmark_privacy_matcher); -criterion_main!(benches); diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md index bd47dc1bc9..a3202e460d 100644 --- a/docs/en/api/01-overview.md +++ b/docs/en/api/01-overview.md @@ -523,6 +523,7 @@ This catalog follows the routes actually mounted by the server. Each group headi | POST | `/api/v1/privacy-configs/scan` | Scan or repair registered values in content | | GET | `/api/v1/privacy-configs/{category}` | List category targets | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | Get the active configuration | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | List safe active-field metadata without values | | GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | Get a field reference or explicitly reveal its value | | POST | `/api/v1/privacy-configs/resolve` | Explicitly reveal a field by reference or URI | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | List configuration versions | diff --git a/docs/en/api/05-sessions.md b/docs/en/api/05-sessions.md index 7fb0128ac8..1fe91fd97a 100644 --- a/docs/en/api/05-sessions.md +++ b/docs/en/api/05-sessions.md @@ -1333,7 +1333,7 @@ ov add-memory '[{"role":"user","content":"Hello"},{"role":"assistant","content": Commit a session. Message archiving (Phase 1) completes before the response returns. Summary generation and memory extraction (Phase 2) run asynchronously in the background when messages are archived. Archived commits return `status: "accepted"` with a `task_id`; no-op commits return `status: "skipped"` with `task_id: null`. -The standalone `POST /api/v1/sessions/{session_id}/extract` endpoint has been removed. Use `commit` for memory extraction. This archives messages according to the retention policy; use the returned `task_id` to check the background extraction result. An `accepted` response does not mean extraction is complete. +The former standalone session `extract` endpoint has been removed. Use `commit` for memory extraction. This archives messages according to the retention policy; use the returned `task_id` to check the background extraction result. An `accepted` response does not mean extraction is complete. **Two-Phase Commit Flow:** - **Phase 1 (Synchronous)**: Split messages according to the retention policy, persist the archive and recovery records, enqueue Phase 2, and write retained messages back to the live session diff --git a/docs/en/api/10-privacy.md b/docs/en/api/10-privacy.md index 37f51e10db..eb28267bc4 100644 --- a/docs/en/api/10-privacy.md +++ b/docs/en/api/10-privacy.md @@ -17,12 +17,12 @@ Activating a version changes only the values stored in OpenViking. It does not r | GET | `/api/v1/privacy-configs/{category}` | List targets under a category | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | Get active config (`meta + current`) | | GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | List safe active-field metadata without values | -| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | Get a field reference; optionally reveal its value | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field?field={field}` | Get a field reference; optionally reveal its value | | POST | `/api/v1/privacy-configs/resolve` | Explicitly reveal a field by reference or URI | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | Upsert and activate a new/current version | | PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | Atomically merge fields; create a missing target | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | Delete a target and every version | -| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | Remove a field from the active config | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field?field={field}` | Remove a field from the active config | | DELETE | `/api/v1/privacy-configs/selector` | Delete by target URI or field reference/URI | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | List version numbers | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | Get a specific version snapshot | diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md index e5598a6b8b..afbaf3e950 100644 --- a/docs/zh/api/01-overview.md +++ b/docs/zh/api/01-overview.md @@ -518,6 +518,7 @@ JSON 输出 - 错误： | POST | `/api/v1/privacy-configs/scan` | 扫描或修复正文中的登记值 | | GET | `/api/v1/privacy-configs/{category}` | 列出分类目标 | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | 获取生效配置 | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | 列出不含明文值的安全字段元数据 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | 获取单字段引用或显式读取值 | | POST | `/api/v1/privacy-configs/resolve` | 按字段 reference/URI 显式读取值 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | 列出配置版本 | diff --git a/docs/zh/api/05-sessions.md b/docs/zh/api/05-sessions.md index fd06c2c2fc..74a1cef2af 100644 --- a/docs/zh/api/05-sessions.md +++ b/docs/zh/api/05-sessions.md @@ -1332,7 +1332,7 @@ ov add-memory '[{"role":"user","content":"Hello"},{"role":"assistant","content": 提交会话。归档消息（Phase 1）在响应返回前完成；有消息被归档时，摘要生成和记忆提取（Phase 2）在后台异步执行。产生归档的 commit 返回 `status: "accepted"` 和 `task_id`；没有可归档内容的 no-op commit 返回 `status: "skipped"` 和 `task_id: null`。 -原 `POST /api/v1/sessions/{session_id}/extract` 接口已移除，记忆提取统一使用 `commit`。迁移后消息会按保留策略归档；请通过返回的 `task_id` 查询后台提取结果，不能将 `accepted` 当作提取完成。 +原独立 session `extract` 接口已移除，记忆提取统一使用 `commit`。迁移后消息会按保留策略归档；请通过返回的 `task_id` 查询后台提取结果，不能将 `accepted` 当作提取完成。 **两阶段提交流程**： - **Phase 1（同步）**: 按保留策略拆分消息，写入归档和恢复记录，入队 Phase 2，并将保留消息写回 live session diff --git a/docs/zh/api/10-privacy.md b/docs/zh/api/10-privacy.md index 39cdf391c9..bf6b4e1a84 100644 --- a/docs/zh/api/10-privacy.md +++ b/docs/zh/api/10-privacy.md @@ -26,12 +26,12 @@ | GET | `/api/v1/privacy-configs/{category}` | 列出分类下目标 | | GET | `/api/v1/privacy-configs/{category}/{target_key}` | 获取当前生效配置（meta + current） | | GET | `/api/v1/privacy-configs/{category}/{target_key}/fields` | 列出不含明文值的安全字段元数据 | -| GET | `/api/v1/privacy-configs/{category}/{target_key}/field` | 获取单字段引用；显式选择时返回值 | +| GET | `/api/v1/privacy-configs/{category}/{target_key}/field?field={field}` | 获取单字段引用；显式选择时返回值 | | POST | `/api/v1/privacy-configs/resolve` | 按字段 reference/URI 显式读取值 | | POST | `/api/v1/privacy-configs/{category}/{target_key}` | 写入新版本并激活 | | PATCH | `/api/v1/privacy-configs/{category}/{target_key}` | 原子合并字段；目标不存在时创建 | | DELETE | `/api/v1/privacy-configs/{category}/{target_key}` | 删除目标及全部版本 | -| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field` | 删除当前生效配置中的字段 | +| DELETE | `/api/v1/privacy-configs/{category}/{target_key}/field?field={field}` | 删除当前生效配置中的字段 | | DELETE | `/api/v1/privacy-configs/selector` | 按 target URI 或字段 reference/URI 删除 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions` | 列出版本号 | | GET | `/api/v1/privacy-configs/{category}/{target_key}/versions/{version}` | 获取指定版本详情 | diff --git a/openviking/privacy/__init__.py b/openviking/privacy/__init__.py index 0d7dc6b3d0..e01fb95f79 100644 --- a/openviking/privacy/__init__.py +++ b/openviking/privacy/__init__.py @@ -2,26 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0 """Privacy config utilities for OpenViking.""" -from openviking.privacy.models import ( - UserPrivacyConfigMeta, - UserPrivacyConfigVersion, - UserPrivacyFieldDeleteResult, -) -from openviking.privacy.reference import ( - PrivacyFieldReference, - PrivacyTargetReference, - build_privacy_field_reference, - parse_privacy_field_reference, - parse_privacy_field_selector, - parse_privacy_target_selector, -) -from openviking.privacy.sanitizer import ( - DEFAULT_MATCHER_CACHE_TTL_SECONDS, - MIN_PROTECTED_VALUE_CHARS, - PrivacyReplacement, - PrivacySanitizationResult, - PrivacySanitizer, -) +from openviking.privacy.models import UserPrivacyConfigMeta, UserPrivacyConfigVersion from openviking.privacy.service import UserPrivacyConfigService from openviking.privacy.skill_extractor import ( SkillPrivacyExtractionResult, @@ -38,19 +19,7 @@ __all__ = [ "UserPrivacyConfigMeta", "UserPrivacyConfigVersion", - "UserPrivacyFieldDeleteResult", "UserPrivacyConfigService", - "PrivacyFieldReference", - "PrivacyTargetReference", - "build_privacy_field_reference", - "parse_privacy_field_reference", - "parse_privacy_field_selector", - "parse_privacy_target_selector", - "PrivacyReplacement", - "MIN_PROTECTED_VALUE_CHARS", - "DEFAULT_MATCHER_CACHE_TTL_SECONDS", - "PrivacySanitizationResult", - "PrivacySanitizer", "SkillPrivacyExtractionResult", "extract_skill_privacy_values", "SkillPrivacyPlaceholderizationResult", diff --git a/openviking/privacy/sanitizer.py b/openviking/privacy/sanitizer.py index f0efc83efc..91d4811274 100644 --- a/openviking/privacy/sanitizer.py +++ b/openviking/privacy/sanitizer.py @@ -12,6 +12,7 @@ from openviking.privacy.reference import build_privacy_field_reference from openviking.pyagfs import get_privacy_matcher_class from openviking.server.identity import RequestContext +from openviking_cli.exceptions import ResourceExhaustedError if TYPE_CHECKING: from openviking.privacy.service import UserPrivacyConfigService @@ -19,6 +20,8 @@ _AMBIGUOUS_PLACEHOLDER = "{{ov_privacy:ambiguous}}" MIN_PROTECTED_VALUE_CHARS = 6 DEFAULT_MATCHER_CACHE_TTL_SECONDS = 3600.0 +DEFAULT_SANITIZER_MAX_CONTENT_BYTES = 16 * 1024 * 1024 +DEFAULT_SANITIZER_MAX_REPLACEMENTS = 10000 @dataclass(frozen=True) @@ -48,18 +51,31 @@ def __init__( privacy: "UserPrivacyConfigService", *, cache_ttl_seconds: float = DEFAULT_MATCHER_CACHE_TTL_SECONDS, + max_content_bytes: int = DEFAULT_SANITIZER_MAX_CONTENT_BYTES, + max_replacements: int = DEFAULT_SANITIZER_MAX_REPLACEMENTS, monotonic_clock: Callable[[], float] = time.monotonic, ): if cache_ttl_seconds <= 0: raise ValueError("cache_ttl_seconds must be positive") + if max_content_bytes <= 0: + raise ValueError("max_content_bytes must be positive") + if max_replacements <= 0: + raise ValueError("max_replacements must be positive") self._privacy = privacy self._cache_ttl_seconds = float(cache_ttl_seconds) + self._max_content_bytes = int(max_content_bytes) + self._max_replacements = int(max_replacements) self._monotonic_clock = monotonic_clock matcher_class: Any = get_privacy_matcher_class() self._matcher_class = matcher_class self._cache: dict[tuple[str, str, int], tuple[float, Any]] = {} self._cache_lock = asyncio.Lock() + @property + def max_content_bytes(self) -> int: + """Maximum UTF-8 input size accepted by this sanitizer.""" + return self._max_content_bytes + async def sanitize( self, content: str, @@ -67,22 +83,40 @@ async def sanitize( ctx: RequestContext, uri: str = "", max_replacements: int | None = None, + fail_on_match_limit: bool = True, ) -> PrivacySanitizationResult: if not content or self._is_privacy_storage_uri(uri, ctx): return PrivacySanitizationResult(content=content) + content_bytes = len(content.encode("utf-8")) + if content_bytes > self._max_content_bytes: + raise ResourceExhaustedError( + "Content exceeds the Privacy sanitizer size limit", + details={ + "uri": uri, + "size_bytes": content_bytes, + "max_bytes": self._max_content_bytes, + }, + ) matcher = await self._matcher(ctx) - match_limit_exceeded = False - if max_replacements is None: - rewritten, raw_replacements, raw_ambiguous = matcher.sanitize(content) - else: - if max_replacements < 0: - raise ValueError("max_replacements must be non-negative") - ( - rewritten, - raw_replacements, - raw_ambiguous, - match_limit_exceeded, - ) = matcher.sanitize_limited(content, max_replacements) + effective_max_replacements = ( + self._max_replacements if max_replacements is None else max_replacements + ) + if effective_max_replacements <= 0: + raise ValueError("max_replacements must be positive") + ( + rewritten, + raw_replacements, + raw_ambiguous, + match_limit_exceeded, + ) = matcher.sanitize_limited(content, effective_max_replacements) + if match_limit_exceeded and fail_on_match_limit: + raise ResourceExhaustedError( + "Content exceeds the Privacy sanitizer replacement limit", + details={ + "uri": uri, + "max_replacements": effective_max_replacements, + }, + ) replacements = tuple( PrivacyReplacement(start=start, end=end, reference=reference) for start, end, reference in raw_replacements @@ -103,6 +137,11 @@ async def _matcher(self, ctx: RequestContext) -> Any: return cached[1] async with self._cache_lock: now = self._monotonic_clock() + self._cache = { + key: item + for key, item in self._cache.items() + if now - item[0] < self._cache_ttl_seconds + } cached = self._cache.get(cache_key) if cached is not None and now - cached[0] < self._cache_ttl_seconds: return cached[1] diff --git a/openviking/privacy/scan.py b/openviking/privacy/scan.py index f27cd628c7..e24631655d 100644 --- a/openviking/privacy/scan.py +++ b/openviking/privacy/scan.py @@ -196,7 +196,9 @@ def skip(reason: str) -> None: max_bytes=PRIVACY_SCAN_MAX_FILE_BYTES, ) last_read_at = time.monotonic() - except ResourceExhaustedError: + except ResourceExhaustedError as exc: + if exc.details.get("reason") != "read_limit_exceeded": + raise skip("file_too_large") continue except (NotFoundError, FileNotFoundError): @@ -219,6 +221,7 @@ def skip(reason: str) -> None: ctx=ctx, uri=file_uri, max_replacements=PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE, + fail_on_match_limit=False, ) if result.match_limit_exceeded: skip("match_limit_exceeded") diff --git a/openviking/privacy/service.py b/openviking/privacy/service.py index 70651a6490..57bebf8cae 100644 --- a/openviking/privacy/service.py +++ b/openviking/privacy/service.py @@ -184,7 +184,7 @@ async def _save_meta( async def get_current( self, ctx: RequestContext, category: str, target_key: str - ) -> Optional[UserPrivacyFieldDeleteResult]: + ) -> Optional[UserPrivacyConfigVersion]: try: content = await self._viking_fs.read_file( current_uri(self._user_space(ctx), category, target_key), ctx=ctx diff --git a/openviking/server/routers/privacy_configs.py b/openviking/server/routers/privacy_configs.py index 27316e4584..728ac29b3a 100644 --- a/openviking/server/routers/privacy_configs.py +++ b/openviking/server/routers/privacy_configs.py @@ -100,7 +100,7 @@ async def scan_privacy_configs( result = await scan_privacy_content( service=service, ctx=_ctx, - sanitizer=PrivacySanitizer(privacy), + sanitizer=service.privacy_sanitizer or PrivacySanitizer(privacy), uri=request.uri, fix=request.fix, node_limit=request.node_limit, diff --git a/openviking/service/core.py b/openviking/service/core.py index 4b0f868fd3..08c8698bc0 100644 --- a/openviking/service/core.py +++ b/openviking/service/core.py @@ -12,7 +12,8 @@ from typing import TYPE_CHECKING, Any, Optional from openviking.core.directories import DirectoryInitializer -from openviking.privacy import PrivacySanitizer, UserPrivacyConfigService +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.privacy.service import UserPrivacyConfigService from openviking.resource.uri_mutation_coordinator import UriMutationCoordinator from openviking.resource.watch_scheduler import WatchScheduler from openviking.server.identity import RequestContext, Role @@ -107,6 +108,7 @@ def __init__( self._expiry_runtime: Optional["ExpiryRuntime"] = None self._encryptor: Optional[Any] = None self._privacy_config_service: Optional[UserPrivacyConfigService] = None + self._privacy_sanitizer: Optional[PrivacySanitizer] = None self._runtime_config_manager: Optional[Any] = None self._vlm_resolver: Optional["VLMResolver"] = None self._vector_config_resolver: Optional[Any] = None @@ -415,6 +417,11 @@ def privacy_configs(self) -> Optional[UserPrivacyConfigService]: """Get UserPrivacyConfigService instance.""" return self._privacy_config_service + @property + def privacy_sanitizer(self) -> Optional[PrivacySanitizer]: + """Get the shared PrivacySanitizer instance.""" + return self._privacy_sanitizer + @property def debug(self) -> DebugService: """Get DebugService instance.""" @@ -519,14 +526,15 @@ async def initialize(self) -> None: user_count, ) self._privacy_config_service = UserPrivacyConfigService(self._viking_fs) - privacy_sanitizer = PrivacySanitizer(self._privacy_config_service) - self._viking_fs.set_privacy_sanitizer(privacy_sanitizer) - self._resource_processor.set_privacy_sanitizer(privacy_sanitizer) + self._privacy_sanitizer = PrivacySanitizer(self._privacy_config_service) + self._viking_fs.set_privacy_sanitizer(self._privacy_sanitizer) + self._resource_processor.set_privacy_sanitizer(self._privacy_sanitizer) # Initialize processors self._skill_processor = SkillProcessor( vikingdb=self._vikingdb_manager, privacy_config_service=self._privacy_config_service, + privacy_sanitizer=self._privacy_sanitizer, vlm_resolver=self._vlm_resolver, ) self._session_compressor = create_session_compressor( @@ -547,6 +555,7 @@ async def initialize(self) -> None: viking_fs=self._viking_fs, vikingdb=self._vikingdb_manager, privacy_config_service=self._privacy_config_service, + privacy_sanitizer=self._privacy_sanitizer, resource_memory_link_service=self._resource_memory_link_service, watch_scheduler=self._watch_scheduler, uri_mutation_coordinator=self._uri_mutation_coordinator, diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py index fb281cf313..89c453a79b 100644 --- a/openviking/service/fs_service.py +++ b/openviking/service/fs_service.py @@ -19,12 +19,9 @@ uri_leaf_name, uri_parts, ) -from openviking.privacy import ( - PrivacySanitizer, - UserPrivacyConfigService, - get_skill_name_from_uri, - restore_skill_content, -) +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.privacy.service import UserPrivacyConfigService +from openviking.privacy.skill_restore import get_skill_name_from_uri, restore_skill_content from openviking.resource.uri_mutation_coordinator import UriMutationCoordinator from openviking.resource.watch_storage import is_watch_task_control_uri from openviking.server.identity import RequestContext @@ -57,6 +54,7 @@ InvalidArgumentError, NotFoundError, NotInitializedError, + ResourceExhaustedError, ) from openviking_cli.utils import VikingURI, get_logger from openviking_cli.utils.config import get_openviking_config @@ -104,11 +102,12 @@ def __init__( resource_memory_link_service: Optional["ResourceMemoryLinkService"] = None, watch_scheduler: Optional["WatchScheduler"] = None, uri_mutation_coordinator: Optional[UriMutationCoordinator] = None, + privacy_sanitizer: Optional[PrivacySanitizer] = None, ): self._viking_fs = viking_fs self._vikingdb = vikingdb self._privacy_config_service = privacy_config_service - self._privacy_sanitizer = ( + self._privacy_sanitizer = privacy_sanitizer or ( PrivacySanitizer(privacy_config_service) if privacy_config_service is not None else None ) self._resource_memory_link_service = resource_memory_link_service @@ -123,12 +122,13 @@ def set_dependencies( resource_memory_link_service: Optional["ResourceMemoryLinkService"] = None, watch_scheduler: Optional["WatchScheduler"] = None, uri_mutation_coordinator: Optional[UriMutationCoordinator] = None, + privacy_sanitizer: Optional[PrivacySanitizer] = None, ) -> None: """Set service dependencies (for deferred initialization).""" self._viking_fs = viking_fs self._vikingdb = vikingdb self._privacy_config_service = privacy_config_service - self._privacy_sanitizer = ( + self._privacy_sanitizer = privacy_sanitizer or ( PrivacySanitizer(privacy_config_service) if privacy_config_service is not None else None ) self._resource_memory_link_service = resource_memory_link_service @@ -1123,6 +1123,23 @@ async def read_visible( # Resolve id to URI before classify_uri / visible_content which expect # a real viking:// URI. resolved_uri = await self._resolve_uri(uri, ctx) + if self._privacy_sanitizer is not None: + stat = await self._ensure_initialized().stat( + resolved_uri, + ctx=ctx, + skip_count=True, + ) + size = stat.get("size") if isinstance(stat, dict) else None + if isinstance(size, int) and size > self._privacy_sanitizer.max_content_bytes: + raise ResourceExhaustedError( + "File exceeds the Privacy visible-read size limit", + details={ + "uri": resolved_uri, + "reason": "privacy_visible_read_limit_exceeded", + "size_bytes": size, + "max_bytes": self._privacy_sanitizer.max_content_bytes, + }, + ) content = await self.read(resolved_uri, ctx=ctx) if classify_uri(resolved_uri).is_memory: content = visible_content(content, uri=resolved_uri) diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py index d97325a51f..7bba4648c7 100644 --- a/openviking/storage/viking_fs/_ops.py +++ b/openviking/storage/viking_fs/_ops.py @@ -2103,16 +2103,26 @@ async def read_file_bytes( size = stat.get("size") if isinstance(stat, dict) else None if isinstance(size, int) and size > max_bytes: raise ResourceExhaustedError( - "File exceeds the Privacy scan size limit", - details={"uri": uri, "size_bytes": size, "max_bytes": max_bytes}, + "File exceeds the configured read size limit", + details={ + "uri": uri, + "reason": "read_limit_exceeded", + "size_bytes": size, + "max_bytes": max_bytes, + }, ) try: read_size = -1 if max_bytes is None else max_bytes + 1 raw = self._handle_agfs_read(await self._async_agfs.read(path, size=read_size)) if max_bytes is not None and len(raw) > max_bytes: raise ResourceExhaustedError( - "File exceeds the Privacy scan size limit", - details={"uri": uri, "size_bytes": len(raw), "max_bytes": max_bytes}, + "File exceeds the configured read size limit", + details={ + "uri": uri, + "reason": "read_limit_exceeded", + "size_bytes": len(raw), + "max_bytes": max_bytes, + }, ) return raw except Exception as exc: diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py index 7e6e0407ce..d75b1982ae 100644 --- a/openviking/utils/resource_processor.py +++ b/openviking/utils/resource_processor.py @@ -59,6 +59,17 @@ from openviking.parse.vlm import VLMProcessor logger = get_logger(__name__) + +_PARSE_ARTIFACT_INTERNAL_NAMES = frozenset( + { + ".abstract.md", + ".overview.md", + ".relations.json", + ".source.json", + ".image_mappings.json", + ".artifact_manifest.json", + } +) _MAX_FILE_VECTORIZATION_CONCURRENCY = 64 VECTORDB_MAX_QUERY_LIMIT = 100_000 @@ -178,7 +189,9 @@ async def walk(directory: str) -> None: if entry.is_dir: await walk(entry.rel_path) continue - if entry.name.startswith("."): + if entry.name in _PARSE_ARTIFACT_INTERNAL_NAMES or is_storage_internal_name( + entry.name + ): continue relative = ( entry.rel_path[len(prefix) :] diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.py index 9a4c56165c..944c76be72 100644 --- a/openviking/utils/skill_processor.py +++ b/openviking/utils/skill_processor.py @@ -21,11 +21,9 @@ from openviking.core.mcp_converter import is_mcp_format, mcp_to_skill from openviking.core.namespace import canonical_user_root from openviking.core.skill_loader import SkillLoader -from openviking.privacy import ( - PrivacySanitizer, - UserPrivacyConfigService, - extract_skill_privacy_values, -) +from openviking.privacy.sanitizer import PrivacySanitizer +from openviking.privacy.service import UserPrivacyConfigService +from openviking.privacy.skill_extractor import extract_skill_privacy_values from openviking.server.identity import RequestContext from openviking.server.local_input_guard import deny_direct_local_skill_input from openviking.service.task_tracker_concurrency import run_to_completion @@ -104,6 +102,7 @@ def __init__( vikingdb: VikingDBManager, privacy_config_service: Optional[UserPrivacyConfigService] = None, vlm_resolver: Optional[VLMResolver] = None, + privacy_sanitizer: Optional[PrivacySanitizer] = None, ): """Initialize skill processor.""" if privacy_config_service is not None and vlm_resolver is None: @@ -112,7 +111,7 @@ def __init__( ) self.vikingdb = vikingdb self._privacy_config_service = privacy_config_service - self._privacy_sanitizer = ( + self._privacy_sanitizer = privacy_sanitizer or ( PrivacySanitizer(privacy_config_service) if isinstance(privacy_config_service, UserPrivacyConfigService) else None @@ -489,23 +488,31 @@ async def prepare_skill_privacy( if not self._privacy_config_service: return skill_dict, {} + skill_uri = ( + f"{canonical_user_root(ctx)}/skills/" + f"{skill_dict.get('name', '')}/SKILL.md" + ) content = skill_dict.get("content", "") sanitized_by_catalog = await self._sanitize_registered_privacy( content, ctx=ctx, - uri=( - f"{canonical_user_root(ctx)}/skills/" - f"{skill_dict.get('name', '')}/SKILL.md" - ), + uri=skill_uri, + ) + description = skill_dict.get("description", "") + sanitized_description = await self._sanitize_registered_privacy( + description, + ctx=ctx, + uri=skill_uri, ) prepared_skill = skill_dict - if sanitized_by_catalog != content: + if sanitized_by_catalog != content or sanitized_description != description: prepared_skill = deepcopy(skill_dict) prepared_skill["content"] = sanitized_by_catalog + prepared_skill["description"] = sanitized_description content = sanitized_by_catalog extraction_kwargs = { - "skill_name": skill_dict.get("name", ""), - "skill_description": skill_dict.get("description", ""), + "skill_name": prepared_skill.get("name", ""), + "skill_description": prepared_skill.get("description", ""), "content": content, } if self._vlm_resolver is None: diff --git a/tests/server/test_privacy_sanitizer_writes.py b/tests/server/test_privacy_sanitizer_writes.py index 2df4c9528d..a1eef6985a 100644 --- a/tests/server/test_privacy_sanitizer_writes.py +++ b/tests/server/test_privacy_sanitizer_writes.py @@ -1,9 +1,12 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 +from types import SimpleNamespace + import pytest from openviking.privacy.reference import build_privacy_field_reference +from openviking.privacy.skill_extractor import SkillPrivacyExtractionResult from openviking.server.identity import RequestContext, Role from openviking_cli.exceptions import InvalidArgumentError from openviking_cli.session.user_id import UserIdentifier @@ -35,6 +38,123 @@ async def test_user_and_public_content_writes_replace_registered_privacy_values( assert stored == f"credential={reference}" +async def test_service_components_share_one_privacy_sanitizer(service): + assert service.privacy_sanitizer is not None + assert service.fs._privacy_sanitizer is service.privacy_sanitizer + assert service._skill_processor._privacy_sanitizer is service.privacy_sanitizer + assert service._resource_processor._privacy_sanitizer is service.privacy_sanitizer + + +async def test_skill_description_is_sanitized_before_privacy_vlm( + service, + monkeypatch, +): + ctx = RequestContext(user=service.user, role=Role.USER) + secret = "registered-description-value" + await service.privacy_configs.upsert( + ctx, + "credential", + "description-service", + {"key": secret}, + updated_by=ctx.user.user_id, + ) + captured: dict[str, str] = {} + + async def fake_extract(**kwargs): + captured["description"] = kwargs["skill_description"] + return SkillPrivacyExtractionResult( + values={}, + original_content=kwargs["content"], + sanitized_content=kwargs["content"], + ) + + monkeypatch.setattr( + "openviking.utils.skill_processor.extract_skill_privacy_values", + fake_extract, + ) + prepared, values = await service._skill_processor.prepare_skill_privacy( + { + "name": "description-privacy", + "description": f"token={secret}", + "content": "No credentials in the body.", + }, + ctx, + ) + + reference = build_privacy_field_reference( + ctx.user.user_id, + "credential", + "description-service", + "key", + ) + assert captured["description"] == f"token={reference}" + assert prepared["description"] == f"token={reference}" + assert values == {} + + +async def test_resource_artifact_sanitizes_user_dotfiles_but_skips_internal_sidecars( + service, + monkeypatch, +): + ctx = RequestContext(user=service.user, role=Role.USER) + secret = "registered-dotfile-value" + await service.privacy_configs.upsert( + ctx, + "credential", + "dotfile-service", + {"key": secret}, + updated_by=ctx.user.user_id, + ) + + class ArtifactStore: + def __init__(self): + self.writes: dict[str, bytes] = {} + + async def list(self, _artifact_ref, _directory): + return [ + SimpleNamespace(is_dir=False, name=".env", rel_path=".env"), + SimpleNamespace( + is_dir=False, + name=".artifact_manifest.json", + rel_path=".artifact_manifest.json", + ), + ] + + async def read_bytes(self, _artifact_ref, path): + if path == ".artifact_manifest.json": + raise AssertionError("internal sidecar must not be sanitized") + return f"TOKEN={secret}".encode() + + async def write_bytes(self, _artifact_ref, path, data): + self.writes[path] = data + + async def read_manifest(_store, _artifact_ref): + return {} + + async def write_manifest(_store, _artifact_ref, _manifest): + return None + + monkeypatch.setattr("openviking.parse.output.read_artifact_manifest", read_manifest) + monkeypatch.setattr("openviking.parse.output.write_artifact_manifest", write_manifest) + store = ArtifactStore() + await service._resource_processor._sanitize_parse_artifact( + store=store, + artifact_ref=SimpleNamespace(root_type="dir"), + doc_rel="", + target_root_uri="viking://resources/dotfile-test", + ctx=ctx, + ) + + rewritten = store.writes[".env"].decode() + assert secret not in rewritten + assert build_privacy_field_reference( + ctx.user.user_id, + "credential", + "dotfile-service", + "key", + ) in rewritten + + async def test_content_write_rejects_registered_value_with_ambiguous_references(service): ctx = RequestContext(user=service.user, role=Role.USER) duplicate = "same-registered-value" @@ -105,7 +225,11 @@ async def test_visible_read_sanitizes_legacy_content_that_bypassed_content_write updated_by=ctx.user.user_id, ) uri = f"viking://user/{ctx.user.user_id}/legacy-content.md" - await service.viking_fs.write_file(uri, f"before {secret} after", ctx=ctx) + raw_path = service.viking_fs._uri_to_path(uri, ctx=ctx) + await service.viking_fs._async_agfs.write( + raw_path, + f"before {secret} after".encode("utf-8"), + ) visible = await service.fs.read_visible(uri, ctx=ctx) assert secret not in visible diff --git a/tests/unit/test_privacy_sanitizer.py b/tests/unit/test_privacy_sanitizer.py index fca6f335ac..a0014ec6be 100644 --- a/tests/unit/test_privacy_sanitizer.py +++ b/tests/unit/test_privacy_sanitizer.py @@ -3,6 +3,8 @@ from types import SimpleNamespace +import pytest + from openviking.privacy.reference import ( PrivacyFieldReference, build_privacy_field_reference, @@ -12,6 +14,7 @@ ) from openviking.privacy.sanitizer import PrivacySanitizer from openviking.server.identity import RequestContext, Role +from openviking_cli.exceptions import ResourceExhaustedError from openviking_cli.session.user_id import UserIdentifier @@ -151,6 +154,7 @@ async def test_sanitizer_bounds_dense_match_metadata_for_scans(): "secret secret secret", ctx=_ctx(), max_replacements=2, + fail_on_match_limit=False, ) assert result.match_limit_exceeded @@ -158,6 +162,18 @@ async def test_sanitizer_bounds_dense_match_metadata_for_scans(): assert not result.replacements +async def test_sanitizer_rejects_oversized_or_excessively_dense_writes(): + catalog = _PrivacyCatalog({"credential": {"service": {"key": "secret"}}}) + + size_limited = PrivacySanitizer(catalog, max_content_bytes=5) + with pytest.raises(ResourceExhaustedError, match="size limit"): + await size_limited.sanitize("secret", ctx=_ctx()) + + match_limited = PrivacySanitizer(catalog, max_replacements=2) + with pytest.raises(ResourceExhaustedError, match="replacement limit"): + await match_limited.sanitize("secret secret secret", ctx=_ctx()) + + async def test_sanitizer_cache_rebuilds_only_after_catalog_revision_changes(): catalog = _PrivacyCatalog({"credential": {"service": {"key": "first1"}}}) sanitizer = PrivacySanitizer(catalog) diff --git a/tests/unit/test_privacy_scan.py b/tests/unit/test_privacy_scan.py index ebf0cee981..380ff2b44e 100644 --- a/tests/unit/test_privacy_scan.py +++ b/tests/unit/test_privacy_scan.py @@ -55,11 +55,20 @@ async def read_file_bytes(self, uri, *, ctx, max_bytes): # noqa: ANN001 class _Sanitizer: - async def sanitize(self, content, *, ctx, uri, max_replacements): # noqa: ANN001 + async def sanitize( # noqa: ANN001 + self, + content, + *, + ctx, + uri, + max_replacements, + fail_on_match_limit, + ): del ctx assert content == "prefix secret" assert uri == "viking://resources/demo/content.md" assert max_replacements == privacy_scan.PRIVACY_SCAN_MAX_REPLACEMENTS_PER_FILE + assert fail_on_match_limit is False return PrivacySanitizationResult( content="prefix reference", replacements=(PrivacyReplacement(7, 13, "reference"),), From 40c7e6316de46636d8f0e6c57ba3469aab69e3c4 Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Fri, 16 Oct 2026 14:45:48 +0800 Subject: [PATCH 5/6] feat: update ov privacy commands for agent use --- docs/en/api/10-privacy.md | 2 +- docs/zh/api/10-privacy.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/api/10-privacy.md b/docs/en/api/10-privacy.md index eb28267bc4..dc2563fa4d 100644 --- a/docs/en/api/10-privacy.md +++ b/docs/en/api/10-privacy.md @@ -279,7 +279,7 @@ POST /api/v1/privacy-configs/resolve Field URIs may use `viking://~/privacy/...#/values/...` or an explicit `viking://user/{user_id}/privacy/...#/values/...`. An explicit owner that differs from the current identity returns `PERMISSION_DENIED`. -References generated, stored, and returned by OpenViking always use the whitespace-free canonical form. The parser accepts spaces or tabs immediately after `{{` and before `}}`, such as `{{ ov_privacy:alice:... }}`, then normalizes the result to `{{ov_privacy:alice:...}}`. Whitespace is not accepted inside the owner, URI, or path tokens. +References generated, stored, and returned by OpenViking always use the whitespace-free canonical form. The parser accepts spaces or tabs immediately inside the outer braces, such as {{ ov_privacy:alice:... }}, then normalizes the result to {{ov_privacy:alice:...}}. Whitespace is not accepted inside the owner, URI, or path tokens. --- diff --git a/docs/zh/api/10-privacy.md b/docs/zh/api/10-privacy.md index bf6b4e1a84..057cc74aa4 100644 --- a/docs/zh/api/10-privacy.md +++ b/docs/zh/api/10-privacy.md @@ -283,7 +283,7 @@ POST /api/v1/privacy-configs/resolve 字段 URI 支持 `viking://~/privacy/...#/values/...` 和显式 `viking://user/{user_id}/privacy/...#/values/...`。显式 owner 与当前身份不一致时返回 `PERMISSION_DENIED`。 -OpenViking 生成、保存和返回的 canonical reference 始终不带空格。解析器兼容 `{{` 后和 `}}` 前的空格或 Tab，例如 `{{ ov_privacy:alice:... }}`，解析后仍规范化为 `{{ov_privacy:alice:...}}`；owner、URI 和路径字段内部不允许插入空白。 +OpenViking 生成、保存和返回的 canonical reference 始终不带空格。解析器兼容双左花括号后和双右花括号前的空格或 Tab，例如 {{ ov_privacy:alice:... }}，解析后仍规范化为 {{ov_privacy:alice:...}}；owner、URI 和路径字段内部不允许插入空白。 --- From 4760825cfd6080dd14a6c558ee132f79be40b2b6 Mon Sep 17 00:00:00 2001 From: DuTao Date: Fri, 16 Oct 2026 15:45:28 +0800 Subject: [PATCH 6/6] fix(privacy): preserve referenced config when updating skills --- openviking/privacy/service.py | 11 ++++++++++ openviking/utils/skill_processor.py | 33 ++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/openviking/privacy/service.py b/openviking/privacy/service.py index 57bebf8cae..b7b1edbd82 100644 --- a/openviking/privacy/service.py +++ b/openviking/privacy/service.py @@ -246,10 +246,21 @@ async def upsert( labels: Optional[dict[str, Any]] = None, *, owner_lease_ref: Optional[dict[str, Any]] = None, + preserve_fields: Optional[set[str]] = None, ) -> UserPrivacyConfigVersion: async with self._config_lock( ctx, category, target_key, owner_lease_ref=owner_lease_ref ) as lease: + if preserve_fields: + # Resolve retained references under the write lock so a Skill + # prepared before a credential rotation keeps the latest value. + current = await self.get_current(ctx, category, target_key) + retained = ( + {key: value for key, value in current.values.items() if key in preserve_fields} + if current is not None + else {} + ) + values = {**retained, **values} return await self._upsert_locked( ctx=ctx, category=category, diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.py index 944c76be72..cdd1d24294 100644 --- a/openviking/utils/skill_processor.py +++ b/openviking/utils/skill_processor.py @@ -6,6 +6,7 @@ Handles skill parsing, LLM generation, and storage operations. """ +import re import shutil import tempfile import time @@ -21,6 +22,7 @@ from openviking.core.mcp_converter import is_mcp_format, mcp_to_skill from openviking.core.namespace import canonical_user_root from openviking.core.skill_loader import SkillLoader +from openviking.privacy.reference import parse_privacy_field_reference from openviking.privacy.sanitizer import PrivacySanitizer from openviking.privacy.service import UserPrivacyConfigService from openviking.privacy.skill_extractor import extract_skill_privacy_values @@ -488,10 +490,7 @@ async def prepare_skill_privacy( if not self._privacy_config_service: return skill_dict, {} - skill_uri = ( - f"{canonical_user_root(ctx)}/skills/" - f"{skill_dict.get('name', '')}/SKILL.md" - ) + skill_uri = f"{canonical_user_root(ctx)}/skills/{skill_dict.get('name', '')}/SKILL.md" content = skill_dict.get("content", "") sanitized_by_catalog = await self._sanitize_registered_privacy( content, @@ -549,6 +548,28 @@ async def _sanitize_registered_privacy( ) return result.content + @staticmethod + def _referenced_privacy_fields(skill_dict: Dict[str, Any], ctx: RequestContext) -> set[str]: + """Find references to this Skill's own config without resolving secrets.""" + skill_name = skill_dict["name"] + legacy_prefix = f"{{{{ov_privacy:skill:{skill_name}:" + fields: set[str] = set() + for text in (skill_dict.get("content", ""), skill_dict.get("description", "")): + for placeholder in re.findall(r"\{\{[^{}\r\n]*\}\}", text or ""): + reference = parse_privacy_field_reference(placeholder) + if reference is not None: + if ( + reference.owner_user_id == ctx.user.user_id + and reference.category == "skill" + and reference.target_key == skill_name + ): + fields.add(reference.field) + elif placeholder.startswith(legacy_prefix): + field_name = placeholder[len(legacy_prefix) : -2] + if field_name and ":" not in field_name: + fields.add(field_name) + return fields + async def apply_skill_privacy( self, skill_dict: Dict[str, Any], @@ -562,7 +583,8 @@ async def apply_skill_privacy( if not self._privacy_config_service: return skill_dict - if privacy_values: + referenced_fields = self._referenced_privacy_fields(skill_dict, ctx) + if privacy_values or referenced_fields: await self._privacy_config_service.upsert( ctx=ctx, category="skill", @@ -571,6 +593,7 @@ async def apply_skill_privacy( updated_by=ctx.user.user_id, change_reason=change_reason, **({"owner_lease_ref": owner_lease_ref} if owner_lease_ref is not None else {}), + **({"preserve_fields": referenced_fields} if referenced_fields else {}), ) return skill_dict