From 6154cc33d1460e4c547308e2d0c2e888fcd6961f Mon Sep 17 00:00:00 2001 From: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Date: Tue, 25 Aug 2026 16:13:07 -0400 Subject: [PATCH 1/2] fix(db): exclude kind:30179 ciphertext from brownfield FTS Kind 30179 (NIP-PMA private managed agent) joined AUTHOR_ONLY_KINDS in #4593, but only fresh installs stopped indexing it: migration 0008's positive allowlist never touches populated databases, and 0014 wrapped the retained brownfield expression for 30350 alone. A relay upgraded in place still tokenizes the NIP-44 ciphertext into events.search_tsv. Add migration 0033 in the 0014 shape: capture the current generated expression and re-add search_tsv wrapped with kind = 30179 -> NULL, so every other kind keeps whatever policy the database already had. Mirror the exclusion in schema/schema.sql. Tests: extend the populated-upgrade migration test with a 30179 row and a pre-0033 checkpoint (fails without the new file); assert 0033's shape in the embedded-migrator test; apply 0033 in the buzz-search FTS setup. Co-authored-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Signed-off-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> --- crates/buzz-db/src/migration.rs | 42 +++++++++++++++---- crates/buzz-search/tests/fts_integration.rs | 5 +++ migrations/0033_private_managed_agent_fts.sql | 36 ++++++++++++++++ schema/schema.sql | 4 +- 4 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 migrations/0033_private_managed_agent_fts.sql diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 9df02c8abf9..ca84b9ff8a1 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -645,7 +645,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 32); + assert_eq!(migrations.len(), 33); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -844,6 +844,16 @@ mod tests { assert!(migrations[13].sql.as_str().contains("search_tsv")); assert!(!migrations[0].sql.as_str().contains("30350")); + // NIP-PMA kind:30179 FTS exclusion (0033): same wrap-the-existing- + // expression shape as 0014 so brownfield databases stop tokenizing + // private managed-agent ciphertext without a heap rewrite. + assert_eq!(migrations[32].version, 33); + assert!(migrations[32].sql.as_str().contains("kind = 30179")); + assert!(migrations[32].sql.as_str().contains("search_tsv")); + assert!(!migrations[0].sql.as_str().contains("30179")); + assert!(include_str!("../../../schema/schema.sql") + .contains("kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200)")); + // Public push-gateway authority is intentionally deployment-global and // durable: immediate revocation and hostile-relay admission cannot be // honestly provided by a stateless gateway. @@ -1915,7 +1925,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn populated_upgrade_preserves_search_policy_except_for_push_leases() { + async fn populated_upgrade_preserves_search_policy_except_for_private_kinds() { let pool = connect_test_pool().await; reset_public_schema(&pool).await; MIGRATOR @@ -1931,7 +1941,7 @@ mod tests { .await .expect("insert community"); - for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32)] { + for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32), (3_u8, 30_179_i32)] { sqlx::query( "INSERT INTO events \ (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at) \ @@ -1958,19 +1968,37 @@ mod tests { .fetch_all(&pool) .await .expect("read pre-push search behavior"); - assert_eq!(before, vec![(1, true), (30_350, true)]); + assert_eq!(before, vec![(1, true), (30_179, true), (30_350, true)]); + + // 0014 fixes 30350 only. A brownfield database that stopped here still + // tokenized kind:30179 ciphertext — the gap 0033 closes. + MIGRATOR + .run_to(32, &pool) + .await + .expect("apply migrations through 32"); + let pre_0033: Vec<(i32, Option)> = sqlx::query_as( + "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ + FROM events ORDER BY kind", + ) + .fetch_all(&pool) + .await + .expect("read pre-0033 search behavior"); + assert_eq!( + pre_0033, + vec![(1, Some(true)), (30_179, Some(true)), (30_350, None)] + ); run_migrations(&pool) .await - .expect("apply push migrations to populated database"); + .expect("apply remaining migrations to populated database"); let after: Vec<(i32, Option)> = sqlx::query_as( "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ FROM events ORDER BY kind", ) .fetch_all(&pool) .await - .expect("read post-push search behavior"); - assert_eq!(after, vec![(1, Some(true)), (30_350, None)]); + .expect("read post-upgrade search behavior"); + assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]); } #[tokio::test] diff --git a/crates/buzz-search/tests/fts_integration.rs b/crates/buzz-search/tests/fts_integration.rs index e7c196ee3e8..175a01aaaa3 100644 --- a/crates/buzz-search/tests/fts_integration.rs +++ b/crates/buzz-search/tests/fts_integration.rs @@ -28,6 +28,8 @@ const MIGRATION_0007_SQL: &str = include_str!("../../../migrations/0007_nip_rs_r const MIGRATION_0008_SQL: &str = include_str!("../../../migrations/0008_fresh_install_search_allowlist.sql"); const MIGRATION_0014_SQL: &str = include_str!("../../../migrations/0014_push_lease_fts.sql"); +const MIGRATION_0033_SQL: &str = + include_str!("../../../migrations/0033_private_managed_agent_fts.sql"); async fn setup() -> (PgPool, String) { let url = std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string()); @@ -81,6 +83,9 @@ async fn setup() -> (PgPool, String) { pool.execute(MIGRATION_0014_SQL) .await .expect("apply 0014 migration"); + pool.execute(MIGRATION_0033_SQL) + .await + .expect("apply 0033 migration"); (pool, schema) } diff --git a/migrations/0033_private_managed_agent_fts.sql b/migrations/0033_private_managed_agent_fts.sql new file mode 100644 index 00000000000..4e85081ec70 --- /dev/null +++ b/migrations/0033_private_managed_agent_fts.sql @@ -0,0 +1,36 @@ +-- NIP-PMA kind:30179 carries the owner's private managed-agent payload +-- (agent nsec, env vars, prompt) as NIP-44 ciphertext and is author-only. +-- Exclude it from full-text search without changing the search policy of +-- existing installations. Fresh installs are already safe via the positive +-- allowlist from migration 0008; this closes the brownfield gap where a +-- populated database still runs the legacy negative skip-set (0001/0005) +-- and would tokenize the ciphertext into search_tsv. +-- +-- Same shape as 0014 (kind:30350): PostgreSQL cannot alter a generated +-- expression in place, so capture the current expression, drop the column, +-- and re-add it wrapped with the new exclusion. Every other kind keeps +-- whatever policy the database had before. +DO $$ +DECLARE + existing_expression TEXT; +BEGIN + SELECT pg_get_expr(d.adbin, d.adrelid) + INTO existing_expression + FROM pg_attrdef d + JOIN pg_attribute a + ON a.attrelid = d.adrelid + AND a.attnum = d.adnum + WHERE d.adrelid = 'events'::regclass + AND a.attname = 'search_tsv'; + + IF existing_expression IS NULL THEN + RAISE EXCEPTION 'events.search_tsv generated expression not found'; + END IF; + + ALTER TABLE events DROP COLUMN search_tsv; + EXECUTE format( + 'ALTER TABLE events ADD COLUMN search_tsv TSVECTOR GENERATED ALWAYS AS (CASE WHEN kind = 30179 THEN NULL::tsvector ELSE (%s) END) STORED', + existing_expression + ); + CREATE INDEX idx_events_search_tsv ON events USING GIN (search_tsv); +END $$; diff --git a/schema/schema.sql b/schema/schema.sql index 6e14e6be1bf..c461039485f 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -219,9 +219,9 @@ CREATE TABLE events ( -- Privacy: encrypted/private routing wrappers and p-gated membership notices -- must never be discoverable through NIP-50 full-text search. NULL tsvector -- never matches `@@`. - -- Keep in sync with migrations (final state: 0001 + 0005 + 0009). + -- Keep in sync with migrations (final state: 0001 + 0005 + 0014 + 0033). search_tsv TSVECTOR GENERATED ALWAYS AS ( - CASE WHEN kind IN (1059, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector + CASE WHEN kind IN (1059, 30179, 30300, 30350, 30622, 44100, 44101, 44200) THEN NULL::tsvector ELSE to_tsvector('simple', content) END ) STORED, From 8687abc3cce9d3ad1bfff56a45a2ac7ffcfcf363 Mon Sep 17 00:00:00 2001 From: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Wed, 26 Aug 2026 14:45:53 -0400 Subject: [PATCH 2/2] docs(db): state the operational cost of migration 0033 honestly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 0033 header claimed the wrap-the-expression shape avoids a heap rewrite. It avoids a *policy* rewrite; it does not avoid a heap rewrite. DROP COLUMN + ADD ... GENERATED ... STORED recomputes every row under ACCESS EXCLUSIVE and rebuilds the GIN index in-transaction (no CONCURRENTLY), with no lock_timeout — downtime proportional to events size, and the index is recreated from the stock definition only. Say so in the migration header so operators of large brownfield databases can schedule a window instead of discovering it live. Comment-only change; no SQL semantics touched. Raised by Carl in review of #6822. Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- crates/buzz-db/src/migration.rs | 4 +++- migrations/0033_private_managed_agent_fts.sql | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index ca84b9ff8a1..75201afba85 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -846,7 +846,9 @@ mod tests { // NIP-PMA kind:30179 FTS exclusion (0033): same wrap-the-existing- // expression shape as 0014 so brownfield databases stop tokenizing - // private managed-agent ciphertext without a heap rewrite. + // private managed-agent ciphertext without a policy rewrite. (The + // migration itself still rewrites the events heap and rebuilds the + // GIN index — see the 0033 header for the operational cost.) assert_eq!(migrations[32].version, 33); assert!(migrations[32].sql.as_str().contains("kind = 30179")); assert!(migrations[32].sql.as_str().contains("search_tsv")); diff --git a/migrations/0033_private_managed_agent_fts.sql b/migrations/0033_private_managed_agent_fts.sql index 4e85081ec70..d2c29813966 100644 --- a/migrations/0033_private_managed_agent_fts.sql +++ b/migrations/0033_private_managed_agent_fts.sql @@ -10,6 +10,16 @@ -- expression in place, so capture the current expression, drop the column, -- and re-add it wrapped with the new exclusion. Every other kind keeps -- whatever policy the database had before. +-- +-- Operational cost: this is not free on large databases. DROP COLUMN + +-- ADD ... GENERATED ... STORED rewrites the entire events heap and then +-- rebuilds the GIN index, all under an ACCESS EXCLUSIVE lock inside the +-- migration transaction (CREATE INDEX CONCURRENTLY is not possible +-- here), with no lock_timeout. Expect relay downtime proportional to +-- the size of events. The index is recreated from the stock definition +-- below; any non-stock indexes or storage parameters on search_tsv are +-- not captured or replayed. 0014 set this precedent on smaller tables; +-- operators with large brownfield databases should schedule a window. DO $$ DECLARE existing_expression TEXT;