From 9522d5b2dfb8897aaa3d7a20be974dd2cf9fadf4 Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Mon, 8 Jun 2026 14:15:58 +0000 Subject: [PATCH 1/2] fix(postgres): create content trigram index with fastupdate=off idx_messages_content_trgm is created with PostgreSQL's default fastupdate=on. The resulting GIN pending list is only merged into the index by VACUUM, so under continuous `pg sync` ingest where long-lived transactions pin the xmin horizon and starve autovacuum, it grows without bound. In one deployment the index bloated to 283 GB for ~213k rows and filled the disk, crashing PostgreSQL with ENOSPC. Disabling fastupdate merges entries directly into the tree, keeping the index bounded and predictable at a small per-insert cost. --- internal/postgres/schema.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index 414851fe2..665bcf113 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -690,9 +690,13 @@ func createContentSearchIndexesPG(ctx context.Context, db *sql.DB) { log.Printf("pg schema: invalid pg_trgm schema %q: %v", extSchema, err) return } + // fastupdate=off keeps the index bounded: the default fastupdate=on + // buffers inserts into a pending list that only VACUUM merges, which grows + // unbounded when continuous ingest starves autovacuum. if _, err := db.ExecContext(ctx, fmt.Sprintf( `CREATE INDEX IF NOT EXISTS idx_messages_content_trgm - ON messages USING gin (content %s.gin_trgm_ops)`, quotedExt, + ON messages USING gin (content %s.gin_trgm_ops) + WITH (fastupdate = off)`, quotedExt, )); err != nil { log.Printf( "pg schema: creating messages.content trigram index failed: %v", err, From 3f36eef0f14ba634dc0151343596a269b224f37f Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Mon, 8 Jun 2026 14:26:21 +0000 Subject: [PATCH 2/2] fix(postgres): reapply fastupdate=off on existing trigram index CREATE INDEX IF NOT EXISTS only applies WITH (fastupdate = off) on first creation, so stores upgraded from an earlier schema retained the default fastupdate=on. Reapply idempotently with ALTER INDEX on every schema bootstrap, and assert reloptions in the pgtest suite. Addresses review feedback on #606. --- internal/postgres/schema.go | 12 ++++++++++++ .../postgres/search_content_pgtest_test.go | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index 665bcf113..127972454 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -701,6 +701,18 @@ func createContentSearchIndexesPG(ctx context.Context, db *sql.DB) { log.Printf( "pg schema: creating messages.content trigram index failed: %v", err, ) + return + } + // CREATE INDEX IF NOT EXISTS only applies WITH (fastupdate = off) on + // first creation. Re-apply on every boot so stores upgraded from a + // prior schema (which left fastupdate=on) also get the bounded index. + if _, err := db.ExecContext(ctx, + `ALTER INDEX idx_messages_content_trgm SET (fastupdate = off)`, + ); err != nil { + log.Printf( + "pg schema: disabling fastupdate on messages.content trigram index failed: %v", + err, + ) } } diff --git a/internal/postgres/search_content_pgtest_test.go b/internal/postgres/search_content_pgtest_test.go index bb0397c77..df278f48e 100644 --- a/internal/postgres/search_content_pgtest_test.go +++ b/internal/postgres/search_content_pgtest_test.go @@ -481,6 +481,24 @@ func TestPGContentSearchTrigramIndex(t *testing.T) { ).Scan(&hasIdx), "query pg_indexes") assert.True(t, hasIdx, "idx_messages_content_trgm missing after EnsureSchema") + + // fastupdate=off must be set so the pending list cannot grow + // unbounded under continuous ingest. The schema bootstrap applies + // this on every boot (including stores upgraded from an earlier + // schema that created the index with the default fastupdate=on). + var fastupdateOff bool + require.NoError(t, store.DB().QueryRowContext(ctx, + `SELECT EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 + AND c.relname = 'idx_messages_content_trgm' + AND 'fastupdate=off' = ANY(c.reloptions) + )`, contentSearchSchema, + ).Scan(&fastupdateOff), "query pg_class.reloptions") + assert.True(t, fastupdateOff, + "idx_messages_content_trgm must have fastupdate=off") } // TestPGSearchContentRegex verifies regex mode.