From d7613128afa5af16d12bd61a3a47e6f27c684b24 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:38:13 -0700 Subject: [PATCH 1/2] feat(calibration): self-host Postgres parity for every calibration surface + pg-capable maintainer CLIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins #8082's calibration system against real Postgres (#8171): a PG_TEST_URL-gated integration suite exercises the capture writers (incl. raw-context metadata), threshold-backtest persistence, the calibration trend, the satisfaction-floor status surface, and the loosening loop's system_flags write path through the selfhost shim — asserting non-empty results, since every one of these paths degrades silently on a dialect gap. The suite caught two real self-host bugs, both fixed in pg-dialect: - date() reached Postgres as a date-typed value node-pg parses into a JS Date, so every day-bucketed trend read bucketed nothing; now translated to a TEXT day like SQLite returns. - nested json_extract paths ($.comparison.verdict) were untranslated hard errors swallowed by fail-safe reads; now translated to jsonb #>>. The three maintainer CLIs (corpus export, track record, backfill) accept --pg (bare --pg falls back to DATABASE_URL, mirroring the selfhost stack's own driver pick) and reuse createPgAdapter — same dialect translation as the deployed Worker, pure cores untouched. The backfill wrapper's confidence projection now coerces the text ->> returns instead of type-gating it. Verified end-to-end against a real Postgres: dry-run report, idempotent apply, checksummed export (manifest names the database, never credentials), track-record read. Closes #8171 --- .github/workflows/selfhost.yml | 2 +- .../content/docs/backtest-calibration.mdx | 23 +++ scripts/backfill-calibration-corpus.ts | 55 +++-- scripts/backtest-corpus-export.ts | 35 +++- scripts/backtest-track-record.ts | 39 +++- scripts/pg-cli.ts | 50 +++++ src/selfhost/pg-dialect.ts | 12 +- .../selfhost-pg-calibration.test.ts | 191 ++++++++++++++++++ test/unit/pg-cli.test.ts | 38 ++++ test/unit/selfhost-pg-dialect.test.ts | 15 ++ 10 files changed, 429 insertions(+), 31 deletions(-) create mode 100644 scripts/pg-cli.ts create mode 100644 test/integration/selfhost-pg-calibration.test.ts create mode 100644 test/unit/pg-cli.test.ts diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index 6827511739..9d66fe6799 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -94,7 +94,7 @@ jobs: run: npm run build --workspace @loopover/engine - name: Postgres integration test (real PG) - run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/loopover npx vitest run test/integration/selfhost-pg.test.ts + run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/loopover npx vitest run test/integration/selfhost-pg.test.ts test/integration/selfhost-pg-calibration.test.ts - name: Build the self-host bundle run: node --experimental-strip-types scripts/build-selfhost.ts diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 41c7a89eae..f36bf38819 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -79,3 +79,26 @@ npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.jso ``` Both CLIs are strictly read-only against the database. + +## Self-hosting + +On a self-host deployment the same calibration system runs against your Postgres instead of D1 — +the capture writers, the persisted backtest runs, the trend and satisfaction-floor endpoints, and +the loosening loop all go through the storage shim, so the corpus lives in your own +`audit_events` table and never leaves your infrastructure. Parity is pinned by a real-Postgres +integration suite that runs in CI. + +All three maintainer CLIs accept `--pg` to run against a self-host Postgres instead of D1. An +explicit connection string wins; a bare `--pg` falls back to `DATABASE_URL`, the same variable the +self-host stack itself boots from: + +```bash +npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.json --pg "$DATABASE_URL" +npx tsx scripts/backtest-track-record.ts --pg +npx tsx scripts/backfill-calibration-corpus.ts --pg # dry-run; add --apply to write +``` + +The pg path reuses the deployment's own dialect adapter, so the backfill keeps its +`INSERT OR IGNORE` idempotency contract, and exported corpus manifests name only the database — +never the credentials in your connection string. Each deployment's corpus is its own: there is no +cross-store sync between a self-host Postgres and any other deployment. diff --git a/scripts/backfill-calibration-corpus.ts b/scripts/backfill-calibration-corpus.ts index ede9073b21..ee3fbfc33c 100644 --- a/scripts/backfill-calibration-corpus.ts +++ b/scripts/backfill-calibration-corpus.ts @@ -7,11 +7,14 @@ // wrapper — mirrors backtest-corpus-export.ts's identical split. // // tsx scripts/backfill-calibration-corpus.ts --db loopover [--remote] [--apply] +// … --pg postgres://… runs against a self-host Postgres instead (#8171; bare --pg uses DATABASE_URL) // -// Deployment note (#8157): source AND destination are the same D1 — the ledger of record for this -// deployment. Self-host operators' corpora live in their own Postgres; a pg-driver variant is an explicit -// non-goal here. +// Deployment note (#8157): source AND destination are the same database — the ledger of record for +// WHICHEVER deployment the flags select: the D1 default, or a self-host Postgres via --pg (#8171). The +// pg path rides the same selfhost adapter the Worker uses, so INSERT OR IGNORE keeps its idempotency +// contract through the dialect translation. import { spawnSync } from "node:child_process"; +import { openPgDatabase, resolvePgConnection, type PgCliSession } from "./pg-cli.js"; import { buildBackfillInsertStatements, renderBackfillReport, @@ -19,15 +22,19 @@ import { type ReviewTargetDecisionRow, } from "./backfill-calibration-corpus-core.js"; -type Args = { db: string; remote: boolean; apply: boolean }; +type Args = { db: string; remote: boolean; apply: boolean; pgPresent: boolean; pgValue: string | undefined }; function parseArgs(argv: string[]): Args { - const args: Args = { db: "loopover", remote: false, apply: false }; + const args: Args = { db: "loopover", remote: false, apply: false, pgPresent: false, pgValue: undefined }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (flag === "--remote") args.remote = true; else if (flag === "--apply") args.apply = true; else if (flag === "--db") args.db = argv[++i]!; + else if (flag === "--pg") { + args.pgPresent = true; + if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i]; + } } return args; } @@ -46,12 +53,31 @@ function d1Execute(db: string, remote: boolean, sql: string): Array { + const execute = async (sql: string): Promise>> => + pgSession ? ((await pgSession.db.prepare(sql).all>()).results ?? []) : d1Execute(args.db, args.remote, sql); - const rows = d1Execute( - args.db, - args.remote, + const rows = await execute( `SELECT repo, number, verdict, status, json_extract(decision_json, '$.confidence') AS confidence, terminal_at FROM review_targets WHERE kind = 'pull_request'`, ); @@ -60,7 +86,9 @@ function main() { number: typeof row.number === "number" ? row.number : Number(row.number ?? 0), verdict: typeof row.verdict === "string" ? row.verdict : null, status: typeof row.status === "string" ? row.status : null, - confidence: typeof row.confidence === "number" ? row.confidence : null, + // D1's json_extract returns a JSON number as a number; Postgres's translated `->>` ALWAYS returns + // text (the same #4997 semantics the dialect suite pins) — coerce, don't type-gate (#8171). + confidence: toFiniteNumber(row.confidence), terminalAt: typeof row.terminal_at === "string" ? row.terminal_at : null, })); @@ -74,11 +102,14 @@ function main() { const statements = buildBackfillInsertStatements(report.rows); let written = 0; for (const statement of statements) { - d1Execute(args.db, args.remote, statement); + await execute(statement); written += 1; console.error(`applied statement ${written}/${statements.length}`); } console.error(`backfill applied: ${report.rows.length} row(s) across ${statements.length} statement(s) (re-runs are no-ops).`); } -main(); +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/backtest-corpus-export.ts b/scripts/backtest-corpus-export.ts index 3309417b09..1d4ab40524 100644 --- a/scripts/backtest-corpus-export.ts +++ b/scripts/backtest-corpus-export.ts @@ -6,6 +6,7 @@ // backtest-corpus-export-core.ts (unit-tested); this file is the thin IO wrapper — mirrors export-d1-data.ts. // // tsx scripts/backtest-corpus-export.ts --rule-id --output [--remote] [--since-date ] [--db loopover] +// … --pg postgres://… reads a self-host Postgres instead (#8171; bare --pg uses DATABASE_URL) // // --remote reads the deployed D1 (default is the local miniflare DB). --since-date does an INCREMENTAL export // (rows whose created_at is >= the date); omit it for a full history. NEVER pass a write command. ORB only — @@ -14,6 +15,7 @@ import { writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "@loopover/engine"; import { buildBacktestCorpusManifest } from "./backtest-corpus-export-core.js"; +import { openPgDatabase, pgDatabaseLabel, resolvePgConnection } from "./pg-cli.js"; type D1Row = Record; @@ -23,13 +25,15 @@ type Args = { remote: boolean; sinceDate: string | undefined; db: string; + pgPresent: boolean; + pgValue: string | undefined; }; const RULE_FIRED_EVENT_TYPE_PREFIX = "signal.rule_fired:"; const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:"; function parseArgs(argv: string[]): Args { - const args: Args = { ruleId: undefined, output: undefined, remote: false, sinceDate: undefined, db: "loopover" }; + const args: Args = { ruleId: undefined, output: undefined, remote: false, sinceDate: undefined, db: "loopover", pgPresent: false, pgValue: undefined }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (flag === "--remote") args.remote = true; @@ -37,6 +41,10 @@ function parseArgs(argv: string[]): Args { else if (flag === "--output") args.output = argv[++i]; else if (flag === "--since-date") args.sinceDate = argv[++i]; else if (flag === "--db") args.db = argv[++i]!; + else if (flag === "--pg") { + args.pgPresent = true; + if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i]; + } } return args; } @@ -113,8 +121,9 @@ function rowEventType(row: D1Row): string { return typeof row.event_type === "string" ? row.event_type : ""; } -function main() { +async function main() { const args = parseArgs(process.argv.slice(2)); + const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL); if (!args.ruleId || !args.output) { console.error( "Usage: tsx scripts/backtest-corpus-export.ts --rule-id --output [--remote] [--since-date ] [--db loopover]", @@ -131,7 +140,7 @@ function main() { sinceClause + ` ORDER BY created_at ASC`; - const rows = d1Query(args.db, args.remote, sql); + const rows = pgConnection ? await pgQuery(pgConnection, sql) : d1Query(args.db, args.remote, sql); const fired: RuleFiredEvent[] = []; const overrides: HumanOverrideEvent[] = []; for (const row of rows) { @@ -148,8 +157,9 @@ function main() { const cases: BacktestCase[] = buildBacktestCorpus(args.ruleId, fired, overrides); const manifest = buildBacktestCorpusManifest(args.ruleId, cases, { generatedAt: new Date().toISOString(), - source: args.remote ? "d1-remote" : "d1-local", - database: args.db, + // pgDatabaseLabel names the database WITHOUT the connection string's credentials. + source: pgConnection ? "postgres" : args.remote ? "d1-remote" : "d1-local", + database: pgConnection ? pgDatabaseLabel(pgConnection) : args.db, incremental: Boolean(args.sinceDate), ...(args.sinceDate ? { sinceDate: args.sinceDate } : {}), }); @@ -157,4 +167,17 @@ function main() { console.error(`exported ${manifest.caseCount} cases for rule ${args.ruleId} (checksum ${manifest.checksum.slice(0, 12)}…) → ${args.output}`); } -main(); +// The pg sibling of d1Query: same fail-loud contract, same shim the Worker runs on (#8171). +async function pgQuery(connection: string, sql: string): Promise { + const session = openPgDatabase(connection); + try { + return (await session.db.prepare(sql).all()).results ?? []; + } finally { + await session.close(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/backtest-track-record.ts b/scripts/backtest-track-record.ts index a4dbf4bc69..6c46683487 100644 --- a/scripts/backtest-track-record.ts +++ b/scripts/backtest-track-record.ts @@ -7,9 +7,11 @@ // (pure, unit-tested); this file is the thin IO wrapper — mirrors backtest-corpus-export.ts's identical split. // // tsx scripts/backtest-track-record.ts --db loopover [--remote] +// tsx scripts/backtest-track-record.ts --pg postgres://… (self-host Postgres — #8171; bare --pg uses DATABASE_URL) // // --remote reads the deployed D1 (default is the local miniflare DB). NEVER pass a write command. import { spawnSync } from "node:child_process"; +import { openPgDatabase, resolvePgConnection } from "./pg-cli.js"; import { computeRegressedVerdictTrackRecord, type BacktestComparison } from "@loopover/engine"; import { LOGIC_BACKTEST_EVENT_TYPE } from "./backtest-logic-check-core.js"; @@ -20,14 +22,18 @@ import { LOGIC_BACKTEST_EVENT_TYPE } from "./backtest-logic-check-core.js"; // module with no Worker-bound imports, so THAT one is imported for real rather than hand-mirrored. const THRESHOLD_BACKTEST_EVENT_TYPE = "calibration.threshold_backtest_run"; -type Args = { db: string | undefined; remote: boolean }; +type Args = { db: string | undefined; remote: boolean; pgPresent: boolean; pgValue: string | undefined }; function parseArgs(argv: string[]): Args { - const args: Args = { db: undefined, remote: false }; + const args: Args = { db: undefined, remote: false, pgPresent: false, pgValue: undefined }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (flag === "--remote") args.remote = true; else if (flag === "--db") args.db = argv[++i]; + else if (flag === "--pg") { + args.pgPresent = true; + if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i]; + } } return args; } @@ -46,17 +52,15 @@ function d1Query(db: string, remote: boolean, sql: string): Array [--remote]"); + const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL); + if (!pgConnection && !args.db) { + console.error("Usage: tsx scripts/backtest-track-record.ts --db [--remote] | --pg "); process.exit(2); } - const rows = d1Query( - args.db, - args.remote, - `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`, - ); + const sql = `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`; + const rows = pgConnection ? await pgQuery(pgConnection, sql) : d1Query(args.db!, args.remote, sql); const comparisons: BacktestComparison[] = []; for (const row of rows) { try { @@ -75,4 +79,17 @@ function main() { } } -main(); +// The pg sibling of d1Query: same fail-loud contract, same shim the Worker runs on (#8171). +async function pgQuery(connection: string, sql: string): Promise>> { + const session = openPgDatabase(connection); + try { + return (await session.db.prepare(sql).all>()).results ?? []; + } finally { + await session.close(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/pg-cli.ts b/scripts/pg-cli.ts new file mode 100644 index 0000000000..a6a63a257d --- /dev/null +++ b/scripts/pg-cli.ts @@ -0,0 +1,50 @@ +// Shared pg-driver plumbing for the calibration CLIs (#8171). Self-host operators' ledgers live in +// Postgres behind the same shim the Worker uses, so the CLIs reuse src/selfhost/pg-adapter.ts's +// createPgAdapter — the SAME dialect translation the deployed code runs, never a second SQL +// implementation — and each script's pure core stays untouched. Driver selection mirrors how the +// selfhost stack itself picks its DB (src/selfhost/preflight.ts reads DATABASE_URL): an explicit +// `--pg ` wins, a bare `--pg` falls back to DATABASE_URL, and without `--pg` the D1/wrangler +// path stays the default — DATABASE_URL alone never silently switches drivers. +import pg from "pg"; +import { createPgAdapter } from "../src/selfhost/pg-adapter.js"; + +/** PURE driver selection: the pg connection string to use, or null for the default D1/wrangler path. + * Throws (fail-loud, like every IO error in these CLIs) when `--pg` was given but no connection string + * can be resolved — a silent fall-through to D1 could read the WRONG deployment's ledger. */ +export function resolvePgConnection(pgFlagPresent: boolean, pgFlagValue: string | undefined, databaseUrl: string | undefined): string | null { + if (!pgFlagPresent) return null; + const explicit = pgFlagValue?.trim(); + if (explicit) return explicit; + const fromEnv = databaseUrl?.trim(); + if (fromEnv) return fromEnv; + throw new Error("--pg was given without a connection string and DATABASE_URL is not set"); +} + +/** PURE label for reports/manifests: names the database WITHOUT the credentials the connection string + * carries (nothing secret-shaped may reach a written artifact). */ +export function pgDatabaseLabel(connection: string): string { + try { + const database = new URL(connection).pathname.replace(/^\//, ""); + return database ? `postgres:${database}` : "postgres"; + } catch { + return "postgres"; + } +} + +export type PgCliSession = { + /** The selfhost adapter: SQLite-dialect SQL in, translated Postgres out — identical to the Worker path. */ + db: D1Database; + close(): Promise; +}; + +/** Open a pooled connection wrapped in the selfhost adapter. Callers own close() (try/finally). */ +export function openPgDatabase(connection: string): PgCliSession { + const pool = new pg.Pool({ connectionString: connection }); + pg.types.setTypeParser(20, (value: string) => Number.parseInt(value, 10)); // int8 (COUNT/SUM) → number, like D1 + return { + db: createPgAdapter(pool), + close: async () => { + await pool.end(); + }, + }; +} diff --git a/src/selfhost/pg-dialect.ts b/src/selfhost/pg-dialect.ts index ab0c3dc491..55737d2ce8 100644 --- a/src/selfhost/pg-dialect.ts +++ b/src/selfhost/pg-dialect.ts @@ -60,7 +60,17 @@ export function translateFunctions(sql: string): string { .replace(/datetime\(\s*'now'\s*\)/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) // CURRENT_TIMESTAMP → SQLite's TEXT format (the columns are TEXT) .replace(/CURRENT_TIMESTAMP/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) - // json_extract(col, '$.key') → (col::jsonb ->> 'key') (single-level paths — all the codebase uses) + // date() → TEXT 'YYYY-MM-DD' like SQLite's date(). Postgres would accept date() via an + // implicit cast, but it returns a `date`-typed value that node-pg parses into a JS Date object — so + // every day-bucketed trend read (rule-calibration-trend.ts, public-accuracy-trend.ts, stats.ts, ...) + // silently bucketed NOTHING on self-host: the JS-side week matching expects the TEXT day D1 returns + // (#8171). `datetime(` cannot match (the regex requires "(" immediately after "date"). + .replace(/(?> '{a,b…}') — the persisted backtest runs + // read `$.comparison.verdict`, which the single-level rule below can't see; untranslated it is a hard + // "function json_extract does not exist" error swallowed by the fail-safe trend reads (#8171). + .replace(/json_extract\(\s*([^,]+?)\s*,\s*'\$\.((?:[A-Za-z0-9_]+\.)+[A-Za-z0-9_]+)'\s*\)/gi, (_m, col, path) => `((${col})::jsonb #>> '{${path.split(".").join(",")}}')`) + // json_extract(col, '$.key') → (col::jsonb ->> 'key') (single-level paths) .replace(/json_extract\(\s*([^,]+?)\s*,\s*'\$\.([A-Za-z0-9_]+)'\s*\)/gi, `(($1)::jsonb ->> '$2')`) // instr(haystack, needle) → strpos(haystack, needle): both are 1-based first-occurrence index, 0 if // absent -- a direct semantic match, no formula adjustment needed. Postgres has no `instr` builtin at diff --git a/test/integration/selfhost-pg-calibration.test.ts b/test/integration/selfhost-pg-calibration.test.ts new file mode 100644 index 0000000000..d6d928ec90 --- /dev/null +++ b/test/integration/selfhost-pg-calibration.test.ts @@ -0,0 +1,191 @@ +// Real-Postgres parity suite for the #8082 calibration surfaces (#8171). The Worker-path calibration code +// all reads/writes through env.DB; on self-host that is Postgres behind the pg shim, and every one of these +// paths degrades SILENTLY on a dialect gap (safeAll / fail-safe catches) — so parity must be pinned by +// asserting non-empty results, never just "didn't throw". Skipped unless PG_TEST_URL is set (same contract +// as selfhost-pg.test.ts); run locally against a real PG: +// docker run -d -e POSTGRES_PASSWORD=devpw -e POSTGRES_DB=loopover -p 55432:5432 postgres:16 +// PG_TEST_URL=postgres://postgres:devpw@localhost:55432/loopover npx vitest run test/integration/selfhost-pg-calibration.test.ts +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import pg from "pg"; +import { splitBacktestCorpus } from "@loopover/engine"; +import { runSelfHostMigrations } from "../../src/selfhost/migrate"; +import { createPgAdapter } from "../../src/selfhost/pg-adapter"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import * as core from "../../src/services/satisfaction-floor-loosening"; +import { + getSatisfactionFloorOverride, + loadSatisfactionFloorStatus, + runSatisfactionFloorLoosening, + SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, + SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY, +} from "../../src/services/satisfaction-floor-loosening-run"; +import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory, THRESHOLD_BACKTEST_EVENT_TYPE } from "../../src/services/threshold-backtest-run"; +import { loadCalibrationTrend } from "../../src/services/rule-calibration-trend"; +import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction"; + +const PG_URL = process.env.PG_TEST_URL; +const suite = PG_URL ? describe : describe.skip; + +// Own database, so this file and selfhost-pg.test.ts (which drops/recreates ITS public schema) can run in +// the same vitest invocation without racing each other. +const CALIB_DB = "loopover_calibration_parity"; + +function withDatabase(base: string, database: string): string { + const url = new URL(base); + url.pathname = `/${database}`; + return url.toString(); +} + +suite("Postgres calibration parity (#8171) — real Postgres", () => { + let pool: pg.Pool; + let env: Env; + + beforeAll(async () => { + pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 + const admin = new pg.Pool({ connectionString: PG_URL }); + await admin.query(`DROP DATABASE IF EXISTS ${CALIB_DB}`); + await admin.query(`CREATE DATABASE ${CALIB_DB}`); + await admin.end(); + + pool = new pg.Pool({ connectionString: withDatabase(PG_URL!, CALIB_DB) }); + const db = createPgAdapter(pool); + expect(await runSelfHostMigrations(db, "migrations")).toBeGreaterThan(50); + // The autotune flag ON is part of the surface under test (the loosening loop's write path). + env = { DB: db, SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" } as unknown as Env; + }, 120_000); // database create + full migration chain legitimately exceeds the 10s hook default + afterAll(async () => { + await pool?.end(); + }); + + it("capture writers round-trip: rule_fired (incl. bounded raw context metadata) + human_override + queryRuleHistory", async () => { + const store = createSignalStore(env); + await store.recordRuleFired({ + ruleId: "ai_consensus_defect", + targetKey: "acme/widgets#900", + outcome: "unaddressed", + occurredAt: new Date(Date.now() - 5000).toISOString(), + // The #8130/#8129 writers put the bounded raw excerpt in metadata — parity means it survives the + // pg JSON round-trip byte-for-byte, not merely that the row inserts. + metadata: { confidence: 0.72, rawContext: { excerpt: "line 12: TODO drop table", truncated: false } }, + }); + await store.recordHumanOverride({ + ruleId: "ai_consensus_defect", + targetKey: "acme/widgets#900", + verdict: "reversed", + occurredAt: new Date().toISOString(), + }); + + const history = await createSignalStore(env).queryRuleHistory("ai_consensus_defect", 0); + expect(history.fired).toHaveLength(1); + expect(history.fired[0]!.metadata).toMatchObject({ confidence: 0.72, rawContext: { excerpt: "line 12: TODO drop table", truncated: false } }); + expect(history.overrides).toHaveLength(1); + expect(history.overrides[0]!).toMatchObject({ targetKey: "acme/widgets#900", verdict: "reversed" }); + }); + + it("threshold backtest: advisory run over a pg-backed corpus produces a comparison and persistThresholdBacktestRuns records it", async () => { + await seedLooseningFriendlyHistory(env); + + const diff = [ + "diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts", + "@@ -980,7 +980,7 @@", + `-export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5;`, + `+export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.2;`, + ].join("\n"); + const { changed, comparisons } = await runThresholdBacktestAdvisory(env, diff); + expect(changed).toHaveLength(1); + expect(comparisons.length).toBeGreaterThan(0); // empty ⇒ the corpus read silently failed on pg + + await persistThresholdBacktestRuns(env, "acme/widgets", 7, changed, comparisons); + const row = await env.DB.prepare("SELECT target_key, metadata_json FROM audit_events WHERE event_type = ? LIMIT 1") + .bind(THRESHOLD_BACKTEST_EVENT_TYPE) + .first<{ target_key: string; metadata_json: string }>(); + expect(row?.target_key).toBe("acme/widgets#7"); + const metadata = JSON.parse(row!.metadata_json) as { comparison: { ruleId: string; verdict: string } }; + expect(metadata.comparison.ruleId).toBe(core.SATISFACTION_FLOOR_RULE_ID); + expect(typeof metadata.comparison.verdict).toBe("string"); + }); + + it("calibration trend re-buckets fired/override/run events from pg with non-zero counts (a dialect gap here degrades silently to zeros)", async () => { + const report = await loadCalibrationTrend(env); + const ruleTrend = report.rules.find((rule) => rule.ruleId === core.SATISFACTION_FLOOR_RULE_ID); + expect(ruleTrend).toBeDefined(); + const fired = ruleTrend!.weeks.reduce((sum, week) => sum + week.fired, 0); + expect(fired).toBeGreaterThan(0); + const runs = report.backtestRuns.reduce((sum, week) => sum + week.improved + week.regressed + week.unchanged, 0); + expect(runs).toBeGreaterThan(0); // the persisted threshold run above must be visible to the trend + }); + + it("loosening loop end-to-end on pg: apply writes the system_flags override + evidence event; status and override reads see both", async () => { + const result = await runSatisfactionFloorLoosening(env); + expect(result.applied).toBe(true); + if (!result.applied) throw new Error("unreachable"); + expect(result.proposal.proposedFloor).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + + // The single consumption-point read. + expect(await getSatisfactionFloorOverride(env)).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + // The raw row the override rides on (INSERT ... ON CONFLICT upsert through the shim). + const flag = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?") + .bind(SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY) + .first<{ value: string }>(); + expect(flag?.value).toBe(String(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0])); + + // The operator status surface (/v1/internal/calibration/satisfaction-floor's whole body). + const status = await loadSatisfactionFloorStatus(env); + expect(status.flagEnabled).toBe(true); + expect(status.storedOverride).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + expect(status.liveFloor).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + expect(status.applied.length).toBeGreaterThan(0); + expect(status.applied[0]!).toMatchObject({ + currentFloor: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR, + proposedFloor: core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0], + }); + + // And the evidence event itself. + const evidence = await env.DB.prepare("SELECT COUNT(*) AS n FROM audit_events WHERE event_type = ?") + .bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE) + .first<{ n: number }>(); + expect(evidence?.n).toBe(1); + }); +}); + +// Same membership-probe seeding as the unit suite (test/unit/satisfaction-floor-loosening-run.test.ts): +// ask the real splitter which keys land in which slice, then seed borderline-confirmed history in both so +// 0.5 → 0.45 clears the visible AND held-out gates, plus one genuinely-reversed deep-low firing per slice. +async function seedLooseningFriendlyHistory(env: Env): Promise { + const pool = Array.from({ length: 120 }, (_, i) => `acme/widgets#${i + 1}`); + const probe = pool.map((targetKey) => ({ + ruleId: core.SATISFACTION_FLOOR_RULE_ID, + targetKey, + outcome: "unaddressed", + label: "confirmed" as const, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + })); + const { visible, heldOut } = splitBacktestCorpus(probe, core.SATISFACTION_FLOOR_HELD_OUT_FRACTION, core.SATISFACTION_FLOOR_SPLIT_SEED); + const store = createSignalStore(env); + const now = Date.now(); + const keys = [ + ...visible.slice(0, core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 4).map((c) => c.targetKey), + ...heldOut.slice(0, core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 2).map((c) => c.targetKey), + ]; + for (const [i, targetKey] of keys.entries()) { + await store.recordRuleFired({ + ruleId: core.SATISFACTION_FLOOR_RULE_ID, + targetKey, + outcome: "unaddressed", + occurredAt: new Date(now - 10_000 - i).toISOString(), + metadata: { confidence: 0.47 }, + }); + await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "confirmed", occurredAt: new Date(now - i).toISOString() }); + } + for (const targetKey of [visible[core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 5]!.targetKey, heldOut[core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 3]!.targetKey]) { + await store.recordRuleFired({ + ruleId: core.SATISFACTION_FLOOR_RULE_ID, + targetKey, + outcome: "unaddressed", + occurredAt: new Date(now - 20_000).toISOString(), + metadata: { confidence: 0.1 }, + }); + await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "reversed", occurredAt: new Date(now - 5000).toISOString() }); + } +} diff --git a/test/unit/pg-cli.test.ts b/test/unit/pg-cli.test.ts new file mode 100644 index 0000000000..6fb33880f0 --- /dev/null +++ b/test/unit/pg-cli.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { pgDatabaseLabel, resolvePgConnection } from "../../scripts/pg-cli"; + +// #8171: the pure driver-selection half of the calibration CLIs' pg path. The IO half (openPgDatabase) +// is exercised for real by the PG_TEST_URL-gated integration suite + the selfhost CI job. + +describe("resolvePgConnection (#8171)", () => { + it("returns null when --pg is absent — DATABASE_URL alone must never silently switch drivers", () => { + expect(resolvePgConnection(false, undefined, "postgres://env/db")).toBeNull(); + }); + + it("prefers the explicit --pg value over DATABASE_URL", () => { + expect(resolvePgConnection(true, "postgres://flag/db", "postgres://env/db")).toBe("postgres://flag/db"); + }); + + it("falls back to DATABASE_URL on a bare --pg (mirroring the selfhost stack's own driver pick)", () => { + expect(resolvePgConnection(true, undefined, "postgres://env/db")).toBe("postgres://env/db"); + expect(resolvePgConnection(true, " ", " postgres://env/db ")).toBe("postgres://env/db"); + }); + + it("fails loud on a bare --pg with no DATABASE_URL — a silent D1 fall-through could read the wrong deployment's ledger", () => { + expect(() => resolvePgConnection(true, undefined, undefined)).toThrow(/DATABASE_URL/); + expect(() => resolvePgConnection(true, "", " ")).toThrow(/DATABASE_URL/); + }); +}); + +describe("pgDatabaseLabel (#8171)", () => { + it("names the database without the credentials the connection string carries", () => { + const label = pgDatabaseLabel("postgres://user:s3cret@host:5432/loopover"); + expect(label).toBe("postgres:loopover"); + expect(label).not.toContain("s3cret"); + }); + + it("degrades to the bare driver name on an unparseable string or a missing database path", () => { + expect(pgDatabaseLabel("not a url")).toBe("postgres"); + expect(pgDatabaseLabel("postgres://host:5432/")).toBe("postgres"); + }); +}); diff --git a/test/unit/selfhost-pg-dialect.test.ts b/test/unit/selfhost-pg-dialect.test.ts index 1ccc7931b8..dac29ddd0c 100644 --- a/test/unit/selfhost-pg-dialect.test.ts +++ b/test/unit/selfhost-pg-dialect.test.ts @@ -31,6 +31,21 @@ describe("pg-dialect (#977 SQLite → Postgres)", () => { expect(translateFunctions("json_extract(meta, '$.mode')")).toBe("((meta)::jsonb ->> 'mode')"); }); + it("translates nested json_extract paths to #>> and date() to a TEXT day (#8171 — both previously silent self-host gaps)", () => { + // Nested path: the persisted backtest runs read $.comparison.verdict — untranslated this is a hard + // "function json_extract does not exist" error swallowed by the fail-safe trend reads. + expect(translateFunctions("json_extract(metadata_json, '$.comparison.verdict')")).toBe("((metadata_json)::jsonb #>> '{comparison,verdict}')"); + expect(translateFunctions("json_extract(m, '$.a.b.c')")).toBe("((m)::jsonb #>> '{a,b,c}')"); + // date(): Postgres's implicit-cast date() returns a `date` node-pg parses into a JS Date object, so + // every day-bucketed trend read bucketed NOTHING on self-host. TEXT parity with SQLite instead. + expect(translateFunctions("SELECT date(created_at) AS day")).toBe("SELECT to_char((created_at)::timestamptz, 'YYYY-MM-DD') AS day"); + expect(translateFunctions("date(pr.merged_at)")).toBe("to_char((pr.merged_at)::timestamptz, 'YYYY-MM-DD')"); + expect(translateFunctions("WHERE date(t.first_seen) >= date(?)")).toBe("WHERE to_char((t.first_seen)::timestamptz, 'YYYY-MM-DD') >= to_char((?)::timestamptz, 'YYYY-MM-DD')"); + // datetime( must NOT match the date( rule, and an unrelated identifier ending in date( must survive. + expect(translateFunctions("datetime('now')")).not.toContain("YYYY-MM-DD')"); + expect(translateFunctions("candidate(x)")).toBe("candidate(x)"); + }); + it("REGRESSION: translates instr(haystack, needle) to Postgres's strpos (SQLite has no `instr` on Postgres)", () => { expect(translateFunctions("instr(x, '#')")).toBe("strpos(x, '#')"); expect(translateFunctions("instr(ra.target_id, '#') > 0")).toBe("strpos(ra.target_id, '#') > 0"); From 6ada01ab8c773b4763f18b2ce10da1e1f684d48e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:55:27 -0700 Subject: [PATCH 2/2] ci(selfhost): single-source the ephemeral PG service container's throwaway password MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scanner flagged the literal in the test step. It is deliberately not a repository secret — the service container is job-scoped, localhost-only, and holds synthetic rows — but the credential now lives in ONE documented job-level env var consumed by both the service definition and the test step, so the two can never drift and the intent is explicit at the definition site. --- .github/workflows/selfhost.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index 9d66fe6799..b8e11d5908 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -61,11 +61,17 @@ jobs: if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true }} runs-on: ubuntu-latest timeout-minutes: 20 + env: + # Throwaway credential for the job-scoped ephemeral Postgres SERVICE CONTAINER below — deliberately + # not a repository secret: the container exists only for this job's lifetime, is reachable only from + # the runner's localhost, and holds nothing but synthetic test rows. Single-sourced here so the + # service env and the test step can never drift. + PG_TEST_PASSWORD: devpw services: postgres: image: postgres:18-alpine env: - POSTGRES_PASSWORD: devpw + POSTGRES_PASSWORD: ${{ env.PG_TEST_PASSWORD }} POSTGRES_DB: loopover ports: - 5432:5432 @@ -94,7 +100,7 @@ jobs: run: npm run build --workspace @loopover/engine - name: Postgres integration test (real PG) - run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/loopover npx vitest run test/integration/selfhost-pg.test.ts test/integration/selfhost-pg-calibration.test.ts + run: PG_TEST_URL=postgres://postgres:${{ env.PG_TEST_PASSWORD }}@localhost:5432/loopover npx vitest run test/integration/selfhost-pg.test.ts test/integration/selfhost-pg-calibration.test.ts - name: Build the self-host bundle run: node --experimental-strip-types scripts/build-selfhost.ts