diff --git a/internal/jobs/deploy_reconcile_integration_test.go b/internal/jobs/deploy_reconcile_integration_test.go new file mode 100644 index 0000000..09dd15e --- /dev/null +++ b/internal/jobs/deploy_reconcile_integration_test.go @@ -0,0 +1,213 @@ +package jobs + +// deploy_reconcile_integration_test.go — REAL-Postgres integration tests for +// the silent-deploy-failure triad (2026-05-30 incident class). These are the +// highest-value worker integration tests per INTEGRATION-COVERAGE-PLAN- +// 2026-06-04.md §5 #2: deploy_status_reconcile + deploy_failure_autopsy were +// fake-tested only, which is exactly the regression class that hides — a sqlmock +// expectation passes whether or not the real SQL is valid against the live +// schema. +// +// Unlike the sibling *_test.go files in this package (which drive the SQL with +// go-sqlmock), these tests seed real rows in a live platform Postgres, run the +// job's Work()/captureDeploymentAutopsy against that DB, and assert the actual +// row state transition + side effects (status flip, deployment_events upsert, +// audit_log deploy.failed emit + its idempotency, error_message stamp). +// +// The k8s leg stays faked (no live cluster) — the fakeDeployStatusK8s / +// fakeAutopsyK8sCov helpers from deploy_lifecycle_coverage_test.go + +// deploy_status_reconcile_job_failed_test.go supply the cluster state. The DB +// leg is REAL. That split is the design the plan prescribes (§3 wave 3): "the +// deploy/k8s jobs use a fake clientset for the k8s leg but a real DB for the +// row mutation." +// +// GATING: testhelpers.SetupTestDB skips when no DB is reachable, so `make gate` +// (deploy.yml) and ci.yml (no DB service) skip these cleanly. They run locally +// against postgres://postgres@localhost:5432/instant_dev_test and wherever a +// TEST_DATABASE_URL is supplied (developer DB, coverage.yml's postgres service). + +import ( + "context" + "testing" + "time" + + "instant.dev/worker/internal/testhelpers" +) + +// TestIntegration_DeployStatusReconcile_JobFailedFlipsToFailed is the PRIMARY +// real-DB guard for the silent-deploy-failure bug class (Bug A of the +// 2026-05-30 triad). Setup mirrors the user's incident: +// +// - a deployments row at status='building' (api goroutine crashed mid-build +// or never stamped the terminal status) +// - the runtime Deployment was never created → GetDeployment returns NotFound +// - the kaniko build Job is Failed (BackoffLimitExceeded) but survives within +// its TTLSecondsAfterFinished window +// +// The fixed reconciler MUST: (1) flip the REAL row to 'failed', and (2) write a +// REAL deployment_events failure_autopsy row (the in-sweep capture). The +// sqlmock sibling pins the SQL string; THIS pins that the SQL is valid against +// the live schema and the row actually transitions — a sqlmock expectation +// would pass even if the UPDATE's WHERE clause silently matched zero rows. +func TestIntegration_DeployStatusReconcile_JobFailedFlipsToFailed(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + teamID := testhelpers.SeedTeam(t, db, "pro") + // provider_id "app-itest1" → namespace "instant-deploy-itest1". + deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "app-itest1") + + k8s := newFakeDeployStatusK8s() + // Runtime Deployment missing (build never reached apply). Build Job Failed. + k8s.jobs["instant-deploy-itest1|build-itest1"] = jobBackoffLimitExceeded() + + w := NewDeployStatusReconciler(db, k8s).WithAutopsyK8s(&fakeAutopsyK8sCov{}) + if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil { + t.Fatalf("Work: %v", err) + } + + gotStatus, _ := testhelpers.DeploymentStatus(t, db, deployID) + if gotStatus != deployStatusFailed { + t.Errorf("deployment status = %q, want %q (build Job Failed must flip the real row)", gotStatus, deployStatusFailed) + } + + // The in-sweep autopsy must have written a REAL deployment_events row. + if _, ok := testhelpers.AutopsyRow(t, db, deployID); !ok { + t.Error("no failure_autopsy deployment_events row written — the in-sweep capture did not round-trip to the DB") + } +} + +// TestIntegration_DeployStatusReconcile_HealthyTransition asserts the happy +// path round-trips: a 'building' row whose runtime Deployment is now healthy +// (AvailableReplicas>=1) is flipped to 'healthy' in the REAL DB. This pins that +// the updateStatus UPDATE's WHERE status IN (...) guard actually matches the +// row (a sqlmock test cannot catch a guard that excludes the live row). +func TestIntegration_DeployStatusReconcile_HealthyTransition(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + teamID := testhelpers.SeedTeam(t, db, "hobby") + deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "app-itest2") + + k8s := newFakeDeployStatusK8s() + k8s.objs["instant-deploy-itest2|app-itest2"] = newHealthyDeployment() + + w := NewDeployStatusReconciler(db, k8s) + if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil { + t.Fatalf("Work: %v", err) + } + + gotStatus, _ := testhelpers.DeploymentStatus(t, db, deployID) + if gotStatus != deployStatusHealthy { + t.Errorf("deployment status = %q, want %q", gotStatus, deployStatusHealthy) + } +} + +// TestIntegration_DeployStatusReconcile_StuckBuildingReaped covers sweep +// finding #5 (Bug A's tier-cap leak): a 'building' row with an EMPTY +// provider_id whose age exceeds stuckBuildingGrace is reaped to 'failed' with +// the stuck-building error_message stamped. This exercises reapStuckBuilding's +// double-guarded UPDATE against the live schema — the WHERE provider_id IS NULL +// OR provider_id = ” guard must match the real NULL row. +func TestIntegration_DeployStatusReconcile_StuckBuildingReaped(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + teamID := testhelpers.SeedTeam(t, db, "hobby") + // Empty provider_id → SeedDeployment writes NULL. + deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "") + + // Age the row past the 15m grace window so the reaper fires. + if _, err := db.Exec( + `UPDATE deployments SET created_at = $1 WHERE id = $2`, + time.Now().Add(-stuckBuildingGrace-time.Minute), deployID, + ); err != nil { + t.Fatalf("age row: %v", err) + } + + // k8s is present but the row has no provider_id, so no namespace is + // derivable — the reaper path runs without any k8s Get. + w := NewDeployStatusReconciler(db, newFakeDeployStatusK8s()) + if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil { + t.Fatalf("Work: %v", err) + } + + gotStatus, gotErr := testhelpers.DeploymentStatus(t, db, deployID) + if gotStatus != deployStatusFailed { + t.Errorf("stuck-building row status = %q, want %q (reaper must free the tier cap)", gotStatus, deployStatusFailed) + } + if !gotErr.Valid || gotErr.String != stuckBuildingReapMessage { + t.Errorf("error_message = %q (valid=%v), want %q", gotErr.String, gotErr.Valid, stuckBuildingReapMessage) + } +} + +// TestIntegration_DeployFailureAutopsy_UpsertAndAuditEmit is the real-DB guard +// for Bug B of the triad. captureDeploymentAutopsy against a live DB must: +// +// 1. UPSERT a deployment_events failure_autopsy row (idempotent on re-run via +// the partial-unique index — a second call must NOT create a duplicate), +// 2. stamp deployments.error_message with ": ", and +// 3. emit an audit_log kind='deploy.failed' row exactly ONCE even across +// repeated ticks (the idempotency guard added in bug bash 2026-06-02 #15). +// +// The ON CONFLICT ... WHERE kind='failure_autopsy' clause + the audit dedup +// probe are SQL that sqlmock cannot validate against the real schema — this is +// where the partial-unique-index gap (the dev-box schema lacked the index) +// would surface as a hard error rather than a green mock. +func TestIntegration_DeployFailureAutopsy_UpsertAndAuditEmit(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + teamID := testhelpers.SeedTeam(t, db, "pro") + deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusFailed, "app-itest3") + + // Autopsy k8s returns OOMKilled pod logs so the captured reason is concrete + // (not Unknown) — proves the k8s→DB plumbing end-to-end. + autopsy := &fakeAutopsyK8sCov{ + logs: []string{"panic: out of memory", "exit status 137"}, + } + + ctx := context.Background() + + // First capture. + captureDeploymentAutopsy(ctx, db, deployID, "app-itest3", autopsy) + + reason, ok := testhelpers.AutopsyRow(t, db, deployID) + if !ok { + t.Fatal("no failure_autopsy row after first capture") + } + if reason == "" { + t.Error("autopsy reason is empty — capture did not populate the row") + } + + // error_message must be stamped (it was NULL on seed). + _, gotErr := testhelpers.DeploymentStatus(t, db, deployID) + if !gotErr.Valid || gotErr.String == "" { + t.Error("deployments.error_message not stamped by autopsy") + } + + // audit_log deploy.failed emitted exactly once. + if n := testhelpers.CountAuditLog(t, db, auditKindDeployFailed, deployID.String()); n != 1 { + t.Errorf("deploy.failed audit rows after first capture = %d, want 1", n) + } + + // Second capture (idempotent re-tick): MUST NOT duplicate either the + // autopsy row or the audit_log row. + captureDeploymentAutopsy(ctx, db, deployID, "app-itest3", autopsy) + + if n := testhelpers.CountAuditLog(t, db, auditKindDeployFailed, deployID.String()); n != 1 { + t.Errorf("deploy.failed audit rows after SECOND capture = %d, want 1 — idempotency guard regressed (duplicate failure emails)", n) + } + + // Still exactly one autopsy row (partial-unique index + ON CONFLICT). + var autopsyRows int + if err := db.QueryRow( + `SELECT count(*) FROM deployment_events WHERE deployment_id = $1 AND kind = 'failure_autopsy'`, + deployID, + ).Scan(&autopsyRows); err != nil { + t.Fatalf("count autopsy rows: %v", err) + } + if autopsyRows != 1 { + t.Errorf("failure_autopsy rows after two captures = %d, want 1 (ON CONFLICT upsert regressed)", autopsyRows) + } +} diff --git a/internal/jobs/entitlement_reconciler_integration_test.go b/internal/jobs/entitlement_reconciler_integration_test.go new file mode 100644 index 0000000..bc2f738 --- /dev/null +++ b/internal/jobs/entitlement_reconciler_integration_test.go @@ -0,0 +1,113 @@ +package jobs + +// entitlement_reconciler_integration_test.go — REAL-Postgres integration test +// for the entitlement reconciler's Postgres connection-cap regrade round-trip. +// +// Per INTEGRATION-COVERAGE-PLAN-2026-06-04.md §5 #1, entitlement_reconciler is +// a Tier-1 (data-adjacent) job whose DB round-trip was fake-tested only. The +// sibling entitlement_reconciler_test.go drives the SELECT/UPDATE through +// go-sqlmock; THIS test seeds a real drifted resources row, runs Work() against +// a live platform Postgres with a stub provisioner regrader, and asserts the +// row's applied_conn_limit was actually persisted by the worker's UPDATE. +// +// The provisioner gRPC leg stays stubbed (stubRegrader returns +// {Applied:true, AppliedConnLimit:5}) — the DB leg (the drift SELECT + the +// UPDATE resources SET applied_conn_limit) is REAL. This is the convergence +// signal the job depends on in production: a sqlmock test passes whether or not +// the UPDATE's WHERE id = $2 matches the live row; this proves it does. +// +// GATING: testhelpers.SetupTestDB skips when no DB is reachable, so the regular +// gate (deploy.yml / ci.yml, no Postgres service) stays green. It runs against a +// real Postgres wherever one is supplied (developer DB, coverage.yml service). + +import ( + "context" + "database/sql" + "testing" + + "instant.dev/worker/internal/testhelpers" +) + +// TestIntegration_EntitlementReconciler_PersistsRegradedConnLimit seeds a +// drifted Postgres resource (applied_conn_limit = NULL, never re-graded) on a +// pro-tier team, runs the reconciler with a stub regrader that reports +// {Applied:true, AppliedConnLimit:5}, and asserts the worker persisted that +// value to the REAL row. +// +// Drift detection: shouldRegrade returns drift=true for a NULL applied limit on +// any non-ephemeral tier. The stub stands in for the provisioner's +// RegradeResource; the worker's `UPDATE resources SET applied_conn_limit = $1 +// WHERE id = $2` is the integration assertion target. +func TestIntegration_EntitlementReconciler_PersistsRegradedConnLimit(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + teamID := testhelpers.SeedTeam(t, db, "pro") + // resource.tier = pro (the per-row snapshot the reconciler resolves caps + // from), applied_conn_limit = NULL → drifts. + resID, _ := testhelpers.SeedResource(t, db, teamID, "postgres", "pro", sql.NullInt64{}) + + // Scope the sweep to ONLY this team so it cannot touch any other rows that + // may exist in a shared local DB (and so the assertion is deterministic). + t.Setenv("ENTITLEMENT_RECONCILE_TEAM", teamID.String()) + + stub := &stubRegrader{} // returns Applied:true, AppliedConnLimit:5 + reg := liveRegistry(t) + w := NewEntitlementReconcilerWorker(db, reg, stub) + + if err := w.Work(context.Background(), fakeEntitlementJob()); err != nil { + t.Fatalf("Work: %v", err) + } + + // The stub must have been asked to regrade our drifted row at least once. + if got := int(stub.calls.Load()); got < 1 { + t.Fatalf("RegradeResource called %d times, want >= 1 (drifted row should have been regraded)", got) + } + + // The REAL row's applied_conn_limit must now equal the stub's reported value. + got := testhelpers.AppliedConnLimit(t, db, resID) + if !got.Valid { + t.Fatal("applied_conn_limit is still NULL — the worker's UPDATE did not persist against the live row") + } + if got.Int64 != 5 { + t.Errorf("applied_conn_limit = %d, want 5 (the value the stub regrader reported)", got.Int64) + } +} + +// TestIntegration_EntitlementReconciler_NoDriftLeavesRowUntouched is the +// complement: a resource whose applied_conn_limit already equals the entitled +// cap for its tier must NOT be re-graded (no drift) and the row is left +// untouched. This pins that the live drift SELECT + shouldRegrade decision +// correctly identify the no-op case against real data — a regression that +// always-regrades would burn provisioner RPCs every 5-minute tick. +func TestIntegration_EntitlementReconciler_NoDriftLeavesRowUntouched(t *testing.T) { + db, cleanup := testhelpers.SetupTestDB(t) + defer cleanup() + + reg := liveRegistry(t) + entitled := reg.ConnectionsLimit("pro", "postgres") + + teamID := testhelpers.SeedTeam(t, db, "pro") + // applied_conn_limit already == entitled → no drift. + resID, _ := testhelpers.SeedResource(t, db, teamID, "postgres", "pro", + sql.NullInt64{Int64: int64(entitled), Valid: true}) + + t.Setenv("ENTITLEMENT_RECONCILE_TEAM", teamID.String()) + + stub := &stubRegrader{} + w := NewEntitlementReconcilerWorker(db, reg, stub) + + if err := w.Work(context.Background(), fakeEntitlementJob()); err != nil { + t.Fatalf("Work: %v", err) + } + + if got := int(stub.calls.Load()); got != 0 { + t.Errorf("RegradeResource called %d times, want 0 — a row at the entitled cap must NOT drift", got) + } + + // Row value unchanged. + got := testhelpers.AppliedConnLimit(t, db, resID) + if !got.Valid || got.Int64 != int64(entitled) { + t.Errorf("applied_conn_limit = %v, want %d (untouched)", got, entitled) + } +} diff --git a/internal/testhelpers/testhelpers.go b/internal/testhelpers/testhelpers.go new file mode 100644 index 0000000..6e11eaf --- /dev/null +++ b/internal/testhelpers/testhelpers.go @@ -0,0 +1,439 @@ +// Package testhelpers provides a real-Postgres integration harness for the +// worker's periodic-job tests. +// +// # WHY THIS EXISTS +// +// Before this package, only 3 of the worker's ~59 job files exercised a real +// database — the rest drove their logic through sqlmock / fake River / fake +// k8s clients. That is *unit* coverage of job logic, not *integration* +// coverage of the trigger→DB-effect round-trip that the job actually performs +// in production. The integration-coverage plan (INTEGRATION-COVERAGE-PLAN- +// 2026-06-04.md §5 #1) flagged the worker as the single biggest integration +// gap on the platform and called for a harness mirroring api's +// testhelpers.SetupTestDB. +// +// # DESIGN +// +// The worker module does NOT import the api module (the platform-DB schema is +// owned by api/internal/db/migrations). So — like the job files themselves +// (deploy_status_reconcile.go, deploy_failure_autopsy.go) which duplicate the +// schema strings rather than importing api — this harness ensures the *subset* +// of the platform schema the integration-tested jobs touch, idempotently, via +// CREATE TABLE / CREATE INDEX IF NOT EXISTS. It is deliberately a subset (not +// the full 66-migration mirror api maintains): only the tables the worker jobs +// under test round-trip against. New jobs that touch new tables extend +// ensureSchema here. +// +// # GATING +// +// SetupTestDB calls t.Skip (NOT t.Fatal) when the DB is unreachable or +// TEST_DATABASE_URL is unset, AND when running under `-short`. This matches the +// worker's two CI workflows: deploy.yml runs `go test ./... -short` (no DB +// service) and ci.yml runs `go test ./... -race` (also no DB service). In both, +// these integration tests SKIP cleanly. They run only where a real Postgres is +// supplied via TEST_DATABASE_URL (developer machine / a future CI DB service). +// This mirrors the existing propagation_runner_integration_test.go gating. +package testhelpers + +import ( + "context" + "database/sql" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/lib/pq" +) + +// tFatalf / tSkipf are indirection seams over (*testing.T).Fatalf / .Skipf. +// They exist solely so the harness's own error/skip arms (a DB that fails to +// open, an INSERT that errors, a scan that fails) are reachable from this +// package's in-package coverage tests — which swap them for recording stubs and +// drive the arms with a deliberately-broken DB. In every real test run they are +// the genuine t.Fatalf / t.Skipf. This is a test seam (per the platform's +// "use test seams, not waivers" coverage rule), NOT a behavioural change: +// production callers see identical fail/skip semantics. The default values are +// reassigned only inside testhelpers_smoke_test.go and restored via t.Cleanup. +var ( + tFatalf = func(t *testing.T, format string, args ...any) { t.Helper(); t.Fatalf(format, args...) } + tSkipf = func(t *testing.T, format string, args ...any) { t.Helper(); t.Skipf(format, args...) } +) + +// isUndefinedColumn reports whether err is a Postgres "column does not exist" +// error (SQLSTATE 42703). Used so SeedDeployment can target the richer prod +// schema (NOT NULL app_id) and fall back to the bare-harness schema when the +// column is absent — without importing the pq error type at the call site. +func isUndefinedColumn(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "42703") || + (strings.Contains(msg, "column") && strings.Contains(msg, "does not exist")) +} + +// DefaultTestDBURL is the local platform-DB DSN used when TEST_DATABASE_URL is +// unset. Matches the integration-coverage plan §1.4 (the api harness uses the +// same host/db). The worker's local dev DB and the test DB share the +// instant_dev_test database. +const DefaultTestDBURL = "postgres://postgres@localhost:5432/instant_dev_test?sslmode=disable" + +// SetupTestDB opens a connection to the platform test database, ensures the +// schema subset the worker integration tests need, and returns the *sql.DB +// plus a cleanup function. +// +// It SKIPS (does not fail) the test when TEST_DATABASE_URL is unset AND the +// default local DB is unreachable. This keeps `make gate` / deploy.yml / ci.yml +// green without a DB (those workflows ship no Postgres service container, so the +// ping below misses and the test skips) while still running the real round-trip +// — and crediting this package's own coverage — wherever a Postgres is provided +// (developer machine, coverage.yml's postgres service). +// +// NOTE: this deliberately does NOT short-circuit on `testing.Short()`. The +// coverage.yml job runs `go test ./... -short` against a real Postgres service; +// a `-short` guard here would skip the harness in that job and leave every line +// of this file uncovered, reding the 100%-patch-coverage gate. Gating purely on +// DB reachability matches api/internal/testhelpers.SetupTestDB and keeps the +// `-short`, no-DB workflows green via the ping skip below. +func SetupTestDB(t *testing.T) (*sql.DB, func()) { + t.Helper() + + dsn := os.Getenv("TEST_DATABASE_URL") + if dsn == "" { + dsn = DefaultTestDBURL + } + + // Build the connector explicitly via pq.NewConnector rather than sql.Open: + // sql.Open only validates the (always-"postgres") driver string and never + // returns an error here, so its error arm would be an untestable dead branch + // under the patch-coverage gate. pq.NewConnector parses the DSN eagerly and + // DOES return an error for a malformed DSN — a reachable, tested skip arm. + connector, err := pq.NewConnector(dsn) + if err != nil { + tSkipf(t, "testhelpers.SetupTestDB: parse DSN %q: %v — set TEST_DATABASE_URL to a valid platform DB", dsn, err) + return nil, func() {} + } + db := sql.OpenDB(connector) + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(5) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + tSkipf(t, "testhelpers.SetupTestDB: ping %q failed: %v — DB not reachable (set TEST_DATABASE_URL or start postgres)", dsn, err) + return nil, func() {} + } + + ensureSchema(t, db) + + return db, func() { _ = db.Close() } +} + +// ensureSchema applies the subset of the platform schema the worker +// integration tests round-trip against. Every statement is idempotent +// (IF NOT EXISTS / ADD COLUMN IF NOT EXISTS) so it is safe against a DB that +// already has the full api migration set applied (the modal local case) AND +// against a bare DB. +// +// The one thing a bare api-migrated DB can lack (it was a real gap on the dev +// box that authored this) is the deployment_events autopsy partial-unique +// index that deploy_failure_autopsy.go's ON CONFLICT clause depends on — so we +// (re)create it here explicitly. +func ensureSchema(t *testing.T, db *sql.DB) { + t.Helper() + + stmts := []string{ + `CREATE EXTENSION IF NOT EXISTS pgcrypto`, + + // teams — minimal shape the deploy + entitlement jobs join against. + `CREATE TABLE IF NOT EXISTS teams ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT, + plan_tier TEXT NOT NULL DEFAULT 'hobby', + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`, + `ALTER TABLE teams ADD COLUMN IF NOT EXISTS plan_tier TEXT NOT NULL DEFAULT 'hobby'`, + `ALTER TABLE teams ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active'`, + + // resources — the entitlement reconciler reads tier / applied_conn_limit. + `CREATE TABLE IF NOT EXISTS resources ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + team_id UUID REFERENCES teams(id) ON DELETE SET NULL, + token UUID UNIQUE NOT NULL DEFAULT gen_random_uuid(), + resource_type TEXT NOT NULL, + tier TEXT NOT NULL DEFAULT 'anonymous', + status TEXT NOT NULL DEFAULT 'active', + provider_resource_id TEXT, + applied_conn_limit INT, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`, + `ALTER TABLE resources ADD COLUMN IF NOT EXISTS applied_conn_limit INT`, + `ALTER TABLE resources ADD COLUMN IF NOT EXISTS provider_resource_id TEXT`, + + // deployments — the status reconciler + failure autopsy round-trip here. + `CREATE TABLE IF NOT EXISTS deployments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + team_id UUID REFERENCES teams(id) ON DELETE SET NULL, + provider_id TEXT, + status TEXT NOT NULL DEFAULT 'building', + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`, + `ALTER TABLE deployments ADD COLUMN IF NOT EXISTS provider_id TEXT`, + `ALTER TABLE deployments ADD COLUMN IF NOT EXISTS error_message TEXT`, + `ALTER TABLE deployments ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()`, + `CREATE INDEX IF NOT EXISTS idx_deployments_status ON deployments(status)`, + + // deployment_events — the autopsy upsert target. The partial-unique + // index is what ON CONFLICT (deployment_id, kind) WHERE + // kind = 'failure_autopsy' resolves against; without it the upsert + // errors "no unique or exclusion constraint matching the ON CONFLICT". + `CREATE TABLE IF NOT EXISTS deployment_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + deployment_id UUID NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, + kind TEXT NOT NULL, + reason TEXT NOT NULL, + exit_code INT, + event TEXT NOT NULL DEFAULT '', + last_lines JSONB NOT NULL DEFAULT '[]'::jsonb, + hint TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`, + `CREATE INDEX IF NOT EXISTS deployment_events_deployment_id_idx + ON deployment_events (deployment_id, created_at DESC)`, + + // audit_log — the failure-autopsy backstop emits deploy.failed here so + // the email forwarder dispatches the failure email. + `CREATE TABLE IF NOT EXISTS audit_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + team_id UUID REFERENCES teams(id) ON DELETE CASCADE, + user_id UUID, + actor TEXT NOT NULL DEFAULT 'agent', + kind TEXT NOT NULL, + resource_type TEXT, + resource_id UUID, + summary TEXT NOT NULL, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`, + `ALTER TABLE audit_log ALTER COLUMN team_id DROP NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_audit_team_at ON audit_log (team_id, created_at DESC)`, + } + + for _, s := range stmts { + if _, err := db.Exec(s); err != nil { + tFatalf(t, "testhelpers.ensureSchema: %v\n SQL: %.140s", err, s) + return + } + } + + ensureAutopsyUniqueIndex(t, db) +} + +// ensureAutopsyUniqueIndex guarantees the public.deployment_events table +// carries a partial-unique index on (deployment_id, kind) WHERE +// kind = 'failure_autopsy' — the constraint deploy_failure_autopsy.go's +// ON CONFLICT clause resolves against. A plain +// `CREATE UNIQUE INDEX IF NOT EXISTS ` is NOT sufficient here: index +// names are schema-scoped, so if the canonical name is already taken by an +// index on a *different* table (observed on a dev box that had a stray +// deployment_events_hidden table owning deployment_events_autopsy_uniq), the +// IF NOT EXISTS turns into a silent no-op and the real table never gets the +// constraint. So we check for a matching index on the actual table first and +// only create one (under a collision-proof name) when none exists. +func ensureAutopsyUniqueIndex(t *testing.T, db *sql.DB) { + t.Helper() + var present bool + if err := db.QueryRow(` + SELECT EXISTS ( + SELECT 1 + FROM pg_index i + JOIN pg_class idx ON idx.oid = i.indexrelid + JOIN pg_class tbl ON tbl.oid = i.indrelid + JOIN pg_namespace n ON n.oid = tbl.relnamespace + WHERE tbl.relname = 'deployment_events' + AND n.nspname = 'public' + AND i.indisunique + AND pg_get_indexdef(i.indexrelid) ILIKE '%failure_autopsy%' + ) + `).Scan(&present); err != nil { + tFatalf(t, "ensureAutopsyUniqueIndex: probe: %v", err) + return + } + if present { + return + } + // Use a harness-specific name that cannot collide with the canonical + // production index name on any other table. + if _, err := db.Exec(` + CREATE UNIQUE INDEX IF NOT EXISTS deployment_events_autopsy_uniq_itest + ON public.deployment_events (deployment_id, kind) + WHERE kind = 'failure_autopsy' + `); err != nil { + tFatalf(t, "ensureAutopsyUniqueIndex: create: %v", err) + return + } +} + +// SeedTeam inserts a team row (plan_tier defaults to "hobby") and returns its +// id. The row is removed by the returned test via t.Cleanup so a re-run of the +// same test against a long-lived local DB does not accumulate rows; the +// ON DELETE CASCADE / SET NULL on dependent tables tidies children. +func SeedTeam(t *testing.T, db *sql.DB, planTier string) uuid.UUID { + t.Helper() + if planTier == "" { + planTier = "hobby" + } + id := uuid.New() + if _, err := db.Exec( + `INSERT INTO teams (id, name, plan_tier, status) VALUES ($1, $2, $3, 'active')`, + id, "itest-"+id.String()[:8], planTier, + ); err != nil { + tFatalf(t, "SeedTeam: %v", err) + return uuid.Nil + } + t.Cleanup(func() { + _, _ = db.Exec(`DELETE FROM teams WHERE id = $1`, id) + }) + return id +} + +// SeedDeployment inserts a deployments row with the given status / provider_id +// and returns its id. providerID may be "" to model the stuck-building case +// (api goroutine died before stamping provider_id). The row (and any +// deployment_events / audit_log children via FK) is cleaned up after the test. +// +// app_id is populated with a unique value: the production deployments table +// (full api migration applied locally) carries a NOT NULL app_id column with no +// default; the bare harness CREATE TABLE above omits it, so SeedDeployment must +// supply it to satisfy both schema shapes. +func SeedDeployment(t *testing.T, db *sql.DB, teamID uuid.UUID, status, providerID string) uuid.UUID { + t.Helper() + id := uuid.New() + var providerArg interface{} + if providerID != "" { + providerArg = providerID + } + // app_id is NOT NULL (no default) in the prod schema; the bare-harness + // table lacks the column. Try the prod shape first, fall back to the bare + // shape so the harness works against either schema. + _, err := db.Exec( + `INSERT INTO deployments (id, team_id, app_id, provider_id, status, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, now(), now())`, + id, teamID, id.String(), providerArg, status, + ) + if err != nil && isUndefinedColumn(err) { + _, err = db.Exec( + `INSERT INTO deployments (id, team_id, provider_id, status, created_at, updated_at) + VALUES ($1, $2, $3, $4, now(), now())`, + id, teamID, providerArg, status, + ) + } + if err != nil { + tFatalf(t, "SeedDeployment: %v", err) + return uuid.Nil + } + t.Cleanup(func() { + _, _ = db.Exec(`DELETE FROM deployments WHERE id = $1`, id) + }) + return id +} + +// SeedResource inserts a resources row (used by the entitlement reconciler +// round-trip) and returns its id + token. appliedConnLimit may be an invalid +// sql.NullInt64 to model the never-re-graded (NULL) case. The row is cleaned +// up after the test. +func SeedResource( + t *testing.T, + db *sql.DB, + teamID uuid.UUID, + resourceType, tier string, + appliedConnLimit sql.NullInt64, +) (uuid.UUID, string) { + t.Helper() + id := uuid.New() + token := uuid.New() + var limitArg interface{} + if appliedConnLimit.Valid { + limitArg = appliedConnLimit.Int64 + } + if _, err := db.Exec( + `INSERT INTO resources + (id, team_id, token, resource_type, tier, status, applied_conn_limit, created_at) + VALUES ($1, $2, $3, $4, $5, 'active', $6, now())`, + id, teamID, token, resourceType, tier, limitArg, + ); err != nil { + tFatalf(t, "SeedResource: %v", err) + return uuid.Nil, "" + } + t.Cleanup(func() { + _, _ = db.Exec(`DELETE FROM resources WHERE id = $1`, id) + }) + return id, token.String() +} + +// DeploymentStatus reads back the status + error_message of a deployments row. +func DeploymentStatus(t *testing.T, db *sql.DB, id uuid.UUID) (status string, errorMessage sql.NullString) { + t.Helper() + if err := db.QueryRow( + `SELECT status, error_message FROM deployments WHERE id = $1`, id, + ).Scan(&status, &errorMessage); err != nil { + tFatalf(t, "DeploymentStatus: %v", err) + return "", sql.NullString{} + } + return status, errorMessage +} + +// AppliedConnLimit reads back resources.applied_conn_limit for a row. +func AppliedConnLimit(t *testing.T, db *sql.DB, id uuid.UUID) sql.NullInt64 { + t.Helper() + var v sql.NullInt64 + if err := db.QueryRow( + `SELECT applied_conn_limit FROM resources WHERE id = $1`, id, + ).Scan(&v); err != nil { + tFatalf(t, "AppliedConnLimit: %v", err) + return sql.NullInt64{} + } + return v +} + +// CountAuditLog returns how many audit_log rows of the given kind reference the +// given deployment id in metadata->>'deploy_id'. Used to assert the +// failure-autopsy deploy.failed emit (and its idempotency). +func CountAuditLog(t *testing.T, db *sql.DB, kind, deployID string) int { + t.Helper() + var n int + if err := db.QueryRow( + `SELECT count(*) FROM audit_log WHERE kind = $1 AND metadata->>'deploy_id' = $2`, + kind, deployID, + ).Scan(&n); err != nil { + tFatalf(t, "CountAuditLog: %v", err) + return 0 + } + return n +} + +// AutopsyRow reads back the failure_autopsy deployment_events row for a +// deployment. Returns ok=false when no such row exists. +func AutopsyRow(t *testing.T, db *sql.DB, deploymentID uuid.UUID) (reason string, ok bool) { + t.Helper() + err := db.QueryRow( + `SELECT reason FROM deployment_events + WHERE deployment_id = $1 AND kind = 'failure_autopsy'`, + deploymentID, + ).Scan(&reason) + if err == sql.ErrNoRows { + return "", false + } + if err != nil { + tFatalf(t, "AutopsyRow: %v", err) + return "", false + } + return reason, true +} diff --git a/internal/testhelpers/testhelpers_smoke_test.go b/internal/testhelpers/testhelpers_smoke_test.go new file mode 100644 index 0000000..f2dd83a --- /dev/null +++ b/internal/testhelpers/testhelpers_smoke_test.go @@ -0,0 +1,385 @@ +package testhelpers + +// testhelpers_smoke_test.go — in-package coverage for the worker integration +// harness itself. +// +// WHY THIS EXISTS +// --------------- +// The Seed*/read helpers + SetupTestDB live in this (non-_test.go) file so the +// jobs-package integration tests can import them. Go's per-package coverage +// attribution only credits the `testhelpers` package when a test in THIS +// package runs — the jobs-package integration tests that call these helpers +// credit `internal/jobs`, not `internal/testhelpers`. Without an in-package +// test, every line of testhelpers.go reads as 0% in diff-cover and reds the +// 100%-patch-coverage gate (the exact failure on PR #87). This mirrors +// api/internal/testhelpers/testapp_smoke_test.go, the established platform +// convention for giving a test-harness package its own coverage. +// +// The harness's own fail/skip arms (DB fails to open/ping, an INSERT errors, a +// scan fails) are exercised by swapping the package's tFatalf/tSkipf seams for +// recording stubs and driving the arm with a deliberately-closed DB. This is a +// test seam (the platform's "use test seams, not waivers" coverage rule), not a +// behavioural change — real callers get genuine t.Fatalf / t.Skipf. +// +// GATING: the DB-backed tests route through SetupTestDB, which skips when no +// Postgres is reachable — so they skip cleanly on the no-DB workflows +// (deploy.yml `-short`, ci.yml `-race`) and run against coverage.yml's postgres +// service + any developer DB. isUndefinedColumn is pure logic and is +// unit-tested unconditionally below. + +import ( + "database/sql" + "errors" + "testing" + + "github.com/google/uuid" +) + +// swapFatalSeams replaces tFatalf/tSkipf with stubs that record the message and +// abort the *current goroutine path* via panic-free early return semantics. The +// helpers under test all `return` immediately after calling the seam, so a +// recording stub that does nothing lets the helper return its zero value while +// the test asserts the arm fired. Restored via t.Cleanup. +func swapFatalSeams(t *testing.T) (fatal, skip *[]string) { + t.Helper() + var fatals, skips []string + origF, origS := tFatalf, tSkipf + tFatalf = func(_ *testing.T, format string, args ...any) { fatals = append(fatals, format) } + tSkipf = func(_ *testing.T, format string, args ...any) { skips = append(skips, format) } + t.Cleanup(func() { tFatalf, tSkipf = origF, origS }) + return &fatals, &skips +} + +// TestSeamDefaults covers the default tFatalf/tSkipf closures (the real +// (*testing.T).Fatalf / .Skipf forwarders). Both abort via runtime.Goexit, so +// each is invoked on a throwaway *testing.T inside its own goroutine: the Goexit +// terminates only that goroutine, never the parent test, and a sentinel set +// AFTER the call proves the call returned only via the seam (Goexit), i.e. the +// forwarder body ran. +func TestSeamDefaults(t *testing.T) { + run := func(name string, call func(*testing.T)) { + reached := false + done := make(chan struct{}) + // A fresh, isolated *testing.T whose pass/fail is intentionally + // discarded (we never call t.Run on it) — we only need a valid receiver + // for the forwarder. The goroutine ends at the seam's runtime.Goexit. + go func() { + defer close(done) + st := &testing.T{} + call(st) + reached = true // unreachable when call() Goexits, as it must. + }() + <-done + if reached { + t.Fatalf("%s: default seam did not abort the goroutine (forwarder body not exercised)", name) + } + } + run("fatal", func(st *testing.T) { tFatalf(st, "default fatal forwarder: %s", "ok") }) + run("skip", func(st *testing.T) { tSkipf(st, "default skip forwarder: %s", "ok") }) +} + +// TestIsUndefinedColumn exercises every branch of the SQLSTATE-42703 detector +// used by SeedDeployment's prod-vs-bare schema fallback. Pure logic, no DB — +// runs on every workflow (unit test, not an Integration test). +func TestIsUndefinedColumn(t *testing.T) { + t.Parallel() + cases := []struct { + name string + err error + want bool + }{ + {"nil error", nil, false}, + {"sqlstate 42703", errors.New(`pq: column "app_id" of relation "deployments" (SQLSTATE 42703)`), true}, + {"column does not exist phrasing", errors.New(`ERROR: column "app_id" does not exist`), true}, + {"column word only, no does-not-exist", errors.New(`ERROR: column "x" is ambiguous`), false}, + {"does-not-exist without column word", errors.New(`relation "foo" does not exist`), false}, + {"unrelated error", errors.New("connection refused"), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isUndefinedColumn(tc.err); got != tc.want { + t.Fatalf("isUndefinedColumn(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +// TestIntegration_SetupTestDB_DefaultDSNFallback covers the +// `TEST_DATABASE_URL unset -> DefaultTestDBURL` fallback branch. CI always +// exports TEST_DATABASE_URL, so only an explicit unset exercises it. The +// default DSN points at the same local/CI Postgres (localhost:5432), so when a +// DB is reachable SetupTestDB succeeds; otherwise it skips via the ping arm — +// either way the fallback assignment line runs. +func TestIntegration_SetupTestDB_DefaultDSNFallback(t *testing.T) { + t.Setenv("TEST_DATABASE_URL", "") // empty -> SetupTestDB falls back to DefaultTestDBURL + db, cleanup := SetupTestDB(t) // skips here if the default DSN is unreachable + defer cleanup() + if db == nil { + t.Fatal("SetupTestDB returned nil db after default-DSN fallback") + } + if err := db.Ping(); err != nil { + t.Fatalf("default-DSN db not usable: %v", err) + } +} + +// TestSetupTestDB_BadDSN covers SetupTestDB's DSN-parse skip arm by pointing +// TEST_DATABASE_URL at a string pq.NewConnector rejects (invalid URL escape). +func TestSetupTestDB_BadDSN(t *testing.T) { + t.Setenv("TEST_DATABASE_URL", "postgres://%zz") + _, skips := swapFatalSeams(t) + db, cleanup := SetupTestDB(t) + if cleanup != nil { + cleanup() + } + if db != nil { + t.Fatal("SetupTestDB returned a non-nil db on DSN parse failure") + } + if len(*skips) == 0 { + t.Fatal("expected a Skipf on DSN parse failure, got none") + } +} + +// TestSetupTestDB_PingError covers the ping-failure skip arm by pointing at a +// valid DSN whose host/port refuses connections. +func TestSetupTestDB_PingError(t *testing.T) { + // Point at a closed port so PingContext fails fast → the ping skip arm. + t.Setenv("TEST_DATABASE_URL", "postgres://postgres@127.0.0.1:1/doesnotexist?sslmode=disable&connect_timeout=1") + _, skips := swapFatalSeams(t) + db, cleanup := SetupTestDB(t) + if cleanup != nil { + cleanup() + } + if db != nil { + t.Fatal("SetupTestDB returned a non-nil db on ping failure") + } + if len(*skips) == 0 { + t.Fatal("expected a Skipf on ping failure, got none") + } +} + +// TestHarnessErrorArms drives every fallible helper against a CLOSED *sql.DB so +// each Exec/QueryRow fails — covering the error arm (via the recording seam) of +// SeedTeam, SeedDeployment, SeedResource, DeploymentStatus, AppliedConnLimit, +// CountAuditLog, AutopsyRow, ensureSchema and ensureAutopsyUniqueIndex. +func TestHarnessErrorArms(t *testing.T) { + closed, err := sql.Open("postgres", "postgres://postgres@127.0.0.1:1/x?sslmode=disable") + if err != nil { + t.Fatalf("open placeholder db: %v", err) + } + _ = closed.Close() // every subsequent Exec/QueryRow now errors. + + fatals, _ := swapFatalSeams(t) + id := uuid.New() + + ensureSchema(t, closed) + ensureAutopsyUniqueIndex(t, closed) + SeedTeam(t, closed, "pro") + SeedDeployment(t, closed, id, "building", "prov") + SeedResource(t, closed, id, "postgres", "pro", sql.NullInt64{Int64: 5, Valid: true}) + DeploymentStatus(t, closed, id) + AppliedConnLimit(t, closed, id) + CountAuditLog(t, closed, "deploy.failed", id.String()) + AutopsyRow(t, closed, id) + + // 9 distinct fallible call paths each recorded at least one Fatalf. + if len(*fatals) < 9 { + t.Fatalf("expected >=9 recorded Fatalf arms against the closed DB, got %d: %v", len(*fatals), *fatals) + } +} + +// TestIntegration_HarnessRoundTrip drives every DB-backed helper against a real +// Postgres so the harness's happy paths + both reachable conditional branches +// (the autopsy-index create branch and the bare-schema SeedDeployment fallback) +// carry real line coverage. +func TestIntegration_HarnessRoundTrip(t *testing.T) { + db, cleanup := SetupTestDB(t) + defer cleanup() + + // ensureSchema is idempotent — re-run drives the IF-NOT-EXISTS no-op path + // and the already-present arm of ensureAutopsyUniqueIndex. + ensureSchema(t, db) + + // --- create-index branch of ensureAutopsyUniqueIndex (250-256) --------- + // Drop any failure_autopsy unique index so the "not present" arm runs. + dropAutopsyIndexes(t, db) + ensureAutopsyUniqueIndex(t, db) + if !autopsyIndexPresent(t, db) { + t.Fatal("ensureAutopsyUniqueIndex did not create the autopsy index when absent") + } + + // SeedTeam: explicit tier + the "" -> default-hobby branch. + teamPro := SeedTeam(t, db, "pro") + teamDefault := SeedTeam(t, db, "") + if teamPro == uuid.Nil || teamDefault == uuid.Nil { + t.Fatal("SeedTeam returned nil uuid") + } + + // SeedDeployment with a provider_id, then read it back. + depID := SeedDeployment(t, db, teamPro, "building", "prov-123") + if status, _ := DeploymentStatus(t, db, depID); status != "building" { + t.Fatalf("DeploymentStatus = %q, want building", status) + } + + // SeedDeployment with an empty provider_id (the stuck-building model). + stuckID := SeedDeployment(t, db, teamPro, "building", "") + if status, _ := DeploymentStatus(t, db, stuckID); status != "building" { + t.Fatalf("stuck DeploymentStatus = %q, want building", status) + } + + // SeedResource: a valid applied_conn_limit and a NULL one. + resGraded, _ := SeedResource(t, db, teamPro, "postgres", "pro", + sql.NullInt64{Int64: 20, Valid: true}) + resUngraded, tok := SeedResource(t, db, teamPro, "redis", "pro", sql.NullInt64{}) + if tok == "" { + t.Fatal("SeedResource returned empty token") + } + if v := AppliedConnLimit(t, db, resGraded); !v.Valid || v.Int64 != 20 { + t.Fatalf("AppliedConnLimit(graded) = %+v, want {20,true}", v) + } + if v := AppliedConnLimit(t, db, resUngraded); v.Valid { + t.Fatalf("AppliedConnLimit(ungraded) = %+v, want NULL", v) + } + + // AutopsyRow not-found arm before any autopsy row exists. + if _, ok := AutopsyRow(t, db, depID); ok { + t.Fatal("AutopsyRow returned ok=true before any autopsy row was written") + } + + // Write a failure_autopsy deployment_events row + a deploy.failed audit_log + // row, then read both back (CountAuditLog + AutopsyRow found arm). + if _, err := db.Exec( + `INSERT INTO deployment_events (deployment_id, kind, reason) + VALUES ($1, 'failure_autopsy', 'BackoffLimitExceeded')`, depID, + ); err != nil { + t.Fatalf("insert autopsy row: %v", err) + } + if _, err := db.Exec( + `INSERT INTO audit_log (team_id, actor, kind, summary, metadata) + VALUES ($1, 'worker', 'deploy.failed', 'deploy failed', + jsonb_build_object('deploy_id', $2::text))`, + teamPro, depID.String(), + ); err != nil { + t.Fatalf("insert audit_log row: %v", err) + } + + if reason, ok := AutopsyRow(t, db, depID); !ok || reason != "BackoffLimitExceeded" { + t.Fatalf("AutopsyRow = (%q, %v), want (BackoffLimitExceeded, true)", reason, ok) + } + if n := CountAuditLog(t, db, "deploy.failed", depID.String()); n != 1 { + t.Fatalf("CountAuditLog = %d, want 1", n) + } + + // --- create-error arm of ensureAutopsyUniqueIndex (272) ---------------- + // Drop the autopsy index, then insert two failure_autopsy rows for one + // deployment so the CREATE UNIQUE INDEX fails on the duplicate. The probe + // returns not-present, so the create branch runs and its error arm fires. + dep2 := SeedDeployment(t, db, teamPro, "building", "p2") + dropAutopsyIndexes(t, db) + for i := 0; i < 2; i++ { + if _, err := db.Exec( + `INSERT INTO deployment_events (deployment_id, kind, reason) + VALUES ($1, 'failure_autopsy', 'dup')`, dep2, + ); err != nil { + t.Fatalf("seed duplicate autopsy row: %v", err) + } + } + func() { + fatals, _ := swapFatalSeams(t) + ensureAutopsyUniqueIndex(t, db) + if len(*fatals) == 0 { + t.Fatal("expected ensureAutopsyUniqueIndex create arm to fail on duplicate rows") + } + }() + // Clean up the duplicate rows so the index can be recreated for other tests. + if _, err := db.Exec(`DELETE FROM deployment_events WHERE deployment_id = $1`, dep2); err != nil { + t.Fatalf("cleanup duplicate autopsy rows: %v", err) + } + ensureAutopsyUniqueIndex(t, db) // restore the index + + // --- bare-schema fallback of SeedDeployment (the isUndefinedColumn arm) - + // Temporarily drop the prod app_id column so the first INSERT 42703s and the + // bare-schema fallback INSERT runs. Restored via cleanup. + withoutAppIDColumn(t, db, func() { + bareID := SeedDeployment(t, db, teamPro, "building", "prov-bare") + if bareID == uuid.Nil { + t.Fatal("SeedDeployment bare-schema fallback returned nil id") + } + if status, _ := DeploymentStatus(t, db, bareID); status != "building" { + t.Fatalf("bare-schema DeploymentStatus = %q, want building", status) + } + }) +} + +// dropAutopsyIndexes drops every unique index on deployment_events whose +// definition mentions failure_autopsy, so the create-index arm of +// ensureAutopsyUniqueIndex runs on the next call. +func dropAutopsyIndexes(t *testing.T, db *sql.DB) { + t.Helper() + rows, err := db.Query(` + SELECT idx.relname + FROM pg_index i + JOIN pg_class idx ON idx.oid = i.indexrelid + JOIN pg_class tbl ON tbl.oid = i.indrelid + WHERE tbl.relname = 'deployment_events' + AND i.indisunique + AND pg_get_indexdef(i.indexrelid) ILIKE '%failure_autopsy%'`) + if err != nil { + t.Fatalf("dropAutopsyIndexes query: %v", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan index name: %v", err) + } + names = append(names, n) + } + for _, n := range names { + if _, err := db.Exec(`DROP INDEX IF EXISTS ` + n); err != nil { + t.Fatalf("drop index %s: %v", n, err) + } + } +} + +func autopsyIndexPresent(t *testing.T, db *sql.DB) bool { + t.Helper() + var present bool + if err := db.QueryRow(` + SELECT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class idx ON idx.oid = i.indexrelid + JOIN pg_class tbl ON tbl.oid = i.indrelid + WHERE tbl.relname = 'deployment_events' + AND i.indisunique + AND pg_get_indexdef(i.indexrelid) ILIKE '%failure_autopsy%')`).Scan(&present); err != nil { + t.Fatalf("autopsyIndexPresent: %v", err) + } + return present +} + +// withoutAppIDColumn drops deployments.app_id for the duration of fn, then +// restores it (nullable — the harness only needs the column to exist). If the +// column was already absent (bare harness DB), fn runs unchanged. +func withoutAppIDColumn(t *testing.T, db *sql.DB, fn func()) { + t.Helper() + var had bool + if err := db.QueryRow(` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'deployments' AND column_name = 'app_id')`).Scan(&had); err != nil { + t.Fatalf("probe app_id column: %v", err) + } + if had { + if _, err := db.Exec(`ALTER TABLE deployments DROP COLUMN app_id`); err != nil { + t.Fatalf("drop app_id: %v", err) + } + defer func() { + if _, err := db.Exec(`ALTER TABLE deployments ADD COLUMN IF NOT EXISTS app_id TEXT`); err != nil { + t.Fatalf("restore app_id: %v", err) + } + }() + } + fn() +}