test(worker): real-DB integration harness + Tier-1 job coverage (toward 100% integration)

internal/jobs/deploy_reconcile_integration_test.go

@@ -0,0 +1,213 @@
+package jobs
+
+// deploy_reconcile_integration_test.go — REAL-Postgres integration tests for
+// the silent-deploy-failure triad (2026-05-30 incident class). These are the
+// highest-value worker integration tests per INTEGRATION-COVERAGE-PLAN-
+// 2026-06-04.md §5 #2: deploy_status_reconcile + deploy_failure_autopsy were
+// fake-tested only, which is exactly the regression class that hides — a sqlmock
+// expectation passes whether or not the real SQL is valid against the live
+// schema.
+//
+// Unlike the sibling *_test.go files in this package (which drive the SQL with
+// go-sqlmock), these tests seed real rows in a live platform Postgres, run the
+// job's Work()/captureDeploymentAutopsy against that DB, and assert the actual
+// row state transition + side effects (status flip, deployment_events upsert,
+// audit_log deploy.failed emit + its idempotency, error_message stamp).
+//
+// The k8s leg stays faked (no live cluster) — the fakeDeployStatusK8s /
+// fakeAutopsyK8sCov helpers from deploy_lifecycle_coverage_test.go +
+// deploy_status_reconcile_job_failed_test.go supply the cluster state. The DB
+// leg is REAL. That split is the design the plan prescribes (§3 wave 3): "the
+// deploy/k8s jobs use a fake clientset for the k8s leg but a real DB for the
+// row mutation."
+//
+// GATING: testhelpers.SetupTestDB skips when no DB is reachable, so `make gate`
+// (deploy.yml) and ci.yml (no DB service) skip these cleanly. They run locally
+// against postgres://postgres@localhost:5432/instant_dev_test and wherever a
+// TEST_DATABASE_URL is supplied (developer DB, coverage.yml's postgres service).
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"instant.dev/worker/internal/testhelpers"
+)
+
+// TestIntegration_DeployStatusReconcile_JobFailedFlipsToFailed is the PRIMARY
+// real-DB guard for the silent-deploy-failure bug class (Bug A of the
+// 2026-05-30 triad). Setup mirrors the user's incident:
+//
+//   - a deployments row at status='building' (api goroutine crashed mid-build
+//     or never stamped the terminal status)
+//   - the runtime Deployment was never created → GetDeployment returns NotFound
+//   - the kaniko build Job is Failed (BackoffLimitExceeded) but survives within
+//     its TTLSecondsAfterFinished window
+//
+// The fixed reconciler MUST: (1) flip the REAL row to 'failed', and (2) write a
+// REAL deployment_events failure_autopsy row (the in-sweep capture). The
+// sqlmock sibling pins the SQL string; THIS pins that the SQL is valid against
+// the live schema and the row actually transitions — a sqlmock expectation
+// would pass even if the UPDATE's WHERE clause silently matched zero rows.
+func TestIntegration_DeployStatusReconcile_JobFailedFlipsToFailed(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	teamID := testhelpers.SeedTeam(t, db, "pro")
+	// provider_id "app-itest1" → namespace "instant-deploy-itest1".
+	deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "app-itest1")
+
+	k8s := newFakeDeployStatusK8s()
+	// Runtime Deployment missing (build never reached apply). Build Job Failed.
+	k8s.jobs["instant-deploy-itest1|build-itest1"] = jobBackoffLimitExceeded()
+
+	w := NewDeployStatusReconciler(db, k8s).WithAutopsyK8s(&fakeAutopsyK8sCov{})
+	if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil {
+		t.Fatalf("Work: %v", err)
+	}
+
+	gotStatus, _ := testhelpers.DeploymentStatus(t, db, deployID)
+	if gotStatus != deployStatusFailed {
+		t.Errorf("deployment status = %q, want %q (build Job Failed must flip the real row)", gotStatus, deployStatusFailed)
+	}
+
+	// The in-sweep autopsy must have written a REAL deployment_events row.
+	if _, ok := testhelpers.AutopsyRow(t, db, deployID); !ok {
+		t.Error("no failure_autopsy deployment_events row written — the in-sweep capture did not round-trip to the DB")
+	}
+}
+
+// TestIntegration_DeployStatusReconcile_HealthyTransition asserts the happy
+// path round-trips: a 'building' row whose runtime Deployment is now healthy
+// (AvailableReplicas>=1) is flipped to 'healthy' in the REAL DB. This pins that
+// the updateStatus UPDATE's WHERE status IN (...) guard actually matches the
+// row (a sqlmock test cannot catch a guard that excludes the live row).
+func TestIntegration_DeployStatusReconcile_HealthyTransition(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	teamID := testhelpers.SeedTeam(t, db, "hobby")
+	deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "app-itest2")
+
+	k8s := newFakeDeployStatusK8s()
+	k8s.objs["instant-deploy-itest2|app-itest2"] = newHealthyDeployment()
+
+	w := NewDeployStatusReconciler(db, k8s)
+	if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil {
+		t.Fatalf("Work: %v", err)
+	}
+
+	gotStatus, _ := testhelpers.DeploymentStatus(t, db, deployID)
+	if gotStatus != deployStatusHealthy {
+		t.Errorf("deployment status = %q, want %q", gotStatus, deployStatusHealthy)
+	}
+}
+
+// TestIntegration_DeployStatusReconcile_StuckBuildingReaped covers sweep
+// finding #5 (Bug A's tier-cap leak): a 'building' row with an EMPTY
+// provider_id whose age exceeds stuckBuildingGrace is reaped to 'failed' with
+// the stuck-building error_message stamped. This exercises reapStuckBuilding's
+// double-guarded UPDATE against the live schema — the WHERE provider_id IS NULL
+// OR provider_id = ” guard must match the real NULL row.
+func TestIntegration_DeployStatusReconcile_StuckBuildingReaped(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	teamID := testhelpers.SeedTeam(t, db, "hobby")
+	// Empty provider_id → SeedDeployment writes NULL.
+	deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusBuilding, "")
+
+	// Age the row past the 15m grace window so the reaper fires.
+	if _, err := db.Exec(
+		`UPDATE deployments SET created_at = $1 WHERE id = $2`,
+		time.Now().Add(-stuckBuildingGrace-time.Minute), deployID,
+	); err != nil {
+		t.Fatalf("age row: %v", err)
+	}
+
+	// k8s is present but the row has no provider_id, so no namespace is
+	// derivable — the reaper path runs without any k8s Get.
+	w := NewDeployStatusReconciler(db, newFakeDeployStatusK8s())
+	if err := w.Work(context.Background(), fakeRiverJob[DeployStatusReconcileArgs]()); err != nil {
+		t.Fatalf("Work: %v", err)
+	}
+
+	gotStatus, gotErr := testhelpers.DeploymentStatus(t, db, deployID)
+	if gotStatus != deployStatusFailed {
+		t.Errorf("stuck-building row status = %q, want %q (reaper must free the tier cap)", gotStatus, deployStatusFailed)
+	}
+	if !gotErr.Valid || gotErr.String != stuckBuildingReapMessage {
+		t.Errorf("error_message = %q (valid=%v), want %q", gotErr.String, gotErr.Valid, stuckBuildingReapMessage)
+	}
+}
+
+// TestIntegration_DeployFailureAutopsy_UpsertAndAuditEmit is the real-DB guard
+// for Bug B of the triad. captureDeploymentAutopsy against a live DB must:
+//
+//  1. UPSERT a deployment_events failure_autopsy row (idempotent on re-run via
+//     the partial-unique index — a second call must NOT create a duplicate),
+//  2. stamp deployments.error_message with "<reason>: <hint snippet>", and
+//  3. emit an audit_log kind='deploy.failed' row exactly ONCE even across
+//     repeated ticks (the idempotency guard added in bug bash 2026-06-02 #15).
+//
+// The ON CONFLICT ... WHERE kind='failure_autopsy' clause + the audit dedup
+// probe are SQL that sqlmock cannot validate against the real schema — this is
+// where the partial-unique-index gap (the dev-box schema lacked the index)
+// would surface as a hard error rather than a green mock.
+func TestIntegration_DeployFailureAutopsy_UpsertAndAuditEmit(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	teamID := testhelpers.SeedTeam(t, db, "pro")
+	deployID := testhelpers.SeedDeployment(t, db, teamID, deployStatusFailed, "app-itest3")
+
+	// Autopsy k8s returns OOMKilled pod logs so the captured reason is concrete
+	// (not Unknown) — proves the k8s→DB plumbing end-to-end.
+	autopsy := &fakeAutopsyK8sCov{
+		logs: []string{"panic: out of memory", "exit status 137"},
+	}
+
+	ctx := context.Background()
+
+	// First capture.
+	captureDeploymentAutopsy(ctx, db, deployID, "app-itest3", autopsy)
+
+	reason, ok := testhelpers.AutopsyRow(t, db, deployID)
+	if !ok {
+		t.Fatal("no failure_autopsy row after first capture")
+	}
+	if reason == "" {
+		t.Error("autopsy reason is empty — capture did not populate the row")
+	}
+
+	// error_message must be stamped (it was NULL on seed).
+	_, gotErr := testhelpers.DeploymentStatus(t, db, deployID)
+	if !gotErr.Valid || gotErr.String == "" {
+		t.Error("deployments.error_message not stamped by autopsy")
+	}
+
+	// audit_log deploy.failed emitted exactly once.
+	if n := testhelpers.CountAuditLog(t, db, auditKindDeployFailed, deployID.String()); n != 1 {
+		t.Errorf("deploy.failed audit rows after first capture = %d, want 1", n)
+	}
+
+	// Second capture (idempotent re-tick): MUST NOT duplicate either the
+	// autopsy row or the audit_log row.
+	captureDeploymentAutopsy(ctx, db, deployID, "app-itest3", autopsy)
+
+	if n := testhelpers.CountAuditLog(t, db, auditKindDeployFailed, deployID.String()); n != 1 {
+		t.Errorf("deploy.failed audit rows after SECOND capture = %d, want 1 — idempotency guard regressed (duplicate failure emails)", n)
+	}
+
+	// Still exactly one autopsy row (partial-unique index + ON CONFLICT).
+	var autopsyRows int
+	if err := db.QueryRow(
+		`SELECT count(*) FROM deployment_events WHERE deployment_id = $1 AND kind = 'failure_autopsy'`,
+		deployID,
+	).Scan(&autopsyRows); err != nil {
+		t.Fatalf("count autopsy rows: %v", err)
+	}
+	if autopsyRows != 1 {
+		t.Errorf("failure_autopsy rows after two captures = %d, want 1 (ON CONFLICT upsert regressed)", autopsyRows)
+	}
+}

internal/jobs/entitlement_reconciler_integration_test.go

@@ -0,0 +1,113 @@
+package jobs
+
+// entitlement_reconciler_integration_test.go — REAL-Postgres integration test
+// for the entitlement reconciler's Postgres connection-cap regrade round-trip.
+//
+// Per INTEGRATION-COVERAGE-PLAN-2026-06-04.md §5 #1, entitlement_reconciler is
+// a Tier-1 (data-adjacent) job whose DB round-trip was fake-tested only. The
+// sibling entitlement_reconciler_test.go drives the SELECT/UPDATE through
+// go-sqlmock; THIS test seeds a real drifted resources row, runs Work() against
+// a live platform Postgres with a stub provisioner regrader, and asserts the
+// row's applied_conn_limit was actually persisted by the worker's UPDATE.
+//
+// The provisioner gRPC leg stays stubbed (stubRegrader returns
+// {Applied:true, AppliedConnLimit:5}) — the DB leg (the drift SELECT + the
+// UPDATE resources SET applied_conn_limit) is REAL. This is the convergence
+// signal the job depends on in production: a sqlmock test passes whether or not
+// the UPDATE's WHERE id = $2 matches the live row; this proves it does.
+//
+// GATING: testhelpers.SetupTestDB skips when no DB is reachable, so the regular
+// gate (deploy.yml / ci.yml, no Postgres service) stays green. It runs against a
+// real Postgres wherever one is supplied (developer DB, coverage.yml service).
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"instant.dev/worker/internal/testhelpers"
+)
+
+// TestIntegration_EntitlementReconciler_PersistsRegradedConnLimit seeds a
+// drifted Postgres resource (applied_conn_limit = NULL, never re-graded) on a
+// pro-tier team, runs the reconciler with a stub regrader that reports
+// {Applied:true, AppliedConnLimit:5}, and asserts the worker persisted that
+// value to the REAL row.
+//
+// Drift detection: shouldRegrade returns drift=true for a NULL applied limit on
+// any non-ephemeral tier. The stub stands in for the provisioner's
+// RegradeResource; the worker's `UPDATE resources SET applied_conn_limit = $1
+// WHERE id = $2` is the integration assertion target.
+func TestIntegration_EntitlementReconciler_PersistsRegradedConnLimit(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	teamID := testhelpers.SeedTeam(t, db, "pro")
+	// resource.tier = pro (the per-row snapshot the reconciler resolves caps
+	// from), applied_conn_limit = NULL → drifts.
+	resID, _ := testhelpers.SeedResource(t, db, teamID, "postgres", "pro", sql.NullInt64{})
+
+	// Scope the sweep to ONLY this team so it cannot touch any other rows that
+	// may exist in a shared local DB (and so the assertion is deterministic).
+	t.Setenv("ENTITLEMENT_RECONCILE_TEAM", teamID.String())
+
+	stub := &stubRegrader{} // returns Applied:true, AppliedConnLimit:5
+	reg := liveRegistry(t)
+	w := NewEntitlementReconcilerWorker(db, reg, stub)
+
+	if err := w.Work(context.Background(), fakeEntitlementJob()); err != nil {
+		t.Fatalf("Work: %v", err)
+	}
+
+	// The stub must have been asked to regrade our drifted row at least once.
+	if got := int(stub.calls.Load()); got < 1 {
+		t.Fatalf("RegradeResource called %d times, want >= 1 (drifted row should have been regraded)", got)
+	}
+
+	// The REAL row's applied_conn_limit must now equal the stub's reported value.
+	got := testhelpers.AppliedConnLimit(t, db, resID)
+	if !got.Valid {
+		t.Fatal("applied_conn_limit is still NULL — the worker's UPDATE did not persist against the live row")
+	}
+	if got.Int64 != 5 {
+		t.Errorf("applied_conn_limit = %d, want 5 (the value the stub regrader reported)", got.Int64)
+	}
+}
+
+// TestIntegration_EntitlementReconciler_NoDriftLeavesRowUntouched is the
+// complement: a resource whose applied_conn_limit already equals the entitled
+// cap for its tier must NOT be re-graded (no drift) and the row is left
+// untouched. This pins that the live drift SELECT + shouldRegrade decision
+// correctly identify the no-op case against real data — a regression that
+// always-regrades would burn provisioner RPCs every 5-minute tick.
+func TestIntegration_EntitlementReconciler_NoDriftLeavesRowUntouched(t *testing.T) {
+	db, cleanup := testhelpers.SetupTestDB(t)
+	defer cleanup()
+
+	reg := liveRegistry(t)
+	entitled := reg.ConnectionsLimit("pro", "postgres")
+
+	teamID := testhelpers.SeedTeam(t, db, "pro")
+	// applied_conn_limit already == entitled → no drift.
+	resID, _ := testhelpers.SeedResource(t, db, teamID, "postgres", "pro",
+		sql.NullInt64{Int64: int64(entitled), Valid: true})
+
+	t.Setenv("ENTITLEMENT_RECONCILE_TEAM", teamID.String())
+
+	stub := &stubRegrader{}
+	w := NewEntitlementReconcilerWorker(db, reg, stub)
+
+	if err := w.Work(context.Background(), fakeEntitlementJob()); err != nil {
+		t.Fatalf("Work: %v", err)
+	}
+
+	if got := int(stub.calls.Load()); got != 0 {
+		t.Errorf("RegradeResource called %d times, want 0 — a row at the entitled cap must NOT drift", got)
+	}
+
+	// Row value unchanged.
+	got := testhelpers.AppliedConnLimit(t, db, resID)
+	if !got.Valid || got.Int64 != int64(entitled) {
+		t.Errorf("applied_conn_limit = %v, want %d (untouched)", got, entitled)
+	}
+}