From c575dcd6eba13debff72e48939b1e13c9cedaea7 Mon Sep 17 00:00:00 2001 From: Manas Srivastava Date: Fri, 5 Jun 2026 11:12:07 +0530 Subject: [PATCH] fix(e2e): prod LIVE provisions + reaps as minted pro account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two prod-only failures from run 26997362193: 1. Anon provisions 402 (free_tier_recycle_requires_claim) — the X-E2E-Test-Token bypass was sending X-Forwarded-For, but the api's fingerprint middleware reads the dedicated X-E2E-Source-IP header for the override (XFF is overwritten by ingress-nginx on prod). So every anon provision collapsed onto one fingerprint that had a recycle_seen marker → 402. 2. Anon reap 401 — there is no unauthed resource-delete on the api, so the ledger reaper's DELETE /api/v1/resources/:id 401'd on anon resources. Fix (decisive: provision + reap AS the minted account on prod): - cohort.ts: anonProvisionHeaders now sends X-E2E-Source-IP (the header the bypass actually reads) with a unique IP per call → fresh fingerprint → no recycle gate. Adds authedProvisionHeaders() + provisionIdentity(): when E2E_SESSION_JWT is set (sanctioned prod run) resources are provisioned AS the minted is_test_cohort PRO account (no recycle gate, tier has headroom) and the provision bearer is recorded on the ledger so the reaper's authed DELETE returns 200, not 401. - live-provision-smoke + live-anon-provision: use provisionIdentity(); assert the identity's tier ('pro' authed / 'anonymous' otherwise); record id.bearer for the authed reap. - live-claim-deploy: anon claim legs get fresh fingerprints via the source-IP fix; the deploy-lifecycle leg now deploys AS the minted pro account directly (deployments_apps=10) instead of skipping on prod — falls back to the dev-only set-tier path on staging. npm run gate green (tsc + build + vitest: 1115 passed, 3 skipped). Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/cohort.ts | 125 +++++++++++++++++++++++++++---- e2e/live-anon-provision.spec.ts | 30 +++++--- e2e/live-claim-deploy.spec.ts | 98 ++++++++++++++++-------- e2e/live-provision-smoke.spec.ts | 24 ++++-- 4 files changed, 212 insertions(+), 65 deletions(-) diff --git a/e2e/cohort.ts b/e2e/cohort.ts index 8b7ec8a..6b42fca 100644 --- a/e2e/cohort.ts +++ b/e2e/cohort.ts @@ -169,35 +169,128 @@ export function mintedSession(): MintedSession | null { } // ── Per-fingerprint bypass for anonymous provisions (ISSUE 1) ──────────────── -// Prod does NOT trust X-Forwarded-For, so the CI runner's many anon provisions -// all share ONE real fingerprint (SHA256(/24 + ASN), rule 6) and trip the -// free-tier recycle/dedup gate → 402 free_tier_recycle_requires_claim. The api -// exposes a real bypass: the X-E2E-Test-Token header (api internal/middleware/ -// fingerprint.go) — a matching token skips the per-fingerprint cap. The token is -// a CI secret (E2E_TEST_TOKEN); when unset (local dev / un-tokened run) we send -// no bypass header and rely on X-Forwarded-For varying the fingerprint instead. - -/** The header name the api's fingerprint middleware honours to skip the cap. */ +// Prod does NOT trust X-Forwarded-For (ingress-nginx overwrites it with the real +// client IP), so the CI runner's many anon provisions all collapse onto ONE real +// fingerprint (SHA256(/24 + ASN), rule 6). That single fingerprint trips the +// free-tier recycle gate → 402 free_tier_recycle_requires_claim the moment a +// previous run's anonymous resource has aged out (recycle_seen: set, no +// active resource — api internal/handlers/provision_helper.go). +// +// The api's REAL bypass (internal/middleware/fingerprint.go): a request bearing +// a valid X-E2E-Test-Token (matched against the cluster's E2E_TEST_TOKEN secret) +// may override the fingerprint source IP via the dedicated **X-E2E-Source-IP** +// header — NOT X-Forwarded-For (which the ingress overwrites). A unique source +// IP per call yields a fresh fingerprint per provision → no recycle_seen marker +// → no 402. Sending only X-Forwarded-For (the previous bug) did nothing on prod +// because the middleware never reads XFF for the override. We now send BOTH the +// token AND X-E2E-Source-IP so the override actually takes on prod; XFF is kept +// for staging/local where the proxy IS trusted. + +/** The header name the api's fingerprint middleware honours to accept the bypass token. */ export const E2E_TEST_TOKEN_HEADER = 'X-E2E-Test-Token' +/** The header the api's fingerprint middleware reads for the override source IP. */ +export const E2E_SOURCE_IP_HEADER = 'X-E2E-Source-IP' /** - * Headers an anonymous provision should carry: a unique X-Forwarded-For (varies - * the fingerprint where the proxy IS trusted — staging/local) PLUS the - * X-E2E-Test-Token bypass when E2E_TEST_TOKEN is set (the only thing that gets - * past the per-fingerprint recycle gate on prod, which ignores X-Forwarded-For). - * Both are harmless together: the bypass wins on prod, the XFF varies elsewhere. + * Headers an anonymous provision should carry. On prod the load-bearing pair is + * X-E2E-Test-Token (the bypass token) + X-E2E-Source-IP (a unique override IP → + * a fresh fingerprint per call → no recycle gate). X-Forwarded-For is also set + * with the same unique IP for staging/local where the proxy is trusted. All are + * harmless together: on prod the override wins, elsewhere the XFF varies. */ export function anonProvisionHeaders(extra: Record = {}): Record { + const ip = uniqueIP() const headers: Record = { 'Content-Type': 'application/json', - 'X-Forwarded-For': uniqueIP(), + 'X-Forwarded-For': ip, ...extra, } const token = process.env.E2E_TEST_TOKEN - if (token) headers[E2E_TEST_TOKEN_HEADER] = token + if (token) { + headers[E2E_TEST_TOKEN_HEADER] = token + // The override source IP the middleware actually reads (XFF is ignored on + // prod). A unique value per call → a fresh fingerprint → no recycle gate. + headers[E2E_SOURCE_IP_HEADER] = ip + } return headers } +// ── Authed provision (workflow-minted pro account) ─────────────────────────── +// On a SANCTIONED prod run the workflow mints an is_test_cohort=true PRO account +// and exports its session JWT (E2E_SESSION_JWT). Provisioning AS that account is +// the decisive fix for both prod anon failures: +// 1. No 402: authed provisioning skips the anonymous recycle gate entirely +// (the gate runs only on the anon path), and tier=pro has ample headroom. +// 2. No 401 on reap: the resource is OWNED by the minted team, so the reaper's +// authed DELETE /api/v1/resources/:id (with the same bearer) returns 200 — +// there is NO unauthed resource-delete on the api, so anon resources could +// only ever be TTL-cleaned (the 401 the last run hit). +// The minted account is reaped wholesale by the workflow teardown (cascade), so +// every resource it owns is double-covered. + +/** + * Headers for an AUTHENTICATED provision as the given bearer (the minted pro + * account's session JWT). A unique X-Forwarded-For keeps per-call fingerprints + * distinct on trusted proxies, but the authed path is not fingerprint-gated. + */ +export function authedProvisionHeaders( + token: string, + extra: Record = {}, +): Record { + return { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + 'X-Forwarded-For': uniqueIP(), + ...extra, + } +} + +/** + * The provision identity a LIVE spec should use for a resource it will reap via + * the authed DELETE surface. When a minted session exists (sanctioned prod run) + * we provision + reap AS that pro account (no 402, authed reap 200). Otherwise + * (staging/local) we use the anonymous path with the fingerprint bypass, and the + * resource is reaped by TTL / the un-authed staging delete path. + * + * - `headers` : the provision request headers (authed or anon-with-bypass). + * - `bearer` : the token to record on the ledger so the reaper's DELETE is + * authorized (undefined on the anon path). + * - `expectTier` : the tier the provision response will echo ('pro' authed, + * 'anonymous' otherwise) — specs assert against this. + * - `authed` : true when provisioning as the minted account. + */ +export interface ProvisionIdentity { + headers: Record + bearer?: string + expectTier: string + authed: boolean +} + +/** Tier echoed by an anonymous provision. */ +export const ANON_TIER = 'anonymous' + +/** + * Resolve the provision identity for a LIVE resource spec. Prefers the minted + * pro account (authed) when E2E_SESSION_JWT is set; else the anon bypass path. + */ +export function provisionIdentity(extra: Record = {}): ProvisionIdentity { + const minted = mintedSession() + if (minted?.token) { + return { + headers: authedProvisionHeaders(minted.token, extra), + bearer: minted.token, + expectTier: minted.tier || 'pro', + authed: true, + } + } + return { + headers: anonProvisionHeaders(extra), + bearer: undefined, + expectTier: ANON_TIER, + authed: false, + } +} + // Unique source IP per call (varies the fingerprint where the proxy is trusted). function uniqueIP(): string { const b = () => Math.floor(Math.random() * 254) + 1 diff --git a/e2e/live-anon-provision.spec.ts b/e2e/live-anon-provision.spec.ts index f437222..55f31b2 100644 --- a/e2e/live-anon-provision.spec.ts +++ b/e2e/live-anon-provision.spec.ts @@ -42,7 +42,7 @@ import { expect, test, type APIRequestContext } from '@playwright/test' -import { cohortName, COHORT_MARKER, assertSafeApiTarget, anonProvisionHeaders } from './cohort' +import { cohortName, COHORT_MARKER, assertSafeApiTarget, provisionIdentity } from './cohort' import { recordEntity, loadLedger, @@ -183,15 +183,19 @@ test.describe('LIVE — every anonymous provision flow → backend-assert → re }) => { const name = cohortName(`anon-${flow.label}`) - // ── Create: real anonymous provision against the live api ────────────── - // anonProvisionHeaders() adds the X-E2E-Test-Token fingerprint-bypass when - // E2E_TEST_TOKEN is set (the only thing that gets past prod's per- - // fingerprint recycle gate, which ignores X-Forwarded-For) plus a unique - // XFF for staging/local. A cohort name is always sent: /vector & /db - // REQUIRE a name (CLAUDE.md) and it is harmless (cohort-tagging) on the rest. + // ── Create: real provision against the live api ──────────────────────── + // On a sanctioned prod run (E2E_SESSION_JWT set) provisionIdentity() + // provisions AS the minted PRO account: authed provisioning skips the + // anonymous recycle gate (no 402) and the resource is team-owned so the + // reaper's authed DELETE returns 200 (no 401). Otherwise (staging/local) + // it uses the anon path with the X-E2E-Test-Token + X-E2E-Source-IP + // fingerprint bypass (a fresh fingerprint per call → no recycle gate). + // A cohort name is always sent: /vector & /db REQUIRE a name (CLAUDE.md) + // and it is harmless (cohort-tagging) on the rest. + const id = provisionIdentity() const resp = await request.fetch(`${API_URL}${flow.path}`, { method: 'POST', - headers: anonProvisionHeaders(), + headers: id.headers, data: JSON.stringify({ name }), failOnStatusCode: false, }) @@ -217,6 +221,10 @@ test.describe('LIVE — every anonymous provision flow → backend-assert → re kind: 'resource', id: token, apiUrl: API_URL, + // Record the provision bearer (minted pro session on prod) so the + // reaper's authed DELETE is authorized → 200, not 401. Undefined on the + // anon path (resource is TTL/staging-reaped). + token: id.bearer, note: `${flow.label} ${name}`, } recordEntity(entity) @@ -236,7 +244,11 @@ test.describe('LIVE — every anonymous provision flow → backend-assert → re expect(body.ok, `${flow.label}/new ok flag`).toBe(true) expect(token, `${flow.path} must return a resource token`).toBeTruthy() expect(body.id, `${flow.path} must return a resource id`).toBeTruthy() - expect(body.tier, `anon ${flow.label} provision is tier=anonymous`).toBe('anonymous') + // Tier reflects the provision identity: 'pro' as the minted account + // (sanctioned prod run), 'anonymous' on the anon path (staging/local). + expect(body.tier, `${flow.label} provision tier should be '${id.expectTier}'`).toBe( + id.expectTier, + ) // env is echoed on every provision (rule 11 — default 'development'). expect(body.env, `${flow.path} must echo the resolved env (rule 11)`).toBeTruthy() // The usable connection/URL — proves a real backend minted a live endpoint. diff --git a/e2e/live-claim-deploy.spec.ts b/e2e/live-claim-deploy.spec.ts index 4a2b99c..875773c 100644 --- a/e2e/live-claim-deploy.spec.ts +++ b/e2e/live-claim-deploy.spec.ts @@ -59,6 +59,7 @@ import { isCohortBranded, assertSafeApiTarget, anonProvisionHeaders, + mintedSession, } from './cohort' import { recordEntity, @@ -478,31 +479,50 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c // shipped 2026-05-30, #200) → DELETE → 404 gone. That is the user-visible // integration boundary; the build-to-live leg is exercised by /instant-e2e // and the per-deploy synthetic, not here. - test.describe('deploy lifecycle — create(202) → events surface → delete → gone [STAGING-ONLY]', () => { + test.describe('deploy lifecycle — create(202) → events surface → delete → gone', () => { test('POST /deploy/new accepted → events timeline → DELETE → gone', async ({ request }) => { - const anon = await provisionAnonCache(request) - const team = await claim(request, anon.upgradeJWT) - - // Elevate the free team to a deployable tier via the dev-only endpoint. - // 404 ⇒ not a dev/staging stack ⇒ skip loudly (never 402-wall or charge). - const setTier = await request.fetch(`${API_URL}${SET_TIER_PATH}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - data: JSON.stringify({ team_id: team.teamID, tier: DEPLOY_TIER }), - failOnStatusCode: false, - }) - test.skip( - setTier.status() === STATUS_NOT_FOUND, - `${SET_TIER_PATH} returned 404 — this is a prod stack (dev-only endpoint not registered). ` + - `The deploy-lifecycle leg is STAGING-ONLY: a free team can't deploy (402) and we must ` + - `never cross a real billing path to elevate it. Reports skipped. Run against a ` + - `dev/staging api (ENVIRONMENT=development) to exercise it.`, - ) - expect( - setTier.status(), - `${SET_TIER_PATH} should return ${STATUS_OK} on a dev/staging stack; got ${setTier.status()}. ` + - `Body: ${await setTier.text().catch(() => '')}`, - ).toBe(STATUS_OK) + // Resolve a team with deployment headroom WITHOUT crossing a billing path: + // - Prod (sanctioned run): the workflow-minted account is already PRO + // (deployments_apps=10), so we deploy AS it directly — no claim, no + // dev-only set-tier. This is the brief's "use the minted pro account". + // - Staging/local: the Brevo-free claim mints a `free` team (0 deploy + // headroom → 402), so we elevate it via the DEV-ONLY set-tier endpoint. + // If that endpoint 404s AND there is no minted session, we skip loudly + // (never 402-wall or charge). + const minted = mintedSession() + let deployBearer: string + let baseResource: { token: string; bearer: string } | null = null + + if (minted?.token) { + // Deploy as the minted pro account — has deployment headroom already. + deployBearer = minted.token + } else { + const anon = await provisionAnonCache(request) + const team = await claim(request, anon.upgradeJWT) + baseResource = { token: anon.token, bearer: team.sessionToken } + + // Elevate the free team to a deployable tier via the dev-only endpoint. + // 404 ⇒ not a dev/staging stack AND no minted account ⇒ skip loudly. + const setTier = await request.fetch(`${API_URL}${SET_TIER_PATH}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + data: JSON.stringify({ team_id: team.teamID, tier: DEPLOY_TIER }), + failOnStatusCode: false, + }) + test.skip( + setTier.status() === STATUS_NOT_FOUND, + `${SET_TIER_PATH} returned 404 — this is a prod stack (dev-only endpoint not registered) ` + + `and no minted pro account is present. The deploy leg needs deployment headroom and we ` + + `must never cross a real billing path to elevate a free team. Reports skipped. Run ` + + `against a dev/staging api (ENVIRONMENT=development) or a sanctioned minted-account run.`, + ) + expect( + setTier.status(), + `${SET_TIER_PATH} should return ${STATUS_OK} on a dev/staging stack; got ${setTier.status()}. ` + + `Body: ${await setTier.text().catch(() => '')}`, + ).toBe(STATUS_OK) + deployBearer = team.sessionToken + } // ── Create: a minimal tarball deploy. We do NOT wait for the Kaniko build // to go live (deferred) — only the accepted contract + events surface + @@ -523,7 +543,7 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c } const create = await request.fetch(`${API_URL}/deploy/new`, { method: 'POST', - headers: { Authorization: `Bearer ${team.sessionToken}` }, + headers: { Authorization: `Bearer ${deployBearer}` }, multipart: form, failOnStatusCode: false, }) @@ -552,7 +572,7 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c kind: 'deployment', id: appID, apiUrl: API_URL, - token: team.sessionToken, + token: deployBearer, note: `W3 deploy ${deployName}`, }) // Accepted-contract assertions: a real building deployment in the env we @@ -584,7 +604,7 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c // surface EXISTS and is team-scoped, not that a specific event landed. ─ const events = await request.fetch( `${API_URL}/api/v1/deployments/${encodeURIComponent(appID)}/events`, - { method: 'GET', headers: { Authorization: `Bearer ${team.sessionToken}` }, failOnStatusCode: false }, + { method: 'GET', headers: { Authorization: `Bearer ${deployBearer}` }, failOnStatusCode: false }, ) expect( events.status(), @@ -617,7 +637,7 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c // so the dashboard would correctly show it as removed. ──────────────── const del = await request.fetch( `${API_URL}/api/v1/deployments/${encodeURIComponent(appID)}`, - { method: 'DELETE', headers: { Authorization: `Bearer ${team.sessionToken}` }, failOnStatusCode: false }, + { method: 'DELETE', headers: { Authorization: `Bearer ${deployBearer}` }, failOnStatusCode: false }, ) expect( del.status() >= 200 && del.status() < 300, @@ -632,7 +652,7 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c // load-bearing assertion is that the team can no longer act on it as live. const afterDelete = await request.fetch( `${API_URL}/api/v1/deployments/${encodeURIComponent(appID)}/events`, - { method: 'GET', headers: { Authorization: `Bearer ${team.sessionToken}` }, failOnStatusCode: false }, + { method: 'GET', headers: { Authorization: `Bearer ${deployBearer}` }, failOnStatusCode: false }, ) expect( [STATUS_OK, STATUS_NOT_FOUND].includes(afterDelete.status()), @@ -641,10 +661,22 @@ test.describe('LIVE — W3 claim/conversion + deploy-lifecycle + env-switcher (c ).toBe(true) // Reap is now idempotent (404 == alreadyGone). Clears the ledger. - const result = await reapEntities(request, [ - { kind: 'deployment', id: appID, apiUrl: API_URL, token: team.sessionToken, note: 'W3 deploy reap', recordedAt: new Date().toISOString() }, - { kind: 'resource', id: anon.token, apiUrl: API_URL, token: team.sessionToken, note: 'W3 deploy base resource', recordedAt: new Date().toISOString() }, - ]) + const reapTargets = [ + { kind: 'deployment' as const, id: appID, apiUrl: API_URL, token: deployBearer, note: 'W3 deploy reap', recordedAt: new Date().toISOString() }, + ] + // The staging path created an anon base resource (claimed → team-owned); + // reap it too. The minted-account path created no base resource. + if (baseResource) { + reapTargets.push({ + kind: 'resource' as const, + id: baseResource.token, + apiUrl: API_URL, + token: baseResource.bearer, + note: 'W3 deploy base resource', + recordedAt: new Date().toISOString(), + }) + } + const result = await reapEntities(request, reapTargets) expect(result.failed.length, `reap failed: ${JSON.stringify(result.failed)}`).toBe(0) clearLedger() }) diff --git a/e2e/live-provision-smoke.spec.ts b/e2e/live-provision-smoke.spec.ts index b6d79dc..0aa01f2 100644 --- a/e2e/live-provision-smoke.spec.ts +++ b/e2e/live-provision-smoke.spec.ts @@ -23,7 +23,7 @@ import { expect, test, type APIRequestContext } from '@playwright/test' -import { cohortName, COHORT_MARKER, assertSafeApiTarget, anonProvisionHeaders } from './cohort' +import { cohortName, COHORT_MARKER, assertSafeApiTarget, provisionIdentity } from './cohort' import { recordEntity, loadLedger, @@ -93,13 +93,17 @@ test.describe('LIVE smoke — anonymous provision → backend-assert → reap', }) => { const name = cohortName('smoke-db') - // ── Create: real anonymous Postgres against the live api ────────────── - // anonProvisionHeaders() carries the X-E2E-Test-Token fingerprint-bypass when - // E2E_TEST_TOKEN is set (prod ignores X-Forwarded-For) + a unique XFF - // elsewhere. /db/new REQUIRES a name (CLAUDE.md) — already sent below. + // ── Create: real Postgres against the live api ──────────────────────── + // On a sanctioned prod run (E2E_SESSION_JWT set) we provision AS the minted + // PRO account — authed provisioning skips the anonymous recycle gate (no + // 402) and the resource is team-owned so the reaper's authed DELETE returns + // 200 (no 401). Otherwise (staging/local) we use the anon path with the + // X-E2E-Test-Token + X-E2E-Source-IP fingerprint bypass. /db/new REQUIRES a + // name (CLAUDE.md) — sent below in both modes. + const id = provisionIdentity() const resp = await request.fetch(`${API_URL}/db/new`, { method: 'POST', - headers: anonProvisionHeaders(), + headers: id.headers, data: JSON.stringify({ name }), failOnStatusCode: false, }) @@ -124,6 +128,10 @@ test.describe('LIVE smoke — anonymous provision → backend-assert → reap', kind: 'resource', id: db.token, apiUrl: API_URL, + // Record the provision bearer (minted pro session on prod) so the reaper's + // authed DELETE /api/v1/resources/:id is authorized → 200, not 401. On the + // anon path this is undefined and the resource is TTL/staging-reaped. + token: id.bearer, note: `postgres ${name}`, } recordEntity(entity) @@ -132,7 +140,9 @@ test.describe('LIVE smoke — anonymous provision → backend-assert → reap', expect(db.ok, 'db/new ok flag').toBe(true) expect(db.token, 'db/new must return a resource token').toBeTruthy() expect(db.id, 'db/new must return a resource id').toBeTruthy() - expect(db.tier, 'anon provision is tier=anonymous').toBe('anonymous') + // Tier reflects the provision identity: 'pro' when provisioned as the minted + // account (sanctioned prod run), 'anonymous' on the anon path (staging/local). + expect(db.tier, `provision tier should be '${id.expectTier}'`).toBe(id.expectTier) expect( db.connection_url, 'db/new must return a usable postgres connection_url (proves a real DB was created)',