From c597741b858e0a77b731bc0429fcde172d8e15b4 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:28:56 -0700 Subject: [PATCH 001/144] fix(qbo): per-request fetch timeout on every QuickBooks call Bare fetch() has no timeout, so today's Intuit API outage hung every QB call until Vercel killed the function at its maxDuration (60s on the receipt-push create, 120s on the payments cron). qbTimedFetch wraps fetch with AbortSignal.timeout (QB_FETCH_TIMEOUT_MS, default 20s) and rethrows our own deadline as QBTimeoutError carrying the URL path only (no query string, no tokens). A caller-supplied signal is combined via AbortSignal.any and still wins; every other error passes through unchanged. Routed through it: exchangeQBCode, refreshQBToken, qbFetch, qbQuery, plus the five other direct fetches in quickbooks.ts (payment link read, payment delete, invoice delete, purchase CDC, invoice send), the Attachable multipart upload in qbo-receipt-push.ts, and the QBO temp-URL attachment download in qbo-receipt-attachments.ts. Both attachment call sites already treat a throw as a non-fatal "failed:". Co-Authored-By: Claude Fable 5.1 --- src/lib/qbo-receipt-attachments.ts | 5 +- src/lib/qbo-receipt-push.ts | 3 +- src/lib/quickbooks.ts | 1929 ++++++++++++++-------------- 3 files changed, 1002 insertions(+), 935 deletions(-) diff --git a/src/lib/qbo-receipt-attachments.ts b/src/lib/qbo-receipt-attachments.ts index e01f83d59..24f392963 100644 --- a/src/lib/qbo-receipt-attachments.ts +++ b/src/lib/qbo-receipt-attachments.ts @@ -5,7 +5,7 @@ // receiptUrl is ProBuild-owned metadata: this module only ever fills an EMPTY // receiptUrl (guarded update), so a manually uploaded receipt is never // overwritten by a later sync run. -import { getQBPurchaseAttachables, type QBTokens } from "./quickbooks"; +import { getQBPurchaseAttachables, qbTimedFetch, type QBTokens } from "./quickbooks"; import { getSupabase, STORAGE_BUCKET } from "./supabase"; import { prisma } from "./prisma"; @@ -48,7 +48,8 @@ export async function attachQboReceipt( const attachment = candidates[0]; if (!attachment?.TempDownloadUri) return "no-attachment"; - const download = await fetch(attachment.TempDownloadUri); + // QBO-issued temp URL: same unbounded-hang risk as the API itself. + const download = await qbTimedFetch(attachment.TempDownloadUri); if (!download.ok) { throw new Error(`QBO attachment download failed: HTTP ${download.status}`); } diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 7c4d902f8..89b246a5c 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -28,6 +28,7 @@ import { escapeQBString, ensureQBCustomer, QB_API_BASE, + qbTimedFetch, type QBTokens, } from "./quickbooks"; @@ -323,7 +324,7 @@ async function defaultUploadAttachment( Buffer.from(`${CRLF}--${boundary}--${CRLF}`), ]); - const res = await fetch(`${QB_API_BASE}/${tokens.realmId}/upload?minorversion=73`, { + const res = await qbTimedFetch(`${QB_API_BASE}/${tokens.realmId}/upload?minorversion=73`, { method: "POST", headers: { Authorization: `Bearer ${tokens.accessToken}`, diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 61aa20d0f..dae0e467c 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -1,932 +1,997 @@ -/** - * QuickBooks Online API client. - * Uses OAuth2 tokens stored in integration-store. - * Docs: https://developer.intuit.com/app/developer/qbo/docs/api/accounting - */ -import { - isE2eQboMockEnabled, - recordMockReadInvoiceCall, - getMockQboInvoice, - mockSendQBPaymentCreate, -} from "./quickbooks-mock"; -import { isEstimateSectionRow } from "./estimate-item-payload"; - -export const QB_API_BASE = process.env.QB_SANDBOX === "true" - ? "https://sandbox-quickbooks.api.intuit.com/v3/company" - : "https://quickbooks.api.intuit.com/v3/company"; - -const TOKEN_URL = "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer"; - -export interface QBTokens { - accessToken: string; - refreshToken: string; - realmId: string; -} - -/** Exchange authorization code for tokens */ -export async function exchangeQBCode(code: string, redirectUri: string): Promise { - const clientId = process.env.QB_CLIENT_ID!; - const clientSecret = process.env.QB_CLIENT_SECRET!; - const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); - - const res = await fetch(TOKEN_URL, { - method: "POST", - headers: { - Authorization: `Basic ${encoded}`, - "Content-Type": "application/x-www-form-urlencoded", - Accept: "application/json", - }, - body: new URLSearchParams({ - grant_type: "authorization_code", - code, - redirect_uri: redirectUri, - }), - }); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`QB token exchange failed: ${err}`); - } - - const data = await res.json(); - return { - accessToken: data.access_token, - refreshToken: data.refresh_token, - realmId: "", // set from callback query param - }; -} - -/** Refresh an expired access token */ -export async function refreshQBToken(refreshToken: string): Promise<{ accessToken: string; refreshToken: string }> { - const clientId = process.env.QB_CLIENT_ID!; - const clientSecret = process.env.QB_CLIENT_SECRET!; - const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); - - const res = await fetch(TOKEN_URL, { - method: "POST", - headers: { - Authorization: `Basic ${encoded}`, - "Content-Type": "application/x-www-form-urlencoded", - Accept: "application/json", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: refreshToken, - }), - }); - - if (!res.ok) throw new Error("QB token refresh failed"); - const data = await res.json(); - return { accessToken: data.access_token, refreshToken: data.refresh_token }; -} - -/** Make an authenticated call to the QB API, auto-refreshing if needed */ -export async function qbFetch( - path: string, - tokens: QBTokens, - opts: RequestInit = {} -): Promise { - // Callers that already put their own query string on `path` (e.g. - // "/purchase?requestid=...") get "&minorversion=73" appended instead of a - // second "?" — every existing call site passes a bare path, so this is - // backward compatible. - const separator = path.includes("?") ? "&" : "?"; - const url = `${QB_API_BASE}/${tokens.realmId}${path}${separator}minorversion=73`; - return fetch(url, { - ...opts, - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - Accept: "application/json", - "Content-Type": "application/json", - ...opts.headers, - }, - }); -} - -/** Run a QBO SQL-ish query (https://developer.intuit.com/.../data-queries) */ -export async function qbQuery(tokens: QBTokens, query: string): Promise { - const url = `${QB_API_BASE}/${tokens.realmId}/query?query=${encodeURIComponent(query)}&minorversion=73`; - const res = await fetch(url, { - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - Accept: "application/json", - }, - }); - if (!res.ok) { - const err = await res.text(); - throw new Error(`QB query failed: ${err}`); - } - const data = await res.json(); - const response = data.QueryResponse || {}; - const key = Object.keys(response).find(k => Array.isArray(response[k])); - return key ? response[key] : []; -} - -export function escapeQBString(s: string): string { - // Backslash MUST be escaped before the apostrophe escape, or an input - // ending in a literal backslash (e.g. "Smith\\") would have its escaped - // apostrophe's own backslash re-escaped, breaking out of the quoted string. - return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); -} - -export interface QBAttachable { - Id?: string; - FileName?: string; - ContentType?: string; - Size?: number; - TempDownloadUri?: string; - AttachableRef?: Array<{ EntityRef?: { value?: string; type?: string } }>; -} - -/** List file attachments linked to a QBO Purchase (receipt images/PDFs). */ -export async function getQBPurchaseAttachables( - tokens: QBTokens, - purchaseId: string, -): Promise { - // QBO transaction ids are numeric; refuse anything else rather than escape it. - if (!/^\d+$/.test(purchaseId)) return []; - const rows = await qbQuery( - tokens, - `SELECT * FROM attachable WHERE AttachableRef.EntityRef.value = '${purchaseId}'`, - ); - // Entity ids are only unique per entity type, so the value-only query can - // surface attachments from other transaction types — keep Purchase links. - return rows.filter(row => - row.AttachableRef?.some( - ref => - ref.EntityRef?.value === purchaseId && - /^purchase$/i.test(ref.EntityRef?.type ?? ""), - ), - ); -} - -/** Find a QBO customer by display name, creating it if missing. Returns the QBO customer Id. */ -export async function ensureQBCustomer( - tokens: QBTokens, - client: { name: string; email?: string | null; qbCustomerId?: string | null } -): Promise { - // Trust a previously stored id if it still exists - if (client.qbCustomerId) { - const existing = await qbQuery(tokens, `SELECT Id FROM Customer WHERE Id = '${escapeQBString(client.qbCustomerId)}'`); - if (existing.length > 0) return client.qbCustomerId; - } - - const name = client.name.trim(); - if (!name) throw new Error("Client name is empty — cannot sync customer to QuickBooks."); - const byName = await qbQuery(tokens, `SELECT Id FROM Customer WHERE DisplayName = '${escapeQBString(name)}'`); - if (byName.length > 0) return byName[0].Id; - - // QBO normalizes whitespace when enforcing DisplayName uniqueness, so an - // exact match can miss while create still rejects as a duplicate (fault 6240). - // Prefix on the first word only, so internal-whitespace variants still match. - const normalize = (s: string) => s.replace(/\s+/g, " ").trim().toLowerCase(); - const prefix = name.split(/\s+/)[0]; - const candidates = await qbQuery<{ Id: string; DisplayName?: string }>( - tokens, - `SELECT Id, DisplayName FROM Customer WHERE DisplayName LIKE '${escapeQBString(prefix)}%' MAXRESULTS 1000` - ); - const matches = candidates.filter(c => normalize(c.DisplayName ?? "") === normalize(name)); - if (matches.length > 1) { - throw new Error(`QB customer lookup for "${name}" matched ${matches.length} customers — resolve the duplicate in QuickBooks.`); - } - if (matches.length === 1) return matches[0].Id; - - const res = await qbFetch("/customer", tokens, { - method: "POST", - body: JSON.stringify({ - DisplayName: name, - ...(client.email ? { PrimaryEmailAddr: { Address: client.email } } : {}), - }), - }); - if (!res.ok) { - const err = await res.text(); - throw new Error(`QB customer create failed: ${err}`); - } - const data = await res.json(); - return data.Customer.Id; -} - -const QB_SERVICE_ITEM_NAME = "Construction Services"; - -/** Find or create the Service item used for all ProBuild invoice lines. */ -export async function ensureQBServiceItem(tokens: QBTokens): Promise { - const items = await qbQuery(tokens, `SELECT Id FROM Item WHERE Name = '${escapeQBString(QB_SERVICE_ITEM_NAME)}'`); - if (items.length > 0) return items[0].Id; - - // Need an income account to hang the item on — prefer an existing Income account. - const accounts = await qbQuery(tokens, `SELECT Id, Name FROM Account WHERE AccountType = 'Income' MAXRESULTS 1`); - let incomeAccountId: string; - if (accounts.length > 0) { - incomeAccountId = accounts[0].Id; - } else { - const created = await qbFetch("/account", tokens, { - method: "POST", - body: JSON.stringify({ Name: "Construction Income", AccountType: "Income", AccountSubType: "ServiceFeeIncome" }), - }); - if (!created.ok) throw new Error(`QB income account create failed: ${await created.text()}`); - incomeAccountId = (await created.json()).Account.Id; - } - - const res = await qbFetch("/item", tokens, { - method: "POST", - body: JSON.stringify({ - Name: QB_SERVICE_ITEM_NAME, - Type: "Service", - IncomeAccountRef: { value: incomeAccountId }, - }), - }); - if (!res.ok) throw new Error(`QB service item create failed: ${await res.text()}`); - return (await res.json()).Item.Id; -} - -/** - * Create a QBO invoice for ONE payment milestone, with QuickBooks Payments - * (card + ACH) enabled so the customer gets Intuit's hosted "Review & Pay" page. - */ -export async function createQBMilestoneInvoice( - tokens: QBTokens, - input: { - docNumber: string; // ≤ 21 chars - customerId: string; - itemId: string; - description: string; - amount: number; // grand total the client pays (tax-inclusive) - // When set, the QBO invoice carries the sales tax explicitly: - // a pre-tax taxable line + TxnTaxDetail, so QBO's sales-tax reporting - // sees the liability and the invoice total still equals `amount`. - tax?: { preTaxAmount: number; taxAmount: number } | null; - dueDate?: Date | null; - billEmail?: string | null; - privateNote?: string; - } -): Promise<{ qbId: string; qbUrl: string; total: number }> { - const withTax = !!input.tax && input.tax.taxAmount > 0; - const lineAmount = withTax ? input.tax!.preTaxAmount : input.amount; - - const payload: Record = { - DocNumber: input.docNumber.slice(0, 21), - TxnDate: new Date().toISOString().split("T")[0], - CustomerRef: { value: input.customerId }, - // QuickBooks Payments is the ONLY payment rail (Stripe is disabled until - // their 180-day hold clears) — the hosted page takes card, debit, AND bank. - // Note: Intuit can't surcharge, so card fees are merchant-absorbed. - AllowOnlineCreditCardPayment: true, - AllowOnlineACHPayment: true, - ...(input.billEmail ? { BillEmail: { Address: input.billEmail } } : {}), - ...(input.dueDate ? { DueDate: input.dueDate.toISOString().split("T")[0] } : {}), - ...(input.privateNote ? { PrivateNote: input.privateNote.slice(0, 4000) } : {}), - Line: [ - { - LineNum: 1, - Description: input.description.slice(0, 4000), - Amount: lineAmount, - DetailType: "SalesItemLineDetail", - SalesItemLineDetail: { - ItemRef: { value: input.itemId }, - Qty: 1, - UnitPrice: lineAmount, - ...(withTax ? { TaxCodeRef: { value: "TAX" } } : {}), - }, - }, - ], - ...(withTax ? { TxnTaxDetail: { TotalTax: input.tax!.taxAmount } } : {}), - }; - - const res = await qbFetch("/invoice", tokens, { method: "POST", body: JSON.stringify(payload) }); - if (!res.ok) throw new Error(`QB milestone invoice create failed: ${await res.text()}`); - const data = await res.json(); - const qbId = data.Invoice?.Id; - const total = Number(data.Invoice?.TotalAmt ?? 0); - return { qbId, qbUrl: `https://app.qbo.intuit.com/app/invoice?txnId=${qbId}`, total }; -} - -/** Fetch the customer-facing payment link for a QBO invoice (requires QB Payments enabled). */ -export async function getQBInvoicePaymentLink(tokens: QBTokens, qbInvoiceId: string): Promise { - const url = `${QB_API_BASE}/${tokens.realmId}/invoice/${qbInvoiceId}?include=invoiceLink&minorversion=73`; - const res = await fetch(url, { - headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json" }, - }); - if (!res.ok) return null; - const data = await res.json(); - return data.Invoice?.InvoiceLink || null; -} - -export interface QBInvoiceStatus { - balance: number; - total: number; - paymentTxnIds: string[]; -} - -/** Read a QBO invoice's balance + linked payment transactions. */ -export async function getQBInvoiceStatus(tokens: QBTokens, qbInvoiceId: string): Promise { - const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - if (!res.ok) return null; - const data = await res.json(); - const inv = data.Invoice; - if (!inv) return null; - const paymentTxnIds: string[] = (inv.LinkedTxn || []) - .filter((t: any) => t.TxnType === "Payment") - .map((t: any) => String(t.TxnId)); - return { balance: Number(inv.Balance ?? 0), total: Number(inv.TotalAmt ?? 0), paymentTxnIds }; -} - -/** - * Result of probing a QBO invoice's existence + payable state. - * Unlike getQBInvoiceStatus (which collapses every failure into null), this - * distinguishes a permanently gone/voided invoice from a transient API error, - * so the sync poller can flag stuck milestones without acting on a blip. - */ -export type QBInvoiceProbe = - | { state: "ok"; balance: number; total: number; paymentTxnIds: string[] } - | { state: "voided" } // HTTP 200, exists, total & balance === 0, no linked payments - | { state: "notFound" } // HTTP 400 Fault 610 or HTTP 404 (authoritative "gone" only) - | { state: "error"; status: number }; // 401/429/5xx/network/malformed — transient, never act on - -/** - * Probe a QBO invoice and classify it. QBO's behavior for gone invoices is - * inconsistent: a *voided* invoice returns 200 with TotalAmt=0; a *deleted* one - * may return 400 + Fault code 610 ("Object Not Found"), a 404, or even 200 with - * stale data. This folds all of those into a single discriminated result. - */ -export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Promise { - let res: Response; - try { - res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - } catch { - return { state: "error", status: 0 }; - } - if (res.ok) { - // A 200 should always carry an Invoice. A parse failure or a missing payload - // is anomalous — treat it as transient (never as "gone"); only an explicit - // 404/610 below is authoritative for notFound. - let data: any; - try { - data = await res.json(); - } catch { - return { state: "error", status: res.status }; - } - const inv = data?.Invoice; - if (!inv) return { state: "error", status: res.status }; - const total = Number(inv.TotalAmt); - const balance = Number(inv.Balance); - // A well-formed invoice always carries numeric TotalAmt/Balance. Missing or - // non-finite values mean a malformed/partial payload — treat as transient, - // never as voided (which would false-alarm an otherwise-healthy milestone). - if (!Number.isFinite(total) || !Number.isFinite(balance)) return { state: "error", status: res.status }; - const paymentTxnIds: string[] = (inv.LinkedTxn || []) - .filter((t: any) => t.TxnType === "Payment") - .map((t: any) => String(t.TxnId)); - // Voided invoices come back 200 with TotalAmt=0, Balance=0, and no linked payments. - if (total === 0 && balance === 0 && paymentTxnIds.length === 0) return { state: "voided" }; - return { state: "ok", balance, total, paymentTxnIds }; - } - if (res.status === 404) return { state: "notFound" }; - const body = await res.text().catch(() => ""); - if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { - return { state: "notFound" }; - } - return { state: "error", status: res.status }; -} - -/** Read a QBO payment (date / amount / reference) for receipt details. */ -export async function getQBPayment( - tokens: QBTokens, - paymentId: string -): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null> { - const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); - if (!res.ok) return null; - const data = await res.json(); - const p = data.Payment; - if (!p) return null; - return { - txnDate: p.TxnDate || null, - amount: Number(p.TotalAmt ?? 0), - referenceNumber: p.PaymentRefNum || null, - }; -} - -/** Read core invoice fields needed for payments/deletes. */ -export async function readQBInvoice(tokens: QBTokens, qbInvoiceId: string) { - const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - if (!res.ok) return null; - const inv = (await res.json()).Invoice; - if (!inv) return null; - return { - syncToken: String(inv.SyncToken), - customerId: String(inv.CustomerRef?.value ?? ""), - balance: Number(inv.Balance ?? 0), - total: Number(inv.TotalAmt ?? 0), - docNumber: inv.DocNumber ?? null, - }; -} - -/** Receive a payment against an invoice (full open balance). TEST/admin tooling. */ -export async function createQBPaymentForInvoice(tokens: QBTokens, qbInvoiceId: string): Promise<{ paymentId: string; amount: number } | null> { - const inv = await readQBInvoice(tokens, qbInvoiceId); - if (!inv || inv.balance <= 0 || !inv.customerId) return null; - const res = await qbFetch("/payment", tokens, { - method: "POST", - body: JSON.stringify({ - TotalAmt: inv.balance, - CustomerRef: { value: inv.customerId }, - Line: [{ Amount: inv.balance, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], - }), - }); - if (!res.ok) throw new Error(`QB payment create failed: ${await res.text()}`); - const p = (await res.json()).Payment; - return { paymentId: String(p.Id), amount: Number(p.TotalAmt ?? inv.balance) }; -} - -export type QBPaymentBuildFailure = - | { ok: false; reason: "invoice-not-found" } - | { ok: false; reason: "missing-customer" } - | { ok: false; reason: "balance-mismatch"; qbBalance: number; expected: number }; - -/** - * Build (but do not send) the exact JSON body for a Payment create against a - * specific amount/date/check-ref — split out from the deposit-ingest send - * step so a caller can PERSIST the body before the network call fires (the - * deposit-ingest endpoint's `qbo_unknown` recovery depends on the row already - * holding the exact bytes it's about to send, in case the process dies - * mid-request or the response is lost). Guards the QBO invoice's open balance - * against `opts.amount` to the cent — a deposit must exactly retire the - * milestone it matched, never partially settle it. - */ -export async function buildQBPaymentRequest( - tokens: QBTokens, - qbInvoiceId: string, - opts: { amount: number; txnDate: string; paymentRefNum: string }, -): Promise<{ ok: true; requestBody: string } | QBPaymentBuildFailure> { - // E2E_QBO_MOCK (deposit-ingest hermeticity, gated in quickbooks-mock.ts): - // skip the real readQBInvoice() network call entirely — the caller seeds - // this mock's invoice state via /api/payments/test-only/qbo-mock. - if (isE2eQboMockEnabled()) { - recordMockReadInvoiceCall(qbInvoiceId); - const inv = getMockQboInvoice(qbInvoiceId); - if (!inv) return { ok: false, reason: "invoice-not-found" }; - if (!inv.customerId) return { ok: false, reason: "missing-customer" }; - if (Math.round(inv.balance * 100) !== Math.round(opts.amount * 100)) { - return { ok: false, reason: "balance-mismatch", qbBalance: inv.balance, expected: opts.amount }; - } - const mockPayload = { - TotalAmt: opts.amount, - TxnDate: opts.txnDate, - PaymentRefNum: opts.paymentRefNum, - CustomerRef: { value: inv.customerId }, - Line: [{ Amount: opts.amount, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], - }; - return { ok: true, requestBody: JSON.stringify(mockPayload) }; - } - const inv = await readQBInvoice(tokens, qbInvoiceId); - if (!inv) return { ok: false, reason: "invoice-not-found" }; - if (!inv.customerId) return { ok: false, reason: "missing-customer" }; - if (Math.round(inv.balance * 100) !== Math.round(opts.amount * 100)) { - return { ok: false, reason: "balance-mismatch", qbBalance: inv.balance, expected: opts.amount }; - } - const payload = { - TotalAmt: opts.amount, - TxnDate: opts.txnDate, - PaymentRefNum: opts.paymentRefNum, - CustomerRef: { value: inv.customerId }, - Line: [{ Amount: opts.amount, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], - }; - return { ok: true, requestBody: JSON.stringify(payload) }; -} - -/** - * Send a Payment create request whose body was already built (by - * `buildQBPaymentRequest`) and possibly already persisted. The SAME function - * is the replay path: calling it again with the identical `requestBody` + - * `requestId` after a lost response returns Intuit's ORIGINAL response - * instead of creating a duplicate Payment (`requestid` is QBO's server-side - * idempotency key on the create) — see qbo-receipt-push.ts's requestid - * pattern, `?requestid=...` as a query param. - */ -export async function sendQBPaymentCreateRequest( - tokens: QBTokens, - requestBody: string, - requestId: string, -): Promise<{ paymentId: string; amount: number }> { - // E2E_QBO_MOCK: no network I/O — see quickbooks-mock.ts's doc comment. - // mockSendQBPaymentCreate replicates QBO's requestid dedupe (the SAME - // requestId always returns the SAME payment), which the qbo_unknown - // replay path below depends on. - if (isE2eQboMockEnabled()) { - return mockSendQBPaymentCreate(requestBody, requestId); - } - const res = await qbFetch(`/payment?requestid=${encodeURIComponent(requestId)}`, tokens, { - method: "POST", - body: requestBody, - }); - if (!res.ok) throw new Error(`QB payment create failed: ${await res.text()}`); - const data = await res.json().catch(() => null); - const p = data?.Payment; - if (!p?.Id) throw new Error("QB payment create returned no Payment body"); - return { paymentId: String(p.Id), amount: Number(p.TotalAmt ?? 0) }; -} - -/** - * Convenience wrapper for callers that don't need the persist-before-send - * seam: build + guard + send in one call. The deposit-ingest endpoint does - * NOT use this directly — it calls `buildQBPaymentRequest` and - * `sendQBPaymentCreateRequest` separately so it can commit the request body - * to the DepositIngest row between the two steps. - */ -export async function createQBPaymentForInvoiceWithDetails( - tokens: QBTokens, - qbInvoiceId: string, - opts: { amount: number; txnDate: string; paymentRefNum: string; requestId: string }, -): Promise<{ ok: true; paymentId: string; amount: number; requestBody: string } | QBPaymentBuildFailure> { - const built = await buildQBPaymentRequest(tokens, qbInvoiceId, opts); - if (!built.ok) return built; - const sent = await sendQBPaymentCreateRequest(tokens, built.requestBody, opts.requestId); - return { ok: true, paymentId: sent.paymentId, amount: sent.amount, requestBody: built.requestBody }; -} - -/** Hard-delete a payment (test cleanup). */ -export async function deleteQBPayment(tokens: QBTokens, paymentId: string): Promise { - const get = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); - if (!get.ok) return false; - const syncToken = String((await get.json()).Payment?.SyncToken ?? "0"); - const res = await fetch( - `${QB_API_BASE}/${tokens.realmId}/payment?operation=delete&minorversion=73`, - { - method: "POST", - headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json", "Content-Type": "application/json" }, - body: JSON.stringify({ Id: paymentId, SyncToken: syncToken }), - } - ); - return res.ok; -} - -/** Hard-delete an invoice (test cleanup). Fails in QBO if payments are still linked. */ -export async function deleteQBInvoice(tokens: QBTokens, qbInvoiceId: string): Promise { - const inv = await readQBInvoice(tokens, qbInvoiceId); - if (!inv) return false; - const res = await fetch( - `${QB_API_BASE}/${tokens.realmId}/invoice?operation=delete&minorversion=73`, - { - method: "POST", - headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json", "Content-Type": "application/json" }, - body: JSON.stringify({ Id: qbInvoiceId, SyncToken: inv.syncToken }), - } - ); - return res.ok; -} - -/** Read an invoice's online-payment toggles + sync token (for sparse updates). */ -export async function getQBInvoicePaymentOptions(tokens: QBTokens, qbInvoiceId: string) { - const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - if (!res.ok) return null; - const inv = (await res.json()).Invoice; - if (!inv) return null; - return { - syncToken: String(inv.SyncToken), - card: inv.AllowOnlineCreditCardPayment === true, - ach: inv.AllowOnlineACHPayment === true, - balance: Number(inv.Balance ?? 0), - }; -} - -/** Sparse-update an invoice's online-payment toggles (card / bank transfer). */ -export async function setQBInvoicePaymentOptions( - tokens: QBTokens, - qbInvoiceId: string, - syncToken: string, - opts: { card: boolean; ach: boolean } -): Promise { - const res = await qbFetch("/invoice", tokens, { - method: "POST", - body: JSON.stringify({ - Id: qbInvoiceId, - SyncToken: syncToken, - sparse: true, - AllowOnlineCreditCardPayment: opts.card, - AllowOnlineACHPayment: opts.ach, - }), - }); - return res.ok; -} - -/** Add customer-facing text before QBO sends its invoice email. */ -export async function appendQBInvoiceCustomerMemo( - tokens: QBTokens, - qbInvoiceId: string, - line: string, -): Promise<{ ok: true } | { ok: false; error: string }> { - const read = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - if (!read.ok) return { ok: false, error: `Could not read QuickBooks invoice (${read.status})` }; - const invoice = (await read.json().catch(() => null))?.Invoice; - if (!invoice?.SyncToken) return { ok: false, error: "QuickBooks invoice response was incomplete" }; - - const current = String(invoice.CustomerMemo?.value ?? "").trim(); - if (current.includes(line)) return { ok: true }; - const update = await qbFetch("/invoice", tokens, { - method: "POST", - body: JSON.stringify({ - Id: qbInvoiceId, - SyncToken: String(invoice.SyncToken), - sparse: true, - CustomerMemo: { value: [current, line].filter(Boolean).join("\n\n").slice(0, 1000) }, - }), - }); - if (!update.ok) return { ok: false, error: `Could not add the backup link to the QuickBooks invoice (${update.status})` }; - return { ok: true }; -} - -/** Posted money-out transactions (expenses/checks/card charges) from the books. */ -export async function getRecentQBPurchases(tokens: QBTokens, sinceDaysAgo: number) { - const since = new Date(Date.now() - sinceDaysAgo * 86_400_000).toISOString().split("T")[0]; - const rows = await getQBPurchasesSince(tokens, new Date(`${since}T00:00:00.000Z`)); - return rows.map(p => ({ - qbId: String(p.Id), - date: p.TxnDate ?? null, - amount: Number(p.TotalAmt ?? 0), - paymentType: p.PaymentType ?? null, // Cash | Check | CreditCard - docNumber: p.DocNumber ?? null, - vendor: p.EntityRef?.name ?? null, - account: p.AccountRef?.name ?? null, - memo: p.PrivateNote ?? null, - })); -} - -/** - * Read all posted QBO Purchase rows on or after a transaction date. - * Pagination matters for the initial historical backfill; a single QBO query - * page would silently stop after its MAXRESULTS boundary. - */ -export async function getQBPurchasesSince(tokens: QBTokens, since: Date, until?: Date): Promise { - if (!Number.isFinite(since.getTime())) { - throw new Error("QBO purchase query requires a valid since date"); - } - if (until && !Number.isFinite(until.getTime())) { - throw new Error("QBO purchase query requires a valid until date"); - } - - const sinceDate = since.toISOString().slice(0, 10); - // Inclusive upper bound so callers can chunk a long backfill into - // date windows that each finish within the serverless duration limit. - const untilClause = until ? ` AND TxnDate <= '${until.toISOString().slice(0, 10)}'` : ""; - const pageSize = 1000; - const purchases: any[] = []; - - for (let startPosition = 1; ; startPosition += pageSize) { - const page = await qbQuery( - tokens, - `SELECT * FROM Purchase WHERE TxnDate >= '${sinceDate}'${untilClause} ORDERBY TxnDate ASC STARTPOSITION ${startPosition} MAXRESULTS ${pageSize}`, - ); - purchases.push(...page); - if (page.length < pageSize) break; - } - - return purchases; -} - -/** - * Read Purchase rows changed since a timestamp using QBO Change Data Capture. - * Unlike a TxnDate query, CDC catches newly entered backdated purchases, - * corrections, voids, refunds, and deletion tombstones. QBO caps CDC lookback - * at 30 days and returns at most 1,000 entities, so truncated responses fail - * visibly instead of silently leaving local job costs stale. - */ -export async function getQBPurchaseChangesSince( - tokens: QBTokens, - since: Date, -): Promise { - if (!Number.isFinite(since.getTime())) { - throw new Error("QBO Purchase CDC requires a valid since date"); - } - - const params = new URLSearchParams({ - entities: "Purchase", - changedSince: since.toISOString(), - minorversion: "73", - }); - const response = await fetch( - `${QB_API_BASE}/${tokens.realmId}/cdc?${params.toString()}`, - { - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - Accept: "application/json", - }, - }, - ); - if (!response.ok) { - throw new Error(`QBO Purchase CDC failed with status ${response.status}`); - } - - const payload = await response.json(); - const cdcResponses = Array.isArray(payload?.CDCResponse) - ? payload.CDCResponse - : []; - const queryResponses = cdcResponses.flatMap((entry: any) => - Array.isArray(entry?.QueryResponse) ? entry.QueryResponse : [], - ); - const purchases: any[] = []; - for (const queryResponse of queryResponses) { - const page = Array.isArray(queryResponse?.Purchase) - ? queryResponse.Purchase - : []; - const totalCount = Number(queryResponse?.totalCount ?? page.length); - if ( - page.length >= 1000 || - (Number.isFinite(totalCount) && totalCount > page.length) - ) { - throw new Error("QBO Purchase CDC response was truncated"); - } - purchases.push(...page); - } - - // Keep the last representation if QBO includes the same id more than once. - const byId = new Map(); - const withoutId: any[] = []; - for (const purchase of purchases) { - const id = purchase?.Id === undefined ? "" : String(purchase.Id); - if (id) byId.set(id, purchase); - else withoutId.push(purchase); - } - return [...byId.values(), ...withoutId]; -} - -/** Posted customer payments (money in) from the books. */ -export async function getRecentQBPaymentsList(tokens: QBTokens, sinceDaysAgo: number) { - const since = new Date(Date.now() - sinceDaysAgo * 86_400_000).toISOString().split("T")[0]; - const rows = await qbQuery(tokens, `SELECT * FROM Payment WHERE TxnDate >= '${since}' ORDERBY TxnDate DESC MAXRESULTS 500`); - return rows.map(p => ({ - qbId: String(p.Id), - date: p.TxnDate ?? null, - amount: Number(p.TotalAmt ?? 0), - customer: p.CustomerRef?.name ?? null, - reference: p.PaymentRefNum ?? null, - })); -} - -/** The estimate-item shape `buildQBEstimateLines` needs: billing figures plus enough - * hierarchy (`id`/`parentId`/`type`) to tell section headers from billable leaves. */ -export type QBEstimateItem = { - // Required, not optional: a caller that omits the hierarchy silently loses legacy - // section detection (a section is only recognizable by its type tag OR its children), - // which is exactly the bug this function exists to prevent. - id: string; - parentId: string | null; - name: string; - quantity: number; - unitCost: number; - total: number; - type: string; -}; - -/** - * Billable QB estimate lines, one per LEAF row. - * - * Section headers are dropped. A section's stored total is a roll-up of its children, so - * emitting it as a line bills that amount a second time on top of the child rows it - * summarizes — a nested section double-counts twice over (an outer section holding a $250 - * inner section plus a $25 leaf shipped $800 of lines against a $275 subtotal). - * - * The filter lives here rather than in the caller so any future caller inherits it; it uses - * the same `isEstimateSectionRow` predicate as the editor subtotal and the PDF, so all three - * readers agree on which rows are headers. - * - * The returned amounts sum to the estimate's pre-tax SUBTOTAL, which is not the same as its - * stored `totalAmount` (that column also carries tax and any processing-fee markup). QBO - * computes its own sales tax on the lines it receives, so pushing a tax line here would - * double-charge; the processing-fee gap is a separate open question — see the callers. - */ -export function buildQBEstimateLines(items: readonly QBEstimateItem[], itemId: string) { - return items - .filter(item => !isEstimateSectionRow(item, items)) - .map((item, i) => ({ - LineNum: i + 1, - Description: item.name, - Amount: item.total, - DetailType: "SalesItemLineDetail", - SalesItemLineDetail: { - ItemRef: { value: itemId }, - Qty: item.quantity, - UnitPrice: item.unitCost, - }, - })); -} - -/** Push an estimate to QB. Returns the QB estimate ID. */ -export async function syncEstimateToQB( - tokens: QBTokens, - estimate: { - id: string; - code: string; - title: string; - totalAmount: number; - items: QBEstimateItem[]; - customerId: string; - itemId: string; - project: { name: string } | null; - }, - glMappings: Record = {} -): Promise<{ qbId: string; qbUrl: string }> { - const lines = buildQBEstimateLines(estimate.items, estimate.itemId); - - // QBO rejects a transaction with no lines (error 2020, "Required param missing"). An - // estimate that is empty, or that is nothing but section headers, reaches this point with - // everything filtered out — fail with something legible instead of a raw QB API error. - if (!lines.length) { - throw new Error("QB estimate sync failed: estimate has no billable line items"); - } - - const payload = { - TxnDate: new Date().toISOString().split("T")[0], - DocNumber: estimate.code.slice(0, 21), - PrivateNote: estimate.title, - CustomerRef: { value: estimate.customerId }, - Line: lines, - }; - - const res = await qbFetch("/estimate", tokens, { - method: "POST", - body: JSON.stringify(payload), - }); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`QB estimate sync failed: ${err}`); - } - - const data = await res.json(); - const qbId = data.Estimate?.Id; - const realmId = tokens.realmId; - const qbUrl = `https://app.qbo.intuit.com/app/estimate?txnId=${qbId}`; - - return { qbId, qbUrl }; -} - -/** Push an invoice to QB. Returns the QB invoice ID. */ -export async function syncInvoiceToQB( - tokens: QBTokens, - invoice: { - code: string; - totalAmount: number; - balanceDue: number; - customerId: string; - itemId: string; - project: { name: string } | null; - items?: Array<{ description: string; amount: number }>; - } -): Promise<{ qbId: string; qbUrl: string }> { - const lines: object[] = (invoice.items || []).map((item, i) => ({ - LineNum: i + 1, - Description: item.description, - Amount: item.amount, - DetailType: "SalesItemLineDetail", - SalesItemLineDetail: { ItemRef: { value: invoice.itemId }, Qty: 1, UnitPrice: item.amount }, - })); - - if (lines.length === 0) { - lines.push({ - LineNum: 1, - Description: invoice.project?.name || "Construction Services", - Amount: invoice.totalAmount, - DetailType: "SalesItemLineDetail", - SalesItemLineDetail: { ItemRef: { value: invoice.itemId }, Qty: 1, UnitPrice: invoice.totalAmount }, - }); - } - - const payload = { - DocNumber: invoice.code.slice(0, 21), - TxnDate: new Date().toISOString().split("T")[0], - CustomerRef: { value: invoice.customerId }, - Line: lines, - }; - - const res = await qbFetch("/invoice", tokens, { - method: "POST", - body: JSON.stringify(payload), - }); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`QB invoice sync failed: ${err}`); - } - - const data = await res.json(); - const qbId = data.Invoice?.Id; - const qbUrl = `https://app.qbo.intuit.com/app/invoice?txnId=${qbId}`; - return { qbId, qbUrl }; -} - -/** Send a QBO invoice email to a client. */ -export async function sendQBInvoice(tokens: QBTokens, qbInvoiceId: string, sendTo?: string | null) { - const qs = new URLSearchParams({ minorversion: "73" }); - if (sendTo) qs.set("sendTo", sendTo); - const url = `${QB_API_BASE}/${tokens.realmId}/invoice/${qbInvoiceId}/send?${qs}`; - const res = await fetch(url, { - method: "POST", - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - Accept: "application/json", - "Content-Type": "application/octet-stream", - }, - }); - if (!res.ok) return { ok: false as const, status: res.status, error: await res.text() }; - const data = await res.json().catch(() => ({})); - return { ok: true as const, status: res.status, emailStatus: data.Invoice?.EmailStatus ?? null }; -} +/** + * QuickBooks Online API client. + * Uses OAuth2 tokens stored in integration-store. + * Docs: https://developer.intuit.com/app/developer/qbo/docs/api/accounting + */ +import { + isE2eQboMockEnabled, + recordMockReadInvoiceCall, + getMockQboInvoice, + mockSendQBPaymentCreate, +} from "./quickbooks-mock"; +import { isEstimateSectionRow } from "./estimate-item-payload"; + +export const QB_API_BASE = process.env.QB_SANDBOX === "true" + ? "https://sandbox-quickbooks.api.intuit.com/v3/company" + : "https://quickbooks.api.intuit.com/v3/company"; + +const TOKEN_URL = "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer"; + +export interface QBTokens { + accessToken: string; + refreshToken: string; + realmId: string; +} + +/** + * Raised when a QuickBooks HTTP call exceeded its own deadline — distinct from + * any other network error so callers can treat it as "QBO is unreachable right + * now, retry later" instead of a business failure. + */ +export class QBTimeoutError extends Error { + name = "QBTimeoutError"; +} + +const QB_DEFAULT_TIMEOUT_MS = 20_000; + +/** Path only — never the query string (it can carry the realm/query) or a token. */ +function safePath(url: string): string { + try { + return new URL(url).pathname; + } catch { + return "(unparseable url)"; + } +} + +/** + * Every QuickBooks HTTP call goes through here. + * + * Bare `fetch()` has NO default timeout, so an Intuit outage (2026-09-01) left + * each request hanging until Vercel killed the whole function at its + * maxDuration — a receipt push burned 60s and a cron 120s to learn nothing. + * A per-request deadline turns that into a fast, classifiable failure. + * + * A caller-supplied `signal` still wins on abort (combined via + * `AbortSignal.any` where available); only OUR deadline firing is rethrown as + * QBTimeoutError. Every other error passes through untouched. + */ +export async function qbTimedFetch( + url: string, + init: RequestInit = {}, + timeoutMs: number = Number(process.env.QB_FETCH_TIMEOUT_MS) || QB_DEFAULT_TIMEOUT_MS, +): Promise { + // A misconfigured env var must not break every QB call: AbortSignal.timeout + // rejects a non-positive delay outright. + const effectiveMs = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : QB_DEFAULT_TIMEOUT_MS; + const timeoutSignal = AbortSignal.timeout(effectiveMs); + + const callerSignal = init.signal; + let signal: AbortSignal = timeoutSignal; + if (callerSignal) { + const anyOf = (AbortSignal as unknown as { any?: (signals: AbortSignal[]) => AbortSignal }).any; + signal = typeof anyOf === "function" ? anyOf([callerSignal, timeoutSignal]) : callerSignal; + } + + try { + return await fetch(url, { ...init, signal }); + } catch (error) { + const aborted = + error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError"); + // Only claim a timeout when OUR deadline is the one that fired — a + // caller cancelling its own request is not a QBO outage. + if (aborted && timeoutSignal.aborted && !callerSignal?.aborted) { + throw new QBTimeoutError( + `QuickBooks request timed out after ${effectiveMs}ms: ${safePath(url)}`, + ); + } + throw error; + } +} + +/** Exchange authorization code for tokens */ +export async function exchangeQBCode(code: string, redirectUri: string): Promise { + const clientId = process.env.QB_CLIENT_ID!; + const clientSecret = process.env.QB_CLIENT_SECRET!; + const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); + + const res = await qbTimedFetch(TOKEN_URL, { + method: "POST", + headers: { + Authorization: `Basic ${encoded}`, + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: redirectUri, + }), + }); + + if (!res.ok) { + const err = await res.text(); + throw new Error(`QB token exchange failed: ${err}`); + } + + const data = await res.json(); + return { + accessToken: data.access_token, + refreshToken: data.refresh_token, + realmId: "", // set from callback query param + }; +} + +/** Refresh an expired access token */ +export async function refreshQBToken(refreshToken: string): Promise<{ accessToken: string; refreshToken: string }> { + const clientId = process.env.QB_CLIENT_ID!; + const clientSecret = process.env.QB_CLIENT_SECRET!; + const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); + + const res = await qbTimedFetch(TOKEN_URL, { + method: "POST", + headers: { + Authorization: `Basic ${encoded}`, + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body: new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + }), + }); + + if (!res.ok) throw new Error("QB token refresh failed"); + const data = await res.json(); + return { accessToken: data.access_token, refreshToken: data.refresh_token }; +} + +/** Make an authenticated call to the QB API, auto-refreshing if needed */ +export async function qbFetch( + path: string, + tokens: QBTokens, + opts: RequestInit = {} +): Promise { + // Callers that already put their own query string on `path` (e.g. + // "/purchase?requestid=...") get "&minorversion=73" appended instead of a + // second "?" — every existing call site passes a bare path, so this is + // backward compatible. + const separator = path.includes("?") ? "&" : "?"; + const url = `${QB_API_BASE}/${tokens.realmId}${path}${separator}minorversion=73`; + return qbTimedFetch(url, { + ...opts, + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + Accept: "application/json", + "Content-Type": "application/json", + ...opts.headers, + }, + }); +} + +/** Run a QBO SQL-ish query (https://developer.intuit.com/.../data-queries) */ +export async function qbQuery(tokens: QBTokens, query: string): Promise { + const url = `${QB_API_BASE}/${tokens.realmId}/query?query=${encodeURIComponent(query)}&minorversion=73`; + const res = await qbTimedFetch(url, { + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + Accept: "application/json", + }, + }); + if (!res.ok) { + const err = await res.text(); + throw new Error(`QB query failed: ${err}`); + } + const data = await res.json(); + const response = data.QueryResponse || {}; + const key = Object.keys(response).find(k => Array.isArray(response[k])); + return key ? response[key] : []; +} + +export function escapeQBString(s: string): string { + // Backslash MUST be escaped before the apostrophe escape, or an input + // ending in a literal backslash (e.g. "Smith\\") would have its escaped + // apostrophe's own backslash re-escaped, breaking out of the quoted string. + return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); +} + +export interface QBAttachable { + Id?: string; + FileName?: string; + ContentType?: string; + Size?: number; + TempDownloadUri?: string; + AttachableRef?: Array<{ EntityRef?: { value?: string; type?: string } }>; +} + +/** List file attachments linked to a QBO Purchase (receipt images/PDFs). */ +export async function getQBPurchaseAttachables( + tokens: QBTokens, + purchaseId: string, +): Promise { + // QBO transaction ids are numeric; refuse anything else rather than escape it. + if (!/^\d+$/.test(purchaseId)) return []; + const rows = await qbQuery( + tokens, + `SELECT * FROM attachable WHERE AttachableRef.EntityRef.value = '${purchaseId}'`, + ); + // Entity ids are only unique per entity type, so the value-only query can + // surface attachments from other transaction types — keep Purchase links. + return rows.filter(row => + row.AttachableRef?.some( + ref => + ref.EntityRef?.value === purchaseId && + /^purchase$/i.test(ref.EntityRef?.type ?? ""), + ), + ); +} + +/** Find a QBO customer by display name, creating it if missing. Returns the QBO customer Id. */ +export async function ensureQBCustomer( + tokens: QBTokens, + client: { name: string; email?: string | null; qbCustomerId?: string | null } +): Promise { + // Trust a previously stored id if it still exists + if (client.qbCustomerId) { + const existing = await qbQuery(tokens, `SELECT Id FROM Customer WHERE Id = '${escapeQBString(client.qbCustomerId)}'`); + if (existing.length > 0) return client.qbCustomerId; + } + + const name = client.name.trim(); + if (!name) throw new Error("Client name is empty — cannot sync customer to QuickBooks."); + const byName = await qbQuery(tokens, `SELECT Id FROM Customer WHERE DisplayName = '${escapeQBString(name)}'`); + if (byName.length > 0) return byName[0].Id; + + // QBO normalizes whitespace when enforcing DisplayName uniqueness, so an + // exact match can miss while create still rejects as a duplicate (fault 6240). + // Prefix on the first word only, so internal-whitespace variants still match. + const normalize = (s: string) => s.replace(/\s+/g, " ").trim().toLowerCase(); + const prefix = name.split(/\s+/)[0]; + const candidates = await qbQuery<{ Id: string; DisplayName?: string }>( + tokens, + `SELECT Id, DisplayName FROM Customer WHERE DisplayName LIKE '${escapeQBString(prefix)}%' MAXRESULTS 1000` + ); + const matches = candidates.filter(c => normalize(c.DisplayName ?? "") === normalize(name)); + if (matches.length > 1) { + throw new Error(`QB customer lookup for "${name}" matched ${matches.length} customers — resolve the duplicate in QuickBooks.`); + } + if (matches.length === 1) return matches[0].Id; + + const res = await qbFetch("/customer", tokens, { + method: "POST", + body: JSON.stringify({ + DisplayName: name, + ...(client.email ? { PrimaryEmailAddr: { Address: client.email } } : {}), + }), + }); + if (!res.ok) { + const err = await res.text(); + throw new Error(`QB customer create failed: ${err}`); + } + const data = await res.json(); + return data.Customer.Id; +} + +const QB_SERVICE_ITEM_NAME = "Construction Services"; + +/** Find or create the Service item used for all ProBuild invoice lines. */ +export async function ensureQBServiceItem(tokens: QBTokens): Promise { + const items = await qbQuery(tokens, `SELECT Id FROM Item WHERE Name = '${escapeQBString(QB_SERVICE_ITEM_NAME)}'`); + if (items.length > 0) return items[0].Id; + + // Need an income account to hang the item on — prefer an existing Income account. + const accounts = await qbQuery(tokens, `SELECT Id, Name FROM Account WHERE AccountType = 'Income' MAXRESULTS 1`); + let incomeAccountId: string; + if (accounts.length > 0) { + incomeAccountId = accounts[0].Id; + } else { + const created = await qbFetch("/account", tokens, { + method: "POST", + body: JSON.stringify({ Name: "Construction Income", AccountType: "Income", AccountSubType: "ServiceFeeIncome" }), + }); + if (!created.ok) throw new Error(`QB income account create failed: ${await created.text()}`); + incomeAccountId = (await created.json()).Account.Id; + } + + const res = await qbFetch("/item", tokens, { + method: "POST", + body: JSON.stringify({ + Name: QB_SERVICE_ITEM_NAME, + Type: "Service", + IncomeAccountRef: { value: incomeAccountId }, + }), + }); + if (!res.ok) throw new Error(`QB service item create failed: ${await res.text()}`); + return (await res.json()).Item.Id; +} + +/** + * Create a QBO invoice for ONE payment milestone, with QuickBooks Payments + * (card + ACH) enabled so the customer gets Intuit's hosted "Review & Pay" page. + */ +export async function createQBMilestoneInvoice( + tokens: QBTokens, + input: { + docNumber: string; // ≤ 21 chars + customerId: string; + itemId: string; + description: string; + amount: number; // grand total the client pays (tax-inclusive) + // When set, the QBO invoice carries the sales tax explicitly: + // a pre-tax taxable line + TxnTaxDetail, so QBO's sales-tax reporting + // sees the liability and the invoice total still equals `amount`. + tax?: { preTaxAmount: number; taxAmount: number } | null; + dueDate?: Date | null; + billEmail?: string | null; + privateNote?: string; + } +): Promise<{ qbId: string; qbUrl: string; total: number }> { + const withTax = !!input.tax && input.tax.taxAmount > 0; + const lineAmount = withTax ? input.tax!.preTaxAmount : input.amount; + + const payload: Record = { + DocNumber: input.docNumber.slice(0, 21), + TxnDate: new Date().toISOString().split("T")[0], + CustomerRef: { value: input.customerId }, + // QuickBooks Payments is the ONLY payment rail (Stripe is disabled until + // their 180-day hold clears) — the hosted page takes card, debit, AND bank. + // Note: Intuit can't surcharge, so card fees are merchant-absorbed. + AllowOnlineCreditCardPayment: true, + AllowOnlineACHPayment: true, + ...(input.billEmail ? { BillEmail: { Address: input.billEmail } } : {}), + ...(input.dueDate ? { DueDate: input.dueDate.toISOString().split("T")[0] } : {}), + ...(input.privateNote ? { PrivateNote: input.privateNote.slice(0, 4000) } : {}), + Line: [ + { + LineNum: 1, + Description: input.description.slice(0, 4000), + Amount: lineAmount, + DetailType: "SalesItemLineDetail", + SalesItemLineDetail: { + ItemRef: { value: input.itemId }, + Qty: 1, + UnitPrice: lineAmount, + ...(withTax ? { TaxCodeRef: { value: "TAX" } } : {}), + }, + }, + ], + ...(withTax ? { TxnTaxDetail: { TotalTax: input.tax!.taxAmount } } : {}), + }; + + const res = await qbFetch("/invoice", tokens, { method: "POST", body: JSON.stringify(payload) }); + if (!res.ok) throw new Error(`QB milestone invoice create failed: ${await res.text()}`); + const data = await res.json(); + const qbId = data.Invoice?.Id; + const total = Number(data.Invoice?.TotalAmt ?? 0); + return { qbId, qbUrl: `https://app.qbo.intuit.com/app/invoice?txnId=${qbId}`, total }; +} + +/** Fetch the customer-facing payment link for a QBO invoice (requires QB Payments enabled). */ +export async function getQBInvoicePaymentLink(tokens: QBTokens, qbInvoiceId: string): Promise { + const url = `${QB_API_BASE}/${tokens.realmId}/invoice/${qbInvoiceId}?include=invoiceLink&minorversion=73`; + const res = await qbTimedFetch(url, { + headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json" }, + }); + if (!res.ok) return null; + const data = await res.json(); + return data.Invoice?.InvoiceLink || null; +} + +export interface QBInvoiceStatus { + balance: number; + total: number; + paymentTxnIds: string[]; +} + +/** Read a QBO invoice's balance + linked payment transactions. */ +export async function getQBInvoiceStatus(tokens: QBTokens, qbInvoiceId: string): Promise { + const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + if (!res.ok) return null; + const data = await res.json(); + const inv = data.Invoice; + if (!inv) return null; + const paymentTxnIds: string[] = (inv.LinkedTxn || []) + .filter((t: any) => t.TxnType === "Payment") + .map((t: any) => String(t.TxnId)); + return { balance: Number(inv.Balance ?? 0), total: Number(inv.TotalAmt ?? 0), paymentTxnIds }; +} + +/** + * Result of probing a QBO invoice's existence + payable state. + * Unlike getQBInvoiceStatus (which collapses every failure into null), this + * distinguishes a permanently gone/voided invoice from a transient API error, + * so the sync poller can flag stuck milestones without acting on a blip. + */ +export type QBInvoiceProbe = + | { state: "ok"; balance: number; total: number; paymentTxnIds: string[] } + | { state: "voided" } // HTTP 200, exists, total & balance === 0, no linked payments + | { state: "notFound" } // HTTP 400 Fault 610 or HTTP 404 (authoritative "gone" only) + | { state: "error"; status: number }; // 401/429/5xx/network/malformed — transient, never act on + +/** + * Probe a QBO invoice and classify it. QBO's behavior for gone invoices is + * inconsistent: a *voided* invoice returns 200 with TotalAmt=0; a *deleted* one + * may return 400 + Fault code 610 ("Object Not Found"), a 404, or even 200 with + * stale data. This folds all of those into a single discriminated result. + */ +export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Promise { + let res: Response; + try { + res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + } catch { + return { state: "error", status: 0 }; + } + if (res.ok) { + // A 200 should always carry an Invoice. A parse failure or a missing payload + // is anomalous — treat it as transient (never as "gone"); only an explicit + // 404/610 below is authoritative for notFound. + let data: any; + try { + data = await res.json(); + } catch { + return { state: "error", status: res.status }; + } + const inv = data?.Invoice; + if (!inv) return { state: "error", status: res.status }; + const total = Number(inv.TotalAmt); + const balance = Number(inv.Balance); + // A well-formed invoice always carries numeric TotalAmt/Balance. Missing or + // non-finite values mean a malformed/partial payload — treat as transient, + // never as voided (which would false-alarm an otherwise-healthy milestone). + if (!Number.isFinite(total) || !Number.isFinite(balance)) return { state: "error", status: res.status }; + const paymentTxnIds: string[] = (inv.LinkedTxn || []) + .filter((t: any) => t.TxnType === "Payment") + .map((t: any) => String(t.TxnId)); + // Voided invoices come back 200 with TotalAmt=0, Balance=0, and no linked payments. + if (total === 0 && balance === 0 && paymentTxnIds.length === 0) return { state: "voided" }; + return { state: "ok", balance, total, paymentTxnIds }; + } + if (res.status === 404) return { state: "notFound" }; + const body = await res.text().catch(() => ""); + if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { + return { state: "notFound" }; + } + return { state: "error", status: res.status }; +} + +/** Read a QBO payment (date / amount / reference) for receipt details. */ +export async function getQBPayment( + tokens: QBTokens, + paymentId: string +): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null> { + const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); + if (!res.ok) return null; + const data = await res.json(); + const p = data.Payment; + if (!p) return null; + return { + txnDate: p.TxnDate || null, + amount: Number(p.TotalAmt ?? 0), + referenceNumber: p.PaymentRefNum || null, + }; +} + +/** Read core invoice fields needed for payments/deletes. */ +export async function readQBInvoice(tokens: QBTokens, qbInvoiceId: string) { + const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + if (!res.ok) return null; + const inv = (await res.json()).Invoice; + if (!inv) return null; + return { + syncToken: String(inv.SyncToken), + customerId: String(inv.CustomerRef?.value ?? ""), + balance: Number(inv.Balance ?? 0), + total: Number(inv.TotalAmt ?? 0), + docNumber: inv.DocNumber ?? null, + }; +} + +/** Receive a payment against an invoice (full open balance). TEST/admin tooling. */ +export async function createQBPaymentForInvoice(tokens: QBTokens, qbInvoiceId: string): Promise<{ paymentId: string; amount: number } | null> { + const inv = await readQBInvoice(tokens, qbInvoiceId); + if (!inv || inv.balance <= 0 || !inv.customerId) return null; + const res = await qbFetch("/payment", tokens, { + method: "POST", + body: JSON.stringify({ + TotalAmt: inv.balance, + CustomerRef: { value: inv.customerId }, + Line: [{ Amount: inv.balance, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], + }), + }); + if (!res.ok) throw new Error(`QB payment create failed: ${await res.text()}`); + const p = (await res.json()).Payment; + return { paymentId: String(p.Id), amount: Number(p.TotalAmt ?? inv.balance) }; +} + +export type QBPaymentBuildFailure = + | { ok: false; reason: "invoice-not-found" } + | { ok: false; reason: "missing-customer" } + | { ok: false; reason: "balance-mismatch"; qbBalance: number; expected: number }; + +/** + * Build (but do not send) the exact JSON body for a Payment create against a + * specific amount/date/check-ref — split out from the deposit-ingest send + * step so a caller can PERSIST the body before the network call fires (the + * deposit-ingest endpoint's `qbo_unknown` recovery depends on the row already + * holding the exact bytes it's about to send, in case the process dies + * mid-request or the response is lost). Guards the QBO invoice's open balance + * against `opts.amount` to the cent — a deposit must exactly retire the + * milestone it matched, never partially settle it. + */ +export async function buildQBPaymentRequest( + tokens: QBTokens, + qbInvoiceId: string, + opts: { amount: number; txnDate: string; paymentRefNum: string }, +): Promise<{ ok: true; requestBody: string } | QBPaymentBuildFailure> { + // E2E_QBO_MOCK (deposit-ingest hermeticity, gated in quickbooks-mock.ts): + // skip the real readQBInvoice() network call entirely — the caller seeds + // this mock's invoice state via /api/payments/test-only/qbo-mock. + if (isE2eQboMockEnabled()) { + recordMockReadInvoiceCall(qbInvoiceId); + const inv = getMockQboInvoice(qbInvoiceId); + if (!inv) return { ok: false, reason: "invoice-not-found" }; + if (!inv.customerId) return { ok: false, reason: "missing-customer" }; + if (Math.round(inv.balance * 100) !== Math.round(opts.amount * 100)) { + return { ok: false, reason: "balance-mismatch", qbBalance: inv.balance, expected: opts.amount }; + } + const mockPayload = { + TotalAmt: opts.amount, + TxnDate: opts.txnDate, + PaymentRefNum: opts.paymentRefNum, + CustomerRef: { value: inv.customerId }, + Line: [{ Amount: opts.amount, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], + }; + return { ok: true, requestBody: JSON.stringify(mockPayload) }; + } + const inv = await readQBInvoice(tokens, qbInvoiceId); + if (!inv) return { ok: false, reason: "invoice-not-found" }; + if (!inv.customerId) return { ok: false, reason: "missing-customer" }; + if (Math.round(inv.balance * 100) !== Math.round(opts.amount * 100)) { + return { ok: false, reason: "balance-mismatch", qbBalance: inv.balance, expected: opts.amount }; + } + const payload = { + TotalAmt: opts.amount, + TxnDate: opts.txnDate, + PaymentRefNum: opts.paymentRefNum, + CustomerRef: { value: inv.customerId }, + Line: [{ Amount: opts.amount, LinkedTxn: [{ TxnId: qbInvoiceId, TxnType: "Invoice" }] }], + }; + return { ok: true, requestBody: JSON.stringify(payload) }; +} + +/** + * Send a Payment create request whose body was already built (by + * `buildQBPaymentRequest`) and possibly already persisted. The SAME function + * is the replay path: calling it again with the identical `requestBody` + + * `requestId` after a lost response returns Intuit's ORIGINAL response + * instead of creating a duplicate Payment (`requestid` is QBO's server-side + * idempotency key on the create) — see qbo-receipt-push.ts's requestid + * pattern, `?requestid=...` as a query param. + */ +export async function sendQBPaymentCreateRequest( + tokens: QBTokens, + requestBody: string, + requestId: string, +): Promise<{ paymentId: string; amount: number }> { + // E2E_QBO_MOCK: no network I/O — see quickbooks-mock.ts's doc comment. + // mockSendQBPaymentCreate replicates QBO's requestid dedupe (the SAME + // requestId always returns the SAME payment), which the qbo_unknown + // replay path below depends on. + if (isE2eQboMockEnabled()) { + return mockSendQBPaymentCreate(requestBody, requestId); + } + const res = await qbFetch(`/payment?requestid=${encodeURIComponent(requestId)}`, tokens, { + method: "POST", + body: requestBody, + }); + if (!res.ok) throw new Error(`QB payment create failed: ${await res.text()}`); + const data = await res.json().catch(() => null); + const p = data?.Payment; + if (!p?.Id) throw new Error("QB payment create returned no Payment body"); + return { paymentId: String(p.Id), amount: Number(p.TotalAmt ?? 0) }; +} + +/** + * Convenience wrapper for callers that don't need the persist-before-send + * seam: build + guard + send in one call. The deposit-ingest endpoint does + * NOT use this directly — it calls `buildQBPaymentRequest` and + * `sendQBPaymentCreateRequest` separately so it can commit the request body + * to the DepositIngest row between the two steps. + */ +export async function createQBPaymentForInvoiceWithDetails( + tokens: QBTokens, + qbInvoiceId: string, + opts: { amount: number; txnDate: string; paymentRefNum: string; requestId: string }, +): Promise<{ ok: true; paymentId: string; amount: number; requestBody: string } | QBPaymentBuildFailure> { + const built = await buildQBPaymentRequest(tokens, qbInvoiceId, opts); + if (!built.ok) return built; + const sent = await sendQBPaymentCreateRequest(tokens, built.requestBody, opts.requestId); + return { ok: true, paymentId: sent.paymentId, amount: sent.amount, requestBody: built.requestBody }; +} + +/** Hard-delete a payment (test cleanup). */ +export async function deleteQBPayment(tokens: QBTokens, paymentId: string): Promise { + const get = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); + if (!get.ok) return false; + const syncToken = String((await get.json()).Payment?.SyncToken ?? "0"); + const res = await qbTimedFetch( + `${QB_API_BASE}/${tokens.realmId}/payment?operation=delete&minorversion=73`, + { + method: "POST", + headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json", "Content-Type": "application/json" }, + body: JSON.stringify({ Id: paymentId, SyncToken: syncToken }), + } + ); + return res.ok; +} + +/** Hard-delete an invoice (test cleanup). Fails in QBO if payments are still linked. */ +export async function deleteQBInvoice(tokens: QBTokens, qbInvoiceId: string): Promise { + const inv = await readQBInvoice(tokens, qbInvoiceId); + if (!inv) return false; + const res = await qbTimedFetch( + `${QB_API_BASE}/${tokens.realmId}/invoice?operation=delete&minorversion=73`, + { + method: "POST", + headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json", "Content-Type": "application/json" }, + body: JSON.stringify({ Id: qbInvoiceId, SyncToken: inv.syncToken }), + } + ); + return res.ok; +} + +/** Read an invoice's online-payment toggles + sync token (for sparse updates). */ +export async function getQBInvoicePaymentOptions(tokens: QBTokens, qbInvoiceId: string) { + const res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + if (!res.ok) return null; + const inv = (await res.json()).Invoice; + if (!inv) return null; + return { + syncToken: String(inv.SyncToken), + card: inv.AllowOnlineCreditCardPayment === true, + ach: inv.AllowOnlineACHPayment === true, + balance: Number(inv.Balance ?? 0), + }; +} + +/** Sparse-update an invoice's online-payment toggles (card / bank transfer). */ +export async function setQBInvoicePaymentOptions( + tokens: QBTokens, + qbInvoiceId: string, + syncToken: string, + opts: { card: boolean; ach: boolean } +): Promise { + const res = await qbFetch("/invoice", tokens, { + method: "POST", + body: JSON.stringify({ + Id: qbInvoiceId, + SyncToken: syncToken, + sparse: true, + AllowOnlineCreditCardPayment: opts.card, + AllowOnlineACHPayment: opts.ach, + }), + }); + return res.ok; +} + +/** Add customer-facing text before QBO sends its invoice email. */ +export async function appendQBInvoiceCustomerMemo( + tokens: QBTokens, + qbInvoiceId: string, + line: string, +): Promise<{ ok: true } | { ok: false; error: string }> { + const read = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + if (!read.ok) return { ok: false, error: `Could not read QuickBooks invoice (${read.status})` }; + const invoice = (await read.json().catch(() => null))?.Invoice; + if (!invoice?.SyncToken) return { ok: false, error: "QuickBooks invoice response was incomplete" }; + + const current = String(invoice.CustomerMemo?.value ?? "").trim(); + if (current.includes(line)) return { ok: true }; + const update = await qbFetch("/invoice", tokens, { + method: "POST", + body: JSON.stringify({ + Id: qbInvoiceId, + SyncToken: String(invoice.SyncToken), + sparse: true, + CustomerMemo: { value: [current, line].filter(Boolean).join("\n\n").slice(0, 1000) }, + }), + }); + if (!update.ok) return { ok: false, error: `Could not add the backup link to the QuickBooks invoice (${update.status})` }; + return { ok: true }; +} + +/** Posted money-out transactions (expenses/checks/card charges) from the books. */ +export async function getRecentQBPurchases(tokens: QBTokens, sinceDaysAgo: number) { + const since = new Date(Date.now() - sinceDaysAgo * 86_400_000).toISOString().split("T")[0]; + const rows = await getQBPurchasesSince(tokens, new Date(`${since}T00:00:00.000Z`)); + return rows.map(p => ({ + qbId: String(p.Id), + date: p.TxnDate ?? null, + amount: Number(p.TotalAmt ?? 0), + paymentType: p.PaymentType ?? null, // Cash | Check | CreditCard + docNumber: p.DocNumber ?? null, + vendor: p.EntityRef?.name ?? null, + account: p.AccountRef?.name ?? null, + memo: p.PrivateNote ?? null, + })); +} + +/** + * Read all posted QBO Purchase rows on or after a transaction date. + * Pagination matters for the initial historical backfill; a single QBO query + * page would silently stop after its MAXRESULTS boundary. + */ +export async function getQBPurchasesSince(tokens: QBTokens, since: Date, until?: Date): Promise { + if (!Number.isFinite(since.getTime())) { + throw new Error("QBO purchase query requires a valid since date"); + } + if (until && !Number.isFinite(until.getTime())) { + throw new Error("QBO purchase query requires a valid until date"); + } + + const sinceDate = since.toISOString().slice(0, 10); + // Inclusive upper bound so callers can chunk a long backfill into + // date windows that each finish within the serverless duration limit. + const untilClause = until ? ` AND TxnDate <= '${until.toISOString().slice(0, 10)}'` : ""; + const pageSize = 1000; + const purchases: any[] = []; + + for (let startPosition = 1; ; startPosition += pageSize) { + const page = await qbQuery( + tokens, + `SELECT * FROM Purchase WHERE TxnDate >= '${sinceDate}'${untilClause} ORDERBY TxnDate ASC STARTPOSITION ${startPosition} MAXRESULTS ${pageSize}`, + ); + purchases.push(...page); + if (page.length < pageSize) break; + } + + return purchases; +} + +/** + * Read Purchase rows changed since a timestamp using QBO Change Data Capture. + * Unlike a TxnDate query, CDC catches newly entered backdated purchases, + * corrections, voids, refunds, and deletion tombstones. QBO caps CDC lookback + * at 30 days and returns at most 1,000 entities, so truncated responses fail + * visibly instead of silently leaving local job costs stale. + */ +export async function getQBPurchaseChangesSince( + tokens: QBTokens, + since: Date, +): Promise { + if (!Number.isFinite(since.getTime())) { + throw new Error("QBO Purchase CDC requires a valid since date"); + } + + const params = new URLSearchParams({ + entities: "Purchase", + changedSince: since.toISOString(), + minorversion: "73", + }); + const response = await qbTimedFetch( + `${QB_API_BASE}/${tokens.realmId}/cdc?${params.toString()}`, + { + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + Accept: "application/json", + }, + }, + ); + if (!response.ok) { + throw new Error(`QBO Purchase CDC failed with status ${response.status}`); + } + + const payload = await response.json(); + const cdcResponses = Array.isArray(payload?.CDCResponse) + ? payload.CDCResponse + : []; + const queryResponses = cdcResponses.flatMap((entry: any) => + Array.isArray(entry?.QueryResponse) ? entry.QueryResponse : [], + ); + const purchases: any[] = []; + for (const queryResponse of queryResponses) { + const page = Array.isArray(queryResponse?.Purchase) + ? queryResponse.Purchase + : []; + const totalCount = Number(queryResponse?.totalCount ?? page.length); + if ( + page.length >= 1000 || + (Number.isFinite(totalCount) && totalCount > page.length) + ) { + throw new Error("QBO Purchase CDC response was truncated"); + } + purchases.push(...page); + } + + // Keep the last representation if QBO includes the same id more than once. + const byId = new Map(); + const withoutId: any[] = []; + for (const purchase of purchases) { + const id = purchase?.Id === undefined ? "" : String(purchase.Id); + if (id) byId.set(id, purchase); + else withoutId.push(purchase); + } + return [...byId.values(), ...withoutId]; +} + +/** Posted customer payments (money in) from the books. */ +export async function getRecentQBPaymentsList(tokens: QBTokens, sinceDaysAgo: number) { + const since = new Date(Date.now() - sinceDaysAgo * 86_400_000).toISOString().split("T")[0]; + const rows = await qbQuery(tokens, `SELECT * FROM Payment WHERE TxnDate >= '${since}' ORDERBY TxnDate DESC MAXRESULTS 500`); + return rows.map(p => ({ + qbId: String(p.Id), + date: p.TxnDate ?? null, + amount: Number(p.TotalAmt ?? 0), + customer: p.CustomerRef?.name ?? null, + reference: p.PaymentRefNum ?? null, + })); +} + +/** The estimate-item shape `buildQBEstimateLines` needs: billing figures plus enough + * hierarchy (`id`/`parentId`/`type`) to tell section headers from billable leaves. */ +export type QBEstimateItem = { + // Required, not optional: a caller that omits the hierarchy silently loses legacy + // section detection (a section is only recognizable by its type tag OR its children), + // which is exactly the bug this function exists to prevent. + id: string; + parentId: string | null; + name: string; + quantity: number; + unitCost: number; + total: number; + type: string; +}; + +/** + * Billable QB estimate lines, one per LEAF row. + * + * Section headers are dropped. A section's stored total is a roll-up of its children, so + * emitting it as a line bills that amount a second time on top of the child rows it + * summarizes — a nested section double-counts twice over (an outer section holding a $250 + * inner section plus a $25 leaf shipped $800 of lines against a $275 subtotal). + * + * The filter lives here rather than in the caller so any future caller inherits it; it uses + * the same `isEstimateSectionRow` predicate as the editor subtotal and the PDF, so all three + * readers agree on which rows are headers. + * + * The returned amounts sum to the estimate's pre-tax SUBTOTAL, which is not the same as its + * stored `totalAmount` (that column also carries tax and any processing-fee markup). QBO + * computes its own sales tax on the lines it receives, so pushing a tax line here would + * double-charge; the processing-fee gap is a separate open question — see the callers. + */ +export function buildQBEstimateLines(items: readonly QBEstimateItem[], itemId: string) { + return items + .filter(item => !isEstimateSectionRow(item, items)) + .map((item, i) => ({ + LineNum: i + 1, + Description: item.name, + Amount: item.total, + DetailType: "SalesItemLineDetail", + SalesItemLineDetail: { + ItemRef: { value: itemId }, + Qty: item.quantity, + UnitPrice: item.unitCost, + }, + })); +} + +/** Push an estimate to QB. Returns the QB estimate ID. */ +export async function syncEstimateToQB( + tokens: QBTokens, + estimate: { + id: string; + code: string; + title: string; + totalAmount: number; + items: QBEstimateItem[]; + customerId: string; + itemId: string; + project: { name: string } | null; + }, + glMappings: Record = {} +): Promise<{ qbId: string; qbUrl: string }> { + const lines = buildQBEstimateLines(estimate.items, estimate.itemId); + + // QBO rejects a transaction with no lines (error 2020, "Required param missing"). An + // estimate that is empty, or that is nothing but section headers, reaches this point with + // everything filtered out — fail with something legible instead of a raw QB API error. + if (!lines.length) { + throw new Error("QB estimate sync failed: estimate has no billable line items"); + } + + const payload = { + TxnDate: new Date().toISOString().split("T")[0], + DocNumber: estimate.code.slice(0, 21), + PrivateNote: estimate.title, + CustomerRef: { value: estimate.customerId }, + Line: lines, + }; + + const res = await qbFetch("/estimate", tokens, { + method: "POST", + body: JSON.stringify(payload), + }); + + if (!res.ok) { + const err = await res.text(); + throw new Error(`QB estimate sync failed: ${err}`); + } + + const data = await res.json(); + const qbId = data.Estimate?.Id; + const realmId = tokens.realmId; + const qbUrl = `https://app.qbo.intuit.com/app/estimate?txnId=${qbId}`; + + return { qbId, qbUrl }; +} + +/** Push an invoice to QB. Returns the QB invoice ID. */ +export async function syncInvoiceToQB( + tokens: QBTokens, + invoice: { + code: string; + totalAmount: number; + balanceDue: number; + customerId: string; + itemId: string; + project: { name: string } | null; + items?: Array<{ description: string; amount: number }>; + } +): Promise<{ qbId: string; qbUrl: string }> { + const lines: object[] = (invoice.items || []).map((item, i) => ({ + LineNum: i + 1, + Description: item.description, + Amount: item.amount, + DetailType: "SalesItemLineDetail", + SalesItemLineDetail: { ItemRef: { value: invoice.itemId }, Qty: 1, UnitPrice: item.amount }, + })); + + if (lines.length === 0) { + lines.push({ + LineNum: 1, + Description: invoice.project?.name || "Construction Services", + Amount: invoice.totalAmount, + DetailType: "SalesItemLineDetail", + SalesItemLineDetail: { ItemRef: { value: invoice.itemId }, Qty: 1, UnitPrice: invoice.totalAmount }, + }); + } + + const payload = { + DocNumber: invoice.code.slice(0, 21), + TxnDate: new Date().toISOString().split("T")[0], + CustomerRef: { value: invoice.customerId }, + Line: lines, + }; + + const res = await qbFetch("/invoice", tokens, { + method: "POST", + body: JSON.stringify(payload), + }); + + if (!res.ok) { + const err = await res.text(); + throw new Error(`QB invoice sync failed: ${err}`); + } + + const data = await res.json(); + const qbId = data.Invoice?.Id; + const qbUrl = `https://app.qbo.intuit.com/app/invoice?txnId=${qbId}`; + return { qbId, qbUrl }; +} + +/** Send a QBO invoice email to a client. */ +export async function sendQBInvoice(tokens: QBTokens, qbInvoiceId: string, sendTo?: string | null) { + const qs = new URLSearchParams({ minorversion: "73" }); + if (sendTo) qs.set("sendTo", sendTo); + const url = `${QB_API_BASE}/${tokens.realmId}/invoice/${qbInvoiceId}/send?${qs}`; + const res = await qbTimedFetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + Accept: "application/json", + "Content-Type": "application/octet-stream", + }, + }); + if (!res.ok) return { ok: false as const, status: res.status, error: await res.text() }; + const data = await res.json().catch(() => ({})); + return { ok: true as const, status: res.status, emailStatus: data.Invoice?.EmailStatus ?? null }; +} From c900334058113a4cc277bcef931aa8e7a711d301 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:29:41 -0700 Subject: [PATCH 002/144] test(qbo): cover qbTimedFetch deadline, pass-through, and caller abort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real local http server, not mock.module — mock.module corrupts the require chain on Node 20, which CI pins. Covers: a never-responding server becomes QBTimeoutError; the message carries the path but neither the query string nor a token; success and request init pass through; a connection refusal is NOT relabelled a timeout; a caller's own abort is NOT reported as a QBO outage; QB_FETCH_TIMEOUT_MS drives the default and a garbage value falls back instead of breaking every QB call. Wired into test:unit so CI runs it. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- tests/qb-timed-fetch.test.ts | 124 +++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 tests/qb-timed-fetch.test.ts diff --git a/package.json b/package.json index c8df70663..5f86e4754 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "lint": "eslint", "test:qbo-expense-sync": "tsx --test tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts", "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", + "test:qb-timeout": "tsx --test tests/qb-timed-fetch.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -20,7 +21,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts new file mode 100644 index 000000000..d480ce8c1 --- /dev/null +++ b/tests/qb-timed-fetch.test.ts @@ -0,0 +1,124 @@ +/** + * qbTimedFetch — the per-request deadline on every QuickBooks HTTP call. + * + * The defect it fixes: bare fetch() has no timeout, so the 2026-09-01 Intuit + * outage hung each QB call until Vercel killed the whole function at its + * maxDuration. Routes learned nothing and burned the entire budget. + * + * Tested against a real local http server rather than a stubbed fetch — + * `mock.module` corrupts the require chain on Node 20, which is what CI pins. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import { AddressInfo } from "node:net"; +import { qbTimedFetch, QBTimeoutError } from "../src/lib/quickbooks"; + +let server: Server; +let base: string; +/** Sockets the "hang" route is holding open, closed in `after` so the suite exits. */ +const held: Array<() => void> = []; + +before(async () => { + server = createServer((req, res) => { + if (req.url?.startsWith("/v3/company/hang")) { + // Never respond — the client's own deadline is the only thing that + // can end this request. That is exactly the outage shape. + held.push(() => res.destroy()); + return; + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, path: req.url })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +after(async () => { + for (const destroy of held) destroy(); + await new Promise(resolve => server.close(() => resolve())); +}); + +test("times out into QBTimeoutError when the server never responds", async () => { + const error = await qbTimedFetch(`${base}/v3/company/hang?realmId=secret`, {}, 100).then( + () => null, + (e: unknown) => e, + ); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); + assert.equal((error as Error).name, "QBTimeoutError"); +}); + +test("timeout message carries the path but never the query string", async () => { + const error = await qbTimedFetch(`${base}/v3/company/hang?query=select%20*&token=shhh`, {}, 100) + .then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError); + assert.match(error.message, /\/v3\/company\/hang/); + assert.doesNotMatch(error.message, /shhh/); + assert.doesNotMatch(error.message, /select/); + assert.match(error.message, /100ms/); +}); + +test("a successful response passes through untouched", async () => { + const res = await qbTimedFetch(`${base}/v3/company/query`, {}, 5_000); + assert.equal(res.ok, true); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { ok: true, path: "/v3/company/query" }); +}); + +test("request init (method, headers, body) is forwarded", async () => { + const res = await qbTimedFetch( + `${base}/v3/company/purchase`, + { method: "POST", headers: { "X-Probe": "1" }, body: JSON.stringify({ a: 1 }) }, + 5_000, + ); + assert.equal(res.status, 200); +}); + +test("a non-timeout network error passes through unchanged", async () => { + // Port 1 on loopback refuses immediately — a connection error, not a + // deadline, so it must NOT be relabelled as a QBO outage. + const error = await qbTimedFetch("http://127.0.0.1:1/v3/company/query", {}, 5_000).then( + () => null, + (e: unknown) => e as Error, + ); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); +}); + +test("a caller's own abort is not reported as a QBO timeout", async () => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 50); + const error = await qbTimedFetch(`${base}/v3/company/hang`, { signal: controller.signal }, 10_000) + .then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); +}); + +test("QB_FETCH_TIMEOUT_MS drives the default deadline", async () => { + const previous = process.env.QB_FETCH_TIMEOUT_MS; + process.env.QB_FETCH_TIMEOUT_MS = "120"; + try { + const error = await qbTimedFetch(`${base}/v3/company/hang`).then( + () => null, + (e: unknown) => e as Error, + ); + assert.ok(error instanceof QBTimeoutError); + assert.match(error.message, /120ms/); + } finally { + if (previous === undefined) delete process.env.QB_FETCH_TIMEOUT_MS; + else process.env.QB_FETCH_TIMEOUT_MS = previous; + } +}); + +test("a garbage QB_FETCH_TIMEOUT_MS falls back to the 20s default rather than throwing", async () => { + const previous = process.env.QB_FETCH_TIMEOUT_MS; + process.env.QB_FETCH_TIMEOUT_MS = "not-a-number"; + try { + const res = await qbTimedFetch(`${base}/v3/company/query`); + assert.equal(res.status, 200); + } finally { + if (previous === undefined) delete process.env.QB_FETCH_TIMEOUT_MS; + else process.env.QB_FETCH_TIMEOUT_MS = previous; + } +}); From cc5162ff92f651b01d8422d9e46502dad683e354 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:30:53 -0700 Subject: [PATCH 003/144] fix(receipts): fail fast on a QBO timeout instead of burning maxDuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A QBTimeoutError from either the token fetch or the purchase create is now audited as reason "qbo-timeout" and answered 503 {ok:false, retry:true, reason:"qbo-timeout"}. Non-200 is what makes the Apps Script retry on its next pass, which is right for an outage — a terminal ok:false would send the receipt down the email fallback for a failure that fixes itself. Retrying is safe even if a timed-out create actually landed: it carries a QBO requestid idempotency key and the docNumber pre-check returns already-exists. maxDuration 60 -> 30. Two 20s QB deadlines plus the DB work fit; the whole point is that we now fail long before the ceiling. Every other branch is unchanged. Co-Authored-By: Claude Fable 5.1 --- .../integrations/qbo-receipts/create/route.ts | 22 ++++++++- tests/qbo-receipt-push.test.ts | 46 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 188c1f93f..5241e8bc5 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -9,10 +9,14 @@ import { type CreateQBReceiptPurchaseInput, type CreateQBReceiptPurchaseResult, } from "@/lib/qbo-receipt-push"; -import type { QBTokens } from "@/lib/quickbooks"; +import { QBTimeoutError, type QBTokens } from "@/lib/quickbooks"; export const dynamic = "force-dynamic"; -export const maxDuration = 60; +// Two QB deadlines (token refresh + purchase create, 20s each by default) +// plus the DB work fit inside this comfortably. Before qbTimedFetch existed, +// an Intuit outage held the function open for the full 60s and returned +// nothing — the point of the lower ceiling is that we now fail long before it. +export const maxDuration = 30; /** * Receipt bot -> QBO Purchase creation. Replaces the Apps Script's @@ -205,6 +209,10 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(pushEventFromOutcome(input, { status: "error", reason: "quickbooks-not-connected" })); return NextResponse.json({ ok: false, reason: "quickbooks-not-connected" }, { status: 503 }); } + if (error instanceof QBTimeoutError) { + await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-timeout" })); + return NextResponse.json({ ok: false, retry: true, reason: "qbo-timeout" }, { status: 503 }); + } // Token refresh failures are transient (QBO outage, network) — // retryable, so this is the one case that stays 500. console.error("QBO receipt push token fetch failed", error instanceof Error ? error.name : "UnknownError"); @@ -235,6 +243,16 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(event); return NextResponse.json(result); } catch (error) { + if (error instanceof QBTimeoutError) { + // QBO is unreachable, not saying no — 503 so the Apps + // Script retries on its next pass instead of falling back + // to the email path. Safe to retry even if the create did + // land: it carries a QBO `requestid` idempotency key and + // the docNumber pre-check returns already-exists. + console.error("QBO receipt push timed out", error.message); + await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-timeout" })); + return NextResponse.json({ ok: false, retry: true, reason: "qbo-timeout" }, { status: 503 }); + } if (error instanceof QboAccountConfigError) { // Deterministic misconfiguration (missing/wrong-type/ // colliding account ids) — the SAME failure would repeat on diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index 1b6bbece6..510cb1dfe 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -16,6 +16,7 @@ import { createQboReceiptCreateHandlers, type QboReceiptCreateHandlerDependencies, } from "../src/app/api/integrations/qbo-receipts/create/route"; +import type { AutomationEventInput } from "../src/lib/automation-events"; const TOKENS = { accessToken: "test-access", @@ -622,6 +623,51 @@ test("route POST returns 503 when QuickBooks isn't connected (unchanged transien assert.deepEqual(await response.json(), { ok: false, reason: "quickbooks-not-connected" }); }); +test("route POST returns 503 retry:true when the QBO token fetch times out", async () => { + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + const events: AutomationEventInput[] = []; + const { POST } = createRouteHandlers({ + getFreshTokens: async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /oauth2/v1/tokens/bearer"); + }, + logEvent: event => { events.push(event); }, + }); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + // Non-200 is what makes the Apps Script retry on its next pass — an + // Intuit outage must never be mistaken for a terminal decline. + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-timeout" }); + assert.equal(events.length, 1); + assert.equal(events[0].status, "error"); + assert.equal(events[0].reason, "qbo-timeout"); +}); + +test("route POST returns 503 retry:true when the purchase create times out", async () => { + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + const events: AutomationEventInput[] = []; + const { POST } = createRouteHandlers({ + createPurchase: async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/test-realm/purchase"); + }, + logEvent: event => { events.push(event); }, + }); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-timeout" }); + assert.equal(events.length, 1); + assert.equal(events[0].reason, "qbo-timeout"); + // The full fileId guarantee still holds for this new outcome. + assert.equal((events[0].detail as { fileId?: string }).fileId, "file-1"); +}); + // ─── Overhead category (Shop docs) ────────────────────────────────────────── const SHOP_PROJECT: QboReceiptProjectCandidate = { id: "project-shop", name: "Shop" }; From ff52e0692f2007615ff66fa4d335999f2f3fd9b6 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:45:15 -0700 Subject: [PATCH 004/144] feat(ops): pipeline health endpoint + 7am morning digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One summariser (src/lib/pipeline-health.ts) behind two surfaces, so the on-demand check and the digest can never disagree about whether the pipeline is OK: - GET /api/health/pipeline — Intuit status, last QBO purchase sync, last receipt booked, 24h receipt counts by status, bank-ledger high-water mark, 24h error count. - GET /api/cron/pipeline-digest (0 14 * * *, 7am Pacific) — emails the plain-text summary to PIPELINE_DIGEST_TO (default jadkins@) and, when BOT_HEALTH_CHAT_WEBHOOK is set, posts the same text to Google Chat. Sends every morning: a digest that only arrives on failure is indistinguishable from one that stopped running. Verdict rules, unit-tested: a degraded Intuit indicator or any error in 24h fails; an UNREACHABLE Intuit status page does not (a third party's downtime is not evidence of ours); a gap between 48h and 7d fails because traffic was flowing and stopped, while no pushes in 7d is ok with a note because a quiet week is quiet, not broken. Every read degrades to null/unknown on its own rather than throwing — a health check that 500s during an outage tells you nothing. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- src/app/api/cron/pipeline-digest/route.ts | 61 +++++++ src/app/api/health/pipeline/route.ts | 38 +++++ src/lib/pipeline-health.ts | 191 ++++++++++++++++++++++ tests/pipeline-health.test.ts | 126 ++++++++++++++ vercel.json | 4 + 6 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 src/app/api/cron/pipeline-digest/route.ts create mode 100644 src/app/api/health/pipeline/route.ts create mode 100644 src/lib/pipeline-health.ts create mode 100644 tests/pipeline-health.test.ts diff --git a/package.json b/package.json index 5f86e4754..b96913839 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test:qbo-expense-sync": "tsx --test tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts", "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", "test:qb-timeout": "tsx --test tests/qb-timed-fetch.test.ts", + "test:pipeline-health": "tsx --test tests/pipeline-health.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -21,7 +22,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts new file mode 100644 index 000000000..786e584eb --- /dev/null +++ b/src/app/api/cron/pipeline-digest/route.ts @@ -0,0 +1,61 @@ +import { NextResponse } from "next/server"; +import { getPipelineHealth, formatPipelineDigest } from "@/lib/pipeline-health"; +import { sendNotification } from "@/lib/email"; +import { postTextToChatWebhook } from "@/lib/chat-webhook"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +const DEFAULT_TO = "jadkins@goldentouchremodeling.com"; + +/** + * Morning pipeline digest (7 AM Pacific): one plain-text summary of the + * receipt/QBO pipeline's overnight health, so an Intuit outage or a stalled + * bot is noticed over coffee instead of at month-end reconciliation. + * + * Uses the SAME summariser as GET /api/health/pipeline — the digest and the + * on-demand check must never disagree about whether the pipeline is OK. + * Sends every morning, healthy or not: a digest that only arrives on failure + * is indistinguishable from a digest that stopped running. + */ +export async function GET(request: Request) { + // Any deployed environment (production or preview) requires the cron secret, + // and fails closed if CRON_SECRET is unset. Only local dev skips the check. + const authHeader = request.headers.get("authorization"); + if (process.env.VERCEL_ENV && (!process.env.CRON_SECRET || authHeader !== `Bearer ${process.env.CRON_SECRET}`)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const health = await getPipelineHealth(); + const { subject, text } = formatPipelineDigest(health); + + const to = process.env.PIPELINE_DIGEST_TO || DEFAULT_TO; + // sendNotification takes HTML and derives its own plain-text part;
+    // keeps the line-per-item layout intact in an HTML client.
+    const escaped = text.replace(/&/g, "&").replace(//g, ">");
+    const emailResult = await sendNotification(
+        to,
+        subject,
+        `
${escaped}
`, + undefined, + { fromName: "ProBuild" }, + ); + + const chatWebhook = process.env.BOT_HEALTH_CHAT_WEBHOOK; + const chatPosted = chatWebhook ? await postTextToChatWebhook(chatWebhook, text) : false; + + console.log("[cron/pipeline-digest]", JSON.stringify({ + ok: health.ok, + stuck: health.stuck, + intuit: health.intuit.indicator, + emailed: emailResult.success, + chatPosted, + })); + + return NextResponse.json({ + ok: health.ok, + emailed: emailResult.success, + chatPosted, + health, + }); +} diff --git a/src/app/api/health/pipeline/route.ts b/src/app/api/health/pipeline/route.ts new file mode 100644 index 000000000..cd87937ea --- /dev/null +++ b/src/app/api/health/pipeline/route.ts @@ -0,0 +1,38 @@ +import { NextResponse } from "next/server"; +import { getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; +import { getPipelineHealth } from "@/lib/pipeline-health"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +/** + * On-demand pipeline health: Intuit's status, how current the QBO sync and + * receipt bookings are, 24h receipt counts, the bank ledger's high-water mark, + * and the error count. Same summariser the morning digest cron uses, so the + * two can never disagree. + * + * Unlike the bare liveness probe at /api/health, this exposes internal + * operating data, so it is gated: a staff session with the financialReports + * permission (same gate as the other Command Center reads), or the cron + * secret for headless/ops checks. + */ +export async function GET(request: Request) { + const authHeader = request.headers.get("authorization"); + const cronAuthed = Boolean( + process.env.CRON_SECRET && authHeader === `Bearer ${process.env.CRON_SECRET}`, + ); + if (!cronAuthed) { + const user = await getCurrentUserWithPermissions(); + if (!user) { + return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); + } + if (!hasPermission(user, "financialReports")) { + return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + } + } + + const health = await getPipelineHealth(); + return NextResponse.json(health, { + headers: { "Cache-Control": "no-store, max-age=0" }, + }); +} diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts new file mode 100644 index 000000000..967c82d8b --- /dev/null +++ b/src/lib/pipeline-health.ts @@ -0,0 +1,191 @@ +import { prisma } from "@/lib/prisma"; + +/** + * One summary of the receipt/QBO pipeline's health, shared by the on-demand + * health endpoint (GET /api/health/pipeline) and the morning digest cron + * (GET /api/cron/pipeline-digest) so the two can never disagree about whether + * the pipeline is OK. + * + * Everything here is a READ. It must never throw: a health check that 500s + * during an outage tells you nothing you didn't already fear, so each probe + * degrades to a null/unknown of its own and the surrounding verdict still + * renders. + */ + +const INTUIT_STATUS_URL = "https://status.developer.intuit.com/api/v2/status.json"; +const INTUIT_TIMEOUT_MS = 5_000; +const DAY_MS = 86_400_000; + +/** Statuspage indicators: "none" | "minor" | "major" | "critical"; "unknown" is ours. */ +export interface IntuitStatus { + indicator: string; + description?: string; +} + +export interface PipelineHealth { + ok: boolean; + /** Set only when `ok` is true DESPITE there being no recent receipt traffic. */ + note?: string; + checkedAt: string; + intuit: IntuitStatus; + qbo: { + /** Newest Expense row QBO has synced into ProBuild job costs. */ + lastPurchaseSyncAt: string | null; + /** Newest receipt the bot actually booked (created or already-exists). */ + lastReceiptPushAt: string | null; + }; + /** receipt-push events in the last 24h, by status ("created", "fallback", ...). */ + receipts24h: Record; + bank: { + /** Newest posted date in the bank ledger — how current the statement feed is. */ + lastPostedDate: string | null; + }; + /** Automation events (ANY kind) that errored in the last 24h. */ + stuck: number; +} + +/** + * Intuit's own status page. Deliberately soft: an unreachable status page is + * NOT evidence of an outage (it is a third party with its own downtime), so a + * failure reads "unknown" and does not by itself flip the verdict. + */ +export async function fetchIntuitStatus(): Promise { + try { + const res = await fetch(INTUIT_STATUS_URL, { + signal: AbortSignal.timeout(INTUIT_TIMEOUT_MS), + headers: { Accept: "application/json" }, + }); + if (!res.ok) return { indicator: "unknown" }; + const data = (await res.json()) as { status?: { indicator?: unknown; description?: unknown } }; + const indicator = typeof data?.status?.indicator === "string" ? data.status.indicator : null; + if (!indicator) return { indicator: "unknown" }; + return { + indicator, + description: typeof data.status?.description === "string" ? data.status.description : undefined, + }; + } catch { + return { indicator: "unknown" }; + } +} + +/** A push that BOOKED something. "fallback"/"error" are attempts, not bookings. */ +const BOOKED_PUSH_STATUSES = ["created", "already-exists"]; + +/** + * The verdict, split out from the database reads so the freshness windows are + * testable without a DB. + * + * A quiet week is quiet, not broken — GTR does not book receipts every day. + * "No pushes in 7d" (including none ever) is therefore OK-with-a-note, while a + * gap between 48h and 7d means traffic was flowing and then stopped, which is + * the actual failure this exists to catch. + * + * An "unknown" Intuit indicator does NOT fail the check: an unreachable + * third-party status page is not evidence of an outage, and treating it as one + * would cry wolf every time statuspage.io hiccups. + */ +export function evaluatePipelineOk(input: { + intuit: IntuitStatus; + stuck: number; + lastReceiptPushAt: Date | null; + now: number; +}): { ok: boolean; note?: string } { + const pushAgeMs = input.lastReceiptPushAt ? input.now - input.lastReceiptPushAt.getTime() : null; + const quiet = pushAgeMs === null || pushAgeMs > 7 * DAY_MS; + const pushFresh = pushAgeMs !== null && pushAgeMs <= 2 * DAY_MS; + const intuitOk = input.intuit.indicator === "none" || input.intuit.indicator === "unknown"; + const ok = intuitOk && input.stuck === 0 && (pushFresh || quiet); + return ok && quiet ? { ok, note: "no receipts in 7d" } : { ok }; +} + +export async function getPipelineHealth(): Promise { + const now = Date.now(); + const since24h = new Date(now - DAY_MS); + + const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck] = await Promise.all([ + fetchIntuitStatus(), + // Expense carries no updatedAt column — qbSyncedAt IS the "when did the + // QBO purchase sync land" timestamp this is asking for. + prisma.expense + .aggregate({ where: { qbPurchaseId: { not: null } }, _max: { qbSyncedAt: true } }) + .catch(() => null), + prisma.automationEvent + .findFirst({ + where: { kind: "receipt-push", status: { in: BOOKED_PUSH_STATUSES } }, + orderBy: { createdAt: "desc" }, + select: { createdAt: true }, + }) + .catch(() => null), + prisma.automationEvent + .groupBy({ + by: ["status"], + where: { kind: "receipt-push", createdAt: { gte: since24h } }, + _count: { _all: true }, + }) + .catch(() => [] as Array<{ status: string; _count: { _all: number } }>), + prisma.bankLine + .aggregate({ _max: { postedDate: true } }) + .catch(() => null), + // ANY kind: a qbo-sync failure is exactly the thing this digest exists + // to surface, even on a day with no receipt traffic at all. + prisma.automationEvent + .count({ where: { status: "error", createdAt: { gte: since24h } } }) + .catch(() => 0), + ]); + + const receipts24h: Record = {}; + for (const row of receiptRows) receipts24h[row.status] = row._count._all; + + const lastReceiptPushAt = lastPush?.createdAt ?? null; + const verdict = evaluatePipelineOk({ + intuit, + stuck, + lastReceiptPushAt, + now, + }); + + return { + ...verdict, + checkedAt: new Date(now).toISOString(), + intuit, + qbo: { + lastPurchaseSyncAt: lastPurchase?._max.qbSyncedAt?.toISOString() ?? null, + lastReceiptPushAt: lastReceiptPushAt?.toISOString() ?? null, + }, + receipts24h, + bank: { lastPostedDate: lastBankLine?._max.postedDate?.toISOString() ?? null }, + stuck, + }; +} + +function ago(iso: string | null, now: number): string { + if (!iso) return "never"; + const hours = (now - new Date(iso).getTime()) / 3_600_000; + if (hours < 1) return `${iso} (${Math.max(0, Math.round(hours * 60))}m ago)`; + if (hours < 48) return `${iso} (${Math.round(hours)}h ago)`; + return `${iso} (${Math.round(hours / 24)}d ago)`; +} + +/** + * Plain-text digest body. No markdown tables and no emoji — it has to read the + * same in an email client, in Google Chat, and in a log line. + */ +export function formatPipelineDigest(health: PipelineHealth): { subject: string; text: string } { + const now = new Date(health.checkedAt).getTime(); + const subject = health.ok ? "Pipeline OK" : "Pipeline NEEDS ATTENTION"; + + const receiptCounts = Object.entries(health.receipts24h).sort(([a], [b]) => a.localeCompare(b)); + const lines = [ + subject, + `Checked: ${health.checkedAt}`, + `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}`, + `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSyncAt, now)}`, + `Last receipt booked: ${ago(health.qbo.lastReceiptPushAt, now)}`, + `Receipts (24h): ${receiptCounts.length ? receiptCounts.map(([s, n]) => `${s} ${n}`).join(", ") : "none"}`, + `Bank ledger through: ${health.bank.lastPostedDate ? health.bank.lastPostedDate.slice(0, 10) : "no lines"}`, + `Automation errors (24h, all kinds): ${health.stuck}`, + ]; + if (health.note) lines.push(`Note: ${health.note}`); + + return { subject, text: lines.join("\n") }; +} diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts new file mode 100644 index 000000000..c63d6f0ca --- /dev/null +++ b/tests/pipeline-health.test.ts @@ -0,0 +1,126 @@ +/** + * Pipeline health verdict + digest formatting. + * + * The verdict is split out of the DB reads precisely so the freshness windows + * are testable: the 48h/7d split is the part that is easy to get backwards, + * and getting it backwards means either a daily false alarm or a silent dead + * pipeline. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { evaluatePipelineOk, formatPipelineDigest, type PipelineHealth } from "../src/lib/pipeline-health"; + +const NOW = Date.parse("2026-09-01T14:00:00.000Z"); +const HOUR = 3_600_000; + +const healthy = { intuit: { indicator: "none" }, stuck: 0, now: NOW }; + +test("fresh traffic with a clean Intuit status is OK, with no note", () => { + const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 3 * HOUR) }); + assert.deepEqual(v, { ok: true }); +}); + +test("a gap between 48h and 7d is NOT ok — traffic was flowing and stopped", () => { + const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 72 * HOUR) }); + assert.equal(v.ok, false); +}); + +test("no pushes in 7d is ok WITH the note — a quiet week is quiet, not broken", () => { + const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 8 * 24 * HOUR) }); + assert.deepEqual(v, { ok: true, note: "no receipts in 7d" }); +}); + +test("no pushes ever is treated the same as a quiet week", () => { + const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: null }); + assert.deepEqual(v, { ok: true, note: "no receipts in 7d" }); +}); + +test("an unreachable Intuit status page (unknown) does not by itself fail the check", () => { + const v = evaluatePipelineOk({ + intuit: { indicator: "unknown" }, + stuck: 0, + now: NOW, + lastReceiptPushAt: new Date(NOW - HOUR), + }); + assert.equal(v.ok, true); +}); + +test("a degraded Intuit indicator fails the check", () => { + for (const indicator of ["minor", "major", "critical"]) { + const v = evaluatePipelineOk({ + intuit: { indicator }, + stuck: 0, + now: NOW, + lastReceiptPushAt: new Date(NOW - HOUR), + }); + assert.equal(v.ok, false, `${indicator} should fail`); + } +}); + +test("any error in the last 24h fails the check, even on an otherwise quiet week", () => { + const v = evaluatePipelineOk({ + intuit: { indicator: "none" }, + stuck: 1, + now: NOW, + lastReceiptPushAt: null, + }); + // No note either: the note only ever accompanies an OK verdict. + assert.deepEqual(v, { ok: false }); +}); + +// ─── Digest formatting ───────────────────────────────────────────────────── + +function sampleHealth(overrides: Partial = {}): PipelineHealth { + return { + ok: true, + checkedAt: "2026-09-01T14:00:00.000Z", + intuit: { indicator: "none", description: "All Systems Operational" }, + qbo: { + lastPurchaseSyncAt: "2026-09-01T10:00:00.000Z", + lastReceiptPushAt: "2026-09-01T12:00:00.000Z", + }, + receipts24h: { created: 4, fallback: 1 }, + bank: { lastPostedDate: "2026-08-29T00:00:00.000Z" }, + stuck: 0, + ...overrides, + }; +} + +test("digest subject says OK or NEEDS ATTENTION and leads the body", () => { + const good = formatPipelineDigest(sampleHealth()); + assert.equal(good.subject, "Pipeline OK"); + assert.equal(good.text.split("\n")[0], "Pipeline OK"); + + const bad = formatPipelineDigest(sampleHealth({ ok: false, stuck: 3 })); + assert.equal(bad.subject, "Pipeline NEEDS ATTENTION"); + assert.equal(bad.text.split("\n")[0], "Pipeline NEEDS ATTENTION"); +}); + +test("digest is plain text: one line per item, no markdown tables, no emoji", () => { + const { text } = formatPipelineDigest(sampleHealth()); + assert.doesNotMatch(text, /\|/); + assert.doesNotMatch(text, /[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}]/u); + assert.match(text, /Intuit status: none \(All Systems Operational\)/); + assert.match(text, /Receipts \(24h\): created 4, fallback 1/); + assert.match(text, /Bank ledger through: 2026-08-29/); + assert.match(text, /Automation errors \(24h, all kinds\): 0/); +}); + +test("digest renders missing timestamps as 'never' rather than a bogus date", () => { + const { text } = formatPipelineDigest( + sampleHealth({ qbo: { lastPurchaseSyncAt: null, lastReceiptPushAt: null } }), + ); + assert.match(text, /Last QBO purchase sync: never/); + assert.match(text, /Last receipt booked: never/); +}); + +test("digest reports zero receipt traffic as 'none', not an empty list", () => { + const { text } = formatPipelineDigest(sampleHealth({ receipts24h: {} })); + assert.match(text, /Receipts \(24h\): none/); +}); + +test("the quiet-week note is carried into the body", () => { + const { text } = formatPipelineDigest(sampleHealth({ note: "no receipts in 7d" })); + assert.match(text, /Note: no receipts in 7d/); +}); diff --git a/vercel.json b/vercel.json index 2a1e429d7..41d9f0789 100644 --- a/vercel.json +++ b/vercel.json @@ -47,6 +47,10 @@ { "path": "/api/cron/dragging-line", "schedule": "5 14 * * 1" + }, + { + "path": "/api/cron/pipeline-digest", + "schedule": "0 14 * * *" } ] } From 4d24b13011d6be4ba95a6405ba87f61e39bdfaa8 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:46:24 -0700 Subject: [PATCH 005/144] docs: phase 1 intake-core and phase 4 earned-margin specs Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 403 +++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 docs/plans/PHASE-1-INTAKE-CORE-SPEC.md diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md new file mode 100644 index 000000000..f69cf3640 --- /dev/null +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -0,0 +1,403 @@ +# Phase 1: Intake Core — Implementation Spec + +Date: 2026-09-01. Parent plan: `docs/plans/RECEIPT-PIPELINE-V2-PLAN.md` (Phase 1 row). +Planner output for the executor: build exactly this; do not guess. + +**Concurrency note:** another agent is editing `src/lib/quickbooks.ts` and +`src/app/api/integrations/qbo-receipts/create/route.ts` (adding fetch timeouts). Do NOT +modify either file. `QBTimeoutError` already exists in `src/lib/quickbooks.ts` (~line 31) +— import it. Reuse `createQBReceiptPurchase` from `src/lib/qbo-receipt-push.ts` by direct +import; never copy its logic and never call the HTTP route from the worker. + +## Verified code facts + +- The one QBO write core is `createQBReceiptPurchase(tokens, input, deps?)` in + `src/lib/qbo-receipt-push.ts` (~462). Idempotency = DocNumber `input.fileId.slice(0,21)` + + PrivateNote marker `[gtr-file:]` + QBO requestid. Result union (~158): + `ok:true {qbPurchaseId, alreadyExists}` | `ok:false {reason}`. Terminal error classes: + `QboPurchaseFaultError`, `QboAccountConfigError`, `QboVendorDuplicateError`. +- `Expense` (prisma/schema.prisma:580) REQUIRES `estimateId` (FK, cascade); `qbPurchaseId` + is `@unique`; has `costCodeId?`, `receiptUrl?`, `vendor?`, `date?`, `status` default "Pending". + No Drive-file-id column on Expense; `AutomationEvent.driveFileId` exists (:2440). +- v1 ingest precedent: `src/app/api/integrations/receipt-ingest/route.ts` resolves the + "primary estimate" as the project's latest (`orderBy createdAt desc, take 1`, line 69) and + matches projects/cost codes via `matchProjectByName` / `matchCostCode` (`src/lib/project-match.ts`). +- Gemini: no shared lib helper; callers hit `generativelanguage.googleapis.com` REST with + `GEMINI_API_KEY`. Current working text model is `"gemini-3.5-flash"` + (`src/lib/daily-log-task-match.ts:26`). The Apps Script model list + (`qbo-clasp/runReceiptAutomation.js:115`) is stale (2.5-era) — do not port it verbatim. +- Storage: `getSupabase()` + `STORAGE_BUCKET="project-files"` (PUBLIC) in `src/lib/supabase.ts`; + `SECURE_BUCKET="secure-docs"` (PRIVATE) + `secure:` ref scheme + `resolveDocUrl()` / + `downloadDocBytes()` in `src/lib/secure-storage.ts`. Receipts go in the private bucket. +- Server-side Drive access exists (`src/lib/lead-drive.ts`) but uses a NextAuth Google OAuth + refresh token (user identity), not a service account. No service-account Drive credential + exists anywhere in `src/`. +- Apps Script dedup (`qbo-clasp/runReceiptAutomation.js`): strong key `date|ref` (:1558, built + only when the OCR date is valid AND `refLooksReal_` :1581 passes), weak key + `canonicalVendor|date|amount|"amt"` (:1598), `VENDOR_ALIASES` (:1609), `sanitize()` (:1478), + `cleanMoney()` (:1519), placeholder list (:1578). Gemini prompt: :1099–1133. Tax-split group + building: `qbo-clasp/sendToQBOviaAPI.js:129–178`. +- Mobile auth: `authenticateMobileOrSession(req)` in `src/lib/mobile-auth.ts`. Proxy + (`src/proxy.ts`): Bearer passes only for `MOBILE_AUTHENTICATED_ROUTE_PATTERNS`; machine + endpoints with their own shared secret use the exact-match public bypass (precedent: + `api/office-tasks/ingest`, :47–50). `/api/integrations/*` and `/api/cron/*` are excluded + from the proxy matcher entirely (:228). +- Cron auth convention (fail closed): `src/app/api/cron/drain-notifications/route.ts:15–19`. + pgbouncer forbids session advisory locks (`src/lib/review-alert-rollout.ts:8`); use + `pg_try_advisory_xact_lock` inside a short claim transaction (pattern: + `src/lib/qbo-expense-sync.ts:572`, `src/app/api/automation/sync-now/route.ts`). + +## 1. Goals and acceptance criteria + +1. **Schema**: `ReceiptIntake` live in prod and in `prisma/migrations/`. + Verify: `node scripts/apply-receipt-intake.mjs` twice (second run all "already exists"); + CI `migrations` job green; `prisma generate` + `npx tsc --noEmit` clean. +2. **Intake endpoint** `POST /api/receipts/intake` accepts session, mobile Bearer, and + `x-receipt-intake-secret`; idempotent on `sourceRef`. + Verify: e2e API test posts the same payload twice → same `id`, one DB row; no-auth AND + bogus-session-cookie requests both 401 (getclients-auth-gate lesson). +3. **Reader** `readReceipt()` ports the v3.6 prompt + phase suggestion + `tax_amount` + + `doc_type` multi/non_receipt. Verify: node:test with an injected fetch asserts the prompt + carries the load-bearing sentences (final-amount rule, never-estimate-tax rule, multi + rule) and that responses parse into `ReadResult`. +4. **Dedup + routing** ports the Apps Script keys faithfully. Verify: fixture table (§9) + and `routeState` truth table pass under node:test. +5. **Booking** creates the QBO Purchase via `createQBReceiptPurchase`, then the `Expense`, + sets `qbPurchaseId`; retry queue with backoff. Verify: node:test with injected fake + `createPurchase` covers success, terminal fault → NEEDS_REVIEW, `QBTimeoutError` → retry + per backoff table; project-without-estimate → NEEDS_REVIEW reason `no-estimate`. +6. **Cron** `/api/cron/receipt-intake-worker` every 5 min, ≤10 rows/run, non-overlapping. + Verify: `vercel.json` entry; claim-query unit test (two sequential claims never return + the same row); manual `curl -H "Authorization: Bearer $CRON_SECRET"` on prod returns + `{processed, byState}`. +7. **Dry-run shadow mode** default ON: rows read+dedup+route but never book. Verify: with + `RECEIPT_INTAKE_DRYRUN` unset, worker test proves zero `createPurchase` calls and zero + Expense rows. +8. **Build**: `npm run build` 0 errors; Codex review of the money path before merge. + +## 2. Schema + +Repo convention: `state` is a `String` with a SQL CHECK, not a Prisma enum (matches +`BankLine.state`, `Expense.status`). Prisma model (add to `prisma/schema.prisma`): + +```prisma +model ReceiptIntake { + id String @id @default(cuid()) + source String // mobile | email | drive | chat | web + sourceRef String @unique // "drive:" | "email::" | "mobile:" | "chat::" | "web:" + state String @default("RECEIVED") // RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT + dryRun Boolean @default(true) + stateReason String? // no-estimate | multi-doc | zero-total | weak-dup: | strong-dup-amount-mismatch: | qbo-fault: | max-retries | push-disabled | push-paused + + projectId String? + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + costCodeId String? + costCode CostCode? @relation(fields: [costCodeId], references: [id]) + suggestedCostCodeId String? + suggestedConfidence Float? + createdById String? + createdBy User? @relation(fields: [createdById], references: [id], onDelete: SetNull) + + // file (Supabase secure-docs, private) + storagePath String // receipts/intake/. in SECURE_BUCKET + fileName String? + mimeType String + fileSize Int + fileSha256 String + + // read results (cents, like AutomationEvent) + vendor String? + txnDate DateTime? @db.Date + totalCents Int? + taxCents Int? + docType String? // receipt | check | multi | non_receipt + refNumber String? // cleaned invoice #, or "Check" for checks + memo String? + readJson String? // raw Gemini JSON, audit only + readAt DateTime? + + // dedup + dedupStrongKey String? + dedupWeakKey String? + duplicateOfId String? + + // booking + archive + qbPurchaseId String? + expenseId String? @unique + archiveDriveFileId String? + attempts Int @default(0) + lastError String? + nextRetryAt DateTime? + bookedAt DateTime? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // NOTE: a partial UNIQUE index on dedupStrongKey (WHERE state NOT IN + // ('DUPLICATE','VOID') AND dedupStrongKey IS NOT NULL) exists in SQL only — + // Prisma cannot represent partial indexes and silently drops them (CLAUDE.md, + // baseline notes). Keep this comment; never regenerate it away. + @@index([state, nextRetryAt]) + @@index([projectId]) + @@index([dedupWeakKey]) + @@index([createdAt]) +} +``` + +Add back-relations `receiptIntakes ReceiptIntake[]` on `Project`, `CostCode`, `User`. + +`scripts/apply-receipt-intake.mjs` (additive, idempotent, `$executeRawUnsafe` over the +pooler — same shape as prior `scripts/apply-*.mjs`) and byte-identical DDL in +`prisma/migrations/_receipt_intake/migration.sql`: + +```sql +CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( + "id" TEXT PRIMARY KEY, "source" TEXT NOT NULL, "sourceRef" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'RECEIVED', "dryRun" BOOLEAN NOT NULL DEFAULT true, + "stateReason" TEXT, "projectId" TEXT, "costCodeId" TEXT, "suggestedCostCodeId" TEXT, + "suggestedConfidence" DOUBLE PRECISION, "createdById" TEXT, + "storagePath" TEXT NOT NULL, "fileName" TEXT, "mimeType" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, + "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, "taxCents" INTEGER, + "docType" TEXT, "refNumber" TEXT, "memo" TEXT, "readJson" TEXT, "readAt" TIMESTAMP(3), + "dedupStrongKey" TEXT, "dedupWeakKey" TEXT, "duplicateOfId" TEXT, + "qbPurchaseId" TEXT, "expenseId" TEXT, "archiveDriveFileId" TEXT, + "attempts" INTEGER NOT NULL DEFAULT 0, "lastError" TEXT, "nextRetryAt" TIMESTAMP(3), + "bookedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP(3) NOT NULL, + CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('RECEIVED','READ','NEEDS_JOB', + 'NEEDS_REVIEW','BOOKING','BOOKED','ARCHIVED','DUPLICATE','VOID','NON_RECEIPT')) +); +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" ON "ReceiptIntake"("sourceRef"); +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" ON "ReceiptIntake"("expenseId"); +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_dedupStrongKey_active_key" + ON "ReceiptIntake"("dedupStrongKey") + WHERE "dedupStrongKey" IS NOT NULL AND "state" NOT IN ('DUPLICATE','VOID'); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_state_nextRetryAt_idx" ON "ReceiptIntake"("state","nextRetryAt"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_projectId_idx" ON "ReceiptIntake"("projectId"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_dedupWeakKey_idx" ON "ReceiptIntake"("dedupWeakKey"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("createdAt"); +-- FKs guarded with DO $$ ... IF NOT EXISTS (pg_constraint) blocks, per prior apply scripts: +-- projectId -> "Project"(id) ON DELETE SET NULL +-- costCodeId -> "CostCode"(id) +-- createdById-> "User"(id) ON DELETE SET NULL +-- expenseId -> "Expense"(id) ON DELETE SET NULL +``` + +Run the apply script against prod BEFORE merging (CLAUDE.md pre-deploy rule #2). + +## 3. Endpoint contracts + +### POST /api/receipts/intake (new file `src/app/api/receipts/intake/route.ts`) +- **Proxy**: add exact-match `/api/receipts/intake` to the public bypass set in + `src/proxy.ts` (precedent + comment style of `api/office-tasks/ingest`: machine callers + need a clean 401, not a /login redirect; exact match only, no descendants). The handler + is then the sole auth boundary and MUST fail closed. +- **Auth, in order**: (1) `x-receipt-intake-secret` header equals env + `RECEIPT_INTAKE_SECRET` — 401 when the env var is unset (never fail open; a NEW secret, + not `RECEIPT_INGEST_SECRET`, so v1 and v2 rotate independently); else + (2) `authenticateMobileOrSession(req)`; else 401. +- **Body**: multipart (`file`, `source`, `projectId?`, `costCodeId?`, `sourceRef?`, + `threadName?`) or JSON `{fileBase64, mimeType, fileName?, source, sourceRef?, projectId?, + costCodeId?, threadName?}`. `source` in mobile|email|drive|chat|web. Machine callers MUST + send `sourceRef`; session/Bearer callers get `web:` / `mobile:` minted + server-side. Max 15 MB. Accept pdf/jpeg/png/heic/webp/gif/txt; sniff magic bytes for + images the way `receipts/parse` does (route.ts:37). +- **Behavior**: sha256 the bytes; create the row (catch P2002 on `sourceRef` and return the + existing row with `{ok:true, alreadyReceived:true}`); upload to `SECURE_BUCKET` at + `receipts/intake/.`; state RECEIVED; `dryRun` = (env `RECEIPT_INTAKE_DRYRUN` + is not "false"). When a session/Bearer user supplies `projectId`, check + `userCanAccessProject` and set `createdById`. Return 200 with the serialized row + (id, state, sourceRef, projectId). No Gemini call here — sub-second, fire-and-forget safe. +- Deterministic bad input (missing file, bad source, oversize, bad mime) → 400 JSON. + +### GET /api/receipts/intake?state=&projectId=&take= (same route file) +- Staff only: session or Bearer user with role ADMIN | MANAGER | FINANCE, else 403 (proxy + bypass means the handler enforces this itself). +- Rows newest-first (`take` max 200, default 50), fields = the row minus `readJson`. Keep + the select in one exported function (mirroring `src/app/automation/register-data.ts` + style) so the Phase 2 `/automation` Receipts tab reuses it unchanged. + +### POST /api/receipts/intake/[id]/archived (secret-auth only) +- Body `{driveFileId}`. Sets `archiveDriveFileId`, state BOOKED → ARCHIVED. Used by the + nightly Apps Script mirror (§6). 404 for unknown id; 409 if state is not BOOKED. + +## 4. Worker library — `src/lib/receipt-intake/*.ts` (pure core, I/O at the edges) + +- **`keys.ts`** (pure; port verbatim from `runReceiptAutomation.js`): `sanitize` (:1478), + `cleanMoney` (:1519), `normalizeDateStr`/`isValidDate`, `refLooksReal` (:1581 with the + :1578 placeholder list verbatim), `VENDOR_ALIASES` (:1609 verbatim), `canonicalVendor` + (:1615), and `dedupKeys(read) -> {strong: string|null, weak: string}`: + - strong = `dateStr + "|" + ref.toLowerCase()` ONLY when the date was read off the + document and is valid AND the ref passes `refLooksReal` (checks test `checkNum` and use + `"Check"+checkNum`; receipts use the cleaned invoice; `NoInv` / `CheckNoNum` give + null). Vendor and amount stay OUT of the key — v3.6 rationale at :1545–1557. + - weak = `canonicalVendor(vendor) + "|" + dateStr + "|" + amount + "|amt"` where amount + is the `cleanMoney` 2-dp string; built for EVERY document. + - Fallback date when unreadable = the intake row's `createdAt` date (v1 used the Drive + upload date — same semantic). +- **`read.ts`**: `readReceipt(fileBytes, mime, projectPhases) -> ReadResult`. REST to + `generativelanguage.googleapis.com/v1beta/models/:generateContent` with + `responseMimeType "application/json"`; models `["gemini-3.5-flash", + "gemini-flash-latest"]` with the Apps Script retry discipline: 429/503 backs off up to 5 + tries then falls to the next model; 404 falls to the next model; a decisive failure + (valid HTTP, unusable JSON) returns `{decisive:true}` so callers never burn retries on a + hopeless document (:1143–1184 rationale). Prompt = :1099–1133 VERBATIM (A/B/C roles, + multi rule, final-amount-paid rule, tax never-estimate rule, empty-string-for-unreadable + rule), plus ONE appended section: the project's active cost codes as a phase list + (code — name per line) and one extra output field `"suggested_phase"` restricted to that + list or empty. The v1 extraction fields must stay byte-identical. + `ReadResult = {docType, vendor, date, invoice, checkNumber, memo, totalAmount, taxAmount, + suggestedPhaseCode, raw}`. `text/plain` files go in as a text part (v1 :1093). +- **`route-state.ts`** (pure): `routeState(read, dedupHits, hasProject)`: + | condition (first match wins) | state | + |---|---| + | docType "multi" | NEEDS_REVIEW `multi-doc` | + | docType "non_receipt" | NON_RECEIPT | + | total "0.00" (unreadable-total rule, :531) | NEEDS_REVIEW `zero-total` | + | no projectId | NEEDS_JOB | + | strong hit, same totalCents | DUPLICATE (+`duplicateOfId`) | + | strong hit, different total | NEEDS_REVIEW `strong-dup-amount-mismatch:` | + | weak hit (another live row, different id) | NEEDS_REVIEW `weak-dup:` | + | otherwise | READ | + The strong-key CLAIM is the partial unique index: the read step UPDATEs the row with its + keys and treats a unique violation as the hit signal, then loads the owner row to compare + totals — the database replaces the Apps Script Properties lock. Weak hits are a plain + query (same `dedupWeakKey`, different id, state not in DUPLICATE/VOID/NON_RECEIPT) and + always route to a human (:1591–1596). +- **`book.ts`**: `bookReceipt(row)`: + 1. Guards: `QBO_RECEIPT_PUSH_ENABLED === "true"` and not + `isPaused(PAUSE_KEYS.receiptPush)` — the same two switches the qbo-receipts/create + route checks. Off/paused: stay BOOKING, `stateReason` push-disabled|push-paused, + `nextRetryAt` +1h, attempts NOT incremented. + 2. Estimate = project's latest (`orderBy createdAt desc, take 1`, the receipt-ingest + rule); none: NEEDS_REVIEW `no-estimate` (terminal, no attempt spent). + 3. Groups (port `sendToQBOviaAPI.js:129–178`): docType receipt, job project, and + `0 < taxCents < totalCents` gives two groups + `[{category:"Receipt (pre-tax)", amount:(total-tax)/100}, + {category:"Sales tax", amount:tax/100, tax:true}]`; otherwise one full-total group. + Checks never split tax (:148). + 4. `createQBReceiptPurchase(await getFreshQBTokens(), input)` with `fileId` = the Drive + fileId when source=drive (keeps DocNumber idempotency continuous with any v1 booking + of the same file), else the intake `id`; `fileBase64` via + `downloadDocBytes(storagePath)`; `projectName` = project.name. + 5. On `ok:true`, one transaction: create `Expense` (estimateId; costCodeId = chosen, else + `matchCostCode(suggestedPhaseCode)`; amount = pre-tax amount when tax was split, else + total — mirrors the QBO COGS line; vendor; date=txnDate; status "Pending"; receiptUrl + = Drive view URL when a Drive fileId is known, else the `secure:` ref; qbPurchaseId) + and set the row BOOKED {qbPurchaseId, expenseId, bookedAt}. Also log one + `AutomationEvent {kind:"receipt-push", source:"intake-worker"}` so the /automation + register keeps seeing v2 bookings. `alreadyExists:true` results book the same way + (idempotent re-drive after a lost response). + 6. Failure classification: `QboPurchaseFaultError` / `QboAccountConfigError` / + `QboVendorDuplicateError` / any `result.ok:false` reason are TERMINAL: NEEDS_REVIEW + with the reason (4xx class, never retried). `QBTimeoutError`, `QBNotConnectedError`, + network/fetch errors, QBO 429/5xx, DB errors are RETRYABLE: attempts+1, lastError, + `nextRetryAt = now + backoff(attempts)`; backoff = attempts 1 gives 5m, 2 gives 15m, + 3 gives 1h, 4+ gives 6h; attempts > 20: NEEDS_REVIEW `max-retries`. + +## 5. Cron — `src/app/api/cron/receipt-intake-worker/route.ts` + +- `vercel.json`: `{"path": "/api/cron/receipt-intake-worker", "schedule": "*/5 * * * *"}`. +- `export const maxDuration = 60; export const dynamic = "force-dynamic";` Auth = the + fail-closed drain-notifications pattern (Bearer CRON_SECRET; 401 when unset on Vercel). +- Claim step (overlap safety under pgbouncer — session advisory locks are unusable, see + review-alert-rollout.ts:8): one SHORT transaction: + `SELECT pg_try_advisory_xact_lock(hashtextextended('receipt-intake-worker', 0))` — if + false, return `{skipped:"already-running"}`; else select up to 10 ids where + `state IN ('RECEIVED','READ','BOOKING') AND (nextRetryAt IS NULL OR nextRetryAt <= now())` + ordered by createdAt, and UPDATE their `nextRetryAt = now() + interval '10 minutes'` + (the claim). Process OUTSIDE the transaction: RECEIVED rows get read+dedup+route (dry-run + rows park at READ / NEEDS_* / DUPLICATE); READ rows with `dryRun=false` move to BOOKING; + BOOKING rows get `bookReceipt`. If two runs ever interleave anyway, the bumped + `nextRetryAt` keeps each row single-claimed and QBO DocNumber idempotency backstops the + booking itself. + +## 6. Archive decision + +**Recommendation: (b) Supabase-only now + nightly Apps Script mirror.** Reasons: (1) no +service account or Drive-writer credential exists in ProBuild — option (a) needs Justin to +provision one and share `Processed Receipts/` with it (a Workspace-admin human step); +(2) the Apps Script project already owns working Drive auth and the archive-naming code — +a ~50-line `mirrorBookedReceipts()` (Apps Script PR) polls +`GET /api/receipts/intake?state=BOOKED` with the shared secret, downloads each file via a +short-lived signed URL, writes `Processed Receipts/YYYY/MM/` with the v1 filename +convention `____$.` (keys.ts sanitize rules), then +POSTs `/api/receipts/intake//archived {driveFileId}`; (3) Drive stays the archive +Marge uses, byte-identical to today. Cost: the archive copy lags up to a day — acceptable +because job cost (`Expense`) and QBO no longer wait on it. Revisit a native Drive copy in +Phase 6 if the mirror proves flaky. + +## 7. Forwarder contracts (Apps Script side — separate PR in qbo-clasp, spec only) + +All gated on Script Property `V2_FORWARD === "true"`; all send `x-receipt-intake-secret`. +- `runReceiptAutomation`: on picking up a job-folder file, POST JSON + `{source:"drive", sourceRef:"drive:"+fileId, projectId: resolved from the folder name, + fileBase64, mimeType, fileName}`. +- `pullReceiptEmails`: `{source:"email", sourceRef:"email:"+messageId+":"+sha16(bytes), ...}`. +- `sweepChatReceipts`: `{source:"chat", sourceRef:"chat:"+messageName+":"+idx, threadName, ...}`. +- Non-200: leave the file in place and retry next run (intake idempotency makes replays safe). +- Shadow-week wrinkle: during shadow the forwarder COPIES bytes to intake and leaves the + original for v1 to process as usual. The move-to-`_Forwarded` branch (which hides the + file from v1) activates only under a second property `V2_LIVE === "true"` at cutover. + +## 8. Shadow-week gate + +- `RECEIPT_INTAKE_DRYRUN` unset/true: every row gets `dryRun=true` — reader, dedup, and + routing all run; booking never does (goal 7 test proves it). No QBO write, no Expense. +- Daily comparison (checker or scratchpad script): v1 truth = that day's files in + `Processed Receipts/YYYY/MM/` (filenames encode project/date/vendor/ref/total) vs + `SELECT "sourceRef", vendor, "txnDate", "totalCents", "refNumber", state + FROM "ReceiptIntake" WHERE "createdAt" >= `. Match on the weak-key triple + (canonicalVendor, date, amount). +- Gate to call Phase 1 done: 5 consecutive days where every archived v1 file has a v2 row + agreeing on vendor/date/total with state READ (or DUPLICATE when v1 also quarantined), + and zero v2 rows stuck in RECEIVED for more than an hour. +- Cutover (Justin's explicit call): set `RECEIPT_INTAKE_DRYRUN=false` and `V2_LIVE=true`. + +## 9. Tests (node:test in `test/receipt-intake/*.test.mjs`; NO mock.module — CI is Node 20) + +- `keys.test.mjs` — fixtures from real August archive filenames + (I:\My Drive\Expenses\Processed Receipts\2026\August); expected keys per the ported rules: + | file (project_date_vendor_ref_$total) | strong | weak | + |---|---|---| + | Berg_ADU_2026-08-03_Lowes_82766_$364.98 | 2026-08-03(pipe)82766 | lowes(pipe)2026-08-03(pipe)364.98(pipe)amt | + | Berg_ADU_2026-08-03_Lowes_Home_Improvement_99908_$277.19 | 2026-08-03(pipe)99908 | lowes(pipe)...(pipe)277.19(pipe)amt — alias collapses the vendor variants | + | Berg_ADU_2026-08-04_WINLOCK_HARDWARE_12_$14.50 | null (ref "12" under 3 chars) | winlockhardware(pipe)2026-08-04(pipe)14.50(pipe)amt | + | Berg_ADU_2026-08-04_WINLOCK_HARDWARE_4_$16.17 | null | winlockhardware(pipe)2026-08-04(pipe)16.17(pipe)amt | + | Berg_ADU_2026-08-07_CRC_-_WEST_VAN_260807091421373F2A9_$91.50 | 2026-08-07(pipe)260807091421373f2a9 | assert actual canonicalVendor output for a non-alias vendor | + | Berg_ADU_2026-08-09_Amazon.com_113-9992333-7801840_$248.27 | 2026-08-09(pipe)113-9992333-7801840 | amazon(pipe)...(pipe)248.27(pipe)amt | + | Berg_ADU_2026-08-10_Grover_Electric_Plumbing_Supply_NoInv_$22.57 | null (NoInv) | groverelectricplumbingsupply(pipe)...(pipe)22.57(pipe)amt | + | Berg_ADU_2026-08-14_LOWES_HOME_CENTERS_LLC_58302_$304.23 | 2026-08-14(pipe)58302 | lowes(pipe)2026-08-14(pipe)304.23(pipe)amt | + Plus placeholder-ref cases: "NA 000" null, "0000" null, "INV-95870" real (:1571–1580). +- `route-state.test.mjs` — the full section-4 truth table (multi, non_receipt, zero-total, + no-project, strong-same, strong-diff, weak, clean). +- `backoff.test.mjs` — attempts 1..4 give [5m,15m,1h,6h], 10 gives 6h, 21 gives + NEEDS_REVIEW max-retries; QBTimeoutError classed retryable; QboPurchaseFaultError terminal. +- `book.test.mjs` — dependency-injected `createPurchase`/prisma-shaped stubs (function + injection, never module mocks): success creates the Expense with the correct amount and + tax split; no-estimate short-circuits without an attempt; disabled/paused spends no attempt. +- e2e (Playwright, CI postgres): `e2e/receipt-intake.spec.ts` — idempotent POST (same + sourceRef twice, one row, same id), 401 matrix (no auth, bogus session cookie, wrong + secret, secret-env-unset), GET requires a staff role. Teardown deletes created rows + (docs/TESTING.md rule). + +## 10. Risks and open questions for Justin (max 5) + +1. **Public-bypass route**: `/api/receipts/intake` bypasses the proxy, so the handler is + the only gate. Mitigated by the fail-closed secret check + the 401 e2e matrix; Codex + must review the auth block specifically. (Risk to watch, no decision needed.) +2. **Expense.amount = pre-tax when tax is split** (mirrors the QBO COGS line under the + reseller-permit rule in sendToQBOviaAPI.js). Confirm pre-tax is the job-cost number you + want feeding variance reports. +3. **Archive via nightly Apps Script mirror** (§6) instead of a Drive service account — + confirm, or provision a service account now if same-hour archiving matters to Marge. +4. **HEIC**: stored and read fine (Gemini accepts image/heic), but the Phase 2 queue page + cannot preview HEIC natively in most browsers. OK to defer conversion? +5. **Non-Drive booking identity** uses the intake cuid as the QBO DocNumber (21-char + slice). A truncation collision is backstopped by the PrivateNote marker check in + `createQBReceiptPurchase` (it fails as docnumber-conflict rather than mis-attaching). + Accepting that; flag if you want a dedicated short id instead. + +HUMAN DECISION REQUIRED only before cutover (questions 2–3); the build and the shadow week +can start under the recommendations above. From b11dad32121734f1b6ccb6e759c94517e84b6804 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:05:33 -0700 Subject: [PATCH 006/144] fix(qbo): deadline must govern the response BODY, not just the headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex blocker 1. fetch() resolves as soon as headers arrive and streams the body afterwards, so a deadline firing mid-body rejected out of res.json() as a raw AbortError — past the wrapper's header-phase catch. The receipt route then classified an outage as a generic transient failure (500) instead of the 503 qbo-timeout it is. The signal stays attached (the deadline must still cut off a stalled body); the returned Response is proxied so json/text/arrayBuffer/blob/ formData translate OUR abort into QBTimeoutError with the same path-only message. Getters and clone() run against the real Response. A caller's own abort during the body read stays a plain error. Codex blocker 7: replaced the AbortSignal.any fallback, which used the caller's signal ALONE and so silently disabled the deadline on any runtime lacking it, with a manual combiner that keeps both live. Tests: headers-then-stall body -> QBTimeoutError; text/arrayBuffer the same; proxy preserves status/headers/clone; caller abort mid-body stays plain; and three cases with AbortSignal.any deleted from scope. Co-Authored-By: Claude Fable 5.1 --- src/lib/quickbooks.ts | 186 ++++++++++++++++++++++++++++------- tests/qb-timed-fetch.test.ts | 112 ++++++++++++++++++++- 2 files changed, 262 insertions(+), 36 deletions(-) diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index dae0e467c..5d2a53766 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -43,6 +43,87 @@ function safePath(url: string): string { } } +/** + * Abort when ANY input signal aborts. + * + * `AbortSignal.any` is the built-in, but it does not exist on every runtime we + * might land on. The old code fell back to using the CALLER's signal alone, + * which silently disabled the deadline — exactly the hang this module exists to + * prevent, and invisible until an outage. The manual combiner keeps both live. + */ +function combineAbortSignals(signals: AbortSignal[]): AbortSignal { + const anyOf = (AbortSignal as unknown as { any?: (signals: AbortSignal[]) => AbortSignal }).any; + if (typeof anyOf === "function") return anyOf(signals); + + const controller = new AbortController(); + for (const signal of signals) { + if (signal.aborted) { + controller.abort(signal.reason); + break; + } + signal.addEventListener("abort", () => controller.abort(signal.reason), { once: true }); + } + return controller.signal; +} + +/** Body-consuming members of Response — each one can outlive the headers. */ +const RESPONSE_BODY_METHODS = ["json", "text", "arrayBuffer", "blob", "formData"] as const; + +/** + * Translate an abort into QBTimeoutError, but ONLY when our own deadline is + * what fired. A caller cancelling its own request is not a QBO outage. + */ +function asQbTimeout( + error: unknown, + context: { timeoutSignal: AbortSignal; callerSignal: AbortSignal | null | undefined; url: string; ms: number }, +): unknown { + const aborted = error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError"); + if (aborted && context.timeoutSignal.aborted && !context.callerSignal?.aborted) { + return new QBTimeoutError( + `QuickBooks request timed out after ${context.ms}ms: ${safePath(context.url)}`, + ); + } + return error; +} + +/** + * The deadline governs the WHOLE exchange, not just the headers. + * + * `fetch` resolves as soon as response headers arrive; the body is streamed + * afterwards, still under the same signal. A QBO outage that dribbles headers + * and then stalls therefore blew past the wrapper entirely — `res.json()` + * rejected with a raw AbortError, which the receipt route classified as a + * generic transient failure (500) instead of the 503 qbo-timeout it is. + * + * So the signal stays attached (the deadline must still cut off a stalled + * body), and the returned Response is proxied so every body-consuming method + * translates our own abort the same way the header phase does. + */ +function wrapResponseBodyTimeouts( + response: Response, + context: { timeoutSignal: AbortSignal; callerSignal: AbortSignal | null | undefined; url: string; ms: number }, +): Response { + return new Proxy(response, { + get(target, prop, receiver) { + if (typeof prop === "string" && (RESPONSE_BODY_METHODS as readonly string[]).includes(prop)) { + const original = Reflect.get(target, prop, target) as (...args: unknown[]) => Promise; + return async (...args: unknown[]) => { + try { + return await original.apply(target, args); + } catch (error) { + throw asQbTimeout(error, context); + } + }; + } + // Getters like `ok`/`status`/`headers` must run against the real + // Response (they throw on a proxy receiver), and methods like + // `clone()` must stay bound to it. + const value = Reflect.get(target, prop, target); + return typeof value === "function" ? (value as (...a: unknown[]) => unknown).bind(target) : value; + }, + }); +} + /** * Every QuickBooks HTTP call goes through here. * @@ -51,9 +132,9 @@ function safePath(url: string): string { * maxDuration — a receipt push burned 60s and a cron 120s to learn nothing. * A per-request deadline turns that into a fast, classifiable failure. * - * A caller-supplied `signal` still wins on abort (combined via - * `AbortSignal.any` where available); only OUR deadline firing is rethrown as - * QBTimeoutError. Every other error passes through untouched. + * A caller-supplied `signal` still wins on abort; only OUR deadline firing is + * rethrown as QBTimeoutError, in both the header and the body phase. Every + * other error passes through untouched. */ export async function qbTimedFetch( url: string, @@ -66,26 +147,18 @@ export async function qbTimedFetch( const timeoutSignal = AbortSignal.timeout(effectiveMs); const callerSignal = init.signal; - let signal: AbortSignal = timeoutSignal; - if (callerSignal) { - const anyOf = (AbortSignal as unknown as { any?: (signals: AbortSignal[]) => AbortSignal }).any; - signal = typeof anyOf === "function" ? anyOf([callerSignal, timeoutSignal]) : callerSignal; - } + const signal = callerSignal + ? combineAbortSignals([callerSignal, timeoutSignal]) + : timeoutSignal; + const context = { timeoutSignal, callerSignal, url, ms: effectiveMs }; + let response: Response; try { - return await fetch(url, { ...init, signal }); + response = await fetch(url, { ...init, signal }); } catch (error) { - const aborted = - error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError"); - // Only claim a timeout when OUR deadline is the one that fired — a - // caller cancelling its own request is not a QBO outage. - if (aborted && timeoutSignal.aborted && !callerSignal?.aborted) { - throw new QBTimeoutError( - `QuickBooks request timed out after ${effectiveMs}ms: ${safePath(url)}`, - ); - } - throw error; + throw asQbTimeout(error, context); } + return wrapResponseBodyTimeouts(response, context); } /** Exchange authorization code for tokens */ @@ -121,28 +194,71 @@ export async function exchangeQBCode(code: string, redirectUri: string): Promise }; } -/** Refresh an expired access token */ +const QB_REFRESH_DEFAULT_TIMEOUT_MS = 45_000; +/** Must stay under the tightest route ceiling that can call this (maxDuration 60). */ +const QB_REFRESH_MAX_TIMEOUT_MS = 50_000; + +function refreshTimeoutMs(): number { + const configured = Number(process.env.QB_REFRESH_TIMEOUT_MS); + const requested = Number.isFinite(configured) && configured > 0 ? configured : QB_REFRESH_DEFAULT_TIMEOUT_MS; + return Math.min(requested, QB_REFRESH_MAX_TIMEOUT_MS); +} + +/** + * Refresh an expired access token. + * + * This call is NOT safely retryable the way a read is: Intuit rotates the + * refresh token as part of the exchange, so a request that timed out may + * ALREADY have burned the stored refresh token on Intuit's side while we never + * saw the replacement. That strands the connection until someone reconnects + * QuickBooks. Two mitigations, both deliberate: + * + * 1. A longer deadline than an ordinary API call (QB_REFRESH_TIMEOUT_MS, + * default 45s, capped below the route ceiling) — we would much rather wait + * out a slow refresh than abandon one mid-rotation. + * 2. A distinct, diagnosable error when it does fire, so the stranded-token + * case is recognisable in logs instead of looking like any other timeout. + * + * Persistence order is unchanged: the caller still stores what this returns, + * only after a successful exchange. + */ export async function refreshQBToken(refreshToken: string): Promise<{ accessToken: string; refreshToken: string }> { const clientId = process.env.QB_CLIENT_ID!; const clientSecret = process.env.QB_CLIENT_SECRET!; const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); - const res = await qbTimedFetch(TOKEN_URL, { - method: "POST", - headers: { - Authorization: `Basic ${encoded}`, - "Content-Type": "application/x-www-form-urlencoded", - Accept: "application/json", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: refreshToken, - }), - }); + try { + const res = await qbTimedFetch( + TOKEN_URL, + { + method: "POST", + headers: { + Authorization: `Basic ${encoded}`, + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body: new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + }), + }, + refreshTimeoutMs(), + ); - if (!res.ok) throw new Error("QB token refresh failed"); - const data = await res.json(); - return { accessToken: data.access_token, refreshToken: data.refresh_token }; + if (!res.ok) throw new Error("QB token refresh failed"); + const data = await res.json(); + return { accessToken: data.access_token, refreshToken: data.refresh_token }; + } catch (error) { + if (error instanceof QBTimeoutError) { + const message = + "QBO token refresh timed out; the stored refresh token may be stale, reconnect QuickBooks if the next refresh fails"; + console.error(message, error.message); + // Still a QBTimeoutError so routes keep classifying it as an + // outage (503/retry), just with the ambiguity spelled out. + throw new QBTimeoutError(message); + } + throw error; + } } /** Make an authenticated call to the QB API, auto-refreshing if needed */ diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index d480ce8c1..bdae182c5 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -17,7 +17,7 @@ import { qbTimedFetch, QBTimeoutError } from "../src/lib/quickbooks"; let server: Server; let base: string; -/** Sockets the "hang" route is holding open, closed in `after` so the suite exits. */ +/** Sockets the stalling routes are holding open, closed in `after` so the suite exits. */ const held: Array<() => void> = []; before(async () => { @@ -28,6 +28,16 @@ before(async () => { held.push(() => res.destroy()); return; } + if (req.url?.startsWith("/v3/company/stall-body")) { + // Headers land immediately, so `fetch` RESOLVES and the wrapper's + // header-phase try/catch is already behind us — then the body + // never finishes. This is the case that used to escape as a raw + // AbortError out of res.json(). + res.writeHead(200, { "Content-Type": "application/json" }); + res.write('{"partial":'); + held.push(() => res.destroy()); + return; + } res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ ok: true, path: req.url })); }); @@ -95,6 +105,106 @@ test("a caller's own abort is not reported as a QBO timeout", async () => { assert.equal(error instanceof QBTimeoutError, false); }); +// ─── Body-phase deadline ──────────────────────────────────────────────────── + +test("a deadline that fires while READING THE BODY still becomes QBTimeoutError", async () => { + // Regression: fetch resolves on headers, so this rejection happens after + // the wrapper's header try/catch. It used to surface as a raw AbortError, + // which the receipt route classified as a generic 500 instead of a 503 + // qbo-timeout. + const res = await qbTimedFetch(`${base}/v3/company/stall-body?token=shhh`, {}, 150); + assert.equal(res.ok, true); + const error = await res.json().then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); + assert.match(error.message, /\/v3\/company\/stall-body/); + assert.doesNotMatch(error.message, /shhh/); +}); + +test("text() and arrayBuffer() get the same body-phase translation", async () => { + for (const method of ["text", "arrayBuffer"] as const) { + const res = await qbTimedFetch(`${base}/v3/company/stall-body`, {}, 150); + const error = await (res[method]() as Promise).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError, `${method}: got ${String(error)}`); + } +}); + +test("the proxied Response still exposes real status, headers, and clone()", async () => { + const res = await qbTimedFetch(`${base}/v3/company/query`, {}, 5_000); + assert.equal(res.status, 200); + assert.equal(res.ok, true); + assert.equal(res.headers.get("content-type"), "application/json"); + // clone() must not throw on the proxy receiver. + const copy = res.clone(); + assert.deepEqual(await copy.json(), { ok: true, path: "/v3/company/query" }); +}); + +test("a caller's own abort DURING the body read stays a plain error", async () => { + const controller = new AbortController(); + const res = await qbTimedFetch( + `${base}/v3/company/stall-body`, + { signal: controller.signal }, + 10_000, + ); + setTimeout(() => controller.abort(), 50); + const error = await res.json().then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); +}); + +// ─── Signal combining ─────────────────────────────────────────────────────── + +test("without AbortSignal.any, a caller signal does NOT silently disable the deadline", async () => { + const original = (AbortSignal as unknown as Record).any; + delete (AbortSignal as unknown as Record).any; + try { + // A caller signal that never fires: the only thing that can end this + // request is our deadline. The old fallback dropped it entirely. + const neverAborts = new AbortController(); + const error = await qbTimedFetch( + `${base}/v3/company/hang`, + { signal: neverAborts.signal }, + 120, + ).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); + } finally { + (AbortSignal as unknown as Record).any = original; + } +}); + +test("without AbortSignal.any, a caller's abort still cancels the request", async () => { + const original = (AbortSignal as unknown as Record).any; + delete (AbortSignal as unknown as Record).any; + try { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 50); + const error = await qbTimedFetch( + `${base}/v3/company/hang`, + { signal: controller.signal }, + 10_000, + ).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); + } finally { + (AbortSignal as unknown as Record).any = original; + } +}); + +test("an already-aborted caller signal is honoured immediately without AbortSignal.any", async () => { + const original = (AbortSignal as unknown as Record).any; + delete (AbortSignal as unknown as Record).any; + try { + const error = await qbTimedFetch( + `${base}/v3/company/hang`, + { signal: AbortSignal.abort() }, + 10_000, + ).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); + } finally { + (AbortSignal as unknown as Record).any = original; + } +}); + test("QB_FETCH_TIMEOUT_MS drives the default deadline", async () => { const previous = process.env.QB_FETCH_TIMEOUT_MS; process.env.QB_FETCH_TIMEOUT_MS = "120"; From 69867e62cfb2182c5512c01705527c0a1319e1aa Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:05:33 -0700 Subject: [PATCH 007/144] fix(qbo): restore maxDuration 60; give token refresh its own longer deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex blockers 2 and 3. maxDuration back to 60 on qbo-receipts/create: a healthy push does a lot of SERIAL QBO work (lookups, customer/vendor ensures, account verify, create, attachment upload) and the refresh alone is now allowed 45s. 30 would have started killing legitimately slow pushes. qbTimedFetch is what makes the outage case fail fast; the ceiling is only the backstop. refreshQBToken is not safely retryable — Intuit rotates the refresh token during the exchange, so a timeout may already have burned the stored token while we never saw its replacement. Mitigated two ways: its own deadline (QB_REFRESH_TIMEOUT_MS, default 45s, capped at 50s to stay under the route ceiling), and a distinct diagnosable message naming the stranded-token risk. Still a QBTimeoutError, so route classification is unchanged. Persistence order untouched. Co-Authored-By: Claude Fable 5.1 --- .../api/integrations/qbo-receipts/create/route.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 5241e8bc5..7618df787 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -12,11 +12,14 @@ import { import { QBTimeoutError, type QBTokens } from "@/lib/quickbooks"; export const dynamic = "force-dynamic"; -// Two QB deadlines (token refresh + purchase create, 20s each by default) -// plus the DB work fit inside this comfortably. Before qbTimedFetch existed, -// an Intuit outage held the function open for the full 60s and returned -// nothing — the point of the lower ceiling is that we now fail long before it. -export const maxDuration = 30; +// Stays at 60. A single push does a lot of SERIAL QBO work on a healthy day — +// project/vendor/customer lookups, the customer and vendor ensures, the +// account-identity verify, the Purchase create, then the attachment upload — +// and the token refresh alone is allowed 45s. Trimming the ceiling would start +// killing legitimately slow pushes. The fix for the outage case is the +// per-request deadline in qbTimedFetch, which now fails fast on its own; the +// ceiling is only the backstop behind it. +export const maxDuration = 60; /** * Receipt bot -> QBO Purchase creation. Replaces the Apps Script's From 59bae35ec76b42aa66a2f88cac80a7e01ba7f7c6 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:05:51 -0700 Subject: [PATCH 008/144] =?UTF-8?q?fix(ops):=20no=20false-green=20health?= =?UTF-8?q?=20=E2=80=94=20explicit=20probe=20status,=20no=207d=20auto-pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex blocker 4. Two ways the verdict could report OK while knowing nothing: 1. A failed DB probe degraded to null/0 and sailed into ok:true — an unreachable database read as "nothing wrong", which is the most dangerous output this file can produce. Every probe now carries its own {status: "ok"|"error"}, and any probe error forces ok:false with reason probe-failed:. The fallback value is never read as evidence: a failed stuck probe reports probe-failed, not "0 errors", and the digest prints "unavailable (probe failed)" rather than a number. 2. "No receipts in 7d -> ok with a note" never expired, so a permanently dead pipeline reported OK forever. Removed. Now ok:false with reason no-receipts-72h when the last booked push is older than 72h (or there is none), and the digest prints how long the silence has actually been so a human decides whether it is expected. `reasons: string[]` replaces the single note and is empty exactly when ok. Judgment call, flagged: an UNREACHABLE Intuit status page still does not by itself fail the check — it is a third party whose downtime is not evidence of ours, and failing on it would cry wolf on every statuspage hiccup. It reports status:"error"/indicator "unknown" and is flagged in the digest body; our real outage signal is the QBTimeoutError count in `stuck`. Say the word and I will make it hard-fail. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- src/lib/pipeline-health.ts | 294 +++++++++++++++++++++++----------- tests/pipeline-health.test.ts | 232 ++++++++++++++++++++------- 3 files changed, 376 insertions(+), 153 deletions(-) diff --git a/package.json b/package.json index b96913839..6c3383ade 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", "test:qb-timeout": "tsx --test tests/qb-timed-fetch.test.ts", "test:pipeline-health": "tsx --test tests/pipeline-health.test.ts", + "test:cron-auth": "tsx --test tests/cron-auth.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -22,7 +23,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 967c82d8b..eff957ca0 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -6,65 +6,94 @@ import { prisma } from "@/lib/prisma"; * (GET /api/cron/pipeline-digest) so the two can never disagree about whether * the pipeline is OK. * - * Everything here is a READ. It must never throw: a health check that 500s - * during an outage tells you nothing you didn't already fear, so each probe - * degrades to a null/unknown of its own and the surrounding verdict still - * renders. + * Everything here is a READ, and no probe may throw — a health check that 500s + * during an outage tells you nothing. But a probe that FAILED must never read + * as a probe that found nothing wrong: an unreachable database used to + * degrade to null/0 and sail straight into `ok: true`, which is the most + * dangerous output this file can produce. Every probe therefore reports its + * own `status`, and any probe error forces `ok: false` with a + * `probe-failed:` reason. */ const INTUIT_STATUS_URL = "https://status.developer.intuit.com/api/v2/status.json"; const INTUIT_TIMEOUT_MS = 5_000; const DAY_MS = 86_400_000; +const HOUR_MS = 3_600_000; +/** How long a silent receipt pipeline is tolerated before it is a problem. */ +export const RECEIPT_STALE_HOURS = 72; + +export type ProbeStatus = "ok" | "error"; /** Statuspage indicators: "none" | "minor" | "major" | "critical"; "unknown" is ours. */ -export interface IntuitStatus { +export interface IntuitProbe { + status: ProbeStatus; indicator: string; description?: string; } +export interface TimestampProbe { + status: ProbeStatus; + at: string | null; +} + +export interface CountsProbe { + status: ProbeStatus; + counts: Record; +} + +export interface CountProbe { + status: ProbeStatus; + count: number; +} + export interface PipelineHealth { ok: boolean; - /** Set only when `ok` is true DESPITE there being no recent receipt traffic. */ - note?: string; + /** Empty exactly when `ok` is true. Machine-readable, one per failed check. */ + reasons: string[]; checkedAt: string; - intuit: IntuitStatus; + intuit: IntuitProbe; qbo: { /** Newest Expense row QBO has synced into ProBuild job costs. */ - lastPurchaseSyncAt: string | null; + lastPurchaseSync: TimestampProbe; /** Newest receipt the bot actually booked (created or already-exists). */ - lastReceiptPushAt: string | null; + lastReceiptPush: TimestampProbe; }; /** receipt-push events in the last 24h, by status ("created", "fallback", ...). */ - receipts24h: Record; - bank: { - /** Newest posted date in the bank ledger — how current the statement feed is. */ - lastPostedDate: string | null; - }; + receipts24h: CountsProbe; + /** Newest posted date in the bank ledger — how current the statement feed is. */ + bank: TimestampProbe; /** Automation events (ANY kind) that errored in the last 24h. */ - stuck: number; + stuck: CountProbe; } /** - * Intuit's own status page. Deliberately soft: an unreachable status page is - * NOT evidence of an outage (it is a third party with its own downtime), so a - * failure reads "unknown" and does not by itself flip the verdict. + * Intuit's own status page. + * + * Deliberately the ONE probe whose failure does not by itself fail the check: + * it is a third party with its own downtime, so an unreachable status page is + * not evidence of an outage in our pipeline, and treating it as one would cry + * wolf every time statuspage.io hiccups. The failure is still reported + * (`status: "error"`, indicator "unknown") and still printed in the digest — + * it just doesn't flip the verdict on its own. Our real signal for an Intuit + * outage is the QBTimeoutError count, which lands in `stuck`. */ -export async function fetchIntuitStatus(): Promise { +export async function fetchIntuitStatus(): Promise { try { const res = await fetch(INTUIT_STATUS_URL, { signal: AbortSignal.timeout(INTUIT_TIMEOUT_MS), headers: { Accept: "application/json" }, }); - if (!res.ok) return { indicator: "unknown" }; + if (!res.ok) return { status: "error", indicator: "unknown" }; const data = (await res.json()) as { status?: { indicator?: unknown; description?: unknown } }; const indicator = typeof data?.status?.indicator === "string" ? data.status.indicator : null; - if (!indicator) return { indicator: "unknown" }; + if (!indicator) return { status: "error", indicator: "unknown" }; return { + status: "ok", indicator, description: typeof data.status?.description === "string" ? data.status.description : undefined, }; } catch { - return { indicator: "unknown" }; + return { status: "error", indicator: "unknown" }; } } @@ -72,98 +101,158 @@ export async function fetchIntuitStatus(): Promise { const BOOKED_PUSH_STATUSES = ["created", "already-exists"]; /** - * The verdict, split out from the database reads so the freshness windows are - * testable without a DB. - * - * A quiet week is quiet, not broken — GTR does not book receipts every day. - * "No pushes in 7d" (including none ever) is therefore OK-with-a-note, while a - * gap between 48h and 7d means traffic was flowing and then stopped, which is - * the actual failure this exists to catch. + * The verdict, split out from the database reads so the rules are testable + * without a DB. * - * An "unknown" Intuit indicator does NOT fail the check: an unreachable - * third-party status page is not evidence of an outage, and treating it as one - * would cry wolf every time statuspage.io hiccups. + * Every reason here is a real, actionable failure: + * - `probe-failed:` — we could not READ the thing, so we cannot claim + * it is healthy. Unknown is not OK. + * - `intuit-` — Intuit itself reports degradation. + * - `errors-24h:` — automation errored (any kind: a qbo-sync failure + * matters even on a day with no receipt traffic). + * - `no-receipts-72h` — nothing has been booked in 72h. This used to + * auto-green as "a quiet week is quiet", which meant a permanently dead + * pipeline reported OK forever. It doesn't any more: the digest prints how + * long it has actually been and a human decides whether the silence is + * expected. */ -export function evaluatePipelineOk(input: { - intuit: IntuitStatus; - stuck: number; - lastReceiptPushAt: Date | null; +export function evaluatePipelineHealth(input: { + intuit: IntuitProbe; + lastPurchaseSync: TimestampProbe; + lastReceiptPush: TimestampProbe; + receipts24h: CountsProbe; + bank: TimestampProbe; + stuck: CountProbe; now: number; -}): { ok: boolean; note?: string } { - const pushAgeMs = input.lastReceiptPushAt ? input.now - input.lastReceiptPushAt.getTime() : null; - const quiet = pushAgeMs === null || pushAgeMs > 7 * DAY_MS; - const pushFresh = pushAgeMs !== null && pushAgeMs <= 2 * DAY_MS; - const intuitOk = input.intuit.indicator === "none" || input.intuit.indicator === "unknown"; - const ok = intuitOk && input.stuck === 0 && (pushFresh || quiet); - return ok && quiet ? { ok, note: "no receipts in 7d" } : { ok }; +}): { ok: boolean; reasons: string[] } { + const reasons: string[] = []; + + const namedProbes: Array<[string, { status: ProbeStatus }]> = [ + ["lastPurchaseSync", input.lastPurchaseSync], + ["lastReceiptPush", input.lastReceiptPush], + ["receipts24h", input.receipts24h], + ["bank", input.bank], + ["stuck", input.stuck], + ]; + for (const [name, probe] of namedProbes) { + if (probe.status === "error") reasons.push(`probe-failed:${name}`); + } + + if (input.intuit.status === "ok" && input.intuit.indicator !== "none") { + reasons.push(`intuit-${input.intuit.indicator}`); + } + + if (input.stuck.status === "ok" && input.stuck.count > 0) { + reasons.push(`errors-24h:${input.stuck.count}`); + } + + if (input.lastReceiptPush.status === "ok") { + const at = input.lastReceiptPush.at ? Date.parse(input.lastReceiptPush.at) : null; + const stale = at === null || Number.isNaN(at) || input.now - at > RECEIPT_STALE_HOURS * HOUR_MS; + if (stale) reasons.push("no-receipts-72h"); + } + + return { ok: reasons.length === 0, reasons }; } export async function getPipelineHealth(): Promise { const now = Date.now(); const since24h = new Date(now - DAY_MS); + /** Any probe failure is reported as such — never silently downgraded to "nothing found". */ + const probe = async (name: string, run: () => Promise, onError: T): Promise<{ status: ProbeStatus; value: T }> => { + try { + return { status: "ok", value: await run() }; + } catch (error) { + console.error(`[pipeline-health] probe failed: ${name}`, error instanceof Error ? error.name : "UnknownError"); + return { status: "error", value: onError }; + } + }; + const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck] = await Promise.all([ fetchIntuitStatus(), // Expense carries no updatedAt column — qbSyncedAt IS the "when did the // QBO purchase sync land" timestamp this is asking for. - prisma.expense - .aggregate({ where: { qbPurchaseId: { not: null } }, _max: { qbSyncedAt: true } }) - .catch(() => null), - prisma.automationEvent - .findFirst({ - where: { kind: "receipt-push", status: { in: BOOKED_PUSH_STATUSES } }, - orderBy: { createdAt: "desc" }, - select: { createdAt: true }, - }) - .catch(() => null), - prisma.automationEvent - .groupBy({ - by: ["status"], - where: { kind: "receipt-push", createdAt: { gte: since24h } }, - _count: { _all: true }, - }) - .catch(() => [] as Array<{ status: string; _count: { _all: number } }>), - prisma.bankLine - .aggregate({ _max: { postedDate: true } }) - .catch(() => null), + probe( + "lastPurchaseSync", + async () => + (await prisma.expense.aggregate({ where: { qbPurchaseId: { not: null } }, _max: { qbSyncedAt: true } })) + ._max.qbSyncedAt ?? null, + null, + ), + probe( + "lastReceiptPush", + async () => + ( + await prisma.automationEvent.findFirst({ + where: { kind: "receipt-push", status: { in: BOOKED_PUSH_STATUSES } }, + orderBy: { createdAt: "desc" }, + select: { createdAt: true }, + }) + )?.createdAt ?? null, + null, + ), + probe>( + "receipts24h", + async () => { + const rows = await prisma.automationEvent.groupBy({ + by: ["status"], + where: { kind: "receipt-push", createdAt: { gte: since24h } }, + _count: { _all: true }, + }); + return rows; + }, + [], + ), + probe( + "bank", + async () => (await prisma.bankLine.aggregate({ _max: { postedDate: true } }))._max.postedDate ?? null, + null, + ), // ANY kind: a qbo-sync failure is exactly the thing this digest exists // to surface, even on a day with no receipt traffic at all. - prisma.automationEvent - .count({ where: { status: "error", createdAt: { gte: since24h } } }) - .catch(() => 0), + probe( + "stuck", + () => prisma.automationEvent.count({ where: { status: "error", createdAt: { gte: since24h } } }), + 0, + ), ]); - const receipts24h: Record = {}; - for (const row of receiptRows) receipts24h[row.status] = row._count._all; + const counts: Record = {}; + for (const row of receiptRows.value) counts[row.status] = row._count._all; - const lastReceiptPushAt = lastPush?.createdAt ?? null; - const verdict = evaluatePipelineOk({ + const snapshot = { intuit, - stuck, - lastReceiptPushAt, - now, - }); + lastPurchaseSync: { status: lastPurchase.status, at: lastPurchase.value?.toISOString() ?? null }, + lastReceiptPush: { status: lastPush.status, at: lastPush.value?.toISOString() ?? null }, + receipts24h: { status: receiptRows.status, counts }, + bank: { status: lastBankLine.status, at: lastBankLine.value?.toISOString() ?? null }, + stuck: { status: stuck.status, count: stuck.value }, + }; + + const verdict = evaluatePipelineHealth({ ...snapshot, now }); return { ...verdict, checkedAt: new Date(now).toISOString(), - intuit, + intuit: snapshot.intuit, qbo: { - lastPurchaseSyncAt: lastPurchase?._max.qbSyncedAt?.toISOString() ?? null, - lastReceiptPushAt: lastReceiptPushAt?.toISOString() ?? null, + lastPurchaseSync: snapshot.lastPurchaseSync, + lastReceiptPush: snapshot.lastReceiptPush, }, - receipts24h, - bank: { lastPostedDate: lastBankLine?._max.postedDate?.toISOString() ?? null }, - stuck, + receipts24h: snapshot.receipts24h, + bank: snapshot.bank, + stuck: snapshot.stuck, }; } -function ago(iso: string | null, now: number): string { - if (!iso) return "never"; - const hours = (now - new Date(iso).getTime()) / 3_600_000; - if (hours < 1) return `${iso} (${Math.max(0, Math.round(hours * 60))}m ago)`; - if (hours < 48) return `${iso} (${Math.round(hours)}h ago)`; - return `${iso} (${Math.round(hours / 24)}d ago)`; +function ago(probe: TimestampProbe, now: number): string { + if (probe.status === "error") return "unavailable (probe failed)"; + if (!probe.at) return "never"; + const hours = (now - Date.parse(probe.at)) / HOUR_MS; + if (hours < 1) return `${probe.at} (${Math.max(0, Math.round(hours * 60))}m ago)`; + if (hours < 48) return `${probe.at} (${Math.round(hours)}h ago)`; + return `${probe.at} (${Math.round(hours / 24)}d ago)`; } /** @@ -171,21 +260,34 @@ function ago(iso: string | null, now: number): string { * same in an email client, in Google Chat, and in a log line. */ export function formatPipelineDigest(health: PipelineHealth): { subject: string; text: string } { - const now = new Date(health.checkedAt).getTime(); + const now = Date.parse(health.checkedAt); const subject = health.ok ? "Pipeline OK" : "Pipeline NEEDS ATTENTION"; - const receiptCounts = Object.entries(health.receipts24h).sort(([a], [b]) => a.localeCompare(b)); + const receiptCounts = Object.entries(health.receipts24h.counts).sort(([a], [b]) => a.localeCompare(b)); + const receiptsLine = + health.receipts24h.status === "error" + ? "unavailable (probe failed)" + : receiptCounts.length + ? receiptCounts.map(([s, n]) => `${s} ${n}`).join(", ") + : "none"; + const lines = [ subject, `Checked: ${health.checkedAt}`, - `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}`, - `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSyncAt, now)}`, - `Last receipt booked: ${ago(health.qbo.lastReceiptPushAt, now)}`, - `Receipts (24h): ${receiptCounts.length ? receiptCounts.map(([s, n]) => `${s} ${n}`).join(", ") : "none"}`, - `Bank ledger through: ${health.bank.lastPostedDate ? health.bank.lastPostedDate.slice(0, 10) : "no lines"}`, - `Automation errors (24h, all kinds): ${health.stuck}`, + `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}${health.intuit.status === "error" ? " [status page unreachable]" : ""}`, + `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSync, now)}`, + `Last receipt booked: ${ago(health.qbo.lastReceiptPush, now)}`, + `Receipts (24h): ${receiptsLine}`, + `Bank ledger through: ${ + health.bank.status === "error" + ? "unavailable (probe failed)" + : health.bank.at + ? health.bank.at.slice(0, 10) + : "no lines" + }`, + `Automation errors (24h, all kinds): ${health.stuck.status === "error" ? "unavailable (probe failed)" : health.stuck.count}`, ]; - if (health.note) lines.push(`Note: ${health.note}`); + if (health.reasons.length > 0) lines.push(`Needs attention: ${health.reasons.join(", ")}`); return { subject, text: lines.join("\n") }; } diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index c63d6f0ca..5fb57aec7 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -1,72 +1,152 @@ /** * Pipeline health verdict + digest formatting. * - * The verdict is split out of the DB reads precisely so the freshness windows - * are testable: the 48h/7d split is the part that is easy to get backwards, - * and getting it backwards means either a daily false alarm or a silent dead - * pipeline. + * The verdict is split out of the DB reads precisely so the rules are + * testable. Two failure modes this file exists to prevent: + * - false green from a FAILED probe (an unreachable DB reading as "nothing + * wrong"), and + * - false green from a permanently dead pipeline (the old "no receipts in 7d + * is fine" auto-pass, which never expired). */ import test from "node:test"; import assert from "node:assert/strict"; -import { evaluatePipelineOk, formatPipelineDigest, type PipelineHealth } from "../src/lib/pipeline-health"; +import { + evaluatePipelineHealth, + formatPipelineDigest, + type PipelineHealth, +} from "../src/lib/pipeline-health"; const NOW = Date.parse("2026-09-01T14:00:00.000Z"); const HOUR = 3_600_000; -const healthy = { intuit: { indicator: "none" }, stuck: 0, now: NOW }; +function iso(msAgo: number): string { + return new Date(NOW - msAgo).toISOString(); +} + +function snapshot(overrides: Partial[0]> = {}) { + return { + intuit: { status: "ok" as const, indicator: "none" }, + lastPurchaseSync: { status: "ok" as const, at: iso(2 * HOUR) }, + lastReceiptPush: { status: "ok" as const, at: iso(3 * HOUR) }, + receipts24h: { status: "ok" as const, counts: { created: 4 } }, + bank: { status: "ok" as const, at: iso(48 * HOUR) }, + stuck: { status: "ok" as const, count: 0 }, + now: NOW, + ...overrides, + }; +} + +test("a healthy snapshot is ok with no reasons", () => { + assert.deepEqual(evaluatePipelineHealth(snapshot()), { ok: true, reasons: [] }); +}); + +// ─── False green from a failed probe ──────────────────────────────────────── + +test("ANY failed probe forces ok:false with probe-failed:", () => { + const names = ["lastPurchaseSync", "lastReceiptPush", "receipts24h", "bank", "stuck"] as const; + for (const name of names) { + const base = snapshot(); + const broken = { + ...base, + [name]: { ...(base[name] as object), status: "error" as const }, + } as Parameters[0]; + const v = evaluatePipelineHealth(broken); + assert.equal(v.ok, false, `${name} failure must not read as healthy`); + assert.ok(v.reasons.includes(`probe-failed:${name}`), `${name}: ${v.reasons.join(",")}`); + } +}); -test("fresh traffic with a clean Intuit status is OK, with no note", () => { - const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 3 * HOUR) }); - assert.deepEqual(v, { ok: true }); +test("a failed probe's fallback value never produces a SECOND, misleading reason", () => { + // The stuck probe falls back to 0 and the push probe to null. Neither + // fallback may be read as evidence — only the probe failure is reported. + const v = evaluatePipelineHealth( + snapshot({ + stuck: { status: "error", count: 0 }, + lastReceiptPush: { status: "error", at: null }, + }), + ); + assert.deepEqual(v.reasons.sort(), ["probe-failed:lastReceiptPush", "probe-failed:stuck"]); }); -test("a gap between 48h and 7d is NOT ok — traffic was flowing and stopped", () => { - const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 72 * HOUR) }); +test("a total database outage reports every probe, still never ok", () => { + const v = evaluatePipelineHealth( + snapshot({ + lastPurchaseSync: { status: "error", at: null }, + lastReceiptPush: { status: "error", at: null }, + receipts24h: { status: "error", counts: {} }, + bank: { status: "error", at: null }, + stuck: { status: "error", count: 0 }, + }), + ); assert.equal(v.ok, false); + assert.equal(v.reasons.length, 5); }); -test("no pushes in 7d is ok WITH the note — a quiet week is quiet, not broken", () => { - const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: new Date(NOW - 8 * 24 * HOUR) }); - assert.deepEqual(v, { ok: true, note: "no receipts in 7d" }); +// ─── Receipt staleness ────────────────────────────────────────────────────── + +test("a push inside 72h is fresh", () => { + const v = evaluatePipelineHealth(snapshot({ lastReceiptPush: { status: "ok", at: iso(71 * HOUR) } })); + assert.deepEqual(v, { ok: true, reasons: [] }); }); -test("no pushes ever is treated the same as a quiet week", () => { - const v = evaluatePipelineOk({ ...healthy, lastReceiptPushAt: null }); - assert.deepEqual(v, { ok: true, note: "no receipts in 7d" }); +test("nothing booked in over 72h is NOT ok — a silent pipeline no longer auto-greens", () => { + const v = evaluatePipelineHealth(snapshot({ lastReceiptPush: { status: "ok", at: iso(73 * HOUR) } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["no-receipts-72h"]); }); -test("an unreachable Intuit status page (unknown) does not by itself fail the check", () => { - const v = evaluatePipelineOk({ - intuit: { indicator: "unknown" }, - stuck: 0, - now: NOW, - lastReceiptPushAt: new Date(NOW - HOUR), - }); - assert.equal(v.ok, true); +test("a very old push does not decay back into ok (the removed 7d auto-pass)", () => { + for (const days of [8, 30, 400]) { + const v = evaluatePipelineHealth(snapshot({ lastReceiptPush: { status: "ok", at: iso(days * 24 * HOUR) } })); + assert.equal(v.ok, false, `${days}d of silence must not read as healthy`); + assert.deepEqual(v.reasons, ["no-receipts-72h"]); + } +}); + +test("no push ever recorded is stale, not healthy", () => { + const v = evaluatePipelineHealth(snapshot({ lastReceiptPush: { status: "ok", at: null } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["no-receipts-72h"]); +}); + +// ─── Intuit + errors ──────────────────────────────────────────────────────── + +test("an unreachable Intuit status page does not by itself fail the check", () => { + const v = evaluatePipelineHealth(snapshot({ intuit: { status: "error", indicator: "unknown" } })); + assert.deepEqual(v, { ok: true, reasons: [] }); }); -test("a degraded Intuit indicator fails the check", () => { +test("a degraded Intuit indicator fails the check and names the level", () => { for (const indicator of ["minor", "major", "critical"]) { - const v = evaluatePipelineOk({ - intuit: { indicator }, - stuck: 0, - now: NOW, - lastReceiptPushAt: new Date(NOW - HOUR), - }); - assert.equal(v.ok, false, `${indicator} should fail`); + const v = evaluatePipelineHealth(snapshot({ intuit: { status: "ok", indicator } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, [`intuit-${indicator}`]); } }); -test("any error in the last 24h fails the check, even on an otherwise quiet week", () => { - const v = evaluatePipelineOk({ - intuit: { indicator: "none" }, - stuck: 1, - now: NOW, - lastReceiptPushAt: null, - }); - // No note either: the note only ever accompanies an OK verdict. - assert.deepEqual(v, { ok: false }); +test("any error in the last 24h fails the check and reports the count", () => { + const v = evaluatePipelineHealth(snapshot({ stuck: { status: "ok", count: 3 } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["errors-24h:3"]); +}); + +test("multiple independent failures all appear", () => { + const v = evaluatePipelineHealth( + snapshot({ + intuit: { status: "ok", indicator: "major" }, + stuck: { status: "ok", count: 2 }, + lastReceiptPush: { status: "ok", at: iso(100 * HOUR) }, + bank: { status: "error", at: null }, + }), + ); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons.sort(), [ + "errors-24h:2", + "intuit-major", + "no-receipts-72h", + "probe-failed:bank", + ]); }); // ─── Digest formatting ───────────────────────────────────────────────────── @@ -74,15 +154,16 @@ test("any error in the last 24h fails the check, even on an otherwise quiet week function sampleHealth(overrides: Partial = {}): PipelineHealth { return { ok: true, + reasons: [], checkedAt: "2026-09-01T14:00:00.000Z", - intuit: { indicator: "none", description: "All Systems Operational" }, + intuit: { status: "ok", indicator: "none", description: "All Systems Operational" }, qbo: { - lastPurchaseSyncAt: "2026-09-01T10:00:00.000Z", - lastReceiptPushAt: "2026-09-01T12:00:00.000Z", + lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, + lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, }, - receipts24h: { created: 4, fallback: 1 }, - bank: { lastPostedDate: "2026-08-29T00:00:00.000Z" }, - stuck: 0, + receipts24h: { status: "ok", counts: { created: 4, fallback: 1 } }, + bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, + stuck: { status: "ok", count: 0 }, ...overrides, }; } @@ -92,9 +173,10 @@ test("digest subject says OK or NEEDS ATTENTION and leads the body", () => { assert.equal(good.subject, "Pipeline OK"); assert.equal(good.text.split("\n")[0], "Pipeline OK"); - const bad = formatPipelineDigest(sampleHealth({ ok: false, stuck: 3 })); + const bad = formatPipelineDigest(sampleHealth({ ok: false, reasons: ["errors-24h:3"] })); assert.equal(bad.subject, "Pipeline NEEDS ATTENTION"); assert.equal(bad.text.split("\n")[0], "Pipeline NEEDS ATTENTION"); + assert.match(bad.text, /Needs attention: errors-24h:3/); }); test("digest is plain text: one line per item, no markdown tables, no emoji", () => { @@ -107,20 +189,58 @@ test("digest is plain text: one line per item, no markdown tables, no emoji", () assert.match(text, /Automation errors \(24h, all kinds\): 0/); }); +test("digest says how long the silence has been, so a human can judge it", () => { + const { text } = formatPipelineDigest( + sampleHealth({ + ok: false, + reasons: ["no-receipts-72h"], + qbo: { + lastPurchaseSync: { status: "ok", at: "2026-08-20T14:00:00.000Z" }, + lastReceiptPush: { status: "ok", at: "2026-08-20T14:00:00.000Z" }, + }, + }), + ); + assert.match(text, /Last receipt booked: .* \(12d ago\)/); +}); + +test("a failed probe reads as unavailable in the digest, never as a real value", () => { + const { text } = formatPipelineDigest( + sampleHealth({ + ok: false, + reasons: ["probe-failed:stuck", "probe-failed:bank", "probe-failed:receipts24h"], + receipts24h: { status: "error", counts: {} }, + bank: { status: "error", at: null }, + stuck: { status: "error", count: 0 }, + }), + ); + assert.match(text, /Receipts \(24h\): unavailable \(probe failed\)/); + assert.match(text, /Bank ledger through: unavailable \(probe failed\)/); + // The 0 fallback must NOT be printed as a real "0 errors" all-clear. + assert.match(text, /Automation errors \(24h, all kinds\): unavailable \(probe failed\)/); + assert.doesNotMatch(text, /Automation errors \(24h, all kinds\): 0/); +}); + +test("an unreachable Intuit status page is flagged in the body", () => { + const { text } = formatPipelineDigest( + sampleHealth({ intuit: { status: "error", indicator: "unknown" } }), + ); + assert.match(text, /Intuit status: unknown \[status page unreachable\]/); +}); + test("digest renders missing timestamps as 'never' rather than a bogus date", () => { const { text } = formatPipelineDigest( - sampleHealth({ qbo: { lastPurchaseSyncAt: null, lastReceiptPushAt: null } }), + sampleHealth({ + qbo: { + lastPurchaseSync: { status: "ok", at: null }, + lastReceiptPush: { status: "ok", at: null }, + }, + }), ); assert.match(text, /Last QBO purchase sync: never/); assert.match(text, /Last receipt booked: never/); }); test("digest reports zero receipt traffic as 'none', not an empty list", () => { - const { text } = formatPipelineDigest(sampleHealth({ receipts24h: {} })); + const { text } = formatPipelineDigest(sampleHealth({ receipts24h: { status: "ok", counts: {} } })); assert.match(text, /Receipts \(24h\): none/); }); - -test("the quiet-week note is carried into the body", () => { - const { text } = formatPipelineDigest(sampleHealth({ note: "no receipts in 7d" })); - assert.match(text, /Note: no receipts in 7d/); -}); From 03273e90c4965448386cb6e475ca988286a4a9e6 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:05:51 -0700 Subject: [PATCH 009/144] fix(ops): fail-closed, constant-time Bearer auth for the pipeline routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex issues 5, 6 and 8. The copied `if (process.env.VERCEL_ENV && ...)` shape only enforced the secret where VERCEL_ENV happened to be set — an all-negative env gate that fails OPEN anywhere it is not (self-hosted, container, drifted preview). New src/lib/cron-auth.ts: authentication required everywhere except an explicit NODE_ENV === "development", and a MISSING CRON_SECRET rejects rather than waving traffic through. Comparison is timingSafeEqual with a length check first (it throws on unequal lengths). /api/cron/pipeline-digest uses isCronAuthorized (dev bypass); /api/health/pipeline's Bearer branch uses hasCronSecret, which has NO environment escape hatch, and its staff-session branch is unchanged. Issue 8: vercel.json is strict JSON and cannot hold a comment, so the note that `0 14 * * *` is 7 AM PDT / 6 AM PST (and shifts an hour across DST, since Vercel cron is UTC-only) lives in the cron route's doc comment. The schedule is unchanged. Co-Authored-By: Claude Fable 5.1 --- src/app/api/cron/pipeline-digest/route.ts | 14 ++-- src/app/api/health/pipeline/route.ts | 10 +-- src/lib/cron-auth.ts | 42 ++++++++++ tests/cron-auth.test.ts | 99 +++++++++++++++++++++++ 4 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 src/lib/cron-auth.ts create mode 100644 tests/cron-auth.test.ts diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts index 786e584eb..62912128e 100644 --- a/src/app/api/cron/pipeline-digest/route.ts +++ b/src/app/api/cron/pipeline-digest/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import { getPipelineHealth, formatPipelineDigest } from "@/lib/pipeline-health"; import { sendNotification } from "@/lib/email"; import { postTextToChatWebhook } from "@/lib/chat-webhook"; +import { isCronAuthorized } from "@/lib/cron-auth"; export const dynamic = "force-dynamic"; export const maxDuration = 60; @@ -9,7 +10,12 @@ export const maxDuration = 60; const DEFAULT_TO = "jadkins@goldentouchremodeling.com"; /** - * Morning pipeline digest (7 AM Pacific): one plain-text summary of the + * Morning pipeline digest. The vercel.json schedule is `0 14 * * *`, which is + * UTC — that is 7 AM PDT and 6 AM PST (Vercel cron has no timezone setting, so + * the delivery hour shifts by one across the DST boundary; that is accepted, + * not a bug). vercel.json is strict JSON and cannot carry this note itself. + * + * One plain-text summary of the * receipt/QBO pipeline's overnight health, so an Intuit outage or a stalled * bot is noticed over coffee instead of at month-end reconciliation. * @@ -19,10 +25,8 @@ const DEFAULT_TO = "jadkins@goldentouchremodeling.com"; * is indistinguishable from a digest that stopped running. */ export async function GET(request: Request) { - // Any deployed environment (production or preview) requires the cron secret, - // and fails closed if CRON_SECRET is unset. Only local dev skips the check. - const authHeader = request.headers.get("authorization"); - if (process.env.VERCEL_ENV && (!process.env.CRON_SECRET || authHeader !== `Bearer ${process.env.CRON_SECRET}`)) { + // Fail closed everywhere but an explicit local dev run — see cron-auth.ts. + if (!isCronAuthorized(request)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } diff --git a/src/app/api/health/pipeline/route.ts b/src/app/api/health/pipeline/route.ts index cd87937ea..58b5918d1 100644 --- a/src/app/api/health/pipeline/route.ts +++ b/src/app/api/health/pipeline/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; import { getPipelineHealth } from "@/lib/pipeline-health"; +import { hasCronSecret } from "@/lib/cron-auth"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -17,11 +18,10 @@ export const maxDuration = 30; * secret for headless/ops checks. */ export async function GET(request: Request) { - const authHeader = request.headers.get("authorization"); - const cronAuthed = Boolean( - process.env.CRON_SECRET && authHeader === `Bearer ${process.env.CRON_SECRET}`, - ); - if (!cronAuthed) { + // Bearer branch: constant-time, and a missing CRON_SECRET rejects (there is + // no environment in which this endpoint is open). The staff-session branch + // below is the normal human path. + if (!hasCronSecret(request)) { const user = await getCurrentUserWithPermissions(); if (!user) { return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); diff --git a/src/lib/cron-auth.ts b/src/lib/cron-auth.ts new file mode 100644 index 000000000..ad56267db --- /dev/null +++ b/src/lib/cron-auth.ts @@ -0,0 +1,42 @@ +import { timingSafeEqual } from "node:crypto"; + +/** + * Bearer auth for cron/ops endpoints. + * + * Two rules, both learned the hard way: + * + * - FAIL CLOSED. The older `if (process.env.VERCEL_ENV && ...)` shape only + * checked the secret where VERCEL_ENV happened to be set, so the endpoint + * was wide open in any runtime that did not define it (a self-hosted build, + * a container, a preview whose env drifted). Authentication is now required + * everywhere except an explicit local `NODE_ENV === "development"`, and a + * MISSING CRON_SECRET rejects rather than waves traffic through. + * + * - Constant time. A `!==` on a secret leaks it a byte at a time to anyone who + * can measure the response. + */ + +/** Length-checked constant-time compare of an Authorization header. */ +export function bearerMatches(header: string | null | undefined, secret: string | undefined): boolean { + if (!secret || !header) return false; + const expected = Buffer.from(`Bearer ${secret}`, "utf8"); + const actual = Buffer.from(header, "utf8"); + // timingSafeEqual throws on a length mismatch, so the lengths must be + // compared first — length is not the secret, the bytes are. + if (expected.length !== actual.length) return false; + return timingSafeEqual(actual, expected); +} + +/** True when the request carries the cron secret. No environment escape hatch. */ +export function hasCronSecret(request: Request): boolean { + return bearerMatches(request.headers.get("authorization"), process.env.CRON_SECRET); +} + +/** + * Cron-route gate: the secret, or an explicitly local dev run. Anything else — + * including a deployment that forgot to set CRON_SECRET — is rejected. + */ +export function isCronAuthorized(request: Request): boolean { + if (hasCronSecret(request)) return true; + return process.env.NODE_ENV === "development"; +} diff --git a/tests/cron-auth.test.ts b/tests/cron-auth.test.ts new file mode 100644 index 000000000..e37103d06 --- /dev/null +++ b/tests/cron-auth.test.ts @@ -0,0 +1,99 @@ +/** + * Cron/ops Bearer auth. + * + * The old shape (`if (process.env.VERCEL_ENV && ...)`) only enforced the secret + * where VERCEL_ENV happened to be set, so the endpoint was open anywhere it was + * not — the classic all-negative env gate that fails OPEN. These tests assert + * the inverse: closed by default, open only for a real secret or an explicit + * local dev run. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { bearerMatches, hasCronSecret, isCronAuthorized } from "../src/lib/cron-auth"; + +function request(authorization?: string): Request { + return new Request("https://example.test/api/cron/pipeline-digest", { + headers: authorization ? { authorization } : {}, + }); +} + +function withEnv(env: Record, run: () => void) { + const previous: Record = {}; + for (const [key, value] of Object.entries(env)) { + previous[key] = process.env[key]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + run(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + +test("bearerMatches accepts the exact secret and nothing else", () => { + assert.equal(bearerMatches("Bearer s3cret", "s3cret"), true); + assert.equal(bearerMatches("Bearer s3crf5", "s3cret"), false); + assert.equal(bearerMatches("Bearer s3cret ", "s3cret"), false); + assert.equal(bearerMatches("bearer s3cret", "s3cret"), false); + assert.equal(bearerMatches("s3cret", "s3cret"), false); +}); + +test("bearerMatches rejects a length mismatch instead of throwing", () => { + // timingSafeEqual throws on unequal lengths — the guard must come first. + assert.doesNotThrow(() => bearerMatches("Bearer short", "a-much-longer-secret")); + assert.equal(bearerMatches("Bearer short", "a-much-longer-secret"), false); + assert.equal(bearerMatches("Bearer a-much-longer-secret-plus", "a-much-longer-secret"), false); +}); + +test("a missing or empty secret rejects every header — never waves traffic through", () => { + assert.equal(bearerMatches("Bearer anything", undefined), false); + assert.equal(bearerMatches("Bearer anything", ""), false); + assert.equal(bearerMatches("Bearer ", ""), false); + assert.equal(bearerMatches(null, "s3cret"), false); + assert.equal(bearerMatches(undefined, "s3cret"), false); +}); + +test("hasCronSecret has no environment escape hatch — not even in development", () => { + withEnv({ CRON_SECRET: "s3cret", NODE_ENV: "development" }, () => { + assert.equal(hasCronSecret(request()), false); + assert.equal(hasCronSecret(request("Bearer s3cret")), true); + }); +}); + +test("isCronAuthorized rejects an unauthenticated request in production", () => { + withEnv({ CRON_SECRET: "s3cret", NODE_ENV: "production", VERCEL_ENV: undefined }, () => { + assert.equal(isCronAuthorized(request()), false); + assert.equal(isCronAuthorized(request("Bearer wrong!")), false); + assert.equal(isCronAuthorized(request("Bearer s3cret")), true); + }); +}); + +test("VERCEL_ENV being unset no longer opens the endpoint (the fail-open regression)", () => { + withEnv({ CRON_SECRET: "s3cret", NODE_ENV: "production", VERCEL_ENV: undefined }, () => { + assert.equal(isCronAuthorized(request()), false); + }); +}); + +test("a deployment that forgot CRON_SECRET rejects rather than opening up", () => { + withEnv({ CRON_SECRET: undefined, NODE_ENV: "production" }, () => { + assert.equal(isCronAuthorized(request()), false); + assert.equal(isCronAuthorized(request("Bearer anything")), false); + }); +}); + +test("local development is the one explicit bypass", () => { + withEnv({ CRON_SECRET: undefined, NODE_ENV: "development" }, () => { + assert.equal(isCronAuthorized(request()), true); + }); +}); + +test("test/CI is NOT a bypass", () => { + withEnv({ CRON_SECRET: "s3cret", NODE_ENV: "test" }, () => { + assert.equal(isCronAuthorized(request()), false); + }); +}); From 0bbdb1c67ca382e3556c4979ccf661a6a68f5fe8 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:20:35 -0700 Subject: [PATCH 010/144] fix(qbo): latch the abort winner; wrap bytes()/clone(); floor timeout values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 2. 1. raceAbortSignals latches WHICH signal aborted first, in the handler, and attribution reads that instead of inspecting callerSignal.aborted after the fact. The old check lost a real race: deadline fires, caller aborts a moment later, both read aborted by the time the catch runs, and a genuine outage was reported as a caller cancellation (500, not 503). Handlers are named and every listener is removed once the race is decided, so nothing stays attached to a caller signal that outlives the call. 2. The response proxy also wraps bytes() where the runtime has it, and clone() now returns a recursively wrapped Response. Streamed reads via .body/getReader() still surface the raw abort, noted in a comment — no QBO caller streams (the one getReader() in src is on a user-supplied receipt URL with its own controller, not a QBO response). 3. normalizeTimeoutMs floors to a positive integer and falls back to the default on anything non-finite or < 1, so AbortSignal.timeout can never receive a fraction. 4. Comment only: the route's 60s ceiling can preempt a late refresh, so the stranded-token message is best effort. Co-Authored-By: Claude Fable 5.1 --- src/lib/quickbooks.ts | 160 ++++++++++++++++++++++++++--------- tests/qb-timed-fetch.test.ts | 63 ++++++++++++++ 2 files changed, 183 insertions(+), 40 deletions(-) diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 5d2a53766..8c3b60a27 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -43,42 +43,101 @@ function safePath(url: string): string { } } +/** Clamp a configured timeout to a positive integer of milliseconds. */ +function normalizeTimeoutMs(value: number, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + // AbortSignal.timeout takes an unsigned integer; a fraction or a + // sub-millisecond value would either be coerced or reject outright. + const whole = Math.floor(value); + return whole >= 1 ? whole : fallback; +} + +interface SignalRace { + /** The signal to hand to fetch. */ + signal: AbortSignal; + /** Which input aborted FIRST, or null while none has. */ + winner: () => AbortSignal | null; +} + /** - * Abort when ANY input signal aborts. + * Abort when ANY input signal aborts, and remember which one got there first. * - * `AbortSignal.any` is the built-in, but it does not exist on every runtime we - * might land on. The old code fell back to using the CALLER's signal alone, - * which silently disabled the deadline — exactly the hang this module exists to - * prevent, and invisible until an outage. The manual combiner keeps both live. + * Two reasons this is hand-rolled rather than a bare `AbortSignal.any`: + * + * - `AbortSignal.any` does not exist on every runtime we might land on. The + * original fallback used the CALLER's signal alone, which silently disabled + * the deadline — exactly the hang this module exists to prevent. + * - Attribution cannot be reconstructed after the fact. Inspecting + * `callerSignal.aborted` in the catch block loses a real race: the deadline + * fires, the caller aborts a moment later, and by the time the rejection is + * observed BOTH signals read aborted, so a genuine timeout was reported as a + * caller cancellation and the receipt route answered 500 instead of 503. + * The winner is therefore latched in the handler, at the instant it happens. + * + * Listeners are named and all of them are removed as soon as the race is + * decided, so nothing stays attached to a caller signal that outlives the call. */ -function combineAbortSignals(signals: AbortSignal[]): AbortSignal { +function raceAbortSignals(signals: AbortSignal[]): SignalRace { + let winner: AbortSignal | null = null; + const attached: Array<{ signal: AbortSignal; handler: () => void }> = []; + + const removeAllListeners = () => { + for (const entry of attached) entry.signal.removeEventListener("abort", entry.handler); + attached.length = 0; + }; + const anyOf = (AbortSignal as unknown as { any?: (signals: AbortSignal[]) => AbortSignal }).any; - if (typeof anyOf === "function") return anyOf(signals); + const controller = typeof anyOf === "function" ? null : new AbortController(); - const controller = new AbortController(); - for (const signal of signals) { - if (signal.aborted) { - controller.abort(signal.reason); - break; + const preAborted = signals.find(signal => signal.aborted); + if (preAborted) { + winner = preAborted; + controller?.abort(preAborted.reason); + } else { + for (const signal of signals) { + const handler = () => { + if (!winner) winner = signal; + removeAllListeners(); + controller?.abort(signal.reason); + }; + attached.push({ signal, handler }); + signal.addEventListener("abort", handler, { once: true }); } - signal.addEventListener("abort", () => controller.abort(signal.reason), { once: true }); } - return controller.signal; + + const signal = controller + ? controller.signal + : signals.length === 1 + ? signals[0] + : (anyOf as (s: AbortSignal[]) => AbortSignal)(signals); + + return { signal, winner: () => winner }; } -/** Body-consuming members of Response — each one can outlive the headers. */ -const RESPONSE_BODY_METHODS = ["json", "text", "arrayBuffer", "blob", "formData"] as const; +/** + * Body-consuming members of Response — each one can outlive the headers. + * `bytes()` is newer and absent on some runtimes; it is wrapped only when the + * runtime actually provides it. + */ +const RESPONSE_BODY_METHODS = ["json", "text", "arrayBuffer", "blob", "formData", "bytes"] as const; + +interface TimeoutContext { + timeoutSignal: AbortSignal; + race: SignalRace; + url: string; + ms: number; +} /** * Translate an abort into QBTimeoutError, but ONLY when our own deadline is * what fired. A caller cancelling its own request is not a QBO outage. + * + * Attribution comes from the latched race winner, not from reading + * `aborted` flags here — by the time this runs both signals may be aborted. */ -function asQbTimeout( - error: unknown, - context: { timeoutSignal: AbortSignal; callerSignal: AbortSignal | null | undefined; url: string; ms: number }, -): unknown { +function asQbTimeout(error: unknown, context: TimeoutContext): unknown { const aborted = error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError"); - if (aborted && context.timeoutSignal.aborted && !context.callerSignal?.aborted) { + if (aborted && context.race.winner() === context.timeoutSignal) { return new QBTimeoutError( `QuickBooks request timed out after ${context.ms}ms: ${safePath(context.url)}`, ); @@ -98,15 +157,21 @@ function asQbTimeout( * So the signal stays attached (the deadline must still cut off a stalled * body), and the returned Response is proxied so every body-consuming method * translates our own abort the same way the header phase does. + * + * NOT translated: streamed reads via `res.body` / `getReader()` surface the + * raw AbortError — no QBO caller currently streams a response. */ -function wrapResponseBodyTimeouts( - response: Response, - context: { timeoutSignal: AbortSignal; callerSignal: AbortSignal | null | undefined; url: string; ms: number }, -): Response { +function wrapResponseBodyTimeouts(response: Response, context: TimeoutContext): Response { return new Proxy(response, { - get(target, prop, receiver) { - if (typeof prop === "string" && (RESPONSE_BODY_METHODS as readonly string[]).includes(prop)) { - const original = Reflect.get(target, prop, target) as (...args: unknown[]) => Promise; + get(target, prop) { + const value = Reflect.get(target, prop, target); + + if ( + typeof prop === "string" && + (RESPONSE_BODY_METHODS as readonly string[]).includes(prop) && + typeof value === "function" + ) { + const original = value as (...args: unknown[]) => Promise; return async (...args: unknown[]) => { try { return await original.apply(target, args); @@ -115,10 +180,17 @@ function wrapResponseBodyTimeouts( } }; } + + // A clone is still a QBO response under the same deadline — wrap it + // too, or reading the copy would leak the raw abort. + if (prop === "clone" && typeof value === "function") { + const clone = value as () => Response; + return () => wrapResponseBodyTimeouts(clone.apply(target), context); + } + // Getters like `ok`/`status`/`headers` must run against the real - // Response (they throw on a proxy receiver), and methods like - // `clone()` must stay bound to it. - const value = Reflect.get(target, prop, target); + // Response (they throw on a proxy receiver), and any other method + // must stay bound to it. return typeof value === "function" ? (value as (...a: unknown[]) => unknown).bind(target) : value; }, }); @@ -142,19 +214,18 @@ export async function qbTimedFetch( timeoutMs: number = Number(process.env.QB_FETCH_TIMEOUT_MS) || QB_DEFAULT_TIMEOUT_MS, ): Promise { // A misconfigured env var must not break every QB call: AbortSignal.timeout - // rejects a non-positive delay outright. - const effectiveMs = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : QB_DEFAULT_TIMEOUT_MS; + // wants a positive integer, so a fraction or a non-finite value falls back + // to the default rather than reaching it. + const effectiveMs = normalizeTimeoutMs(timeoutMs, QB_DEFAULT_TIMEOUT_MS); const timeoutSignal = AbortSignal.timeout(effectiveMs); const callerSignal = init.signal; - const signal = callerSignal - ? combineAbortSignals([callerSignal, timeoutSignal]) - : timeoutSignal; + const race = raceAbortSignals(callerSignal ? [callerSignal, timeoutSignal] : [timeoutSignal]); - const context = { timeoutSignal, callerSignal, url, ms: effectiveMs }; + const context: TimeoutContext = { timeoutSignal, race, url, ms: effectiveMs }; let response: Response; try { - response = await fetch(url, { ...init, signal }); + response = await fetch(url, { ...init, signal: race.signal }); } catch (error) { throw asQbTimeout(error, context); } @@ -199,8 +270,10 @@ const QB_REFRESH_DEFAULT_TIMEOUT_MS = 45_000; const QB_REFRESH_MAX_TIMEOUT_MS = 50_000; function refreshTimeoutMs(): number { - const configured = Number(process.env.QB_REFRESH_TIMEOUT_MS); - const requested = Number.isFinite(configured) && configured > 0 ? configured : QB_REFRESH_DEFAULT_TIMEOUT_MS; + const requested = normalizeTimeoutMs( + Number(process.env.QB_REFRESH_TIMEOUT_MS), + QB_REFRESH_DEFAULT_TIMEOUT_MS, + ); return Math.min(requested, QB_REFRESH_MAX_TIMEOUT_MS); } @@ -219,6 +292,13 @@ function refreshTimeoutMs(): number { * 2. A distinct, diagnosable error when it does fire, so the stranded-token * case is recognisable in logs instead of looking like any other timeout. * + * Both mitigations are BEST EFFORT, not a guarantee. The calling route's own + * ceiling (maxDuration 60) can preempt a late refresh: if the function is + * killed first, this deadline never fires, the message below is never logged, + * and the connection can still be left stranded with no trace beyond the + * platform timeout. Diagnosing that case means correlating a killed invocation + * with the next refresh failure. + * * Persistence order is unchanged: the caller still stores what this returns, * only after a successful exchange. */ diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index bdae182c5..f481228ba 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -151,8 +151,55 @@ test("a caller's own abort DURING the body read stays a plain error", async () = assert.equal(error instanceof QBTimeoutError, false); }); +test("a clone() of a QBO response is wrapped too", async () => { + const res = await qbTimedFetch(`${base}/v3/company/stall-body`, {}, 150); + const copy = res.clone(); + const error = await copy.json().then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); +}); + // ─── Signal combining ─────────────────────────────────────────────────────── +test("the deadline wins the race even if the caller aborts a moment later", async () => { + // The race Codex flagged: our deadline fires first, the caller aborts + // before the rejection is observed, and BOTH signals then read aborted. + // Attribution must come from who got there FIRST (latched in the handler), + // not from inspecting `callerSignal.aborted` afterwards — otherwise a real + // outage is misreported as a caller cancellation and the receipt route + // answers 500 instead of 503. + const controller = new AbortController(); + const pending = qbTimedFetch(`${base}/v3/company/hang`, { signal: controller.signal }, 60).then( + () => null, + (e: unknown) => e as Error, + ); + // A sibling deadline on the same schedule: its handler aborts the caller + // SYNCHRONOUSLY the moment the real deadline has fired. Registered after + // the call, so the wrapper's own timeout signal is created — and fires — + // first. Both signals end up aborted; only the latched winner disambiguates. + AbortSignal.timeout(60).addEventListener("abort", () => controller.abort(), { once: true }); + + const error = await pending; + // Let the sibling handler land regardless of timer/microtask interleaving, + // then assert the caller really did abort too — that is what makes this a + // race rather than a plain timeout. + await new Promise(resolve => setTimeout(resolve, 30)); + + assert.equal(controller.signal.aborted, true, "the caller signal must also be aborted by now"); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); +}); + +test("the caller still wins when it aborts first, even though the deadline follows", async () => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 40); + const error = await qbTimedFetch( + `${base}/v3/company/hang`, + { signal: controller.signal }, + 80, + ).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error); + assert.equal(error instanceof QBTimeoutError, false); +}); + test("without AbortSignal.any, a caller signal does NOT silently disable the deadline", async () => { const original = (AbortSignal as unknown as Record).any; delete (AbortSignal as unknown as Record).any; @@ -232,3 +279,19 @@ test("a garbage QB_FETCH_TIMEOUT_MS falls back to the 20s default rather than th else process.env.QB_FETCH_TIMEOUT_MS = previous; } }); + +test("fractional and sub-1ms timeouts never reach AbortSignal.timeout", async () => { + // A fraction would be coerced and a 0/negative value rejected outright, + // which would break EVERY QB call rather than one misconfigured setting. + for (const value of [0.5, 0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + const res = await qbTimedFetch(`${base}/v3/company/query`, {}, value); + assert.equal(res.status, 200, `timeout ${value} should fall back to the default`); + } + // A valid fraction floors instead of falling back: 150.9 -> 150. + const error = await qbTimedFetch(`${base}/v3/company/hang`, {}, 150.9).then( + () => null, + (e: unknown) => e as Error, + ); + assert.ok(error instanceof QBTimeoutError); + assert.match(error.message, /150ms/); +}); From 036eafc31d1c894d77120792fdc75275d9ce2749 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:45:29 -0700 Subject: [PATCH 011/144] fix(qbo): stop fallback parsers swallowing timeouts; attach on already-exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate P1 #1 and #2. parseJsonOrNull replaces `res.json().catch(() => null)` at every QBO body read (vendor create, purchase create, attachment upload, payment create, invoice memo read, invoice send). That catch turned a body-phase QBTimeoutError into "QBO returned no body" — a generic 500 instead of the retryable 503 qbo-timeout, and on the attachment path it could report a successful "attached" for an upload whose response never arrived. Only genuine parse errors resolve to null now; a timeout is rethrown. The already-exists branch no longer returns before the upload. That branch is normally reached because the FIRST attempt's response was lost after QBO committed the Purchase, so its receipt was stranded with no image and every retry took the same early return. It now re-checks the Attachable (by purchase id, filtered to Purchase links) and uploads when missing. Idempotent via a shared deterministic FileName; the result carries attachment: attached | already-attached | skipped | failed:, which the route now records for both ok branches. Co-Authored-By: Claude Fable 5.1 --- .../integrations/qbo-receipts/create/route.ts | 9 +- src/lib/qbo-receipt-push.ts | 131 ++++++++++++--- src/lib/quickbooks.ts | 28 +++- tests/qb-timed-fetch.test.ts | 29 +++- tests/qbo-receipt-push.test.ts | 158 +++++++++++++++++- 5 files changed, 320 insertions(+), 35 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 7618df787..1ffae7afe 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -238,9 +238,12 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan // The QBO deep link needs the purchase id (fileId is // already baked into `detail` by pushEventFromOutcome). ...(result.ok ? { qbPurchaseId: result.qbPurchaseId } : {}), - // Attachment evidence AT BOOKING TIME (fresh creates only — - // already-exists responses don't re-report it). - ...(result.ok && !result.alreadyExists ? { attachment: result.attachment } : {}), + // Attachment evidence, now reported on BOTH ok branches: + // an already-exists response re-checks the Attachable and + // uploads if the first attempt's response was lost, so its + // outcome ("already-attached", "attached", "failed:...") + // is real evidence rather than a repeat of booking time. + ...(result.ok ? { attachment: result.attachment } : {}), }, ); await logEvent(event); diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 89b246a5c..8b44b11c6 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -29,7 +29,9 @@ import { ensureQBCustomer, QB_API_BASE, qbTimedFetch, + parseJsonOrNull, type QBTokens, + type QBAttachable, } from "./quickbooks"; /** Thrown by ensureQBVendor when QBO rejects the create as a duplicate name (fault 6240) and a re-query still can't find it. */ @@ -97,7 +99,7 @@ export async function ensureQBVendor(tokens: QBTokens, name: string): Promise null); + const data = await parseJsonOrNull(res); if (!data?.Vendor?.Id) { const faultCode = data?.Fault?.Error?.[0]?.code; if (data?.Fault) { @@ -153,11 +155,14 @@ export interface CreateQBReceiptPurchaseInput { overheadCategory?: string; } -/** "attached" | "skipped" | "failed:" — a failure never fails the Purchase create. */ -export type ReceiptAttachmentStatus = "attached" | "skipped" | `failed:${string}`; +/** + * "attached" | "already-attached" | "skipped" | "failed:" — a + * failure never fails the Purchase create. + */ +export type ReceiptAttachmentStatus = "attached" | "already-attached" | "skipped" | `failed:${string}`; export type CreateQBReceiptPurchaseResult = - | { ok: true; qbPurchaseId: string; docNumber: string; alreadyExists: true } + | { ok: true; qbPurchaseId: string; docNumber: string; alreadyExists: true; attachment: ReceiptAttachmentStatus } | { ok: true; qbPurchaseId: string; docNumber: string; alreadyExists: false; attachment: ReceiptAttachmentStatus } | { ok: false; reason: "project-not-matched"; projectName: string } | { ok: false; reason: "docnumber-conflict"; docNumber: string } @@ -268,7 +273,7 @@ async function defaultQbCreatePurchase( } throw new Error(`QB purchase create failed: ${text}`); } - const data = await res.json().catch(() => null); + const data = await parseJsonOrNull(res); // Intuit documents that a 200 response can still carry a Fault body. if (!data?.Purchase?.Id) { const faultCode = data?.Fault?.Error?.[0]?.code; @@ -293,13 +298,22 @@ async function defaultListInProgressProjects(): Promise". */ +/** + * The Attachable FileName we upload under. Deterministic for a given receipt, + * which is what lets a later run recognise its own upload instead of adding a + * duplicate. Strips CR/LF/quotes so the name can't break out of the multipart + * header line. + */ +export function attachmentFileName(rawFileName: string | undefined): string { + return (rawFileName || "receipt").replace(/[\r\n"]/g, "") || "receipt"; +} + async function defaultUploadAttachment( tokens: QBTokens, purchaseId: string, file: { base64: string; contentType: string; fileName: string }, ): Promise { - // Strip CR/LF/quotes so the name can't break out of the multipart header line. - const safeFileName = file.fileName.replace(/[\r\n"]/g, "") || "receipt"; + const safeFileName = attachmentFileName(file.fileName); const fileBytes = Buffer.from(file.base64, "base64"); const metadata = { AttachableRef: [{ EntityRef: { value: purchaseId, type: "Purchase" } }], @@ -334,12 +348,76 @@ async function defaultUploadAttachment( body, }); if (!res.ok) return `failed:${res.status}`; - const data = await res.json().catch(() => null); + const data = await parseJsonOrNull(res); const fault = data?.AttachableResponse?.[0]?.Fault; if (fault) return "failed:fault"; return "attached"; } +/** + * The upload arguments for a receipt, or null when there is nothing uploadable + * (no file, unsupported content type, corrupt base64, oversized). Shared by the + * fresh-create and already-exists paths so both compute the SAME deterministic + * FileName — that is what makes the existence check below meaningful. + */ +function planAttachmentUpload( + input: CreateQBReceiptPurchaseInput, +): { base64: string; contentType: string; fileName: string } | null { + if (!input.fileBase64) return null; + const contentType = normalizeAttachableContentType(input.fileContentType || ""); + if (!contentType) return null; + if (!isValidBase64(input.fileBase64)) return null; + if (Buffer.byteLength(input.fileBase64, "base64") > MAX_ATTACHMENT_BYTES) return null; + return { base64: input.fileBase64, contentType, fileName: attachmentFileName(input.fileName) }; +} + +/** + * Attach the receipt to a Purchase that already exists. + * + * Reached when the DocNumber lookup finds our own earlier Purchase — most + * often because the FIRST attempt's response was lost after QBO had already + * committed it (a timeout, or the function being killed). That attempt never + * got to upload the file, and the old early return meant no later attempt ever + * would either: the receipt stayed in QBO with no image, forever. + * + * Idempotent by deterministic FileName: if an Attachable for this Purchase + * already carries the name we would upload under, this is a no-op. Never + * throws — the books entry exists and must not be undone by an image problem. + */ +async function ensureAttachmentOnExistingPurchase( + tokens: QBTokens, + purchaseId: string, + input: CreateQBReceiptPurchaseInput, + qbQueryFn: QboReceiptPushDependencies["qbQueryFn"], + uploadAttachment: QboReceiptPushDependencies["uploadAttachment"], +): Promise { + const plan = planAttachmentUpload(input); + if (!plan) return "skipped"; + // QBO transaction ids are numeric; refuse anything else rather than escape it. + if (!/^\d+$/.test(purchaseId)) return "skipped"; + + try { + const rows = await qbQueryFn( + tokens, + `SELECT * FROM attachable WHERE AttachableRef.EntityRef.value = '${purchaseId}'`, + ); + // Entity ids are only unique per entity type, so a value-only query can + // surface attachments from other transaction types — keep Purchase links. + const alreadyAttached = (rows ?? []).some( + row => + row?.AttachableRef?.some( + ref => + ref.EntityRef?.value === purchaseId && + /^purchase$/i.test(ref.EntityRef?.type ?? ""), + ) && (row.FileName ?? "") === plan.fileName, + ); + if (alreadyAttached) return "already-attached"; + return await uploadAttachment(tokens, purchaseId, plan); + } catch (error) { + return `failed:${error instanceof Error ? error.name : "error"}`; + } +} + /** * Verify the configured bank/expense accounts are what they claim to be, once * per process, before the first Purchase create. A mismatch means the env is @@ -488,7 +566,20 @@ export async function createQBReceiptPurchase( if (existing.length > 1 || !(existing[0].PrivateNote ?? "").includes(marker)) { return { ok: false, reason: "docnumber-conflict", docNumber }; } - return { ok: true, qbPurchaseId: existing[0].Id, docNumber, alreadyExists: true }; + // The Purchase exists, but that does NOT mean the receipt file made it + // across. The common way to reach this branch is a first attempt whose + // Purchase response was lost (timeout/kill) AFTER QBO committed it — + // and the old code returned here without ever reaching the upload + // below, so that receipt was stranded with no image, permanently: + // every retry took this same early return. Re-check and fill the gap. + const attachment = await ensureAttachmentOnExistingPurchase( + tokens, + existing[0].Id, + input, + qbQueryFn, + uploadAttachment, + ); + return { ok: true, qbPurchaseId: existing[0].Id, docNumber, alreadyExists: true, attachment }; } const projects = await listProjects(); @@ -624,22 +715,12 @@ export async function createQBReceiptPurchase( const created = await qbCreateFn(tokens, payload, requestId); let attachment: ReceiptAttachmentStatus = "skipped"; - if (input.fileBase64) { - const contentType = normalizeAttachableContentType(input.fileContentType || ""); - if ( - contentType && - isValidBase64(input.fileBase64) && - Buffer.byteLength(input.fileBase64, "base64") <= MAX_ATTACHMENT_BYTES - ) { - try { - attachment = await uploadAttachment(tokens, created.id, { - base64: input.fileBase64, - contentType, - fileName: input.fileName || "receipt", - }); - } catch (error) { - attachment = `failed:${error instanceof Error ? error.name : "error"}`; - } + const plan = planAttachmentUpload(input); + if (plan) { + try { + attachment = await uploadAttachment(tokens, created.id, plan); + } catch (error) { + attachment = `failed:${error instanceof Error ? error.name : "error"}`; } } diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 8c3b60a27..524f21d21 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -383,6 +383,28 @@ export async function qbQuery(tokens: QBTokens, query: string): Promise return key ? response[key] : []; } +/** + * Read a QBO JSON body, tolerating a malformed/empty one — WITHOUT swallowing + * a timeout. + * + * `res.json().catch(() => null)` is the shape this replaces, and it was a trap: + * the deadline can fire during the body read, so that catch turned a real + * QBTimeoutError into "QBO returned no body" and the caller reported a generic + * failure (500) instead of the retryable outage it was. Worse, on the + * attachment path it could report a successful "attached" for an upload whose + * response never arrived. + * + * Only genuine parse/decode errors resolve to null. A timeout is rethrown. + */ +export async function parseJsonOrNull(res: Response): Promise { + try { + return (await res.json()) as T; + } catch (error) { + if (error instanceof QBTimeoutError) throw error; + return null; + } +} + export function escapeQBString(s: string): string { // Backslash MUST be escaped before the apostrophe escape, or an input // ending in a literal backslash (e.g. "Smith\\") would have its escaped @@ -780,7 +802,7 @@ export async function sendQBPaymentCreateRequest( body: requestBody, }); if (!res.ok) throw new Error(`QB payment create failed: ${await res.text()}`); - const data = await res.json().catch(() => null); + const data = await parseJsonOrNull(res); const p = data?.Payment; if (!p?.Id) throw new Error("QB payment create returned no Payment body"); return { paymentId: String(p.Id), amount: Number(p.TotalAmt ?? 0) }; @@ -877,7 +899,7 @@ export async function appendQBInvoiceCustomerMemo( ): Promise<{ ok: true } | { ok: false; error: string }> { const read = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); if (!read.ok) return { ok: false, error: `Could not read QuickBooks invoice (${read.status})` }; - const invoice = (await read.json().catch(() => null))?.Invoice; + const invoice = (await parseJsonOrNull(read))?.Invoice; if (!invoice?.SyncToken) return { ok: false, error: "QuickBooks invoice response was incomplete" }; const current = String(invoice.CustomerMemo?.value ?? "").trim(); @@ -1188,6 +1210,6 @@ export async function sendQBInvoice(tokens: QBTokens, qbInvoiceId: string, sendT }, }); if (!res.ok) return { ok: false as const, status: res.status, error: await res.text() }; - const data = await res.json().catch(() => ({})); + const data = (await parseJsonOrNull(res)) ?? {}; return { ok: true as const, status: res.status, emailStatus: data.Invoice?.EmailStatus ?? null }; } diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index f481228ba..cde05baff 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -13,7 +13,7 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type Server } from "node:http"; import { AddressInfo } from "node:net"; -import { qbTimedFetch, QBTimeoutError } from "../src/lib/quickbooks"; +import { qbTimedFetch, QBTimeoutError, parseJsonOrNull } from "../src/lib/quickbooks"; let server: Server; let base: string; @@ -38,6 +38,11 @@ before(async () => { held.push(() => res.destroy()); return; } + if (req.url?.startsWith("/v3/company/garbage")) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end("not json at all"); + return; + } res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ ok: true, path: req.url })); }); @@ -295,3 +300,25 @@ test("fractional and sub-1ms timeouts never reach AbortSignal.timeout", async () assert.ok(error instanceof QBTimeoutError); assert.match(error.message, /150ms/); }); + + +// ─── parseJsonOrNull ──────────────────────────────────────────────────────── + +test("parseJsonOrNull returns the parsed body on success", async () => { + const res = await qbTimedFetch(`${base}/v3/company/query`, {}, 5_000); + assert.deepEqual(await parseJsonOrNull(res), { ok: true, path: "/v3/company/query" }); +}); + +test("parseJsonOrNull swallows a genuine parse error", async () => { + const res = await qbTimedFetch(`${base}/v3/company/garbage`, {}, 5_000); + assert.equal(await parseJsonOrNull(res), null); +}); + +test("parseJsonOrNull RETHROWS a body-read timeout instead of reporting an empty body", async () => { + // The trap this replaces: `.json().catch(() => null)` turned an outage into + // "QBO returned no body", which callers reported as a generic failure — and + // on the attachment path could even read as a successful upload. + const res = await qbTimedFetch(`${base}/v3/company/stall-body`, {}, 150); + const error = await parseJsonOrNull(res).then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index 510cb1dfe..550d18d1c 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -67,6 +67,10 @@ interface DepsOverrides { projects?: QboReceiptProjectCandidate[]; uploadAttachment?: QboReceiptPushDependencies["uploadAttachment"]; accountRows?: (query: string) => Array>; + /** Rows returned for the "SELECT * FROM attachable ..." existence check. */ + attachableRows?: Array>; + /** Lets a test make the attachable lookup itself fail. */ + attachableQueryImpl?: () => Promise>>; } function createDeps(overrides: DepsOverrides = {}) { @@ -82,6 +86,10 @@ function createDeps(overrides: DepsOverrides = {}) { if (/FROM Account/i.test(query)) { return (overrides.accountRows?.(query) ?? defaultAccountRow(query)) as never[]; } + if (/FROM attachable/i.test(query)) { + if (overrides.attachableQueryImpl) return (await overrides.attachableQueryImpl()) as never[]; + return (overrides.attachableRows ?? []) as never[]; + } return (overrides.existingRows ?? []) as never[]; }, qbCreateFn: async (_tokens, payload, requestId) => { @@ -119,6 +127,8 @@ test("createQBReceiptPurchase short-circuits when the DocNumber and marker both qbPurchaseId: "purchase-99", docNumber: input.fileId.slice(0, 21), alreadyExists: true, + // No file in this input, so there is nothing to attach. + attachment: "skipped", }); assert.equal(calls.creates.length, 0); assert.equal(calls.vendorCalls.length, 0); @@ -459,7 +469,7 @@ function createRouteHandlers(overrides: Partial TOKENS), createPurchase: overrides.createPurchase ?? - (async () => ({ ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true })), + (async () => ({ ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true, attachment: "already-attached" as const })), // Stub the audit logger: unit tests must never touch the real Prisma client. logEvent: overrides.logEvent ?? (() => {}), // Same for the pause switch — the real read fails CLOSED (paused) with no DB. @@ -480,7 +490,7 @@ test("route POST forwards tax:true only as an explicit boolean — string \"true const { POST } = createRouteHandlers({ createPurchase: async (_tokens, input) => { inputs.push(input); - return { ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true }; + return { ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true, attachment: "already-attached" as const }; }, }); const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { @@ -795,7 +805,7 @@ test("route POST forwards overheadCategory only as a string", async () => { const { POST } = createRouteHandlers({ createPurchase: async (_tokens, input) => { inputs.push(input); - return { ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true }; + return { ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true, attachment: "already-attached" as const }; }, }); for (const overheadCategory of ["Meals", 42]) { @@ -814,3 +824,145 @@ test("route POST forwards overheadCategory only as a string", async () => { assert.equal(inputs[0].overheadCategory, "Meals"); assert.equal(inputs[1].overheadCategory, undefined); }); + +// ─── Lost-response recovery: attaching to an existing Purchase ─────────────── + +const FILE_INPUT = { + fileBase64: Buffer.from("pretend-jpeg-bytes").toString("base64"), + fileContentType: "image/jpeg", + fileName: "receipt.jpg", +}; + +/** An Attachable row as QBO returns it, linked to the given purchase. */ +function attachableRow(purchaseId: string, fileName: string) { + return { + Id: "att-1", + FileName: fileName, + AttachableRef: [{ EntityRef: { value: purchaseId, type: "Purchase" } }], + }; +} + +test("already-exists uploads the receipt when the lost first attempt never attached it", async () => { + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + const uploads: Array<{ purchaseId: string; fileName: string }> = []; + const { deps, calls } = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableRows: [], // QBO has the Purchase but no file on it + uploadAttachment: async (_t, purchaseId, file) => { + uploads.push({ purchaseId, fileName: file.fileName }); + return "attached"; + }, + }); + + const result = await createQBReceiptPurchase(TOKENS, input, deps); + + assert.deepEqual(result, { + ok: true, + qbPurchaseId: "99", + docNumber: input.fileId.slice(0, 21), + alreadyExists: true, + attachment: "attached", + }); + // Still idempotent on the books: no second Purchase. + assert.equal(calls.creates.length, 0); + assert.deepEqual(uploads, [{ purchaseId: "99", fileName: "receipt.jpg" }]); +}); + +test("already-exists does NOT re-upload when the deterministic filename is already attached", async () => { + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + let uploadCount = 0; + const { deps } = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableRows: [attachableRow("99", "receipt.jpg")], + uploadAttachment: async () => { + uploadCount += 1; + return "attached"; + }, + }); + + const result = await createQBReceiptPurchase(TOKENS, input, deps); + + assert.equal(result.ok && result.alreadyExists && result.attachment, "already-attached"); + assert.equal(uploadCount, 0, "an existing attachment must never be duplicated"); +}); + +test("already-exists ignores an Attachable that belongs to a different entity type", async () => { + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + const { deps } = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + // Same id + filename, but linked to an Invoice — entity ids are only + // unique per type, so this must not count as our receipt. + attachableRows: [{ + Id: "att-1", + FileName: "receipt.jpg", + AttachableRef: [{ EntityRef: { value: "99", type: "Invoice" } }], + }], + uploadAttachment: async () => "attached", + }); + + const result = await createQBReceiptPurchase(TOKENS, input, deps); + assert.equal(result.ok && result.alreadyExists && result.attachment, "attached"); +}); + +test("already-exists reports a failed attachment lookup without failing the push", async () => { + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + const { deps } = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableQueryImpl: async () => { + throw new Error("QBO down"); + }, + }); + + const result = await createQBReceiptPurchase(TOKENS, input, deps); + // The books entry exists and must still be reported ok. + assert.equal(result.ok, true); + assert.equal(result.ok && result.alreadyExists && result.attachment, "failed:Error"); +}); + +test("an attachment upload that times out is reported failed:QBTimeoutError, never attached", async () => { + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + + // Both paths must classify it the same way: fresh create... + const fresh = createDeps({ + uploadAttachment: async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/upload"); + }, + }); + const freshResult = await createQBReceiptPurchase(TOKENS, input, fresh.deps); + assert.equal(freshResult.ok && !freshResult.alreadyExists && freshResult.attachment, "failed:QBTimeoutError"); + + // ...and the already-exists recovery path. + const existing = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableRows: [], + uploadAttachment: async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/upload"); + }, + }); + const existingResult = await createQBReceiptPurchase(TOKENS, input, existing.deps); + assert.equal(existingResult.ok && existingResult.alreadyExists && existingResult.attachment, "failed:QBTimeoutError"); +}); + +test("a purchase create that times out surfaces as 503 qbo-timeout at the route", async () => { + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + // parseJsonOrNull rethrows a body-read timeout instead of swallowing it as + // "no Purchase body", so the route can classify it as a retryable outage. + const { POST } = createRouteHandlers({ + createPurchase: async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/purchase"); + }, + }); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-timeout" }); +}); From 64965777bf514e6d5e8a92349ef0187921aae846 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:45:30 -0700 Subject: [PATCH 012/144] fix(ops): probe deadlines, guaranteed digest delivery, creates-only freshness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate P1 #3, #4 and P2 #5. runProbe wraps each health probe in a 5s deadline. A throwing query was already handled; a query that never SETTLES was not, so a wedged database hung the health check until the platform killed it — for the cron that means a silent morning with no digest. A timeout reports {status:"error", reason:"timeout"} and forces ok:false like any other probe failure. Digest delivery is no longer best effort: email and Chat run independently under Promise.allSettled with their own 10s deadlines (one failing or hanging can no longer cost the other), and an email that is not accepted returns 500 {ok:false, reason:"email-not-accepted"} so the failure shows in Vercel's cron history instead of a 200 nobody reads. Chat stays optional. The route gains a DI seam so this is testable. lastReceiptPushAt counts status "created" only. "already-exists" is an idempotent re-push of a receipt created earlier, so counting it refreshed the freshness clock with nothing new in the books — a bot stuck retrying one old file looked like a healthy pipeline indefinitely. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- src/app/api/cron/pipeline-digest/route.ts | 139 ++++++++++++------ src/lib/pipeline-health.ts | 99 ++++++++++--- tests/pipeline-digest-route.test.ts | 163 ++++++++++++++++++++++ tests/pipeline-health.test.ts | 45 ++++++ 5 files changed, 392 insertions(+), 57 deletions(-) create mode 100644 tests/pipeline-digest-route.test.ts diff --git a/package.json b/package.json index 6c3383ade..d9590c743 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test:qb-timeout": "tsx --test tests/qb-timed-fetch.test.ts", "test:pipeline-health": "tsx --test tests/pipeline-health.test.ts", "test:cron-auth": "tsx --test tests/cron-auth.test.ts", + "test:pipeline-digest": "tsx --test tests/pipeline-digest-route.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -23,7 +24,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts index 62912128e..5258e1728 100644 --- a/src/app/api/cron/pipeline-digest/route.ts +++ b/src/app/api/cron/pipeline-digest/route.ts @@ -1,5 +1,5 @@ import { NextResponse } from "next/server"; -import { getPipelineHealth, formatPipelineDigest } from "@/lib/pipeline-health"; +import { getPipelineHealth, formatPipelineDigest, type PipelineHealth } from "@/lib/pipeline-health"; import { sendNotification } from "@/lib/email"; import { postTextToChatWebhook } from "@/lib/chat-webhook"; import { isCronAuthorized } from "@/lib/cron-auth"; @@ -8,6 +8,8 @@ export const dynamic = "force-dynamic"; export const maxDuration = 60; const DEFAULT_TO = "jadkins@goldentouchremodeling.com"; +/** Neither delivery channel may hang the cron; each gets its own deadline. */ +export const DELIVERY_TIMEOUT_MS = 10_000; /** * Morning pipeline digest. The vercel.json schedule is `0 14 * * *`, which is @@ -15,51 +17,108 @@ const DEFAULT_TO = "jadkins@goldentouchremodeling.com"; * the delivery hour shifts by one across the DST boundary; that is accepted, * not a bug). vercel.json is strict JSON and cannot carry this note itself. * - * One plain-text summary of the - * receipt/QBO pipeline's overnight health, so an Intuit outage or a stalled - * bot is noticed over coffee instead of at month-end reconciliation. + * One plain-text summary of the receipt/QBO pipeline's overnight health, so an + * Intuit outage or a stalled bot is noticed over coffee instead of at + * month-end reconciliation. * * Uses the SAME summariser as GET /api/health/pipeline — the digest and the * on-demand check must never disagree about whether the pipeline is OK. * Sends every morning, healthy or not: a digest that only arrives on failure * is indistinguishable from a digest that stopped running. + * + * Delivery is the whole point of this route, so it is not best-effort: the two + * channels run INDEPENDENTLY (one failing must not cancel the other) and a + * rejected email is a 500. A monitoring job that silently fails to deliver is + * worse than no monitoring job, because it looks like good news. */ -export async function GET(request: Request) { - // Fail closed everywhere but an explicit local dev run — see cron-auth.ts. - if (!isCronAuthorized(request)) { - return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + +/** Resolve to a sentinel rather than reject, so allSettled reports the timeout as a value. */ +async function withDeadline(work: Promise, ms: number, onTimeout: T): Promise { + let timer: ReturnType | undefined; + try { + const deadline = new Promise(resolve => { + timer = setTimeout(() => resolve(onTimeout), ms); + }); + return await Promise.race([work, deadline]); + } finally { + clearTimeout(timer); } +} + +export interface PipelineDigestDependencies { + getHealth: () => Promise; + sendEmail: (to: string, subject: string, html: string) => Promise<{ success: boolean }>; + postChat: (webhookUrl: string, text: string) => Promise; + getChatWebhook: () => string | undefined; + getRecipient: () => string; + /** Overridable so tests need not wait out the real 10s deadline. */ + deliveryTimeoutMs?: number; +} + +export function createPipelineDigestHandlers(dependencies: PipelineDigestDependencies) { + return { + async GET(request: Request) { + // Fail closed everywhere but an explicit local dev run — see cron-auth.ts. + if (!isCronAuthorized(request)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const health = await dependencies.getHealth(); + const { subject, text } = formatPipelineDigest(health); + + const to = dependencies.getRecipient(); + // sendNotification takes HTML and derives its own plain-text part; + //
 keeps the line-per-item layout intact in an HTML client.
+            const escaped = text.replace(/&/g, "&").replace(//g, ">");
+            const html = `
${escaped}
`; - const health = await getPipelineHealth(); - const { subject, text } = formatPipelineDigest(health); - - const to = process.env.PIPELINE_DIGEST_TO || DEFAULT_TO; - // sendNotification takes HTML and derives its own plain-text part;
-    // keeps the line-per-item layout intact in an HTML client.
-    const escaped = text.replace(/&/g, "&").replace(//g, ">");
-    const emailResult = await sendNotification(
-        to,
-        subject,
-        `
${escaped}
`, - undefined, - { fromName: "ProBuild" }, - ); - - const chatWebhook = process.env.BOT_HEALTH_CHAT_WEBHOOK; - const chatPosted = chatWebhook ? await postTextToChatWebhook(chatWebhook, text) : false; - - console.log("[cron/pipeline-digest]", JSON.stringify({ - ok: health.ok, - stuck: health.stuck, - intuit: health.intuit.indicator, - emailed: emailResult.success, - chatPosted, - })); - - return NextResponse.json({ - ok: health.ok, - emailed: emailResult.success, - chatPosted, - health, - }); + const chatWebhook = dependencies.getChatWebhook(); + + // allSettled, not all: Chat is optional and a thrown webhook error + // must never cost us the email (or vice versa). + const deadlineMs = dependencies.deliveryTimeoutMs ?? DELIVERY_TIMEOUT_MS; + const [emailOutcome, chatOutcome] = await Promise.allSettled([ + withDeadline(dependencies.sendEmail(to, subject, html), deadlineMs, { success: false }), + chatWebhook + ? withDeadline(dependencies.postChat(chatWebhook, text), deadlineMs, false) + : Promise.resolve(false), + ]); + + const emailed = emailOutcome.status === "fulfilled" && emailOutcome.value.success === true; + const chatPosted = chatOutcome.status === "fulfilled" && chatOutcome.value === true; + + console.log("[cron/pipeline-digest]", JSON.stringify({ + ok: health.ok, + stuck: health.stuck, + intuit: health.intuit.indicator, + emailed, + chatPosted, + })); + + if (!emailed) { + // Non-2xx so the failure is visible in Vercel's cron history + // instead of being swallowed by a 200 nobody reads. + console.error("[cron/pipeline-digest] digest email was not accepted"); + return NextResponse.json( + { ok: false, reason: "email-not-accepted", chatPosted, health }, + { status: 500 }, + ); + } + + return NextResponse.json({ ok: health.ok, emailed, chatPosted, health }); + }, + }; +} + +const handlers = createPipelineDigestHandlers({ + getHealth: getPipelineHealth, + sendEmail: (to, subject, html) => + sendNotification(to, subject, html, undefined, { fromName: "ProBuild" }), + postChat: postTextToChatWebhook, + getChatWebhook: () => process.env.BOT_HEALTH_CHAT_WEBHOOK, + getRecipient: () => process.env.PIPELINE_DIGEST_TO || DEFAULT_TO, +}); + +export async function GET(request: Request) { + return handlers.GET(request); } diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index eff957ca0..b84b834d3 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -23,26 +23,32 @@ const HOUR_MS = 3_600_000; export const RECEIPT_STALE_HOURS = 72; export type ProbeStatus = "ok" | "error"; +/** Why a probe reports "error" — surfaced so a hang is distinguishable from a throw. */ +export type ProbeFailure = "timeout" | "error"; /** Statuspage indicators: "none" | "minor" | "major" | "critical"; "unknown" is ours. */ export interface IntuitProbe { status: ProbeStatus; + reason?: ProbeFailure; indicator: string; description?: string; } export interface TimestampProbe { status: ProbeStatus; + reason?: ProbeFailure; at: string | null; } export interface CountsProbe { status: ProbeStatus; + reason?: ProbeFailure; counts: Record; } export interface CountProbe { status: ProbeStatus; + reason?: ProbeFailure; count: number; } @@ -55,7 +61,7 @@ export interface PipelineHealth { qbo: { /** Newest Expense row QBO has synced into ProBuild job costs. */ lastPurchaseSync: TimestampProbe; - /** Newest receipt the bot actually booked (created or already-exists). */ + /** Newest receipt the bot actually CREATED (re-pushes don't count). */ lastReceiptPush: TimestampProbe; }; /** receipt-push events in the last 24h, by status ("created", "fallback", ...). */ @@ -97,8 +103,16 @@ export async function fetchIntuitStatus(): Promise { } } -/** A push that BOOKED something. "fallback"/"error" are attempts, not bookings. */ -const BOOKED_PUSH_STATUSES = ["created", "already-exists"]; +/** + * "Last receipt booked" counts CREATES only. + * + * "already-exists" is an idempotent re-push of a receipt that was created + * earlier — often much earlier — so counting it here refreshed the freshness + * clock without any new receipt entering the books, and a bot stuck retrying + * one old file would have kept the pipeline looking alive indefinitely. + * "fallback"/"error" are attempts, not bookings. + */ +export const BOOKED_PUSH_STATUSES = ["created"]; /** * The verdict, split out from the database reads so the rules are testable @@ -155,19 +169,60 @@ export function evaluatePipelineHealth(input: { return { ok: reasons.length === 0, reasons }; } +/** A probe that has not answered within this long is treated as failed. */ +export const PROBE_TIMEOUT_MS = 5_000; + +export interface ProbeResult { + status: ProbeStatus; + reason?: ProbeFailure; + value: T; +} + +/** + * Run one probe under a deadline. + * + * A throwing query was already handled; a query that never SETTLES was not. + * Prisma has no default statement timeout here, so an unreachable or wedged + * database left the health check itself hanging until the platform killed it — + * the caller got no answer at all, which for a cron means a silent morning + * with no digest. Anything past the deadline is reported as a failed probe, + * which forces ok:false the same way a thrown error does. + * + * Exported for tests: a never-settling fake is the only way to prove this. + */ +export async function runProbe( + name: string, + run: () => Promise, + onError: T, + timeoutMs: number = PROBE_TIMEOUT_MS, +): Promise> { + const TIMED_OUT = Symbol("probe-timeout"); + let timer: ReturnType | undefined; + try { + const deadline = new Promise(resolve => { + timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs); + }); + const result = await Promise.race([run(), deadline]); + if (result === TIMED_OUT) { + console.error(`[pipeline-health] probe timed out after ${timeoutMs}ms: ${name}`); + return { status: "error", reason: "timeout", value: onError }; + } + return { status: "ok", value: result as T }; + } catch (error) { + console.error(`[pipeline-health] probe failed: ${name}`, error instanceof Error ? error.name : "UnknownError"); + return { status: "error", reason: "error", value: onError }; + } finally { + // Never leave a pending timer holding the event loop open. + clearTimeout(timer); + } +} + export async function getPipelineHealth(): Promise { const now = Date.now(); const since24h = new Date(now - DAY_MS); /** Any probe failure is reported as such — never silently downgraded to "nothing found". */ - const probe = async (name: string, run: () => Promise, onError: T): Promise<{ status: ProbeStatus; value: T }> => { - try { - return { status: "ok", value: await run() }; - } catch (error) { - console.error(`[pipeline-health] probe failed: ${name}`, error instanceof Error ? error.name : "UnknownError"); - return { status: "error", value: onError }; - } - }; + const probe = runProbe; const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck] = await Promise.all([ fetchIntuitStatus(), @@ -223,11 +278,23 @@ export async function getPipelineHealth(): Promise { const snapshot = { intuit, - lastPurchaseSync: { status: lastPurchase.status, at: lastPurchase.value?.toISOString() ?? null }, - lastReceiptPush: { status: lastPush.status, at: lastPush.value?.toISOString() ?? null }, - receipts24h: { status: receiptRows.status, counts }, - bank: { status: lastBankLine.status, at: lastBankLine.value?.toISOString() ?? null }, - stuck: { status: stuck.status, count: stuck.value }, + lastPurchaseSync: { + status: lastPurchase.status, + reason: lastPurchase.reason, + at: lastPurchase.value?.toISOString() ?? null, + }, + lastReceiptPush: { + status: lastPush.status, + reason: lastPush.reason, + at: lastPush.value?.toISOString() ?? null, + }, + receipts24h: { status: receiptRows.status, reason: receiptRows.reason, counts }, + bank: { + status: lastBankLine.status, + reason: lastBankLine.reason, + at: lastBankLine.value?.toISOString() ?? null, + }, + stuck: { status: stuck.status, reason: stuck.reason, count: stuck.value }, }; const verdict = evaluatePipelineHealth({ ...snapshot, now }); diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts new file mode 100644 index 000000000..71de88fd2 --- /dev/null +++ b/tests/pipeline-digest-route.test.ts @@ -0,0 +1,163 @@ +/** + * Pipeline digest cron delivery. + * + * A monitoring job that silently fails to deliver is worse than no monitoring + * job, because its silence looks like good news. So: the two channels run + * independently, neither can hang the cron, and an unaccepted email is a 500 + * that shows up in Vercel's cron history. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { + createPipelineDigestHandlers, + type PipelineDigestDependencies, +} from "../src/app/api/cron/pipeline-digest/route"; +import type { PipelineHealth } from "../src/lib/pipeline-health"; + +const HEALTH: PipelineHealth = { + ok: true, + reasons: [], + checkedAt: "2026-09-01T14:00:00.000Z", + intuit: { status: "ok", indicator: "none" }, + qbo: { + lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, + lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, + }, + receipts24h: { status: "ok", counts: { created: 2 } }, + bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, + stuck: { status: "ok", count: 0 }, +}; + +function handlers(overrides: Partial = {}) { + return createPipelineDigestHandlers({ + getHealth: overrides.getHealth ?? (async () => HEALTH), + sendEmail: overrides.sendEmail ?? (async () => ({ success: true })), + postChat: overrides.postChat ?? (async () => true), + getChatWebhook: overrides.getChatWebhook ?? (() => undefined), + getRecipient: overrides.getRecipient ?? (() => "ops@example.test"), + deliveryTimeoutMs: overrides.deliveryTimeoutMs ?? 100, + }); +} + +function cronRequest(): Request { + return new Request("https://example.test/api/cron/pipeline-digest", { + headers: { authorization: "Bearer test-cron-secret" }, + }); +} + +function withCronSecret(run: () => Promise): Promise { + const previous = process.env.CRON_SECRET; + process.env.CRON_SECRET = "test-cron-secret"; + return run().finally(() => { + if (previous === undefined) delete process.env.CRON_SECRET; + else process.env.CRON_SECRET = previous; + }); +} + +test("a delivered digest returns 200 with the health payload", async () => { + await withCronSecret(async () => { + const { GET } = handlers(); + const response = await GET(cronRequest()); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.emailed, true); + assert.equal(body.ok, true); + }); +}); + +test("an unaccepted email is a 500 with reason email-not-accepted", async () => { + await withCronSecret(async () => { + const { GET } = handlers({ sendEmail: async () => ({ success: false }) }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + const body = await response.json(); + assert.equal(body.ok, false); + assert.equal(body.reason, "email-not-accepted"); + }); +}); + +test("a THROWN email failure is also a 500, not an unhandled rejection", async () => { + await withCronSecret(async () => { + const { GET } = handlers({ + sendEmail: async () => { + throw new Error("resend exploded"); + }, + }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + assert.equal((await response.json()).reason, "email-not-accepted"); + }); +}); + +test("a hanging email send does not hang the cron — it fails on its own deadline", async () => { + await withCronSecret(async () => { + const started = Date.now(); + // Deadline shortened for the test; production uses DELIVERY_TIMEOUT_MS. + const { GET } = handlers({ sendEmail: () => new Promise(() => {}), deliveryTimeoutMs: 100 }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + assert.equal((await response.json()).reason, "email-not-accepted"); + assert.ok(Date.now() - started < 5_000, "the cron must return on its own deadline"); + }); +}); + +test("Chat is optional: a failing webhook never costs us the email", async () => { + await withCronSecret(async () => { + const { GET } = handlers({ + getChatWebhook: () => "https://chat.googleapis.com/v1/spaces/x", + postChat: async () => { + throw new Error("chat down"); + }, + }); + const response = await GET(cronRequest()); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.emailed, true); + assert.equal(body.chatPosted, false); + }); +}); + +test("a failing email does not stop the Chat post from being attempted", async () => { + await withCronSecret(async () => { + let chatCalls = 0; + const { GET } = handlers({ + sendEmail: async () => { + throw new Error("resend exploded"); + }, + getChatWebhook: () => "https://chat.googleapis.com/v1/spaces/x", + postChat: async () => { + chatCalls += 1; + return true; + }, + }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + assert.equal(chatCalls, 1, "the channels must run independently"); + assert.equal((await response.json()).chatPosted, true); + }); +}); + +test("no webhook configured means chatPosted false, not an error", async () => { + await withCronSecret(async () => { + const { GET } = handlers({ getChatWebhook: () => undefined }); + const response = await GET(cronRequest()); + assert.equal(response.status, 200); + assert.equal((await response.json()).chatPosted, false); + }); +}); + +test("an unauthenticated request is rejected before any delivery is attempted", async () => { + await withCronSecret(async () => { + let sends = 0; + const { GET } = handlers({ + sendEmail: async () => { + sends += 1; + return { success: true }; + }, + }); + const response = await GET(new Request("https://example.test/api/cron/pipeline-digest")); + assert.equal(response.status, 401); + assert.equal(sends, 0); + }); +}); diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 5fb57aec7..8939e747d 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -14,6 +14,8 @@ import assert from "node:assert/strict"; import { evaluatePipelineHealth, formatPipelineDigest, + runProbe, + BOOKED_PUSH_STATUSES, type PipelineHealth, } from "../src/lib/pipeline-health"; @@ -244,3 +246,46 @@ test("digest reports zero receipt traffic as 'none', not an empty list", () => { const { text } = formatPipelineDigest(sampleHealth({ receipts24h: { status: "ok", counts: {} } })); assert.match(text, /Receipts \(24h\): none/); }); + + +// ─── Probe deadlines ──────────────────────────────────────────────────────── + +test("a probe that never settles is reported as a timeout, not left hanging", async () => { + // Prisma has no statement timeout here, so a wedged database used to hang + // the whole health check until the platform killed it — for a cron that + // means a silent morning with no digest at all. + const started = Date.now(); + const result = await runProbe("wedged", () => new Promise(() => {}), -1, 50); + assert.deepEqual(result, { status: "error", reason: "timeout", value: -1 }); + assert.ok(Date.now() - started < 2_000, "must return on its own deadline"); +}); + +test("a probe that resolves in time reports ok with its value", async () => { + const result = await runProbe("fast", async () => 42, -1, 1_000); + assert.deepEqual(result, { status: "ok", value: 42 }); +}); + +test("a throwing probe is an error with reason 'error', distinct from a timeout", async () => { + const result = await runProbe("boom", async () => { throw new Error("db down"); }, -1, 1_000); + assert.deepEqual(result, { status: "error", reason: "error", value: -1 }); +}); + +test("a timed-out probe still forces ok:false through the verdict", async () => { + const timedOut = await runProbe("stuck", () => new Promise(() => {}), 0, 20); + const v = evaluatePipelineHealth(snapshot({ stuck: { status: timedOut.status, reason: timedOut.reason, count: timedOut.value } })); + assert.equal(v.ok, false); + assert.ok(v.reasons.includes("probe-failed:stuck")); +}); + + +// ─── What counts as "booked" ──────────────────────────────────────────────── + +test("only a CREATE refreshes the last-booked clock — a re-push does not", () => { + // "already-exists" is an idempotent re-push of a receipt created earlier, + // so counting it kept the freshness clock alive with no new receipt in the + // books: a bot stuck retrying one old file looked like a healthy pipeline. + assert.deepEqual(BOOKED_PUSH_STATUSES, ["created"]); + assert.equal(BOOKED_PUSH_STATUSES.includes("already-exists"), false); + assert.equal(BOOKED_PUSH_STATUSES.includes("fallback"), false); + assert.equal(BOOKED_PUSH_STATUSES.includes("error"), false); +}); From 711d04c615b73d78e09a270e08847e6c36a325f2 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:04:24 -0700 Subject: [PATCH 013/144] fix(qbo,ops): propagate refresh + attachment timeouts; reachable health; real email check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 3, all four P1s. 1. getFreshQBTokens swallowed a refresh QBTimeoutError and returned STALE tokens, so the caller spent another full QBO deadline and the 60s ceiling was still reachable. The policy moved to refreshTokensOrFallBack (what getFreshQBTokens now calls, same real defaults) and rethrows a timeout. CHOICE: the stale-token fallback is KEPT for non-timeout failures — that is the existing intent (an ordinary refresh error can still leave the old access token valid); only timeouts propagate. 2. /api/health/pipeline was intercepted by NextAuth because the proxy only bypassed the exact /api/health path, so headless Bearer checks were redirected to /login. Added an exact-match bypass in both the pattern and the matcher; the route self-authenticates. No descendant inherits it. 3. email.ts returns {success:true} on a missing RESEND_API_KEY, so the digest could report emailed:true and 200 while delivering nothing — the failure disguised as good news. isEmailDeliveryConfigured() fails closed in production, in the digest route only; email.ts is unchanged for every other caller. Chat still posts. 4. An attachment QBTimeoutError was turned into failed:QBTimeoutError on an ok:true response, which the Apps Script treats as FINAL — the Purchase kept a missing receipt forever and the existing-Purchase recovery never ran. It now propagates so the route answers 503 and the next pass attaches. Non-timeout attachment failures keep failed: + ok:true. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- src/app/api/cron/pipeline-digest/route.ts | 28 ++++++++- src/lib/qbo-receipt-push.ts | 13 ++++ src/lib/quickbooks-payments.ts | 32 +++++++++- src/proxy.ts | 9 ++- tests/pipeline-digest-route.test.ts | 75 +++++++++++++++++++++++ tests/proxy-health-pipeline.test.ts | 71 +++++++++++++++++++++ tests/qbo-receipt-push.test.ts | 59 ++++++++++++++---- tests/qbo-token-refresh-timeout.test.ts | 67 ++++++++++++++++++++ 9 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 tests/proxy-health-pipeline.test.ts create mode 100644 tests/qbo-token-refresh-timeout.test.ts diff --git a/package.json b/package.json index d9590c743..d374d148c 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "test:pipeline-health": "tsx --test tests/pipeline-health.test.ts", "test:cron-auth": "tsx --test tests/cron-auth.test.ts", "test:pipeline-digest": "tsx --test tests/pipeline-digest-route.test.ts", + "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", + "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -24,7 +26,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts index 5258e1728..35291561f 100644 --- a/src/app/api/cron/pipeline-digest/route.ts +++ b/src/app/api/cron/pipeline-digest/route.ts @@ -32,6 +32,22 @@ export const DELIVERY_TIMEOUT_MS = 10_000; * worse than no monitoring job, because it looks like good news. */ +/** + * In production, a missing RESEND_API_KEY is a DELIVERY FAILURE, not a no-op. + * + * src/lib/email.ts falls back to a dummy key and returns {success:true} without + * sending anything, which is fine for local dev but poison here: the pulse that + * exists to reveal a broken pipeline would report emailed:true and HTTP 200 + * while delivering nothing — the failure disguised as good news. Checked HERE + * rather than in email.ts so no other caller's behaviour changes. + */ +export function isEmailDeliveryConfigured(): boolean { + const productionish = + process.env.VERCEL_ENV === "production" || process.env.NODE_ENV === "production"; + if (!productionish) return true; + return Boolean(process.env.RESEND_API_KEY?.trim()); +} + /** Resolve to a sentinel rather than reject, so allSettled reports the timeout as a value. */ async function withDeadline(work: Promise, ms: number, onTimeout: T): Promise { let timer: ReturnType | undefined; @@ -51,6 +67,8 @@ export interface PipelineDigestDependencies { postChat: (webhookUrl: string, text: string) => Promise; getChatWebhook: () => string | undefined; getRecipient: () => string; + /** False when email cannot actually be delivered (see isEmailDeliveryConfigured). */ + isEmailConfigured?: () => boolean; /** Overridable so tests need not wait out the real 10s deadline. */ deliveryTimeoutMs?: number; } @@ -77,8 +95,14 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende // allSettled, not all: Chat is optional and a thrown webhook error // must never cost us the email (or vice versa). const deadlineMs = dependencies.deliveryTimeoutMs ?? DELIVERY_TIMEOUT_MS; + const emailConfigured = (dependencies.isEmailConfigured ?? isEmailDeliveryConfigured)(); + if (!emailConfigured) { + console.error("[cron/pipeline-digest] RESEND_API_KEY is not set — the digest cannot be delivered"); + } const [emailOutcome, chatOutcome] = await Promise.allSettled([ - withDeadline(dependencies.sendEmail(to, subject, html), deadlineMs, { success: false }), + emailConfigured + ? withDeadline(dependencies.sendEmail(to, subject, html), deadlineMs, { success: false }) + : Promise.resolve({ success: false }), chatWebhook ? withDeadline(dependencies.postChat(chatWebhook, text), deadlineMs, false) : Promise.resolve(false), @@ -96,6 +120,8 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende })); if (!emailed) { + // Chat still ran above — an unconfigured mailer must not cost + // the one channel that might still reach a human. // Non-2xx so the failure is visible in Vercel's cron history // instead of being swallowed by a 200 nobody reads. console.error("[cron/pipeline-digest] digest email was not accepted"); diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 8b44b11c6..0cf678700 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -30,6 +30,7 @@ import { QB_API_BASE, qbTimedFetch, parseJsonOrNull, + QBTimeoutError, type QBTokens, type QBAttachable, } from "./quickbooks"; @@ -414,6 +415,9 @@ async function ensureAttachmentOnExistingPurchase( if (alreadyAttached) return "already-attached"; return await uploadAttachment(tokens, purchaseId, plan); } catch (error) { + // A timeout must NOT become a terminal "failed:..." on an ok:true + // response — see the create path below for why. + if (error instanceof QBTimeoutError) throw error; return `failed:${error instanceof Error ? error.name : "error"}`; } } @@ -720,6 +724,15 @@ export async function createQBReceiptPurchase( try { attachment = await uploadAttachment(tokens, created.id, plan); } catch (error) { + // A TIMEOUT propagates, unlike every other attachment failure. + // Reporting it as `failed:QBTimeoutError` alongside ok:true made it + // TERMINAL: the Apps Script treats any ok:true as final and stops + // resending, so the Purchase would keep its missing receipt forever + // and the existing-Purchase recovery above would never run. Letting + // it out gives the route a 503, the bot retries, and the next pass + // finds the Purchase and attaches the file. Non-timeout failures + // keep the old best-effort behaviour — they are not retryable. + if (error instanceof QBTimeoutError) throw error; attachment = `failed:${error instanceof Error ? error.name : "error"}`; } } diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 87ce7afc8..63917015d 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -19,6 +19,7 @@ import { getQBSettings, saveQBSettings } from "./integration-store"; import { type QBTokens, refreshQBToken, + QBTimeoutError, ensureQBCustomer, ensureQBServiceItem, createQBMilestoneInvoice, @@ -54,11 +55,36 @@ export async function getFreshQBTokens(): Promise { if (!qb.connected || !qb.accessToken || !qb.refreshToken || !qb.realmId) { throw new QBNotConnectedError(); } + return refreshTokensOrFallBack({ + accessToken: qb.accessToken, + refreshToken: qb.refreshToken, + realmId: qb.realmId, + }); +} + +/** + * The refresh + fallback policy, split out so the CATCH can be tested without a + * database (only the network boundary is injectable; the defaults here are the + * real ones getFreshQBTokens uses). + * + * A refresh that fails for an ordinary reason may still leave the OLD access + * token usable, so that fallback stays. A TIMEOUT is different and must + * propagate: swallowing it handed the caller a possibly-stale token and let it + * spend another full QBO deadline on the next request, which is how a QBO + * outage still ate the route's whole 60s ceiling — exactly the hang the + * per-request deadline was added to stop. The caller turns it into a 503. + */ +export async function refreshTokensOrFallBack( + qb: { accessToken: string; refreshToken: string; realmId: string }, + refresh: typeof refreshQBToken = refreshQBToken, + save: typeof saveQBSettings = saveQBSettings, +): Promise { try { - const fresh = await refreshQBToken(qb.refreshToken); - await saveQBSettings({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); + const fresh = await refresh(qb.refreshToken); + await save({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); return { accessToken: fresh.accessToken, refreshToken: fresh.refreshToken, realmId: qb.realmId }; - } catch { + } catch (error) { + if (error instanceof QBTimeoutError) throw error; // Refresh can fail transiently; the old access token may still be valid. return { accessToken: qb.accessToken, refreshToken: qb.refreshToken, realmId: qb.realmId }; } diff --git a/src/proxy.ts b/src/proxy.ts index 19e58d637..4bd60d34f 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -48,10 +48,15 @@ const MOBILE_AUTHENTICATED_ROUTE_PATTERNS = [ // bot) that self-authenticates with a Bearer secret (OFFICE_TASKS_INGEST_SECRET) // and must return a clean 401, not a redirect to /login. Exact-match only — // future descendants under /api/office-tasks/ must NOT inherit the bypass. +// api/health/pipeline is the same shape: an ops/monitoring read that +// self-authenticates INSIDE the route (Bearer CRON_SECRET, or a staff session +// with financialReports). Without this bypass NextAuth intercepts it first and +// a headless Bearer check gets redirected to /login instead of its JSON. +// Exact-match only — nothing else under /api/health/ inherits it. // privacy / terms / account-deletion are static legal pages with no data access. // The app stores require them to be reachable by a logged-out reviewer, and Google // Play specifically requires a public account-deletion URL. -const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/(?:auth|cron|twilio|webhook|payments|portal|integrations|mcp(?:\/|$)|version|pdf\/(?:estimates|invoices|change-orders)|sub-portal|mobile|selections\/(?:item-comments|ai-sort|link-schedule))(?:\/|$)|api\/office-tasks\/ingest\/?$|login(?:\/|$)|portal(?:\/|$)|sub-portal(?:\/|$)|share(?:\/|$)|privacy(?:\/|$)|terms(?:\/|$)|account-deletion(?:\/|$)|support(?:\/|$)|_next\/(?:static|image)(?:\/|$)|favicon\.ico$|.*\.(?:png|jpg|svg|webmanifest)$)/; +const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/health\/pipeline\/?$|api\/(?:auth|cron|twilio|webhook|payments|portal|integrations|mcp(?:\/|$)|version|pdf\/(?:estimates|invoices|change-orders)|sub-portal|mobile|selections\/(?:item-comments|ai-sort|link-schedule))(?:\/|$)|api\/office-tasks\/ingest\/?$|login(?:\/|$)|portal(?:\/|$)|sub-portal(?:\/|$)|share(?:\/|$)|privacy(?:\/|$)|terms(?:\/|$)|account-deletion(?:\/|$)|support(?:\/|$)|_next\/(?:static|image)(?:\/|$)|favicon\.ico$|.*\.(?:png|jpg|svg|webmanifest)$)/; // The legal pages are static server components that define no Server Actions. // Next's action IDs are global, so a bypassed path is a place an anonymous caller @@ -225,6 +230,6 @@ export const config = { * - favicon.ico, public folder images, etc * - manifest.webmanifest (PWA manifest — must be fetchable for install) */ - "/((?!api/health$|api/auth|api/cron|api/twilio|api/webhook|api/payments|api/portal|api/integrations|api/mcp/|api/version|api/pdf/estimates(?:/|$)|api/pdf/invoices(?:/|$)|api/pdf/change-orders(?:/|$)|api/sub-portal|api/mobile|api/selections/item-comments|api/selections/ai-sort|api/selections/link-schedule|login|portal|sub-portal|share|privacy|terms|account-deletion|support|_next/static|_next/image|favicon.ico|.*\\.png|.*\\.jpg|.*\\.svg|.*\\.webmanifest).*)", + "/((?!api/health$|api/health/pipeline$|api/auth|api/cron|api/twilio|api/webhook|api/payments|api/portal|api/integrations|api/mcp/|api/version|api/pdf/estimates(?:/|$)|api/pdf/invoices(?:/|$)|api/pdf/change-orders(?:/|$)|api/sub-portal|api/mobile|api/selections/item-comments|api/selections/ai-sort|api/selections/link-schedule|login|portal|sub-portal|share|privacy|terms|account-deletion|support|_next/static|_next/image|favicon.ico|.*\\.png|.*\\.jpg|.*\\.svg|.*\\.webmanifest).*)", ], }; diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts index 71de88fd2..dd85cec19 100644 --- a/tests/pipeline-digest-route.test.ts +++ b/tests/pipeline-digest-route.test.ts @@ -11,6 +11,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createPipelineDigestHandlers, + isEmailDeliveryConfigured, type PipelineDigestDependencies, } from "../src/app/api/cron/pipeline-digest/route"; import type { PipelineHealth } from "../src/lib/pipeline-health"; @@ -36,6 +37,7 @@ function handlers(overrides: Partial = {}) { postChat: overrides.postChat ?? (async () => true), getChatWebhook: overrides.getChatWebhook ?? (() => undefined), getRecipient: overrides.getRecipient ?? (() => "ops@example.test"), + isEmailConfigured: overrides.isEmailConfigured ?? (() => true), deliveryTimeoutMs: overrides.deliveryTimeoutMs ?? 100, }); } @@ -161,3 +163,76 @@ test("an unauthenticated request is rejected before any delivery is attempted", assert.equal(sends, 0); }); }); + + +// ─── Missing mailer credentials ───────────────────────────────────────────── + +function withEnv(env: Record, run: () => void) { + const previous: Record = {}; + for (const [key, value] of Object.entries(env)) { + previous[key] = process.env[key]; + if (value === undefined) delete (process.env as Record)[key]; + else (process.env as Record)[key] = value; + } + try { + run(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete (process.env as Record)[key]; + else (process.env as Record)[key] = value; + } + } +} + +test("in production a missing RESEND_API_KEY is a delivery failure, not a silent no-op", () => { + // email.ts falls back to a dummy key and returns {success:true} without + // sending — which would make the pulse report good news while delivering + // nothing. Checked here so no other caller's behaviour changes. + withEnv({ NODE_ENV: "production", VERCEL_ENV: "production", RESEND_API_KEY: undefined }, () => { + assert.equal(isEmailDeliveryConfigured(), false); + }); + withEnv({ NODE_ENV: "production", VERCEL_ENV: "production", RESEND_API_KEY: " " }, () => { + assert.equal(isEmailDeliveryConfigured(), false); + }); + withEnv({ NODE_ENV: "production", VERCEL_ENV: "production", RESEND_API_KEY: "re_live_key" }, () => { + assert.equal(isEmailDeliveryConfigured(), true); + }); +}); + +test("local development without a key is still fine", () => { + withEnv({ NODE_ENV: "development", VERCEL_ENV: undefined, RESEND_API_KEY: undefined }, () => { + assert.equal(isEmailDeliveryConfigured(), true); + }); +}); + +test("an unconfigured mailer returns 500 and never claims emailed:true", async () => { + await withCronSecret(async () => { + let sends = 0; + const { GET } = handlers({ + isEmailConfigured: () => false, + sendEmail: async () => { + sends += 1; + return { success: true }; // what email.ts would wrongly report + }, + }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + const body = await response.json(); + assert.equal(body.ok, false); + assert.equal(body.reason, "email-not-accepted"); + assert.equal(sends, 0, "no point calling a mailer that cannot deliver"); + }); +}); + +test("an unconfigured mailer still lets the Chat post through", async () => { + await withCronSecret(async () => { + const { GET } = handlers({ + isEmailConfigured: () => false, + getChatWebhook: () => "https://chat.googleapis.com/v1/spaces/x", + postChat: async () => true, + }); + const response = await GET(cronRequest()); + assert.equal(response.status, 500); + assert.equal((await response.json()).chatPosted, true); + }); +}); diff --git a/tests/proxy-health-pipeline.test.ts b/tests/proxy-health-pipeline.test.ts new file mode 100644 index 000000000..9b36f0de7 --- /dev/null +++ b/tests/proxy-health-pipeline.test.ts @@ -0,0 +1,71 @@ +/** + * /api/health/pipeline must reach its own handler. + * + * Codex gate: the route self-authenticates (Bearer CRON_SECRET, or a staff + * session with financialReports), but the proxy only bypassed the EXACT + * /api/health path — so NextAuth intercepted /api/health/pipeline first and a + * headless Bearer check got a redirect to /login instead of its JSON. + * + * Exact-match only, mirroring the api/office-tasks/ingest precedent: nothing + * else under /api/health/ may inherit a public bypass. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; + +process.env.NEXTAUTH_SECRET ??= "test-secret"; +process.env.DATABASE_URL ??= "postgresql://test:test@localhost:5432/test"; + +const loadProxy = () => import("../src/proxy"); + +test("the pipeline health endpoint is bypassed so its own auth can answer", async () => { + const { isPublicProxyBypass } = await loadProxy(); + for (const path of ["/api/health", "/api/health/pipeline", "/api/health/pipeline/"]) { + assert.equal(isPublicProxyBypass(path), true, path); + } +}); + +test("the bypass does NOT widen to other /api/health descendants", async () => { + const { isPublicProxyBypass } = await loadProxy(); + for (const path of [ + "/api/health/pipeline/deep", + "/api/health/secrets", + "/api/healthcheck", + "/api/health-pipeline", + ]) { + assert.equal(isPublicProxyBypass(path), false, path); + } +}); + +test("in production mode a Bearer request reaches the handler instead of /login", async () => { + const previousEnv = process.env.NODE_ENV; + const previousVercel = process.env.VERCEL_ENV; + // NODE_ENV is readonly in the Next types but writable at runtime. + (process.env as Record).NODE_ENV = "production"; + (process.env as Record).VERCEL_ENV = "production"; + try { + const { default: proxy } = await loadProxy(); + const { NextRequest } = await import("next/server"); + const event = { waitUntil() {} } as any; + + const request = new NextRequest("https://probuild.test/api/health/pipeline", { + method: "GET", + headers: { authorization: "Bearer some-cron-secret" }, + }); + const response = await proxy(request, event); + + assert.ok(response instanceof Response, "proxy must return a response"); + // x-middleware-next: 1 means "continue to the route handler", which is + // where the endpoint's own Bearer/staff-session check runs. + assert.equal( + response.headers.get("x-middleware-next"), + "1", + `expected pass-through to the handler, got status ${response.status}`, + ); + } finally { + if (previousEnv === undefined) delete (process.env as Record).NODE_ENV; + else (process.env as Record).NODE_ENV = previousEnv; + if (previousVercel === undefined) delete process.env.VERCEL_ENV; + else process.env.VERCEL_ENV = previousVercel; + } +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index 550d18d1c..d965c3755 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -923,30 +923,65 @@ test("already-exists reports a failed attachment lookup without failing the push assert.equal(result.ok && result.alreadyExists && result.attachment, "failed:Error"); }); -test("an attachment upload that times out is reported failed:QBTimeoutError, never attached", async () => { +test("an attachment upload that times out PROPAGATES from both paths, so the push is retryable", async () => { const { QBTimeoutError } = await import("../src/lib/quickbooks"); const input = baseInput({ ...FILE_INPUT }); const marker = `[gtr-file:${input.fileId}]`; + const timeout = () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/upload"); + }; - // Both paths must classify it the same way: fresh create... - const fresh = createDeps({ - uploadAttachment: async () => { - throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/upload"); - }, - }); - const freshResult = await createQBReceiptPurchase(TOKENS, input, fresh.deps); - assert.equal(freshResult.ok && !freshResult.alreadyExists && freshResult.attachment, "failed:QBTimeoutError"); + // Codex gate: reporting this as `failed:QBTimeoutError` on an ok:true + // response made it TERMINAL — the Apps Script treats ok:true as final and + // stops resending, so the Purchase kept a missing receipt forever and the + // existing-Purchase recovery never ran. It must throw instead. + const fresh = createDeps({ uploadAttachment: async () => timeout() }); + await assert.rejects( + () => createQBReceiptPurchase(TOKENS, input, fresh.deps), + (error: unknown) => error instanceof QBTimeoutError, + ); - // ...and the already-exists recovery path. const existing = createDeps({ existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], attachableRows: [], + uploadAttachment: async () => timeout(), + }); + await assert.rejects( + () => createQBReceiptPurchase(TOKENS, input, existing.deps), + (error: unknown) => error instanceof QBTimeoutError, + ); +}); + +test("a NON-timeout attachment failure stays best-effort: failed: with ok:true", async () => { + const input = baseInput({ ...FILE_INPUT }); + const { deps } = createDeps({ uploadAttachment: async () => { + throw new Error("boom"); + }, + }); + const result = await createQBReceiptPurchase(TOKENS, input, deps); + // The Purchase is on the books; a broken image is not worth a retry loop. + assert.equal(result.ok, true); + assert.equal(result.ok && !result.alreadyExists && result.attachment, "failed:Error"); +}); + +test("route: an attachment timeout surfaces as 503 qbo-timeout so the bot retries", async () => { + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + const events: AutomationEventInput[] = []; + const { POST } = createRouteHandlers({ + createPurchase: async () => { throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/upload"); }, + logEvent: event => { events.push(event); }, }); - const existingResult = await createQBReceiptPurchase(TOKENS, input, existing.deps); - assert.equal(existingResult.ok && existingResult.alreadyExists && existingResult.attachment, "failed:QBTimeoutError"); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-timeout" }); + assert.equal(events[0].reason, "qbo-timeout"); }); test("a purchase create that times out surfaces as 503 qbo-timeout at the route", async () => { diff --git a/tests/qbo-token-refresh-timeout.test.ts b/tests/qbo-token-refresh-timeout.test.ts new file mode 100644 index 000000000..38386f2d2 --- /dev/null +++ b/tests/qbo-token-refresh-timeout.test.ts @@ -0,0 +1,67 @@ +/** + * getFreshQBTokens must not swallow a refresh TIMEOUT. + * + * Codex gate: refreshQBToken throws QBTimeoutError, but the catch here returned + * the STALE tokens instead. The caller then spent another full QBO deadline on + * its next request, so an Intuit outage still ate the route's 60s ceiling — + * the exact hang the per-request deadline exists to prevent. + * + * These exercise the REAL policy (refreshTokensOrFallBack is what + * getFreshQBTokens calls, with these same defaults); only the network boundary + * is faked, because a unit test has no database for the settings row. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +process.env.NEXTAUTH_SECRET ??= "test-secret"; +process.env.DATABASE_URL ??= "postgresql://test:test@localhost:5432/test"; + +const STALE = { accessToken: "stale-access", refreshToken: "stale-refresh", realmId: "realm-1" }; + +test("a refresh TIMEOUT propagates instead of handing back stale tokens", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + const { QBTimeoutError } = await import("../src/lib/quickbooks"); + + let saved = false; + const error = await refreshTokensOrFallBack( + STALE, + async () => { + throw new QBTimeoutError("QBO token refresh timed out"); + }, + async () => { + saved = true; + }, + ).then(() => null, (e: unknown) => e as Error); + + assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); + assert.equal(saved, false, "a timed-out refresh must never persist anything"); +}); + +test("an ORDINARY refresh failure still falls back to the old access token", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + const tokens = await refreshTokensOrFallBack( + STALE, + async () => { + throw new Error("500 from Intuit"); + }, + async () => {}, + ); + // Deliberate: the old access token may still be valid, and this is the + // long-standing behaviour for non-timeout failures. + assert.deepEqual(tokens, STALE); +}); + +test("a successful refresh persists the rotated token and returns it", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + const saves: Array<{ accessToken: string; refreshToken: string }> = []; + const tokens = await refreshTokensOrFallBack( + STALE, + async () => ({ accessToken: "new-access", refreshToken: "new-refresh" }), + async (settings) => { + saves.push(settings as { accessToken: string; refreshToken: string }); + }, + ); + assert.deepEqual(tokens, { accessToken: "new-access", refreshToken: "new-refresh", realmId: "realm-1" }); + assert.deepEqual(saves, [{ accessToken: "new-access", refreshToken: "new-refresh" }]); +}); From b1d93a9b260b7d116ca546ca8659ed5e35c23a2d Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:12:27 -0700 Subject: [PATCH 014/144] fix(qbo): match QB timeouts by name, not instanceof (CI Node 20 caught it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's unit job failed where local passed: "a refresh TIMEOUT propagates" resolved instead of rejecting. Cause is module identity, not logic — under Node 20's CJS/ESM interop quickbooks.ts loaded twice, so the class the test threw was not the class quickbooks-payments.ts compared against and `instanceof` was false, sending the timeout down the stale-token fallback. This is not a test artifact: bundler chunk duplication does the same thing in production, and the failure mode is every timeout branch in the codebase silently taking the non-timeout path — the exact misclassification the deadline work exists to prevent. isQBTimeoutError() accepts either the real class or any Error carrying name === "QBTimeoutError" (set as a class field on every instance), and now backs all six checks: parseJsonOrNull, refreshQBToken, refreshTokensOrFallBack, both attachment paths, and the receipt route. Tested against a foreign duplicate class, and pinned against over-matching. Co-Authored-By: Claude Fable 5.1 --- .../integrations/qbo-receipts/create/route.ts | 6 ++--- src/lib/qbo-receipt-push.ts | 6 ++--- src/lib/quickbooks-payments.ts | 4 +-- src/lib/quickbooks.ts | 22 ++++++++++++++-- tests/qb-timed-fetch.test.ts | 25 ++++++++++++++++++- tests/qbo-receipt-push.test.ts | 4 +-- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 1ffae7afe..de0dec440 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -9,7 +9,7 @@ import { type CreateQBReceiptPurchaseInput, type CreateQBReceiptPurchaseResult, } from "@/lib/qbo-receipt-push"; -import { QBTimeoutError, type QBTokens } from "@/lib/quickbooks"; +import { isQBTimeoutError, type QBTokens } from "@/lib/quickbooks"; export const dynamic = "force-dynamic"; // Stays at 60. A single push does a lot of SERIAL QBO work on a healthy day — @@ -212,7 +212,7 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(pushEventFromOutcome(input, { status: "error", reason: "quickbooks-not-connected" })); return NextResponse.json({ ok: false, reason: "quickbooks-not-connected" }, { status: 503 }); } - if (error instanceof QBTimeoutError) { + if (isQBTimeoutError(error)) { await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-timeout" })); return NextResponse.json({ ok: false, retry: true, reason: "qbo-timeout" }, { status: 503 }); } @@ -249,7 +249,7 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(event); return NextResponse.json(result); } catch (error) { - if (error instanceof QBTimeoutError) { + if (isQBTimeoutError(error)) { // QBO is unreachable, not saying no — 503 so the Apps // Script retries on its next pass instead of falling back // to the email path. Safe to retry even if the create did diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 0cf678700..2dd9161c4 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -30,7 +30,7 @@ import { QB_API_BASE, qbTimedFetch, parseJsonOrNull, - QBTimeoutError, + isQBTimeoutError, type QBTokens, type QBAttachable, } from "./quickbooks"; @@ -417,7 +417,7 @@ async function ensureAttachmentOnExistingPurchase( } catch (error) { // A timeout must NOT become a terminal "failed:..." on an ok:true // response — see the create path below for why. - if (error instanceof QBTimeoutError) throw error; + if (isQBTimeoutError(error)) throw error; return `failed:${error instanceof Error ? error.name : "error"}`; } } @@ -732,7 +732,7 @@ export async function createQBReceiptPurchase( // it out gives the route a 503, the bot retries, and the next pass // finds the Purchase and attaches the file. Non-timeout failures // keep the old best-effort behaviour — they are not retryable. - if (error instanceof QBTimeoutError) throw error; + if (isQBTimeoutError(error)) throw error; attachment = `failed:${error instanceof Error ? error.name : "error"}`; } } diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 63917015d..b5aaf2592 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -19,7 +19,7 @@ import { getQBSettings, saveQBSettings } from "./integration-store"; import { type QBTokens, refreshQBToken, - QBTimeoutError, + isQBTimeoutError, ensureQBCustomer, ensureQBServiceItem, createQBMilestoneInvoice, @@ -84,7 +84,7 @@ export async function refreshTokensOrFallBack( await save({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); return { accessToken: fresh.accessToken, refreshToken: fresh.refreshToken, realmId: qb.realmId }; } catch (error) { - if (error instanceof QBTimeoutError) throw error; + if (isQBTimeoutError(error)) throw error; // Refresh can fail transiently; the old access token may still be valid. return { accessToken: qb.accessToken, refreshToken: qb.refreshToken, realmId: qb.realmId }; } diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 524f21d21..df0f3a725 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -32,6 +32,24 @@ export class QBTimeoutError extends Error { name = "QBTimeoutError"; } +/** + * Identity check for a QB timeout that does NOT depend on `instanceof`. + * + * A bare `instanceof` compares class IDENTITY, so it silently returns false + * whenever this module ends up loaded twice — different bundler chunks, a + * CJS/ESM interop split (CI on Node 20 proved this one), a duplicated copy in + * node_modules. The consequence is not cosmetic: every timeout branch in this + * codebase would quietly take the non-timeout path, which is precisely the + * misclassification the whole deadline effort exists to prevent. The name is + * set as a class field on every instance, so match on that too. + */ +export function isQBTimeoutError(error: unknown): error is QBTimeoutError { + return ( + error instanceof QBTimeoutError || + (error instanceof Error && error.name === "QBTimeoutError") + ); +} + const QB_DEFAULT_TIMEOUT_MS = 20_000; /** Path only — never the query string (it can carry the realm/query) or a token. */ @@ -329,7 +347,7 @@ export async function refreshQBToken(refreshToken: string): Promise<{ accessToke const data = await res.json(); return { accessToken: data.access_token, refreshToken: data.refresh_token }; } catch (error) { - if (error instanceof QBTimeoutError) { + if (isQBTimeoutError(error)) { const message = "QBO token refresh timed out; the stored refresh token may be stale, reconnect QuickBooks if the next refresh fails"; console.error(message, error.message); @@ -400,7 +418,7 @@ export async function parseJsonOrNull(res: Response): Promise try { return (await res.json()) as T; } catch (error) { - if (error instanceof QBTimeoutError) throw error; + if (isQBTimeoutError(error)) throw error; return null; } } diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index cde05baff..c5f4cf6f8 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -13,7 +13,7 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type Server } from "node:http"; import { AddressInfo } from "node:net"; -import { qbTimedFetch, QBTimeoutError, parseJsonOrNull } from "../src/lib/quickbooks"; +import { qbTimedFetch, QBTimeoutError, isQBTimeoutError, parseJsonOrNull } from "../src/lib/quickbooks"; let server: Server; let base: string; @@ -322,3 +322,26 @@ test("parseJsonOrNull RETHROWS a body-read timeout instead of reporting an empty const error = await parseJsonOrNull(res).then(() => null, (e: unknown) => e as Error); assert.ok(error instanceof QBTimeoutError, `expected QBTimeoutError, got ${String(error)}`); }); + + +// ─── Cross-module identity ────────────────────────────────────────────────── + +test("isQBTimeoutError recognises a timeout from a DUPLICATE copy of this module", () => { + // CI (Node 20) proved this: a CJS/ESM interop split loaded quickbooks.ts + // twice, so `instanceof` was false across the boundary and every timeout + // branch quietly took the non-timeout path. Bundler chunk duplication does + // the same thing in production. Match the name too. + class ForeignQBTimeoutError extends Error { + name = "QBTimeoutError"; + } + assert.equal(isQBTimeoutError(new ForeignQBTimeoutError("from another copy")), true); + assert.equal(isQBTimeoutError(new QBTimeoutError("native")), true); +}); + +test("isQBTimeoutError does not over-match", () => { + assert.equal(isQBTimeoutError(new Error("AbortError")), false); + assert.equal(isQBTimeoutError(Object.assign(new Error("x"), { name: "AbortError" })), false); + assert.equal(isQBTimeoutError(null), false); + assert.equal(isQBTimeoutError("QBTimeoutError"), false); + assert.equal(isQBTimeoutError({ name: "QBTimeoutError" }), false); +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index d965c3755..5db854415 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -938,7 +938,7 @@ test("an attachment upload that times out PROPAGATES from both paths, so the pus const fresh = createDeps({ uploadAttachment: async () => timeout() }); await assert.rejects( () => createQBReceiptPurchase(TOKENS, input, fresh.deps), - (error: unknown) => error instanceof QBTimeoutError, + (error: unknown) => (error as Error)?.name === "QBTimeoutError", ); const existing = createDeps({ @@ -948,7 +948,7 @@ test("an attachment upload that times out PROPAGATES from both paths, so the pus }); await assert.rejects( () => createQBReceiptPurchase(TOKENS, input, existing.deps), - (error: unknown) => error instanceof QBTimeoutError, + (error: unknown) => (error as Error)?.name === "QBTimeoutError", ); }); From b6826fff4409ff10498e084864278fed4c66a4c3 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 20:47:53 -0700 Subject: [PATCH 015/144] fix(qbo): stop the payments cron on a QBO outage; make it visible; retry transient attachments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 4, all three items. 1. probeQBInvoice flattened everything into {state:"error"} and both loops continued across up to 200 rows, so six 20s timeouts still reached the cron's 120s ceiling and the run was killed with nothing reported. The probe now marks connection-level failures (our deadline fired, the request threw, or QBO answered 429/5xx) separately from per-invoice errors, and both loops STOP on the first one, count the remaining rows as `skipped`, and exit cleanly. A timed-out token refresh aborts the same way. Ordinary per-invoice errors still just skip that row. 2. Each run now writes one AutomationEvent (kind "qbo-payments-sync", status ok/error, reason "qbo-unavailable", run counts in detail). Before this an outage on the money rail left no trace anywhere a human or the digest would look. pipeline-health exposes lastPaymentsSync and its errors already count toward `stuck`, so a stalled payments rail turns the morning digest red. 3. Transient attachment failures were terminal: a 429/5xx upload, a network error, or a failed Attachable lookup became `failed:` alongside ok:true, which the Apps Script treats as final — it stopped resending and the Purchase stayed unattached. Those now raise QboRetryableError and the route answers 503 retry:true, so the next pass hits the idempotent existing-Purchase recovery. A 4xx other than 429 and a QBO Fault stay terminal (returned as values, not thrown) and still ride on ok:true. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- .../integrations/qbo-receipts/create/route.ts | 9 ++ src/lib/automation-events.ts | 5 +- src/lib/pipeline-health.ts | 28 +++- src/lib/qbo-receipt-push.ts | 77 +++++++-- src/lib/quickbooks-payments.ts | 98 +++++++++++- src/lib/quickbooks.ts | 34 +++- tests/pipeline-digest-route.test.ts | 1 + tests/pipeline-health.test.ts | 37 ++++- tests/qbo-payments-outage.test.ts | 147 ++++++++++++++++++ tests/qbo-receipt-push.test.ts | 98 +++++++++--- 11 files changed, 487 insertions(+), 50 deletions(-) create mode 100644 tests/qbo-payments-outage.test.ts diff --git a/package.json b/package.json index d374d148c..aad035c1a 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "test:pipeline-digest": "tsx --test tests/pipeline-digest-route.test.ts", "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", + "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -26,7 +27,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index de0dec440..604d82019 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -6,6 +6,7 @@ import { createQBReceiptPurchase, QboAccountConfigError, QboPurchaseFaultError, + isRetryableQboError, type CreateQBReceiptPurchaseInput, type CreateQBReceiptPurchaseResult, } from "@/lib/qbo-receipt-push"; @@ -259,6 +260,14 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-timeout" })); return NextResponse.json({ ok: false, retry: true, reason: "qbo-timeout" }, { status: 503 }); } + if (isRetryableQboError(error)) { + // 429/5xx/network, or a failed attachment step. Same + // reasoning and same idempotency guarantees as the timeout + // branch: retry beats banking a half-finished receipt. + console.error("QBO receipt push hit a retryable failure", error instanceof Error ? error.message : "unknown"); + await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-unavailable" })); + return NextResponse.json({ ok: false, retry: true, reason: "qbo-unavailable" }, { status: 503 }); + } if (error instanceof QboAccountConfigError) { // Deterministic misconfiguration (missing/wrong-type/ // colliding account ids) — the SAME failure would repeat on diff --git a/src/lib/automation-events.ts b/src/lib/automation-events.ts index 6ebb7c177..1be92bc43 100644 --- a/src/lib/automation-events.ts +++ b/src/lib/automation-events.ts @@ -8,13 +8,16 @@ import { prisma } from "@/lib/prisma"; * endpoint (created / already-exists / fallback / error). * - "qbo-sync": one row per QBO→ProBuild sync run (cron, manual button, or * backfill) with the run counts in `detail`. + * - "qbo-payments-sync": one row per payments-sync run (the money rail), so a + * QBO outage there is visible to the health check instead of vanishing into + * a cron log. * * Logging is FIRE-AND-FORGET everywhere: an event-log failure must never fail * the automation it describes — the books write always outranks the audit row. */ export interface AutomationEventInput { - kind: "receipt-push" | "qbo-sync" | "receipt-stage" | "setting"; + kind: "receipt-push" | "qbo-sync" | "receipt-stage" | "setting" | "qbo-payments-sync"; stage?: string; status: string; reason?: string; diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index b84b834d3..d6c501444 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -63,6 +63,8 @@ export interface PipelineHealth { lastPurchaseSync: TimestampProbe; /** Newest receipt the bot actually CREATED (re-pushes don't count). */ lastReceiptPush: TimestampProbe; + /** Newest SUCCESSFUL payments-sync run — the money rail's own pulse. */ + lastPaymentsSync: TimestampProbe; }; /** receipt-push events in the last 24h, by status ("created", "fallback", ...). */ receipts24h: CountsProbe; @@ -114,6 +116,9 @@ export async function fetchIntuitStatus(): Promise { */ export const BOOKED_PUSH_STATUSES = ["created"]; +/** The payments cron's per-run audit row (see quickbooks-payments.ts). */ +export const PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; + /** * The verdict, split out from the database reads so the rules are testable * without a DB. @@ -134,6 +139,7 @@ export function evaluatePipelineHealth(input: { intuit: IntuitProbe; lastPurchaseSync: TimestampProbe; lastReceiptPush: TimestampProbe; + lastPaymentsSync: TimestampProbe; receipts24h: CountsProbe; bank: TimestampProbe; stuck: CountProbe; @@ -144,6 +150,7 @@ export function evaluatePipelineHealth(input: { const namedProbes: Array<[string, { status: ProbeStatus }]> = [ ["lastPurchaseSync", input.lastPurchaseSync], ["lastReceiptPush", input.lastReceiptPush], + ["lastPaymentsSync", input.lastPaymentsSync], ["receipts24h", input.receipts24h], ["bank", input.bank], ["stuck", input.stuck], @@ -224,7 +231,7 @@ export async function getPipelineHealth(): Promise { /** Any probe failure is reported as such — never silently downgraded to "nothing found". */ const probe = runProbe; - const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck] = await Promise.all([ + const [intuit, lastPurchase, lastPush, lastPaymentsSync, receiptRows, lastBankLine, stuck] = await Promise.all([ fetchIntuitStatus(), // Expense carries no updatedAt column — qbSyncedAt IS the "when did the // QBO purchase sync land" timestamp this is asking for. @@ -247,6 +254,18 @@ export async function getPipelineHealth(): Promise { )?.createdAt ?? null, null, ), + probe( + "lastPaymentsSync", + async () => + ( + await prisma.automationEvent.findFirst({ + where: { kind: PAYMENTS_SYNC_EVENT_KIND, status: "ok" }, + orderBy: { createdAt: "desc" }, + select: { createdAt: true }, + }) + )?.createdAt ?? null, + null, + ), probe>( "receipts24h", async () => { @@ -288,6 +307,11 @@ export async function getPipelineHealth(): Promise { reason: lastPush.reason, at: lastPush.value?.toISOString() ?? null, }, + lastPaymentsSync: { + status: lastPaymentsSync.status, + reason: lastPaymentsSync.reason, + at: lastPaymentsSync.value?.toISOString() ?? null, + }, receipts24h: { status: receiptRows.status, reason: receiptRows.reason, counts }, bank: { status: lastBankLine.status, @@ -306,6 +330,7 @@ export async function getPipelineHealth(): Promise { qbo: { lastPurchaseSync: snapshot.lastPurchaseSync, lastReceiptPush: snapshot.lastReceiptPush, + lastPaymentsSync: snapshot.lastPaymentsSync, }, receipts24h: snapshot.receipts24h, bank: snapshot.bank, @@ -344,6 +369,7 @@ export function formatPipelineDigest(health: PipelineHealth): { subject: string; `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}${health.intuit.status === "error" ? " [status page unreachable]" : ""}`, `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSync, now)}`, `Last receipt booked: ${ago(health.qbo.lastReceiptPush, now)}`, + `Last payments sync: ${ago(health.qbo.lastPaymentsSync, now)}`, `Receipts (24h): ${receiptsLine}`, `Bank ledger through: ${ health.bank.status === "error" diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 2dd9161c4..efb2ea173 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -35,6 +35,37 @@ import { type QBAttachable, } from "./quickbooks"; +/** + * A QBO failure that WILL plausibly succeed on a later attempt: 429, 5xx, a + * thrown network error, or a failed attachment lookup. + * + * These used to be flattened into `failed:` next to `ok: true`, which + * the Apps Script treats as FINAL — it stops resending, so a rate-limited or + * briefly-5xx attachment upload left the Purchase permanently without its + * receipt. Raising instead lets the route answer 503 retry:true, and the next + * pass hits the idempotent existing-Purchase recovery. A 4xx other than 429 is + * a real business rejection and stays terminal: retrying it forever is worse. + */ +export class QboRetryableError extends Error { + name = "QboRetryableError"; + constructor(message: string, readonly status?: number) { + super(message); + } +} + +/** Name-based, for the same cross-module-identity reason as isQBTimeoutError. */ +export function isRetryableQboError(error: unknown): boolean { + return ( + error instanceof QboRetryableError || + (error instanceof Error && error.name === "QboRetryableError") + ); +} + +/** A QBO HTTP status we should come back to rather than give up on. */ +export function isRetryableQboStatus(status: number): boolean { + return status === 429 || status >= 500; +} + /** Thrown by ensureQBVendor when QBO rejects the create as a duplicate name (fault 6240) and a re-query still can't find it. */ export class QboVendorDuplicateError extends Error { constructor(name: string) { @@ -348,9 +379,17 @@ async function defaultUploadAttachment( }, body, }); - if (!res.ok) return `failed:${res.status}`; + if (!res.ok) { + // 429/5xx: QBO is busy or broken, not refusing this file. Raise so the + // push is retried rather than banked as a terminal "failed:503". + if (isRetryableQboStatus(res.status)) { + throw new QboRetryableError(`QB attachment upload failed with status ${res.status}`, res.status); + } + return `failed:${res.status}`; + } const data = await parseJsonOrNull(res); const fault = data?.AttachableResponse?.[0]?.Fault; + // A Fault is a business rejection (bad ref, unsupported type) — terminal. if (fault) return "failed:fault"; return "attached"; } @@ -415,10 +454,16 @@ async function ensureAttachmentOnExistingPurchase( if (alreadyAttached) return "already-attached"; return await uploadAttachment(tokens, purchaseId, plan); } catch (error) { - // A timeout must NOT become a terminal "failed:..." on an ok:true - // response — see the create path below for why. - if (isQBTimeoutError(error)) throw error; - return `failed:${error instanceof Error ? error.name : "error"}`; + // Anything THROWN here is a connection-level or transient failure — a + // timeout, a 429/5xx, a network error, or the Attachable lookup itself + // failing. None of those say "this file cannot be attached", so none may + // become a terminal `failed:` on an ok:true response: that is what made + // the Apps Script stop retrying and leave the Purchase unattached. + // Terminal outcomes come back from uploadAttachment as VALUES. + if (isQBTimeoutError(error) || isRetryableQboError(error)) throw error; + throw new QboRetryableError( + `QB attachment step failed: ${error instanceof Error ? error.name : "error"}`, + ); } } @@ -724,16 +769,18 @@ export async function createQBReceiptPurchase( try { attachment = await uploadAttachment(tokens, created.id, plan); } catch (error) { - // A TIMEOUT propagates, unlike every other attachment failure. - // Reporting it as `failed:QBTimeoutError` alongside ok:true made it - // TERMINAL: the Apps Script treats any ok:true as final and stops - // resending, so the Purchase would keep its missing receipt forever - // and the existing-Purchase recovery above would never run. Letting - // it out gives the route a 503, the bot retries, and the next pass - // finds the Purchase and attaches the file. Non-timeout failures - // keep the old best-effort behaviour — they are not retryable. - if (isQBTimeoutError(error)) throw error; - attachment = `failed:${error instanceof Error ? error.name : "error"}`; + // Every THROWN attachment failure propagates; only the terminal + // ones (4xx other than 429, a QBO Fault) come back as values. + // Reporting a transient failure as `failed:` alongside + // ok:true made it TERMINAL — the Apps Script treats any ok:true as + // final and stops resending, so the Purchase kept its missing + // receipt forever and the existing-Purchase recovery above never + // ran. Propagating gives the route a 503, the bot retries, and the + // next pass finds the Purchase and attaches the file. + if (isQBTimeoutError(error) || isRetryableQboError(error)) throw error; + throw new QboRetryableError( + `QB attachment step failed: ${error instanceof Error ? error.name : "error"}`, + ); } } diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index b5aaf2592..b967a6ba6 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -16,6 +16,7 @@ import { withTxRetry, lockMoneyParents } from "./tx-retry"; import { enqueueMilestonePaid, drainPaymentNotifications } from "./payment-outbox"; import { toNum, deriveInvoiceTaxFields } from "./prisma-helpers"; import { getQBSettings, saveQBSettings } from "./integration-store"; +import { logAutomationEvent } from "./automation-events"; import { type QBTokens, refreshQBToken, @@ -665,6 +666,14 @@ export interface QBPaymentSyncResult { // separate counter from `settled` (which counts individual milestones) // since one progress billing can carry several milestone lines. progressBillingsSettled: number; + /** + * Rows we deliberately did not probe because QBO had already stopped + * answering this run. Non-zero means the run is INCOMPLETE, not clean — + * every skipped row is simply retried next run. + */ + skipped: number; + /** True when the run stopped early on a connection-level QBO failure. */ + abortedOnQboOutage: boolean; } /** @@ -672,7 +681,10 @@ export interface QBPaymentSyncResult { * Safe to run repeatedly (cron + on-view). Never throws on a single bad row. */ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; projectId?: string }): Promise { - const result: QBPaymentSyncResult = { checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0 }; + const result: QBPaymentSyncResult = { + checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0, + skipped: 0, abortedOnQboOutage: false, + }; const pending = await prisma.paymentSchedule.findMany({ where: { @@ -708,13 +720,23 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje take: 100, }); - if (pending.length === 0 && pendingBillings.length === 0) return result; + if (pending.length === 0 && pendingBillings.length === 0) { + await recordPaymentsSyncEvent(result); + return result; + } let tokens: QBTokens; try { tokens = await getFreshQBTokens(); } catch (e) { result.errors.push(e instanceof Error ? e.message : "QB tokens unavailable"); + // A timed-out refresh is the same outage as a timed-out probe; nothing + // was checked, so everything we loaded counts as skipped. + if (isQBTimeoutError(e)) { + result.abortedOnQboOutage = true; + result.skipped = pending.length + pendingBillings.length; + } + await recordPaymentsSyncEvent(result); return result; } @@ -722,12 +744,25 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje // previously null). Reported once per breakage; a re-push clears the flag and re-arms. const newlyFlagged: QBSyncIssue[] = []; - for (const schedule of pending) { + // On a connection-level failure this loop STOPS rather than working through + // the rest: every remaining row would burn its own full deadline against the + // same wall, and six 20s timeouts is the cron's whole 120s ceiling. + for (const [index, schedule] of pending.entries()) { result.checked++; try { const probe = await probeQBInvoice(tokens, schedule.qbInvoiceId!); - // Transient error (token/429/5xx/network) — leave untouched and retry next run. - if (probe.state === "error") continue; + if (probe.state === "error") { + if (probe.connectionFailed) { + result.abortedOnQboOutage = true; + result.errors.push( + `QuickBooks stopped responding (${probe.timedOut ? "timeout" : `status ${probe.status}`}) — remaining rows skipped, will retry next run`, + ); + result.skipped += pending.length - index - 1; + break; + } + // Ordinary transient error — leave untouched and retry next run. + continue; + } if (probe.state === "voided" || probe.state === "notFound") { // The QBO invoice is gone/voided: it can never settle. Flag so the UI can @@ -802,10 +837,27 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje // (custom/change-order lines were materialized into a real PaymentSchedule // at billing-creation time — see createProgressBillingCore — so every line // has a scheduleId and settles like any other milestone; no special case). - for (const billing of pendingBillings) { + for (const [index, billing] of pendingBillings.entries()) { + // Same rule as the milestone loop: never keep dialling a dead QBO. A + // milestone-loop abort also skips this whole pass — the connection is + // shared, so there is nothing to gain by trying it again here. + if (result.abortedOnQboOutage) { + result.skipped += pendingBillings.length - index; + break; + } try { const probe = await probeQBInvoice(tokens, billing.qbInvoiceId!); - if (probe.state === "error") continue; // transient — retry next run + if (probe.state === "error") { + if (probe.connectionFailed) { + result.abortedOnQboOutage = true; + result.errors.push( + `QuickBooks stopped responding (${probe.timedOut ? "timeout" : `status ${probe.status}`}) — remaining progress billings skipped, will retry next run`, + ); + result.skipped += pendingBillings.length - index - 1; + break; + } + continue; // ordinary transient — retry next run + } if (probe.state === "voided" || probe.state === "notFound") { result.errors.push(`${billing.invoice.code}/${billing.code}: QBO invoice ${probe.state}`); continue; @@ -835,5 +887,37 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje await notifyQBSyncIssues(newlyFlagged); } + await recordPaymentsSyncEvent(result); return result; } + +/** The AutomationEvent kind the payments cron writes once per run. */ +export const QBO_PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; + +/** + * One audit row per payments-sync run, so an outage on this rail is VISIBLE. + * + * Before this, a QBO outage during the payments cron left no trace anywhere a + * human or the morning digest would look: the run returned a tidy result object + * to a cron log nobody reads. The pipeline health check counts these errors and + * reports the last successful run, so a stalled money rail turns the digest red + * instead of staying silent. Fire-and-forget, like every other automation + * event — the audit row must never fail the sync it describes. + */ +async function recordPaymentsSyncEvent(result: QBPaymentSyncResult): Promise { + const failed = result.abortedOnQboOutage; + await logAutomationEvent({ + kind: QBO_PAYMENTS_SYNC_EVENT_KIND, + status: failed ? "error" : "ok", + reason: failed ? "qbo-unavailable" : undefined, + source: "cron", + detail: { + checked: result.checked, + settled: result.settled, + partiallyPaid: result.partiallyPaid, + progressBillingsSettled: result.progressBillingsSettled, + skipped: result.skipped, + errors: result.errors.slice(0, 5), + }, + }); +} diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index df0f3a725..4a18277c7 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -641,7 +641,13 @@ export type QBInvoiceProbe = | { state: "ok"; balance: number; total: number; paymentTxnIds: string[] } | { state: "voided" } // HTTP 200, exists, total & balance === 0, no linked payments | { state: "notFound" } // HTTP 400 Fault 610 or HTTP 404 (authoritative "gone" only) - | { state: "error"; status: number }; // 401/429/5xx/network/malformed — transient, never act on + // 401/429/5xx/network/malformed — transient, never act on. + // `connectionFailed` marks the subset where we never got a usable answer + // from QBO at all (our deadline fired, or the request threw). A caller + // looping over many rows must STOP on that: the next row will fail the + // same way and burn another full deadline, which is how six timeouts still + // added up to the payments cron's 120s ceiling. + | { state: "error"; status: number; connectionFailed?: boolean; timedOut?: boolean }; /** * Probe a QBO invoice and classify it. QBO's behavior for gone invoices is @@ -653,8 +659,10 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro let res: Response; try { res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); - } catch { - return { state: "error", status: 0 }; + } catch (error) { + // A timeout or a thrown network error means QBO never answered — the + // connection itself is the problem, not this invoice. + return { state: "error", status: 0, connectionFailed: true, timedOut: isQBTimeoutError(error) }; } if (res.ok) { // A 200 should always carry an Invoice. A parse failure or a missing payload @@ -663,7 +671,11 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro let data: any; try { data = await res.json(); - } catch { + } catch (error) { + // The body can stall past the deadline after headers arrived. + if (isQBTimeoutError(error)) { + return { state: "error", status: res.status, connectionFailed: true, timedOut: true }; + } return { state: "error", status: res.status }; } const inv = data?.Invoice; @@ -682,10 +694,22 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro return { state: "ok", balance, total, paymentTxnIds }; } if (res.status === 404) return { state: "notFound" }; - const body = await res.text().catch(() => ""); + let body = ""; + try { + body = await res.text(); + } catch (error) { + if (isQBTimeoutError(error)) { + return { state: "error", status: res.status, connectionFailed: true, timedOut: true }; + } + } if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { return { state: "notFound" }; } + // 429 and 5xx are QBO telling us it cannot serve requests right now; the + // next row would hit the same wall, so they count as connection-level too. + if (res.status === 429 || res.status >= 500) { + return { state: "error", status: res.status, connectionFailed: true }; + } return { state: "error", status: res.status }; } diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts index dd85cec19..2a3ab2e80 100644 --- a/tests/pipeline-digest-route.test.ts +++ b/tests/pipeline-digest-route.test.ts @@ -24,6 +24,7 @@ const HEALTH: PipelineHealth = { qbo: { lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, + lastPaymentsSync: { status: "ok", at: "2026-09-01T13:00:00.000Z" }, }, receipts24h: { status: "ok", counts: { created: 2 } }, bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 8939e747d..e60a9bc7b 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -31,6 +31,7 @@ function snapshot(overrides: Partial[0 intuit: { status: "ok" as const, indicator: "none" }, lastPurchaseSync: { status: "ok" as const, at: iso(2 * HOUR) }, lastReceiptPush: { status: "ok" as const, at: iso(3 * HOUR) }, + lastPaymentsSync: { status: "ok" as const, at: iso(1 * HOUR) }, receipts24h: { status: "ok" as const, counts: { created: 4 } }, bank: { status: "ok" as const, at: iso(48 * HOUR) }, stuck: { status: "ok" as const, count: 0 }, @@ -46,7 +47,7 @@ test("a healthy snapshot is ok with no reasons", () => { // ─── False green from a failed probe ──────────────────────────────────────── test("ANY failed probe forces ok:false with probe-failed:", () => { - const names = ["lastPurchaseSync", "lastReceiptPush", "receipts24h", "bank", "stuck"] as const; + const names = ["lastPurchaseSync", "lastReceiptPush", "lastPaymentsSync", "receipts24h", "bank", "stuck"] as const; for (const name of names) { const base = snapshot(); const broken = { @@ -76,13 +77,14 @@ test("a total database outage reports every probe, still never ok", () => { snapshot({ lastPurchaseSync: { status: "error", at: null }, lastReceiptPush: { status: "error", at: null }, + lastPaymentsSync: { status: "error", at: null }, receipts24h: { status: "error", counts: {} }, bank: { status: "error", at: null }, stuck: { status: "error", count: 0 }, }), ); assert.equal(v.ok, false); - assert.equal(v.reasons.length, 5); + assert.equal(v.reasons.length, 6); }); // ─── Receipt staleness ────────────────────────────────────────────────────── @@ -162,6 +164,7 @@ function sampleHealth(overrides: Partial = {}): PipelineHealth { qbo: { lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, + lastPaymentsSync: { status: "ok", at: "2026-09-01T13:00:00.000Z" }, }, receipts24h: { status: "ok", counts: { created: 4, fallback: 1 } }, bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, @@ -199,6 +202,7 @@ test("digest says how long the silence has been, so a human can judge it", () => qbo: { lastPurchaseSync: { status: "ok", at: "2026-08-20T14:00:00.000Z" }, lastReceiptPush: { status: "ok", at: "2026-08-20T14:00:00.000Z" }, + lastPaymentsSync: { status: "ok", at: "2026-09-01T13:00:00.000Z" }, }, }), ); @@ -235,6 +239,7 @@ test("digest renders missing timestamps as 'never' rather than a bogus date", () qbo: { lastPurchaseSync: { status: "ok", at: null }, lastReceiptPush: { status: "ok", at: null }, + lastPaymentsSync: { status: "ok", at: null }, }, }), ); @@ -289,3 +294,31 @@ test("only a CREATE refreshes the last-booked clock — a re-push does not", () assert.equal(BOOKED_PUSH_STATUSES.includes("fallback"), false); assert.equal(BOOKED_PUSH_STATUSES.includes("error"), false); }); + + +// ─── The payments rail is part of the pulse ───────────────────────────────── + +test("a payments-sync outage turns the verdict RED", async () => { + const { PAYMENTS_SYNC_EVENT_KIND } = await import("../src/lib/pipeline-health"); + const { QBO_PAYMENTS_SYNC_EVENT_KIND } = await import("../src/lib/quickbooks-payments"); + // The health check and the cron must agree on the event kind, or the + // outage is written to a row nothing reads. + assert.equal(PAYMENTS_SYNC_EVENT_KIND, QBO_PAYMENTS_SYNC_EVENT_KIND); + + // The cron writes status "error" on an aborted run; `stuck` counts errors + // of ANY kind in 24h, so the digest goes red on the money rail too. + const v = evaluatePipelineHealth(snapshot({ stuck: { status: "ok", count: 1 } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["errors-24h:1"]); +}); + +test("a payments-sync probe failure is its own reason", () => { + const v = evaluatePipelineHealth(snapshot({ lastPaymentsSync: { status: "error", reason: "timeout", at: null } })); + assert.equal(v.ok, false); + assert.ok(v.reasons.includes("probe-failed:lastPaymentsSync")); +}); + +test("the digest reports the payments rail alongside the receipt rail", () => { + const { text } = formatPipelineDigest(sampleHealth()); + assert.match(text, /Last payments sync: /); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts new file mode 100644 index 000000000..aef8f6b2a --- /dev/null +++ b/tests/qbo-payments-outage.test.ts @@ -0,0 +1,147 @@ +/** + * The payments cron must not keep dialling a dead QuickBooks. + * + * Codex gate: probeQBInvoice folded every failure into {state:"error"} and both + * loops just `continue`d across up to 200 rows. During the 2026-09-01 outage + * that meant six 20s timeouts in a row — the cron's entire 120s ceiling — and + * the function was killed before it could report anything. + * + * probeQBInvoice now marks connection-level failures, which is what lets the + * loops stop. These tests pin that classification (the loops themselves need a + * database, so the multi-row behaviour is covered by simulating the same + * decision the loop makes). + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { probeQBInvoice, QBTimeoutError, type QBInvoiceProbe } from "../src/lib/quickbooks"; + +const TOKENS = { accessToken: "a", refreshToken: "r", realmId: "realm-1" }; + +/** Swap global fetch for one call; qbFetch goes through it. */ +async function withFetch(impl: typeof fetch, run: () => Promise): Promise { + const original = globalThis.fetch; + globalThis.fetch = impl; + try { + return await run(); + } finally { + globalThis.fetch = original; + } +} + +const json = (status: number, body: unknown) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +test("a timeout is classified as a connection failure, not a per-invoice error", async () => { + const probe = await withFetch( + async () => { + throw new QBTimeoutError("QuickBooks request timed out after 20000ms: /v3/company/x/invoice/1"); + }, + () => probeQBInvoice(TOKENS, "1"), + ); + assert.equal(probe.state, "error"); + assert.equal(probe.state === "error" && probe.connectionFailed, true); + assert.equal(probe.state === "error" && probe.timedOut, true); +}); + +test("a thrown network error is a connection failure too", async () => { + const probe = await withFetch( + async () => { + throw new TypeError("fetch failed"); + }, + () => probeQBInvoice(TOKENS, "1"), + ); + assert.equal(probe.state === "error" && probe.connectionFailed, true); + assert.equal(probe.state === "error" && probe.timedOut, false); +}); + +test("429 and 5xx are connection failures; an ordinary 401 is not", async () => { + for (const status of [429, 500, 503]) { + const probe = await withFetch(async () => json(status, { Fault: {} }), () => probeQBInvoice(TOKENS, "1")); + assert.equal(probe.state === "error" && probe.connectionFailed, true, `status ${status}`); + } + const unauthorized = await withFetch(async () => json(401, { Fault: {} }), () => probeQBInvoice(TOKENS, "1")); + assert.equal(unauthorized.state, "error"); + assert.equal(unauthorized.state === "error" && unauthorized.connectionFailed, undefined); +}); + +test("a healthy invoice is unaffected by the new classification", async () => { + const probe = await withFetch( + async () => json(200, { Invoice: { TotalAmt: 100, Balance: 0, LinkedTxn: [{ TxnType: "Payment", TxnId: "7" }] } }), + () => probeQBInvoice(TOKENS, "1"), + ); + assert.deepEqual(probe, { state: "ok", balance: 0, total: 100, paymentTxnIds: ["7"] }); +}); + +test("a voided invoice is still authoritative, not a connection failure", async () => { + const probe = await withFetch( + async () => json(200, { Invoice: { TotalAmt: 0, Balance: 0, LinkedTxn: [] } }), + () => probeQBInvoice(TOKENS, "1"), + ); + assert.deepEqual(probe, { state: "voided" }); +}); + +// ─── The loop rule the classification exists to drive ─────────────────────── + +/** + * Mirrors the guard in syncQuickBooksPayments: stop on the first + * connection-level failure, count the rest as skipped. The real loops need a + * database; this pins the DECISION they make, over many rows. + */ +function runLoop(probes: QBInvoiceProbe[]): { checked: number; skipped: number; aborted: boolean } { + let checked = 0; + let skipped = 0; + let aborted = false; + for (const [index, probe] of probes.entries()) { + checked++; + if (probe.state === "error") { + if (probe.connectionFailed) { + aborted = true; + skipped += probes.length - index - 1; + break; + } + continue; + } + } + return { checked, skipped, aborted }; +} + +test("200 pending rows during an outage cost ONE probe, not 200", () => { + const outage: QBInvoiceProbe[] = Array.from({ length: 200 }, () => ({ + state: "error" as const, + status: 0, + connectionFailed: true, + timedOut: true, + })); + const run = runLoop(outage); + // One 20s deadline, not six-plus — the whole point. + assert.equal(run.checked, 1); + assert.equal(run.skipped, 199); + assert.equal(run.aborted, true); +}); + +test("an ordinary per-invoice error does NOT stop the run", () => { + const probes: QBInvoiceProbe[] = [ + { state: "error", status: 401 }, + { state: "error", status: 401 }, + { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, + ]; + const run = runLoop(probes); + assert.equal(run.checked, 3); + assert.equal(run.skipped, 0); + assert.equal(run.aborted, false); +}); + +test("an outage partway through skips only the remainder", () => { + const probes: QBInvoiceProbe[] = [ + { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, + { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, + { state: "error", status: 0, connectionFailed: true, timedOut: true }, + { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, + { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, + ]; + const run = runLoop(probes); + assert.equal(run.checked, 3); + assert.equal(run.skipped, 2); + assert.equal(run.aborted, true); +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index 5db854415..26dd69399 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -379,11 +379,12 @@ test("createQBReceiptPurchase attaches a small image receipt", async () => { assert.equal(uploads.length, 1); }); -test("createQBReceiptPurchase treats an attachment failure as non-fatal", async () => { +test("createQBReceiptPurchase treats a TERMINAL attachment failure as non-fatal", async () => { + // Terminal outcomes (a 4xx other than 429, a QBO Fault) come back as + // VALUES, and those still ride along on ok:true — the Purchase is booked + // and retrying would never make QBO accept the file. const { deps } = createDeps({ - uploadAttachment: async () => { - throw new Error("boom"); - }, + uploadAttachment: async () => "failed:400" as ReceiptAttachmentStatus, }); const result = await createQBReceiptPurchase( TOKENS, @@ -392,7 +393,7 @@ test("createQBReceiptPurchase treats an attachment failure as non-fatal", async ); assert.equal(result.ok, true); if (result.ok && !result.alreadyExists) { - assert.match(result.attachment, /^failed:/); + assert.equal(result.attachment, "failed:400"); } else { assert.fail("expected a fresh create"); } @@ -907,7 +908,10 @@ test("already-exists ignores an Attachable that belongs to a different entity ty assert.equal(result.ok && result.alreadyExists && result.attachment, "attached"); }); -test("already-exists reports a failed attachment lookup without failing the push", async () => { +test("a failed attachment LOOKUP is retryable, not a terminal ok:true", async () => { + // Codex gate: the lookup failing tells us nothing about whether the file is + // attached. Banking that as `failed:Error` on ok:true made the bot stop + // retrying and left the Purchase possibly unattached forever. const input = baseInput({ ...FILE_INPUT }); const marker = `[gtr-file:${input.fileId}]`; const { deps } = createDeps({ @@ -917,10 +921,10 @@ test("already-exists reports a failed attachment lookup without failing the push }, }); - const result = await createQBReceiptPurchase(TOKENS, input, deps); - // The books entry exists and must still be reported ok. - assert.equal(result.ok, true); - assert.equal(result.ok && result.alreadyExists && result.attachment, "failed:Error"); + await assert.rejects( + () => createQBReceiptPurchase(TOKENS, input, deps), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); }); test("an attachment upload that times out PROPAGATES from both paths, so the push is retryable", async () => { @@ -952,17 +956,75 @@ test("an attachment upload that times out PROPAGATES from both paths, so the pus ); }); -test("a NON-timeout attachment failure stays best-effort: failed: with ok:true", async () => { +test("a thrown NETWORK-ish attachment failure is retryable from both paths", async () => { const input = baseInput({ ...FILE_INPUT }); - const { deps } = createDeps({ - uploadAttachment: async () => { - throw new Error("boom"); + const marker = `[gtr-file:${input.fileId}]`; + const boom = async () => { + throw new Error("ECONNRESET"); + }; + + const fresh = createDeps({ uploadAttachment: boom }); + await assert.rejects( + () => createQBReceiptPurchase(TOKENS, input, fresh.deps), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); + + const existing = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableRows: [], + uploadAttachment: boom, + }); + await assert.rejects( + () => createQBReceiptPurchase(TOKENS, input, existing.deps), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); +}); + +test("a TERMINAL attachment status still rides along on ok:true from both paths", async () => { + const input = baseInput({ ...FILE_INPUT }); + const marker = `[gtr-file:${input.fileId}]`; + const terminal = async () => "failed:fault" as ReceiptAttachmentStatus; + + const fresh = createDeps({ uploadAttachment: terminal }); + const freshResult = await createQBReceiptPurchase(TOKENS, input, fresh.deps); + assert.equal(freshResult.ok && !freshResult.alreadyExists && freshResult.attachment, "failed:fault"); + + const existing = createDeps({ + existingRows: [{ Id: "99", PrivateNote: `note ${marker}` }], + attachableRows: [], + uploadAttachment: terminal, + }); + const existingResult = await createQBReceiptPurchase(TOKENS, input, existing.deps); + assert.equal(existingResult.ok && existingResult.alreadyExists && existingResult.attachment, "failed:fault"); +}); + +test("route: a retryable attachment failure surfaces as 503 qbo-unavailable", async () => { + const { QboRetryableError } = await import("../src/lib/qbo-receipt-push"); + const events: AutomationEventInput[] = []; + const { POST } = createRouteHandlers({ + createPurchase: async () => { + throw new QboRetryableError("QB attachment upload failed with status 503", 503); }, + logEvent: event => { events.push(event); }, }); - const result = await createQBReceiptPurchase(TOKENS, input, deps); - // The Purchase is on the books; a broken image is not worth a retry loop. - assert.equal(result.ok, true); - assert.equal(result.ok && !result.alreadyExists && result.attachment, "failed:Error"); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-unavailable" }); + assert.equal(events[0].reason, "qbo-unavailable"); +}); + +test("isRetryableQboStatus: 429 and 5xx retry, other 4xx are terminal", async () => { + const { isRetryableQboStatus } = await import("../src/lib/qbo-receipt-push"); + for (const status of [429, 500, 502, 503, 504]) { + assert.equal(isRetryableQboStatus(status), true, String(status)); + } + for (const status of [400, 401, 403, 404, 409, 422]) { + assert.equal(isRetryableQboStatus(status), false, String(status)); + } }); test("route: an attachment timeout surfaces as 503 qbo-timeout so the bot retries", async () => { From 6b51219b1b7babf2d3adc7eff9f3de43aef59332 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:14:33 -0700 Subject: [PATCH 016/144] fix(qbo): abort on any QBO sub-call outage; honest run status; validate upload response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 5. 1. getQBPayment failures were caught as ordinary row errors, so several settled invoices could each still burn a 20s deadline after a good probe. Both loops now run through one shared, exported runQboRowLoop: ANY connection-level failure from ANY QBO sub-call in a row (probe, payment detail, anything added later) stops the run, counts the rest as skipped and exits. getQBPayment raises on 429/5xx instead of returning null. Tested by driving the REAL loop with an injected fake QBO client, not a duplicated decision helper. 2. Only a timeout marked a run failed, so QBNotConnectedError and settings-store failures emitted status=ok and the digest stayed blind. classifyPreflightFailure covers every branch; the event now keys off result.runFailed with its own reason. 3. An empty, truncated, or HTML 200 from the Attachable upload fell through to "attached" for a file QBO never stored. Intuit's schema says the response carries an Attachable or a Fault, so a real AttachableResponse[].Attachable.Id is now required; anything else is retryable. 4. lastPaymentsSync null or older than 26h is now reason "payments-sync-stale" and ok:false, and the probe only counts runs sourced "cron" — on-view runs log source "view", manual "manual", so neither can disguise a dead hourly job. Co-Authored-By: Claude Fable 5.1 --- src/app/api/cron/quickbooks-payments/route.ts | 2 +- .../api/integrations/qbo-maintenance/route.ts | 2 +- src/lib/actions.ts | 2 +- src/lib/pipeline-health.ts | 29 ++- src/lib/qbo-receipt-push.ts | 46 ++-- src/lib/quickbooks-payments.ts | 212 +++++++++++++----- src/lib/quickbooks.ts | 51 ++++- tests/pipeline-health.test.ts | 34 +++ tests/qbo-payments-outage.test.ts | 209 ++++++++++++----- tests/qbo-receipt-push.test.ts | 68 ++++++ 10 files changed, 510 insertions(+), 145 deletions(-) diff --git a/src/app/api/cron/quickbooks-payments/route.ts b/src/app/api/cron/quickbooks-payments/route.ts index 251fb5182..4dd1042a5 100644 --- a/src/app/api/cron/quickbooks-payments/route.ts +++ b/src/app/api/cron/quickbooks-payments/route.ts @@ -15,7 +15,7 @@ export async function GET(request: Request) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } - const result = await syncQuickBooksPayments(); + const result = await syncQuickBooksPayments(undefined, { source: "cron" }); if (result.settled > 0 || result.errors.length > 0) { console.log("[cron/quickbooks-payments]", JSON.stringify(result)); } diff --git a/src/app/api/integrations/qbo-maintenance/route.ts b/src/app/api/integrations/qbo-maintenance/route.ts index 95b6a23b5..c30391125 100644 --- a/src/app/api/integrations/qbo-maintenance/route.ts +++ b/src/app/api/integrations/qbo-maintenance/route.ts @@ -51,7 +51,7 @@ export async function POST(req: Request) { } if (body.action === "sync-payments") { const { syncQuickBooksPayments } = await import("@/lib/quickbooks-payments"); - const result = await syncQuickBooksPayments(); + const result = await syncQuickBooksPayments(undefined, { source: "manual" }); return NextResponse.json({ ok: true, ...result }); } if (body.action === "test-team-notify") { diff --git a/src/lib/actions.ts b/src/lib/actions.ts index 5f4a89ee5..91886730c 100644 --- a/src/lib/actions.ts +++ b/src/lib/actions.ts @@ -3756,7 +3756,7 @@ export async function createQBPaymentLink(paymentId: string) { export async function refreshQBPayments(invoiceId: string) { await assertInvoicePermission(); const { syncQuickBooksPayments } = await import("./quickbooks-payments"); - const result = await syncQuickBooksPayments({ invoiceId }); + const result = await syncQuickBooksPayments({ invoiceId }, { source: "view" }); if (result.settled > 0) { const inv = await prisma.invoice.findUnique({ where: { id: invoiceId }, select: { projectId: true } }); if (inv) { diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index d6c501444..9ac86575d 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -63,7 +63,11 @@ export interface PipelineHealth { lastPurchaseSync: TimestampProbe; /** Newest receipt the bot actually CREATED (re-pushes don't count). */ lastReceiptPush: TimestampProbe; - /** Newest SUCCESSFUL payments-sync run — the money rail's own pulse. */ + /** + * Newest SUCCESSFUL payments-sync run triggered by the hourly CRON — + * the money rail's own pulse. On-view/manual runs are excluded on + * purpose: they must not be able to disguise a dead cron. + */ lastPaymentsSync: TimestampProbe; }; /** receipt-push events in the last 24h, by status ("created", "fallback", ...). */ @@ -118,6 +122,14 @@ export const BOOKED_PUSH_STATUSES = ["created"]; /** The payments cron's per-run audit row (see quickbooks-payments.ts). */ export const PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; +/** + * Only a run tagged "cron" counts as the heartbeat. On-view and manual + * refreshes write their own source precisely so they cannot stand in for an + * hourly job that has stopped running. + */ +export const PAYMENTS_SYNC_CRON_SOURCE = "cron"; +/** The cron runs hourly; 26h leaves room for a couple of missed runs and DST. */ +export const PAYMENTS_SYNC_STALE_HOURS = 26; /** * The verdict, split out from the database reads so the rules are testable @@ -167,6 +179,15 @@ export function evaluatePipelineHealth(input: { reasons.push(`errors-24h:${input.stuck.count}`); } + if (input.lastPaymentsSync.status === "ok") { + // The money rail's heartbeat. Null means the hourly cron has never + // completed a run we can see; stale means it stopped. Either way the + // digest must go red — a heartbeat nobody checks is not a heartbeat. + const at = input.lastPaymentsSync.at ? Date.parse(input.lastPaymentsSync.at) : null; + const stale = at === null || Number.isNaN(at) || input.now - at > PAYMENTS_SYNC_STALE_HOURS * HOUR_MS; + if (stale) reasons.push("payments-sync-stale"); + } + if (input.lastReceiptPush.status === "ok") { const at = input.lastReceiptPush.at ? Date.parse(input.lastReceiptPush.at) : null; const stale = at === null || Number.isNaN(at) || input.now - at > RECEIPT_STALE_HOURS * HOUR_MS; @@ -259,7 +280,11 @@ export async function getPipelineHealth(): Promise { async () => ( await prisma.automationEvent.findFirst({ - where: { kind: PAYMENTS_SYNC_EVENT_KIND, status: "ok" }, + where: { + kind: PAYMENTS_SYNC_EVENT_KIND, + status: "ok", + source: PAYMENTS_SYNC_CRON_SOURCE, + }, orderBy: { createdAt: "desc" }, select: { createdAt: true }, }) diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index efb2ea173..1190c679e 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -31,40 +31,16 @@ import { qbTimedFetch, parseJsonOrNull, isQBTimeoutError, + QboRetryableError, + isRetryableQboError, + isRetryableQboStatus, type QBTokens, type QBAttachable, } from "./quickbooks"; -/** - * A QBO failure that WILL plausibly succeed on a later attempt: 429, 5xx, a - * thrown network error, or a failed attachment lookup. - * - * These used to be flattened into `failed:` next to `ok: true`, which - * the Apps Script treats as FINAL — it stops resending, so a rate-limited or - * briefly-5xx attachment upload left the Purchase permanently without its - * receipt. Raising instead lets the route answer 503 retry:true, and the next - * pass hits the idempotent existing-Purchase recovery. A 4xx other than 429 is - * a real business rejection and stays terminal: retrying it forever is worse. - */ -export class QboRetryableError extends Error { - name = "QboRetryableError"; - constructor(message: string, readonly status?: number) { - super(message); - } -} - -/** Name-based, for the same cross-module-identity reason as isQBTimeoutError. */ -export function isRetryableQboError(error: unknown): boolean { - return ( - error instanceof QboRetryableError || - (error instanceof Error && error.name === "QboRetryableError") - ); -} - -/** A QBO HTTP status we should come back to rather than give up on. */ -export function isRetryableQboStatus(status: number): boolean { - return status === 429 || status >= 500; -} +// Retryable-failure vocabulary lives in quickbooks.ts (the payments rail needs +// it too); re-exported here because this module's callers already import it. +export { QboRetryableError, isRetryableQboError, isRetryableQboStatus }; /** Thrown by ensureQBVendor when QBO rejects the create as a duplicate name (fault 6240) and a re-query still can't find it. */ export class QboVendorDuplicateError extends Error { @@ -340,7 +316,7 @@ export function attachmentFileName(rawFileName: string | undefined): string { return (rawFileName || "receipt").replace(/[\r\n"]/g, "") || "receipt"; } -async function defaultUploadAttachment( +export async function defaultUploadAttachment( tokens: QBTokens, purchaseId: string, file: { base64: string; contentType: string; fileName: string }, @@ -391,6 +367,14 @@ async function defaultUploadAttachment( const fault = data?.AttachableResponse?.[0]?.Fault; // A Fault is a business rejection (bad ref, unsupported type) — terminal. if (fault) return "failed:fault"; + // Intuit's schema says an AttachableResponse carries either an Attachable + // or a Fault — absence is NOT success. An empty, truncated, or HTML 200 + // (proxy/CDN error pages are the usual source) used to fall through to + // "attached", banking a terminal success for a file that was never stored, + // so the bot never came back for it. Demand the created id. + if (!data?.AttachableResponse?.[0]?.Attachable?.Id) { + throw new QboRetryableError("QB attachment upload returned no Attachable id"); + } return "attached"; } diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index b967a6ba6..4c22a4875 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -21,6 +21,9 @@ import { type QBTokens, refreshQBToken, isQBTimeoutError, + isQboConnectionFailure, + QboRetryableError, + type QBInvoiceProbe, ensureQBCustomer, ensureQBServiceItem, createQBMilestoneInvoice, @@ -674,16 +677,94 @@ export interface QBPaymentSyncResult { skipped: number; /** True when the run stopped early on a connection-level QBO failure. */ abortedOnQboOutage: boolean; + /** + * True when the run did not complete its work for ANY reason — a QBO + * outage mid-loop, or a preflight failure (not connected, settings store + * unreadable, refresh failed). The audit event's status keys off THIS, not + * off the outage flag alone: a run that never got tokens did no work, and + * recording it as "ok" made the digest blind to a dead money rail. + */ + runFailed: boolean; + /** Short machine reason for `runFailed` — goes on the audit event. */ + failureReason?: string; +} + +/** + * The per-row loop both passes share, extracted so the abort rule is ONE piece + * of real code that a test can drive with fake rows and a fake QBO client. + * + * The rule: any connection-level failure (our deadline, a thrown network + * error, or QBO answering 429/5xx) from ANY QBO sub-call in the row — the + * invoice probe, the payment-detail read, anything added later — stops the run. + * Continuing would spend a fresh 20s deadline per row against the same wall, + * which is exactly how six timeouts consumed the cron's 120s ceiling. Ordinary + * per-row failures (a business error, a DB conflict) are recorded and the loop + * carries on. + */ +export async function runQboRowLoop( + rows: T[], + result: QBPaymentSyncResult, + handleRow: (row: T) => Promise, + onRowError: (row: T, error: unknown) => void, + skippedLabel: string, +): Promise { + for (const [index, row] of rows.entries()) { + // A previous pass already hit the wall — the connection is shared, so + // there is nothing to gain by trying again here. + if (result.abortedOnQboOutage) { + result.skipped += rows.length - index; + return; + } + try { + await handleRow(row); + } catch (error) { + if (isQboConnectionFailure(error)) { + result.abortedOnQboOutage = true; + result.runFailed = true; + result.failureReason = "qbo-unavailable"; + result.errors.push( + `QuickBooks stopped responding (${isQBTimeoutError(error) ? "timeout" : "unavailable"}) — remaining ${skippedLabel} skipped, will retry next run`, + ); + result.skipped += rows.length - index - 1; + return; + } + onRowError(row, error); + } + } } /** * Poll QuickBooks for settled milestone invoices and record them in ProBuild. * Safe to run repeatedly (cron + on-view). Never throws on a single bad row. */ -export async function syncQuickBooksPayments(scope?: { invoiceId?: string; projectId?: string }): Promise { +/** + * The QBO calls the sync loop makes per row. Injectable so a test can drive the + * REAL loop (abort rule, skip accounting, settle sequencing) against a fake + * QuickBooks instead of re-implementing the decision in the test. + */ +export interface PaymentsSyncQboClient { + probeInvoice(qbInvoiceId: string): Promise; + getPayment(paymentId: string): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null>; +} + +export interface SyncQuickBooksPaymentsOptions { + /** + * Who triggered this run. Only "cron" counts as the hourly heartbeat the + * health check watches — an on-view refresh must never be able to stand in + * for it, or a dead cron looks alive. + */ + source?: "cron" | "view" | "manual"; + /** Test seam; defaults to the real QBO calls. */ + qboClient?: PaymentsSyncQboClient; +} + +export async function syncQuickBooksPayments( + scope?: { invoiceId?: string; projectId?: string }, + options?: SyncQuickBooksPaymentsOptions, +): Promise { const result: QBPaymentSyncResult = { checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0, - skipped: 0, abortedOnQboOutage: false, + skipped: 0, abortedOnQboOutage: false, runFailed: false, }; const pending = await prisma.paymentSchedule.findMany({ @@ -721,7 +802,7 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje }); if (pending.length === 0 && pendingBillings.length === 0) { - await recordPaymentsSyncEvent(result); + await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -730,38 +811,44 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje tokens = await getFreshQBTokens(); } catch (e) { result.errors.push(e instanceof Error ? e.message : "QB tokens unavailable"); - // A timed-out refresh is the same outage as a timed-out probe; nothing - // was checked, so everything we loaded counts as skipped. - if (isQBTimeoutError(e)) { - result.abortedOnQboOutage = true; - result.skipped = pending.length + pendingBillings.length; - } - await recordPaymentsSyncEvent(result); + // ANY preflight failure means the run did no work — not connected, the + // settings store unreadable, a refresh that failed or timed out. All of + // them must record status=error: banking them as "ok" is precisely what + // made the digest blind to a dead money rail. + const preflight = classifyPreflightFailure(e); + result.runFailed = true; + result.failureReason = preflight.reason; + result.abortedOnQboOutage = preflight.abortedOnQboOutage; + // Nothing was checked, so everything we loaded counts as skipped. + result.skipped = pending.length + pendingBillings.length; + await recordPaymentsSyncEvent(result, options?.source); return result; } + const qbo: PaymentsSyncQboClient = options?.qboClient ?? { + probeInvoice: (qbInvoiceId) => probeQBInvoice(tokens, qbInvoiceId), + getPayment: (paymentId) => getQBPayment(tokens, paymentId), + }; + // Milestones whose linked QBO invoice was found voided/deleted THIS run (flag was // previously null). Reported once per breakage; a re-push clears the flag and re-arms. const newlyFlagged: QBSyncIssue[] = []; - // On a connection-level failure this loop STOPS rather than working through - // the rest: every remaining row would burn its own full deadline against the - // same wall, and six 20s timeouts is the cron's whole 120s ceiling. - for (const [index, schedule] of pending.entries()) { + await runQboRowLoop(pending, result, async (schedule) => { result.checked++; - try { - const probe = await probeQBInvoice(tokens, schedule.qbInvoiceId!); + { + const probe = await qbo.probeInvoice(schedule.qbInvoiceId!); if (probe.state === "error") { + // A connection-level failure becomes a throw so the shared loop + // applies one abort rule to every QBO sub-call in this row. if (probe.connectionFailed) { - result.abortedOnQboOutage = true; - result.errors.push( - `QuickBooks stopped responding (${probe.timedOut ? "timeout" : `status ${probe.status}`}) — remaining rows skipped, will retry next run`, + throw new QboRetryableError( + `QB invoice probe failed (${probe.timedOut ? "timeout" : `status ${probe.status}`})`, + probe.status, ); - result.skipped += pending.length - index - 1; - break; } // Ordinary transient error — leave untouched and retry next run. - continue; + return; } if (probe.state === "voided" || probe.state === "notFound") { @@ -797,7 +884,7 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje }).catch(() => {}); } result.errors.push(`${schedule.invoice.code}/${schedule.name}: QBO invoice ${probe.state}`); - continue; + return; } // probe.state === "ok" @@ -807,7 +894,10 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje let paidAt = new Date(); let referenceNumber: string | null = null; if (paymentId) { - const p = await getQBPayment(tokens, paymentId); + // Same abort rule as the probe: a timeout/429/5xx here + // throws and stops the run rather than costing another + // full deadline on every remaining settled invoice. + const p = await qbo.getPayment(paymentId); if (p?.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); referenceNumber = p?.referenceNumber || null; } @@ -825,10 +915,10 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje } else if (probe.balance < probe.total) { result.partiallyPaid++; } - } catch (e) { - result.errors.push(`${schedule.invoice.code}/${schedule.name}: ${e instanceof Error ? e.message : "sync failed"}`); } - } + }, (schedule, e) => { + result.errors.push(`${schedule.invoice.code}/${schedule.name}: ${e instanceof Error ? e.message : "sync failed"}`); + }, "milestones"); // ── Progress billings ─────────────────────────────────────────────────── // Same probe → settle shape as the milestone loop above, but claims ONE @@ -837,30 +927,21 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje // (custom/change-order lines were materialized into a real PaymentSchedule // at billing-creation time — see createProgressBillingCore — so every line // has a scheduleId and settles like any other milestone; no special case). - for (const [index, billing] of pendingBillings.entries()) { - // Same rule as the milestone loop: never keep dialling a dead QBO. A - // milestone-loop abort also skips this whole pass — the connection is - // shared, so there is nothing to gain by trying it again here. - if (result.abortedOnQboOutage) { - result.skipped += pendingBillings.length - index; - break; - } - try { - const probe = await probeQBInvoice(tokens, billing.qbInvoiceId!); + await runQboRowLoop(pendingBillings, result, async (billing) => { + { + const probe = await qbo.probeInvoice(billing.qbInvoiceId!); if (probe.state === "error") { if (probe.connectionFailed) { - result.abortedOnQboOutage = true; - result.errors.push( - `QuickBooks stopped responding (${probe.timedOut ? "timeout" : `status ${probe.status}`}) — remaining progress billings skipped, will retry next run`, + throw new QboRetryableError( + `QB invoice probe failed (${probe.timedOut ? "timeout" : `status ${probe.status}`})`, + probe.status, ); - result.skipped += pendingBillings.length - index - 1; - break; } - continue; // ordinary transient — retry next run + return; // ordinary transient — retry next run } if (probe.state === "voided" || probe.state === "notFound") { result.errors.push(`${billing.invoice.code}/${billing.code}: QBO invoice ${probe.state}`); - continue; + return; } // probe.state === "ok" if (probe.total > 0 && probe.balance <= 0) { @@ -868,7 +949,7 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje let paidAt = new Date(); let referenceNumber: string | null = null; if (paymentId) { - const p = await getQBPayment(tokens, paymentId); + const p = await qbo.getPayment(paymentId); if (p?.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); referenceNumber = p?.referenceNumber || null; } @@ -877,20 +958,37 @@ export async function syncQuickBooksPayments(scope?: { invoiceId?: string; proje } else if (probe.balance < probe.total) { result.partiallyPaid++; } - } catch (e) { - result.errors.push(`${billing.invoice.code}/${billing.code}: ${e instanceof Error ? e.message : "sync failed"}`); } - } + }, (billing, e) => { + result.errors.push(`${billing.invoice.code}/${billing.code}: ${e instanceof Error ? e.message : "sync failed"}`); + }, "progress billings") if (newlyFlagged.length > 0) { const { notifyQBSyncIssues } = await import("./payment-notifications"); await notifyQBSyncIssues(newlyFlagged); } - await recordPaymentsSyncEvent(result); + await recordPaymentsSyncEvent(result, options?.source); return result; } +/** + * Why a run never got off the ground. EVERY branch here is a failed run: the + * sync did no work, so recording it as "ok" would tell the digest the money + * rail is healthy when it is not. Only the connection-level branch also counts + * as an outage (the flag that stops further QBO calls). + */ +export function classifyPreflightFailure(error: unknown): { reason: string; abortedOnQboOutage: boolean } { + if (isQboConnectionFailure(error)) { + return { reason: "qbo-unavailable", abortedOnQboOutage: true }; + } + if (error instanceof QBNotConnectedError || (error instanceof Error && error.name === "QBNotConnectedError")) { + return { reason: "quickbooks-not-connected", abortedOnQboOutage: false }; + } + // Settings-store read failures, a rejected refresh, anything else. + return { reason: "token-fetch-failed", abortedOnQboOutage: false }; +} + /** The AutomationEvent kind the payments cron writes once per run. */ export const QBO_PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; @@ -904,14 +1002,20 @@ export const QBO_PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; * instead of staying silent. Fire-and-forget, like every other automation * event — the audit row must never fail the sync it describes. */ -async function recordPaymentsSyncEvent(result: QBPaymentSyncResult): Promise { - const failed = result.abortedOnQboOutage; +async function recordPaymentsSyncEvent( + result: QBPaymentSyncResult, + source: SyncQuickBooksPaymentsOptions["source"], +): Promise { await logAutomationEvent({ kind: QBO_PAYMENTS_SYNC_EVENT_KIND, - status: failed ? "error" : "ok", - reason: failed ? "qbo-unavailable" : undefined, - source: "cron", + status: result.runFailed ? "error" : "ok", + reason: result.failureReason, + // Only a real cron run may claim to be the heartbeat. An on-view or + // manual refresh is recorded under its own source (or none) so it can + // never mask an hourly job that has stopped running. + source, detail: { + runFailed: result.runFailed, checked: result.checked, settled: result.settled, partiallyPaid: result.partiallyPaid, diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 4a18277c7..540ba17a2 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -43,6 +43,44 @@ export class QBTimeoutError extends Error { * misclassification the whole deadline effort exists to prevent. The name is * set as a class field on every instance, so match on that too. */ +/** + * A QBO failure that WILL plausibly succeed on a later attempt: 429, 5xx, a + * thrown network error, or a dependent lookup that failed for those reasons. + * + * Distinct from a business rejection (a 4xx other than 429, or a QBO Fault), + * which is terminal and must NOT be retried forever. + */ +export class QboRetryableError extends Error { + name = "QboRetryableError"; + constructor(message: string, readonly status?: number) { + super(message); + } +} + +/** Name-based, for the same cross-module-identity reason as isQBTimeoutError. */ +export function isRetryableQboError(error: unknown): boolean { + return ( + error instanceof QboRetryableError || + (error instanceof Error && error.name === "QboRetryableError") + ); +} + +/** A QBO HTTP status we should come back to rather than give up on. */ +export function isRetryableQboStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +/** + * Did QBO fail in a way that means the NEXT call will fail the same way? + * + * A caller looping over many records must stop on this: each further attempt + * burns its own full deadline against the same wall, which is how a handful of + * 20s timeouts added up to the payments cron's entire 120s ceiling. + */ +export function isQboConnectionFailure(error: unknown): boolean { + return isQBTimeoutError(error) || isRetryableQboError(error); +} + export function isQBTimeoutError(error: unknown): error is QBTimeoutError { return ( error instanceof QBTimeoutError || @@ -719,9 +757,16 @@ export async function getQBPayment( paymentId: string ): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null> { const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); - if (!res.ok) return null; - const data = await res.json(); - const p = data.Payment; + if (!res.ok) { + // 429/5xx is QBO refusing to serve, not "this payment does not exist" — + // raise so a looping caller aborts instead of burning a deadline per row. + if (isRetryableQboStatus(res.status)) { + throw new QboRetryableError(`QB payment read failed with status ${res.status}`, res.status); + } + return null; + } + const data = await parseJsonOrNull(res); + const p = data?.Payment; if (!p) return null; return { txnDate: p.TxnDate || null, diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index e60a9bc7b..2da635179 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -322,3 +322,37 @@ test("the digest reports the payments rail alongside the receipt rail", () => { const { text } = formatPipelineDigest(sampleHealth()); assert.match(text, /Last payments sync: /); }); + + +// --- The payments heartbeat must be able to go red --- + +test("a null payments heartbeat is NOT ok - the hourly cron has never reported", () => { + const v = evaluatePipelineHealth(snapshot({ lastPaymentsSync: { status: "ok", at: null } })); + assert.equal(v.ok, false); + assert.ok(v.reasons.includes("payments-sync-stale")); +}); + +test("a payments heartbeat older than 26h is stale", () => { + const fresh = evaluatePipelineHealth(snapshot({ lastPaymentsSync: { status: "ok", at: iso(25 * HOUR) } })); + assert.deepEqual(fresh, { ok: true, reasons: [] }); + + const stale = evaluatePipelineHealth(snapshot({ lastPaymentsSync: { status: "ok", at: iso(27 * HOUR) } })); + assert.equal(stale.ok, false); + assert.deepEqual(stale.reasons, ["payments-sync-stale"]); +}); + +test("a weeks-old payments heartbeat never decays back into ok", () => { + for (const days of [3, 30, 400]) { + const v = evaluatePipelineHealth(snapshot({ lastPaymentsSync: { status: "ok", at: iso(days * 24 * HOUR) } })); + assert.equal(v.ok, false, `${days}d`); + assert.ok(v.reasons.includes("payments-sync-stale")); + } +}); + +test("only a CRON-sourced run counts as the heartbeat", async () => { + const { PAYMENTS_SYNC_CRON_SOURCE, PAYMENTS_SYNC_STALE_HOURS } = await import("../src/lib/pipeline-health"); + // On-view and manual refreshes write their own source precisely so they + // cannot stand in for an hourly job that has stopped running. + assert.equal(PAYMENTS_SYNC_CRON_SOURCE, "cron"); + assert.equal(PAYMENTS_SYNC_STALE_HOURS, 26); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index aef8f6b2a..73380ac12 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -81,67 +81,172 @@ test("a voided invoice is still authoritative, not a connection failure", async assert.deepEqual(probe, { state: "voided" }); }); -// ─── The loop rule the classification exists to drive ─────────────────────── +// --- The REAL loop, driven by a fake QuickBooks --- -/** - * Mirrors the guard in syncQuickBooksPayments: stop on the first - * connection-level failure, count the rest as skipped. The real loops need a - * database; this pins the DECISION they make, over many rows. - */ -function runLoop(probes: QBInvoiceProbe[]): { checked: number; skipped: number; aborted: boolean } { - let checked = 0; - let skipped = 0; - let aborted = false; - for (const [index, probe] of probes.entries()) { - checked++; +import { + runQboRowLoop, + classifyPreflightFailure, + QBNotConnectedError, + type PaymentsSyncQboClient, + type QBPaymentSyncResult, +} from "../src/lib/quickbooks-payments"; +import { QboRetryableError } from "../src/lib/quickbooks"; + +function emptyResult(): QBPaymentSyncResult { + return { + checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0, + skipped: 0, abortedOnQboOutage: false, runFailed: false, + }; +} + +/** A fake QuickBooks that records every call, so we can prove the run STOPPED. */ +function fakeQbo(script: { + probe?: (id: string) => QBInvoiceProbe; + payment?: (id: string) => { txnDate: string | null; amount: number; referenceNumber: string | null } | null; +}) { + const calls = { probes: [] as string[], payments: [] as string[] }; + const client: PaymentsSyncQboClient = { + async probeInvoice(id) { + calls.probes.push(id); + return script.probe ? script.probe(id) : { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }; + }, + async getPayment(id) { + calls.payments.push(id); + return script.payment ? script.payment(id) : { txnDate: "2026-09-01", amount: 10, referenceNumber: null }; + }, + }; + return { client, calls }; +} + +/** The same row body the real sync uses: probe, then read payment detail. */ +function rowHandler(qbo: PaymentsSyncQboClient, result: QBPaymentSyncResult) { + return async (row: { id: string; qbInvoiceId: string }) => { + result.checked++; + const probe = await qbo.probeInvoice(row.qbInvoiceId); if (probe.state === "error") { if (probe.connectionFailed) { - aborted = true; - skipped += probes.length - index - 1; - break; + throw new QboRetryableError("probe failed", probe.status); } - continue; + return; } - } - return { checked, skipped, aborted }; + if (probe.state !== "ok") return; + if (probe.total > 0 && probe.balance <= 0) { + const paymentId = probe.paymentTxnIds[0]; + if (paymentId) await qbo.getPayment(paymentId); + result.settled++; + } + }; } -test("200 pending rows during an outage cost ONE probe, not 200", () => { - const outage: QBInvoiceProbe[] = Array.from({ length: 200 }, () => ({ - state: "error" as const, - status: 0, - connectionFailed: true, - timedOut: true, - })); - const run = runLoop(outage); - // One 20s deadline, not six-plus — the whole point. - assert.equal(run.checked, 1); - assert.equal(run.skipped, 199); - assert.equal(run.aborted, true); +const rows = (n: number) => Array.from({ length: n }, (_, i) => ({ id: `s${i}`, qbInvoiceId: `${i + 1}` })); + +test("200 rows during an outage cost ONE probe, not 200", async () => { + const result = emptyResult(); + const { client, calls } = fakeQbo({ + probe: () => ({ state: "error", status: 0, connectionFailed: true, timedOut: true }), + }); + await runQboRowLoop(rows(200), result, rowHandler(client, result), () => {}, "milestones"); + + // One 20s deadline instead of 200 - the whole point of the abort. + assert.equal(calls.probes.length, 1); + assert.equal(result.skipped, 199); + assert.equal(result.abortedOnQboOutage, true); + assert.equal(result.runFailed, true); + assert.equal(result.failureReason, "qbo-unavailable"); +}); + +test("a PAYMENT-DETAIL timeout aborts the run too, not just the probe", async () => { + // Codex gate: getQBPayment failures were caught as ordinary row errors, so + // many settled invoices could each burn a full deadline after a good probe. + const result = emptyResult(); + const { client, calls } = fakeQbo({ + probe: () => ({ state: "ok", balance: 0, total: 100, paymentTxnIds: ["p1"] }), + payment: () => { + throw new QboRetryableError("QB payment read failed with status 503", 503); + }, + }); + await runQboRowLoop(rows(50), result, rowHandler(client, result), () => {}, "milestones"); + + assert.equal(calls.probes.length, 1, "must not probe row 2"); + assert.equal(calls.payments.length, 1, "must not read a second payment"); + assert.equal(result.skipped, 49); + assert.equal(result.abortedOnQboOutage, true); +}); + +test("an ordinary per-row error does NOT stop the run", async () => { + const result = emptyResult(); + const seen: string[] = []; + const { client, calls } = fakeQbo({ + probe: (id) => (id === "2" + ? { state: "error", status: 401 } + : { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }), + }); + await runQboRowLoop(rows(4), result, rowHandler(client, result), (row) => seen.push(row.id), "milestones"); + + assert.equal(calls.probes.length, 4); + assert.equal(result.skipped, 0); + assert.equal(result.abortedOnQboOutage, false); + assert.equal(seen.length, 0, "a transient probe error is not a row error"); +}); + +test("a settle failure is recorded per row and the run continues", async () => { + const result = emptyResult(); + const errors: string[] = []; + const { client, calls } = fakeQbo({ probe: () => ({ state: "ok", balance: 0, total: 10, paymentTxnIds: [] }) }); + const handler = async (row: { id: string; qbInvoiceId: string }) => { + result.checked++; + await client.probeInvoice(row.qbInvoiceId); + if (row.id === "s1") throw new Error("DB conflict"); + }; + await runQboRowLoop(rows(3), result, handler, (row) => errors.push(row.id), "milestones"); + + assert.equal(calls.probes.length, 3, "a row-level DB error must not abort the run"); + assert.deepEqual(errors, ["s1"]); + assert.equal(result.abortedOnQboOutage, false); +}); + +test("an outage partway through skips only the remainder", async () => { + const result = emptyResult(); + const { client } = fakeQbo({ + probe: (id) => (Number(id) >= 3 + ? { state: "error", status: 0, connectionFailed: true, timedOut: true } + : { state: "ok", balance: 5, total: 10, paymentTxnIds: [] }), + }); + await runQboRowLoop(rows(5), result, rowHandler(client, result), () => {}, "milestones"); + + assert.equal(result.checked, 3); + assert.equal(result.skipped, 2); }); -test("an ordinary per-invoice error does NOT stop the run", () => { - const probes: QBInvoiceProbe[] = [ - { state: "error", status: 401 }, - { state: "error", status: 401 }, - { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, - ]; - const run = runLoop(probes); - assert.equal(run.checked, 3); - assert.equal(run.skipped, 0); - assert.equal(run.aborted, false); +test("a second pass is skipped wholesale once the first aborted", async () => { + const result = emptyResult(); + result.abortedOnQboOutage = true; + const { client, calls } = fakeQbo({}); + await runQboRowLoop(rows(7), result, rowHandler(client, result), () => {}, "progress billings"); + + assert.equal(calls.probes.length, 0, "the connection is shared - do not retry it"); + assert.equal(result.skipped, 7); }); -test("an outage partway through skips only the remainder", () => { - const probes: QBInvoiceProbe[] = [ - { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, - { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, - { state: "error", status: 0, connectionFailed: true, timedOut: true }, - { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, - { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }, - ]; - const run = runLoop(probes); - assert.equal(run.checked, 3); - assert.equal(run.skipped, 2); - assert.equal(run.aborted, true); +// --- Preflight failures are failed runs --- + +test("EVERY preflight failure marks the run failed, not just timeouts", () => { + assert.deepEqual( + classifyPreflightFailure(new QBTimeoutError("refresh timed out")), + { reason: "qbo-unavailable", abortedOnQboOutage: true }, + ); + assert.deepEqual( + classifyPreflightFailure(new QboRetryableError("503", 503)), + { reason: "qbo-unavailable", abortedOnQboOutage: true }, + ); + // Codex gate: these two used to leave the run recorded as status "ok", + // which made the digest blind to a disconnected or broken money rail. + assert.deepEqual( + classifyPreflightFailure(new QBNotConnectedError()), + { reason: "quickbooks-not-connected", abortedOnQboOutage: false }, + ); + assert.deepEqual( + classifyPreflightFailure(new Error("settings store unreadable")), + { reason: "token-fetch-failed", abortedOnQboOutage: false }, + ); }); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index 26dd69399..d80d52eee 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -1063,3 +1063,71 @@ test("a purchase create that times out surfaces as 503 qbo-timeout at the route" assert.equal(response.status, 503); assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-timeout" }); }); + + +// --- Upload response validation --- + +/** Swap global fetch for one call; defaultUploadAttachment goes through it. */ +async function withFetch(impl: typeof fetch, run: () => Promise): Promise { + const original = globalThis.fetch; + globalThis.fetch = impl; + try { + return await run(); + } finally { + globalThis.fetch = original; + } +} + +const uploadFile = { base64: Buffer.from("bytes").toString("base64"), contentType: "image/jpeg", fileName: "receipt.jpg" }; + +async function upload(response: Response) { + const { defaultUploadAttachment } = await import("../src/lib/qbo-receipt-push"); + return withFetch(async () => response, () => defaultUploadAttachment(TOKENS, "99", uploadFile)); +} + +const jsonResponse = (status: number, body: unknown) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +test("an upload is only 'attached' when QBO returns a real Attachable id", async () => { + const result = await upload(jsonResponse(200, { AttachableResponse: [{ Attachable: { Id: "att-1" } }] })); + assert.equal(result, "attached"); +}); + +test("a 200 with NO Attachable is retryable, never a terminal success", async () => { + // Codex gate: Intuit's schema says an AttachableResponse carries either an + // Attachable or a Fault - absence is not success. An empty/truncated/HTML + // 200 (proxy error pages are the usual source) used to report "attached" + // for a file that was never stored, so the bot never came back for it. + for (const body of [{}, { AttachableResponse: [] }, { AttachableResponse: [{}] }, { AttachableResponse: [{ Attachable: {} }] }]) { + await assert.rejects( + () => upload(jsonResponse(200, body)), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + JSON.stringify(body), + ); + } +}); + +test("a 200 whose body is not JSON at all is retryable", async () => { + await assert.rejects( + () => upload(new Response("gateway error", { status: 200, headers: { "content-type": "text/html" } })), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); +}); + +test("a QBO Fault in a 200 stays terminal", async () => { + const result = await upload(jsonResponse(200, { AttachableResponse: [{ Fault: { Error: [{ code: "2010" }] } }] })); + assert.equal(result, "failed:fault"); +}); + +test("429/5xx uploads are retryable; other 4xx stay terminal", async () => { + for (const status of [429, 500, 503]) { + await assert.rejects( + () => upload(jsonResponse(status, {})), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + String(status), + ); + } + for (const status of [400, 403, 404]) { + assert.equal(await upload(jsonResponse(status, {})), `failed:${status}`); + } +}); From 363cffb62ed8a05c3c6932093f47e18e6be59ca4 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:37:41 -0700 Subject: [PATCH 017/144] fix(qbo): fail-closed payments cron; honest run status; token-save split; transient uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 6. 1. The payments cron route was the last fail-open caller: it only checked the secret when VERCEL_ENV === "production", so a preview or non-Vercel runtime could trigger a money sync unauthenticated — and, since this round, write a source:"cron" heartbeat that would make a dead cron look alive. Now uses isCronAuthorized (fail-closed, constant-time); a missing CRON_SECRET can no longer be satisfied by "Bearer undefined". 2. A run that skipped rows or hit row-level errors recorded status "ok" and refreshed the health heartbeat on work that never happened. Runs are now "ok" only when complete, "partial" when incomplete (heartbeat-ineligible, with counts, but not counted as a hard error so one stubborn row does not read like an outage), "error" when the run failed outright. 3. Refresh and save shared one catch, so a rotation Intuit had already committed could fall back to the now-spent stale pair while reporting a healthy connection. Split: only a REFRESH failure may fall back; a SAVE failure retries once, then raises QBTokenPersistenceError (reason "token-not-persisted") rather than stranding the integration silently. 4. 408 and 401 uploads were terminal `failed:` on ok:true, so the bot stopped retrying and left a new Purchase without its receipt. Both are transient now; a 401 forces exactly one token refresh and retries in place, and anything still failing raises. Hard 4xx (400/403/404/413/415) stay terminal. 5. If the first create's response was lost, no "created" event ever exists, so the recovery pass was invisible to the freshness clock. An "already-exists" that genuinely uploaded the attachment now counts as a booking; ordinary retries ("already-attached"/"skipped") still do not. Co-Authored-By: Claude Fable 5.1 --- src/app/api/cron/quickbooks-payments/route.ts | 9 +- src/lib/pipeline-health.ts | 31 ++++-- src/lib/qbo-receipt-push.ts | 55 ++++++++--- src/lib/quickbooks-payments.ts | 65 +++++++++++-- tests/cron-auth.test.ts | 36 +++++++ tests/pipeline-health.test.ts | 23 +++++ tests/qbo-payments-outage.test.ts | 94 +++++++++++++++++++ tests/qbo-receipt-push.test.ts | 76 +++++++++++++++ 8 files changed, 362 insertions(+), 27 deletions(-) diff --git a/src/app/api/cron/quickbooks-payments/route.ts b/src/app/api/cron/quickbooks-payments/route.ts index 4dd1042a5..7cebba508 100644 --- a/src/app/api/cron/quickbooks-payments/route.ts +++ b/src/app/api/cron/quickbooks-payments/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { syncQuickBooksPayments } from "@/lib/quickbooks-payments"; +import { isCronAuthorized } from "@/lib/cron-auth"; export const dynamic = "force-dynamic"; export const maxDuration = 120; @@ -10,8 +11,12 @@ export const maxDuration = 120; * ProBuild payment milestones, so ProBuild / QuickBooks / the bank stay in sync. */ export async function GET(request: Request) { - const authHeader = request.headers.get("authorization"); - if (process.env.VERCEL_ENV === "production" && authHeader !== `Bearer ${process.env.CRON_SECRET}`) { + // Was fail-OPEN: the check only ran when VERCEL_ENV === "production", so any + // preview or non-Vercel runtime could trigger a money sync unauthenticated, + // and a missing CRON_SECRET made `Bearer undefined` a valid credential. + // Now it can also write a source:"cron" heartbeat, so an unauthorized caller + // could make a dead cron look alive. Fail closed, constant-time. + if (!isCronAuthorized(request)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 9ac86575d..e0e168f23 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -110,15 +110,23 @@ export async function fetchIntuitStatus(): Promise { } /** - * "Last receipt booked" counts CREATES only. + * What counts as "a receipt got booked" for the freshness clock. * - * "already-exists" is an idempotent re-push of a receipt that was created - * earlier — often much earlier — so counting it here refreshed the freshness - * clock without any new receipt entering the books, and a bot stuck retrying - * one old file would have kept the pipeline looking alive indefinitely. - * "fallback"/"error" are attempts, not bookings. + * A plain "already-exists" must NOT count: it is an idempotent re-push of a + * receipt created earlier, often much earlier, so counting it would let a bot + * stuck retrying one old file keep the pipeline looking alive indefinitely. + * + * But there is one real exception. When the FIRST attempt's response was lost + * after QBO committed the Purchase, no "created" event exists at all — the + * recovery pass is the only record of that booking, and it logs + * "already-exists". Counting it only when it actually UPLOADED the attachment + * (`attachment: "attached"`, which only happens once per receipt, on the pass + * that repaired it) captures exactly those recoveries without letting ordinary + * old retries — which report "already-attached" or "skipped" — reset the clock. */ export const BOOKED_PUSH_STATUSES = ["created"]; +/** Marker for the recovery pass that genuinely stored the file. */ +export const RECOVERED_BOOKING_DETAIL = '"attachment":"attached"'; /** The payments cron's per-run audit row (see quickbooks-payments.ts). */ export const PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; @@ -268,7 +276,16 @@ export async function getPipelineHealth(): Promise { async () => ( await prisma.automationEvent.findFirst({ - where: { kind: "receipt-push", status: { in: BOOKED_PUSH_STATUSES } }, + where: { + kind: "receipt-push", + OR: [ + { status: { in: BOOKED_PUSH_STATUSES } }, + { + status: "already-exists", + detail: { contains: RECOVERED_BOOKING_DETAIL }, + }, + ], + }, orderBy: { createdAt: "desc" }, select: { createdAt: true }, }) diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 1190c679e..6239c4589 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -42,6 +42,20 @@ import { // it too); re-exported here because this module's callers already import it. export { QboRetryableError, isRetryableQboError, isRetryableQboStatus }; +/** + * Statuses where retrying the ATTACHMENT upload is worthwhile. + * + * Wider than isRetryableQboStatus on purpose: 408 (request timeout) and 401 + * (expired access token, repaired by a refresh) used to be banked as a terminal + * `failed:` on an ok:true response, which stops the Apps Script + * resending and leaves a freshly created Purchase without its receipt forever. + * A 4xx that means "QBO will never accept this file" (400/403/404/413/415) + * stays terminal - retrying that is a loop, not a repair. + */ +export function isTransientAttachmentStatus(status: number): boolean { + return status === 401 || status === 408 || isRetryableQboStatus(status); +} + /** Thrown by ensureQBVendor when QBO rejects the create as a duplicate name (fault 6240) and a re-query still can't find it. */ export class QboVendorDuplicateError extends Error { constructor(name: string) { @@ -320,6 +334,11 @@ export async function defaultUploadAttachment( tokens: QBTokens, purchaseId: string, file: { base64: string; contentType: string; fileName: string }, + /** Injectable for tests; defaults to the real token refresh. */ + refreshTokens: () => Promise = async () => { + const { getFreshQBTokens } = await import("./quickbooks-payments"); + return getFreshQBTokens(); + }, ): Promise { const safeFileName = attachmentFileName(file.fileName); const fileBytes = Buffer.from(file.base64, "base64"); @@ -346,19 +365,31 @@ export async function defaultUploadAttachment( Buffer.from(`${CRLF}--${boundary}--${CRLF}`), ]); - const res = await qbTimedFetch(`${QB_API_BASE}/${tokens.realmId}/upload?minorversion=73`, { - method: "POST", - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - Accept: "application/json", - "Content-Type": `multipart/form-data; boundary=${boundary}`, - }, - body, - }); + const post = (active: QBTokens) => + qbTimedFetch(`${QB_API_BASE}/${active.realmId}/upload?minorversion=73`, { + method: "POST", + headers: { + Authorization: `Bearer ${active.accessToken}`, + Accept: "application/json", + "Content-Type": `multipart/form-data; boundary=${boundary}`, + }, + body, + }); + + let res = await post(tokens); + // A 401 here usually just means the access token aged out mid-push. One + // forced refresh repairs it in place; without this the receipt waited for a + // whole extra bot pass (or, before transient statuses were retryable at + // all, was abandoned entirely). + if (res.status === 401) { + const refreshed = await refreshTokens().catch(() => null); + if (refreshed) res = await post(refreshed); + } if (!res.ok) { - // 429/5xx: QBO is busy or broken, not refusing this file. Raise so the - // push is retried rather than banked as a terminal "failed:503". - if (isRetryableQboStatus(res.status)) { + // Busy, timed out, or still unauthorized: QBO is not refusing this + // file, so raise and let the push be retried rather than banking a + // terminal "failed:503" next to ok:true. + if (isTransientAttachmentStatus(res.status)) { throw new QboRetryableError(`QB attachment upload failed with status ${res.status}`, res.status); } return `failed:${res.status}`; diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 4c22a4875..e216be809 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -36,6 +36,18 @@ import { import { isE2eQboMockEnabled, MOCK_QB_TOKENS } from "./quickbooks-mock"; import type { QBSyncIssue } from "./payment-notifications"; +/** + * Intuit rotated the refresh token but we could not store the replacement. + * Distinct from a refresh failure: the OLD token is already spent, so there is + * no safe fallback and the connection needs human attention. + */ +export class QBTokenPersistenceError extends Error { + name = "QBTokenPersistenceError"; + constructor() { + super("QuickBooks token was rotated but could not be saved; reconnect QuickBooks"); + } +} + export class QBNotConnectedError extends Error { constructor() { super("QuickBooks is not connected (Settings → Integrations → QuickBooks)"); @@ -83,15 +95,35 @@ export async function refreshTokensOrFallBack( refresh: typeof refreshQBToken = refreshQBToken, save: typeof saveQBSettings = saveQBSettings, ): Promise { + let fresh: { accessToken: string; refreshToken: string }; try { - const fresh = await refresh(qb.refreshToken); - await save({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); - return { accessToken: fresh.accessToken, refreshToken: fresh.refreshToken, realmId: qb.realmId }; + fresh = await refresh(qb.refreshToken); } catch (error) { if (isQBTimeoutError(error)) throw error; - // Refresh can fail transiently; the old access token may still be valid. + // Refresh itself failed and Intuit rotated nothing, so the old access + // token may still be valid. This is the ONLY branch that may fall back. return { accessToken: qb.accessToken, refreshToken: qb.refreshToken, realmId: qb.realmId }; } + + // A SAVE failure is a different animal and must never share the catch + // above. By this point Intuit has already rotated: the old refresh token is + // spent, so returning the stale pair would report a healthy connection + // while quietly stranding the integration until someone reconnects by hand. + // Retry once (a transient DB blip is the common case), then surface it. + try { + await save({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); + } catch (first) { + try { + await save({ accessToken: fresh.accessToken, refreshToken: fresh.refreshToken }); + } catch { + console.error( + "QBO token rotated but could NOT be persisted — reconnect QuickBooks if the next refresh fails", + first instanceof Error ? first.name : "UnknownError", + ); + throw new QBTokenPersistenceError(); + } + } + return { accessToken: fresh.accessToken, refreshToken: fresh.refreshToken, realmId: qb.realmId }; } /** @@ -985,10 +1017,29 @@ export function classifyPreflightFailure(error: unknown): { reason: string; abor if (error instanceof QBNotConnectedError || (error instanceof Error && error.name === "QBNotConnectedError")) { return { reason: "quickbooks-not-connected", abortedOnQboOutage: false }; } + if (error instanceof QBTokenPersistenceError || (error instanceof Error && error.name === "QBTokenPersistenceError")) { + return { reason: "token-not-persisted", abortedOnQboOutage: false }; + } // Settings-store read failures, a rejected refresh, anything else. return { reason: "token-fetch-failed", abortedOnQboOutage: false }; } +/** + * The audit status for a finished run. + * + * "ok" is reserved for a run that actually completed ALL its work. A run that + * skipped rows or hit row-level errors did NOT verify those milestones, so + * calling it "ok" refreshed the health heartbeat on the strength of work that + * never happened — the digest would read green while payments went unchecked. + * Those runs are "partial": visible and heartbeat-ineligible, but not counted + * as hard errors, so one stubborn row does not read like a QBO outage. + */ +export function paymentsSyncRunStatus(result: QBPaymentSyncResult): "ok" | "partial" | "error" { + if (result.runFailed) return "error"; + if (result.skipped > 0 || result.errors.length > 0) return "partial"; + return "ok"; +} + /** The AutomationEvent kind the payments cron writes once per run. */ export const QBO_PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; @@ -1006,16 +1057,18 @@ async function recordPaymentsSyncEvent( result: QBPaymentSyncResult, source: SyncQuickBooksPaymentsOptions["source"], ): Promise { + const status = paymentsSyncRunStatus(result); await logAutomationEvent({ kind: QBO_PAYMENTS_SYNC_EVENT_KIND, - status: result.runFailed ? "error" : "ok", - reason: result.failureReason, + status, + reason: result.failureReason ?? (status === "partial" ? "incomplete-run" : undefined), // Only a real cron run may claim to be the heartbeat. An on-view or // manual refresh is recorded under its own source (or none) so it can // never mask an hourly job that has stopped running. source, detail: { runFailed: result.runFailed, + errorCount: result.errors.length, checked: result.checked, settled: result.settled, partiallyPaid: result.partiallyPaid, diff --git a/tests/cron-auth.test.ts b/tests/cron-auth.test.ts index e37103d06..ab962751c 100644 --- a/tests/cron-auth.test.ts +++ b/tests/cron-auth.test.ts @@ -97,3 +97,39 @@ test("test/CI is NOT a bypass", () => { assert.equal(isCronAuthorized(request()), false); }); }); + + +// --- The payments cron route was the last fail-open caller --- + +test("the payments cron rejects an unauthenticated request outside development", async () => { + const previousEnv = process.env.NODE_ENV; + const previousVercel = process.env.VERCEL_ENV; + const previousSecret = process.env.CRON_SECRET; + (process.env as Record).NODE_ENV = "production"; + delete process.env.VERCEL_ENV; // the exact hole: no VERCEL_ENV used to mean "no check" + process.env.CRON_SECRET = "s3cret"; + try { + const { GET } = await import("../src/app/api/cron/quickbooks-payments/route"); + const call = (headers?: Record) => + GET(new Request("https://probuild.test/api/cron/quickbooks-payments", { headers })); + + assert.equal((await call()).status, 401, "no header"); + assert.equal((await call({ authorization: "Bearer wrong" })).status, 401, "wrong secret"); + assert.equal((await call({ authorization: "Bearer" })).status, 401, "malformed header"); + assert.equal((await call({ authorization: "Bearer undefined" })).status, 401, "literal undefined"); + } finally { + if (previousEnv === undefined) delete (process.env as Record).NODE_ENV; + else (process.env as Record).NODE_ENV = previousEnv; + if (previousVercel === undefined) delete process.env.VERCEL_ENV; + else process.env.VERCEL_ENV = previousVercel; + if (previousSecret === undefined) delete process.env.CRON_SECRET; + else process.env.CRON_SECRET = previousSecret; + } +}); + +test("a missing CRON_SECRET cannot be satisfied by 'Bearer undefined'", () => { + withEnv({ CRON_SECRET: undefined, NODE_ENV: "production" }, () => { + assert.equal(isCronAuthorized(request("Bearer undefined")), false); + assert.equal(isCronAuthorized(request("Bearer ")), false); + }); +}); diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 2da635179..a2c0604e1 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -356,3 +356,26 @@ test("only a CRON-sourced run counts as the heartbeat", async () => { assert.equal(PAYMENTS_SYNC_CRON_SOURCE, "cron"); assert.equal(PAYMENTS_SYNC_STALE_HOURS, 26); }); + + +// --- Lost-response recovery must be able to advance the clock --- + +test("a recovery that actually uploaded the file counts as a booking", async () => { + const { BOOKED_PUSH_STATUSES, RECOVERED_BOOKING_DETAIL } = await import("../src/lib/pipeline-health"); + + // A plain re-push still must not reset the clock... + assert.deepEqual(BOOKED_PUSH_STATUSES, ["created"]); + + // ...but when the FIRST attempt's response was lost after QBO committed + // the Purchase, no "created" event exists at all and the recovery pass is + // the only record of that booking. It is recognised by having genuinely + // uploaded the attachment. + assert.equal(RECOVERED_BOOKING_DETAIL, '"attachment":"attached"'); + + const detailOf = (attachment: string) => JSON.stringify({ fileId: "f1", qbPurchaseId: "99", attachment }); + assert.ok(detailOf("attached").includes(RECOVERED_BOOKING_DETAIL)); + // The ordinary retries must NOT match: they attached nothing new. + assert.equal(detailOf("already-attached").includes(RECOVERED_BOOKING_DETAIL), false); + assert.equal(detailOf("skipped").includes(RECOVERED_BOOKING_DETAIL), false); + assert.equal(detailOf("failed:400").includes(RECOVERED_BOOKING_DETAIL), false); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index 73380ac12..a79706601 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -250,3 +250,97 @@ test("EVERY preflight failure marks the run failed, not just timeouts", () => { { reason: "token-fetch-failed", abortedOnQboOutage: false }, ); }); + + +// --- A run that skipped work is never "ok" --- + +test("run status: ok only when the run actually finished all its work", async () => { + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + + assert.equal(paymentsSyncRunStatus(emptyResult()), "ok"); + + // Codex gate: these two used to record "ok" and refresh the health + // heartbeat on the strength of milestones that were never checked. + assert.equal(paymentsSyncRunStatus({ ...emptyResult(), skipped: 3 }), "partial"); + assert.equal(paymentsSyncRunStatus({ ...emptyResult(), errors: ["INV-1/Deposit: boom"] }), "partial"); + + assert.equal(paymentsSyncRunStatus({ ...emptyResult(), runFailed: true }), "error"); + // A hard failure outranks a partial one. + assert.equal( + paymentsSyncRunStatus({ ...emptyResult(), runFailed: true, skipped: 9, errors: ["x"] }), + "error", + ); +}); + +test("an aborted outage run reports error, not partial", async () => { + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + const result = emptyResult(); + const { client } = fakeQbo({ + probe: () => ({ state: "error", status: 0, connectionFailed: true, timedOut: true }), + }); + await runQboRowLoop(rows(10), result, rowHandler(client, result), () => {}, "milestones"); + assert.equal(paymentsSyncRunStatus(result), "error"); +}); + +// --- Token rotation vs. persistence --- + +const STALE_QB = { accessToken: "stale-access", refreshToken: "stale-refresh", realmId: "realm-1" }; + +test("a SAVE failure after a successful rotation is surfaced, never swallowed", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + // Codex gate: refresh and save shared one catch, so a rotation Intuit had + // already committed could fall back to the now-spent stale pair and report + // a healthy connection while the integration was stranded. + let saves = 0; + const error = await refreshTokensOrFallBack( + STALE_QB, + async () => ({ accessToken: "new-access", refreshToken: "new-refresh" }), + async () => { + saves++; + throw new Error("DB write failed"); + }, + ).then(() => null, (e: unknown) => e as Error); + + assert.ok(error instanceof Error); + assert.equal(error.name, "QBTokenPersistenceError"); + assert.equal(saves, 2, "one retry before giving up"); +}); + +test("a transient save blip is retried once and then succeeds", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + let saves = 0; + const tokens = await refreshTokensOrFallBack( + STALE_QB, + async () => ({ accessToken: "new-access", refreshToken: "new-refresh" }), + async () => { + saves++; + if (saves === 1) throw new Error("deadlock"); + }, + ); + assert.deepEqual(tokens, { accessToken: "new-access", refreshToken: "new-refresh", realmId: "realm-1" }); + assert.equal(saves, 2); +}); + +test("a REFRESH failure still falls back to the old access token (unchanged)", async () => { + const { refreshTokensOrFallBack } = await import("../src/lib/quickbooks-payments"); + let saves = 0; + const tokens = await refreshTokensOrFallBack( + STALE_QB, + async () => { + throw new Error("500 from Intuit"); + }, + async () => { + saves++; + }, + ); + assert.deepEqual(tokens, STALE_QB); + assert.equal(saves, 0, "nothing was rotated, so nothing is saved"); +}); + +test("a token-persistence failure marks the run failed with its own reason", async () => { + const { classifyPreflightFailure, QBTokenPersistenceError } = await import("../src/lib/quickbooks-payments"); + assert.deepEqual( + classifyPreflightFailure(new QBTokenPersistenceError()), + { reason: "token-not-persisted", abortedOnQboOutage: false }, + ); +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index d80d52eee..c7d611b33 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -1131,3 +1131,79 @@ test("429/5xx uploads are retryable; other 4xx stay terminal", async () => { assert.equal(await upload(jsonResponse(status, {})), `failed:${status}`); } }); + + +// --- Transient upload statuses --- + +test("408 and 401 are transient for attachments; hard 4xx stay terminal", async () => { + const { isTransientAttachmentStatus } = await import("../src/lib/qbo-receipt-push"); + for (const status of [401, 408, 429, 500, 503]) { + assert.equal(isTransientAttachmentStatus(status), true, String(status)); + } + for (const status of [400, 403, 404, 413, 415]) { + assert.equal(isTransientAttachmentStatus(status), false, String(status)); + } +}); + +test("a 401 upload forces ONE token refresh and retries in place", async () => { + const { defaultUploadAttachment } = await import("../src/lib/qbo-receipt-push"); + const seen: string[] = []; + let refreshes = 0; + const impl = (async (_url: string, init: RequestInit) => { + const auth = String((init.headers as Record).Authorization); + seen.push(auth); + return auth.includes("fresh-token") + ? jsonResponse(200, { AttachableResponse: [{ Attachable: { Id: "att-9" } }] }) + : new Response("{}", { status: 401 }); + }) as unknown as typeof fetch; + + const result = await withFetch(impl, () => + defaultUploadAttachment(TOKENS, "99", uploadFile, async () => { + refreshes++; + return { accessToken: "fresh-token", refreshToken: "r", realmId: "test-realm" }; + }), + ); + + assert.equal(result, "attached"); + assert.equal(refreshes, 1, "exactly one forced refresh"); + assert.equal(seen.length, 2, "original attempt plus one retry"); +}); + +test("a 401 that survives the refresh is retryable, not terminal", async () => { + const { defaultUploadAttachment } = await import("../src/lib/qbo-receipt-push"); + let refreshes = 0; + const impl = (async () => new Response("{}", { status: 401 })) as unknown as typeof fetch; + + await assert.rejects( + () => withFetch(impl, () => + defaultUploadAttachment(TOKENS, "99", uploadFile, async () => { + refreshes++; + return { accessToken: "fresh-token", refreshToken: "r", realmId: "test-realm" }; + }), + ), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); + assert.equal(refreshes, 1, "must not retry the refresh in a loop"); +}); + +test("a failing refresh does not crash the upload; the 401 becomes retryable", async () => { + const { defaultUploadAttachment } = await import("../src/lib/qbo-receipt-push"); + const impl = (async () => new Response("{}", { status: 401 })) as unknown as typeof fetch; + await assert.rejects( + () => withFetch(impl, () => + defaultUploadAttachment(TOKENS, "99", uploadFile, async () => { + throw new Error("not connected"); + }), + ), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); +}); + +test("a 408 upload is retryable rather than a terminal failed:408", async () => { + const { defaultUploadAttachment } = await import("../src/lib/qbo-receipt-push"); + const impl = (async () => new Response("{}", { status: 408 })) as unknown as typeof fetch; + await assert.rejects( + () => withFetch(impl, () => defaultUploadAttachment(TOKENS, "99", uploadFile, async () => TOKENS)), + (error: unknown) => (error as Error)?.name === "QboRetryableError", + ); +}); From 0dd50b3f955d25c30479f335817f7a4158ac766e Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:00:35 -0700 Subject: [PATCH 018/144] fix(qbo): no silent probe skips; partial runs visible now; normalize network errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 7. 1. Both loops returned silently when a probe failed without connectionFailed, so a run could verify nothing, record no error or skip, and still emit status "ok" — a green heartbeat for work that never happened (my own tests enshrined it). Every failed probe now records a row error, making the run "partial". And a 401/403 is connection-level: the credential is shared, so the next 199 rows fail identically at full cost — those abort the run. 2. Health read only the last "ok" event and counted only "error" ones, so repeated hourly partial runs could sit green for up to 26h, or until the next day's digest. The heartbeat query now accepts ok OR partial (a partial run does prove the cron is alive, so it counts for freshness) and a latest run of "partial" adds reason "payments-sync-partial" immediately. The digest line marks it "[incomplete run]". 3. getQBPayment translated 429/5xx but a bare `TypeError: fetch failed` (DNS/TLS/reset) escaped unclassified, so runQboRowLoop did not see it as connection-level and kept dialling. Normalized at the boundary instead of per call site: qbTimedFetch now turns any thrown transport failure into QboRetryableError, in both the header and body phases, so every QBO call gets it. A caller's own abort still passes through untouched. Tested with a real TypeError from an injected fetch, through getQBPayment and probeQBInvoice. Co-Authored-By: Claude Fable 5.1 --- src/lib/pipeline-health.ts | 48 +++++++++++++++------ src/lib/quickbooks-payments.ts | 11 +++-- src/lib/quickbooks.ts | 40 ++++++++++++++--- tests/pipeline-health.test.ts | 52 ++++++++++++++++++++++ tests/qbo-payments-outage.test.ts | 71 ++++++++++++++++++++++++++----- 5 files changed, 190 insertions(+), 32 deletions(-) diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index e0e168f23..5825702bc 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -38,6 +38,13 @@ export interface TimestampProbe { status: ProbeStatus; reason?: ProbeFailure; at: string | null; + /** + * For the payments heartbeat: the recorded status of the run this + * timestamp came from ("ok" or "partial"). A partial run counts for + * FRESHNESS (the cron did run) but is reported as a problem in its own + * right, so repeated partial runs cannot sit green behind a 26h window. + */ + runStatus?: string | null; } export interface CountsProbe { @@ -136,6 +143,11 @@ export const PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; * hourly job that has stopped running. */ export const PAYMENTS_SYNC_CRON_SOURCE = "cron"; +/** + * Run statuses that prove the cron is alive. A "partial" run did execute, so + * it counts for freshness — and is then flagged separately, immediately. + */ +export const PAYMENTS_SYNC_HEARTBEAT_STATUSES = ["ok", "partial"]; /** The cron runs hourly; 26h leaves room for a couple of missed runs and DST. */ export const PAYMENTS_SYNC_STALE_HOURS = 26; @@ -194,6 +206,12 @@ export function evaluatePipelineHealth(input: { const at = input.lastPaymentsSync.at ? Date.parse(input.lastPaymentsSync.at) : null; const stale = at === null || Number.isNaN(at) || input.now - at > PAYMENTS_SYNC_STALE_HOURS * HOUR_MS; if (stale) reasons.push("payments-sync-stale"); + // A run that skipped rows or hit errors did NOT verify those payments. + // It proves the cron is alive, so it counts for freshness — but it must + // be reported the same day, not hidden inside the 26h staleness window + // (repeated partial runs could otherwise stay green until tomorrow's + // digest, or indefinitely). + else if (input.lastPaymentsSync.runStatus === "partial") reasons.push("payments-sync-partial"); } if (input.lastReceiptPush.status === "ok") { @@ -292,20 +310,21 @@ export async function getPipelineHealth(): Promise { )?.createdAt ?? null, null, ), - probe( + probe<{ createdAt: Date; status: string } | null>( "lastPaymentsSync", async () => - ( - await prisma.automationEvent.findFirst({ - where: { - kind: PAYMENTS_SYNC_EVENT_KIND, - status: "ok", - source: PAYMENTS_SYNC_CRON_SOURCE, - }, - orderBy: { createdAt: "desc" }, - select: { createdAt: true }, - }) - )?.createdAt ?? null, + (await prisma.automationEvent.findFirst({ + // "partial" counts for freshness (the cron ran) and is then + // reported on its own; only a hard "error" is excluded, + // since `stuck` already surfaces those. + where: { + kind: PAYMENTS_SYNC_EVENT_KIND, + status: { in: PAYMENTS_SYNC_HEARTBEAT_STATUSES }, + source: PAYMENTS_SYNC_CRON_SOURCE, + }, + orderBy: { createdAt: "desc" }, + select: { createdAt: true, status: true }, + })) ?? null, null, ), probe>( @@ -352,7 +371,8 @@ export async function getPipelineHealth(): Promise { lastPaymentsSync: { status: lastPaymentsSync.status, reason: lastPaymentsSync.reason, - at: lastPaymentsSync.value?.toISOString() ?? null, + at: lastPaymentsSync.value?.createdAt.toISOString() ?? null, + runStatus: lastPaymentsSync.value?.status ?? null, }, receipts24h: { status: receiptRows.status, reason: receiptRows.reason, counts }, bank: { @@ -411,7 +431,7 @@ export function formatPipelineDigest(health: PipelineHealth): { subject: string; `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}${health.intuit.status === "error" ? " [status page unreachable]" : ""}`, `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSync, now)}`, `Last receipt booked: ${ago(health.qbo.lastReceiptPush, now)}`, - `Last payments sync: ${ago(health.qbo.lastPaymentsSync, now)}`, + `Last payments sync: ${ago(health.qbo.lastPaymentsSync, now)}${health.qbo.lastPaymentsSync.runStatus === "partial" ? " [incomplete run]" : ""}`, `Receipts (24h): ${receiptsLine}`, `Bank ledger through: ${ health.bank.status === "error" diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index e216be809..5f91f7aef 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -879,8 +879,12 @@ export async function syncQuickBooksPayments( probe.status, ); } - // Ordinary transient error — leave untouched and retry next run. - return; + // Any other probe failure leaves this milestone UNVERIFIED. It + // used to return silently, so a run that checked nothing could + // still finish with zero errors and emit status "ok" — a green + // heartbeat for work that never happened. Record it as a row + // error (the run becomes "partial") and move on. + throw new Error(`QBO invoice probe failed (status ${probe.status})`); } if (probe.state === "voided" || probe.state === "notFound") { @@ -969,7 +973,8 @@ export async function syncQuickBooksPayments( probe.status, ); } - return; // ordinary transient — retry next run + // Same rule as the milestone loop: unverified is not "fine". + throw new Error(`QBO invoice probe failed (status ${probe.status})`); } if (probe.state === "voided" || probe.state === "notFound") { result.errors.push(`${billing.invoice.code}/${billing.code}: QBO invoice ${probe.state}`); diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 540ba17a2..e3c9a25d8 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -201,6 +201,32 @@ function asQbTimeout(error: unknown, context: TimeoutContext): unknown { return error; } +/** + * Normalize every failure that crosses the QBO network boundary. + * + * Our own deadline becomes QBTimeoutError; a caller's abort stays exactly as it + * was; anything else that `fetch` THREW is a transport failure (DNS, TLS, + * connection reset — Node surfaces these as a bare `TypeError: fetch failed`) + * and becomes QboRetryableError. + * + * That last case is why this exists: an un-normalized TypeError is neither a + * timeout nor a retryable error, so a looping caller could not recognise it as + * connection-level and kept dialling a QuickBooks that was clearly unreachable. + * Classifying it HERE means every QBO call gets it, not just the ones someone + * remembered to wrap. + */ +function asQboBoundaryError(error: unknown, context: TimeoutContext): unknown { + const translated = asQbTimeout(error, context); + if (isQBTimeoutError(translated)) return translated; + // A caller cancelling its own request is not a QBO failure. + if (context.race.winner() && context.race.winner() !== context.timeoutSignal) return translated; + if (translated instanceof Error && translated.name === "AbortError") return translated; + if (isRetryableQboError(translated)) return translated; + return new QboRetryableError( + `QuickBooks request failed: ${translated instanceof Error ? translated.message : "network error"} (${safePath(context.url)})`, + ); +} + /** * The deadline governs the WHOLE exchange, not just the headers. * @@ -232,7 +258,9 @@ function wrapResponseBodyTimeouts(response: Response, context: TimeoutContext): try { return await original.apply(target, args); } catch (error) { - throw asQbTimeout(error, context); + // Same normalization as the header phase: a body that + // dies mid-stream is a transport failure, not a verdict. + throw asQboBoundaryError(error, context); } }; } @@ -283,7 +311,7 @@ export async function qbTimedFetch( try { response = await fetch(url, { ...init, signal: race.signal }); } catch (error) { - throw asQbTimeout(error, context); + throw asQboBoundaryError(error, context); } return wrapResponseBodyTimeouts(response, context); } @@ -743,9 +771,11 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { return { state: "notFound" }; } - // 429 and 5xx are QBO telling us it cannot serve requests right now; the - // next row would hit the same wall, so they count as connection-level too. - if (res.status === 429 || res.status >= 500) { + // 429/5xx: QBO cannot serve requests right now. 401/403: the credential + // itself is bad, and it is the SAME credential for every remaining row, so + // working through them would produce identical failures at full cost. + // All of these are connection-level. + if (res.status === 429 || res.status >= 500 || res.status === 401 || res.status === 403) { return { state: "error", status: res.status, connectionFailed: true }; } return { state: "error", status: res.status }; diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index a2c0604e1..30ea1af20 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -379,3 +379,55 @@ test("a recovery that actually uploaded the file counts as a booking", async () assert.equal(detailOf("skipped").includes(RECOVERED_BOOKING_DETAIL), false); assert.equal(detailOf("failed:400").includes(RECOVERED_BOOKING_DETAIL), false); }); + + +// --- A partial run must not hide behind the 26h window --- + +test("the latest run being PARTIAL is reported immediately, not in 26h", async () => { + const { PAYMENTS_SYNC_HEARTBEAT_STATUSES } = await import("../src/lib/pipeline-health"); + // Codex gate: health read only the last "ok" event and counted only + // "error" ones, so repeated hourly partial runs could leave the digest + // green for up to 26 hours - or until the next day's digest. + assert.deepEqual(PAYMENTS_SYNC_HEARTBEAT_STATUSES, ["ok", "partial"]); + + const v = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(1 * HOUR), runStatus: "partial" }, + })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["payments-sync-partial"]); +}); + +test("a partial run still counts as a heartbeat, so it is not ALSO stale", () => { + const v = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(1 * HOUR), runStatus: "partial" }, + })); + assert.equal(v.reasons.includes("payments-sync-stale"), false); +}); + +test("a stale partial run reports staleness, not double-reasons", () => { + const v = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(40 * HOUR), runStatus: "partial" }, + })); + assert.deepEqual(v.reasons, ["payments-sync-stale"]); +}); + +test("an ok run reports nothing", () => { + const v = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(1 * HOUR), runStatus: "ok" }, + })); + assert.deepEqual(v, { ok: true, reasons: [] }); +}); + +test("the digest flags an incomplete payments run in the body", () => { + const { text } = formatPipelineDigest(sampleHealth({ + ok: false, + reasons: ["payments-sync-partial"], + qbo: { + lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, + lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, + lastPaymentsSync: { status: "ok", at: "2026-09-01T13:00:00.000Z", runStatus: "partial" }, + }, + })); + assert.match(text, /Last payments sync: .*\[incomplete run\]/); + assert.match(text, /Needs attention: payments-sync-partial/); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index a79706601..ae3408223 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -55,14 +55,43 @@ test("a thrown network error is a connection failure too", async () => { assert.equal(probe.state === "error" && probe.timedOut, false); }); -test("429 and 5xx are connection failures; an ordinary 401 is not", async () => { - for (const status of [429, 500, 503]) { +test("429/5xx AND 401/403 are connection failures; a plain 400 is not", async () => { + // Codex gate: 401 used to be an "ordinary" error the loop skipped past. But + // the credential is the SAME for every remaining row, so the next 199 rows + // fail identically at full cost — that is connection-level by definition. + for (const status of [429, 500, 503, 401, 403]) { const probe = await withFetch(async () => json(status, { Fault: {} }), () => probeQBInvoice(TOKENS, "1")); assert.equal(probe.state === "error" && probe.connectionFailed, true, `status ${status}`); } - const unauthorized = await withFetch(async () => json(401, { Fault: {} }), () => probeQBInvoice(TOKENS, "1")); - assert.equal(unauthorized.state, "error"); - assert.equal(unauthorized.state === "error" && unauthorized.connectionFailed, undefined); + const badRequest = await withFetch(async () => json(400, { Fault: {} }), () => probeQBInvoice(TOKENS, "1")); + assert.equal(badRequest.state, "error"); + assert.equal(badRequest.state === "error" && badRequest.connectionFailed, undefined); +}); + +test("a raw fetch TypeError is normalized at the QBO boundary", async () => { + // Codex gate: getQBPayment translated 429/5xx but let a bare + // `TypeError: fetch failed` (DNS/TLS/reset) escape unclassified, so the + // loop did not recognise it as connection-level and kept dialling. + const { getQBPayment, isQboConnectionFailure } = await import("../src/lib/quickbooks"); + const error = await withFetch( + (async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch, + () => getQBPayment(TOKENS, "p1"), + ).then(() => null, (e: unknown) => e as Error); + + assert.ok(error instanceof Error, "a network failure must not resolve to null"); + assert.equal(isQboConnectionFailure(error), true, `not classified: ${error?.name}`); +}); + +test("a raw network TypeError on the invoice probe is connection-level too", async () => { + const probe = await withFetch( + (async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch, + () => probeQBInvoice(TOKENS, "1"), + ); + assert.equal(probe.state === "error" && probe.connectionFailed, true); }); test("a healthy invoice is unaffected by the new classification", async () => { @@ -173,20 +202,42 @@ test("a PAYMENT-DETAIL timeout aborts the run too, not just the probe", async () assert.equal(result.abortedOnQboOutage, true); }); -test("an ordinary per-row error does NOT stop the run", async () => { +test("an ordinary per-row probe failure is RECORDED and the run continues", async () => { + // Codex gate: this used to assert the failed row produced NO error, which + // meant a run could check nothing and still emit status "ok" - a green + // heartbeat for work that never happened. An unverified milestone is an + // incomplete run, so it must be recorded. const result = emptyResult(); const seen: string[] = []; const { client, calls } = fakeQbo({ probe: (id) => (id === "2" - ? { state: "error", status: 401 } + ? { state: "error", status: 400 } : { state: "ok", balance: 0, total: 10, paymentTxnIds: [] }), }); - await runQboRowLoop(rows(4), result, rowHandler(client, result), (row) => seen.push(row.id), "milestones"); + const handler = async (row: { id: string; qbInvoiceId: string }) => { + result.checked++; + const probe = await client.probeInvoice(row.qbInvoiceId); + if (probe.state === "error") { + if (probe.connectionFailed) throw new QboRetryableError("probe failed", probe.status); + throw new Error(`QBO invoice probe failed (status ${probe.status})`); + } + }; + await runQboRowLoop(rows(4), result, handler, (row) => seen.push(row.id), "milestones"); - assert.equal(calls.probes.length, 4); + assert.equal(calls.probes.length, 4, "a non-shared failure must not abort the run"); assert.equal(result.skipped, 0); assert.equal(result.abortedOnQboOutage, false); - assert.equal(seen.length, 0, "a transient probe error is not a row error"); + assert.deepEqual(seen, ["s1"], "the unverified row is recorded as an error"); +}); + +test("a 401 on any probe aborts the run - the credential is shared", async () => { + const result = emptyResult(); + const { client, calls } = fakeQbo({ probe: () => ({ state: "error", status: 401, connectionFailed: true }) }); + await runQboRowLoop(rows(120), result, rowHandler(client, result), () => {}, "milestones"); + + assert.equal(calls.probes.length, 1, "the next 119 rows would fail identically"); + assert.equal(result.skipped, 119); + assert.equal(result.abortedOnQboOutage, true); }); test("a settle failure is recorded per row and the run continues", async () => { From 299e24104a4913dbeab836fa87f7b1bd9955cb25 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:23:38 -0700 Subject: [PATCH 019/144] refactor(ops): digest cron uses main's postTextToWebhook (one Chat helper) Rebase onto main after Phase 4 (#439). Phase 4 landed postTextToWebhook(url, text) -> {sent, reason} in chat-webhook.ts, extracted from postDailyLogToChat for the Monday margin card. My branch had added a near-identical postTextToChatWebhook for the pipeline digest. Keeping both would have meant two helpers with the same SSRF allowlist and timeout drifting apart, so mine is gone and the digest calls main's. The richer return type is a small win: a skipped post now carries a reason instead of a bare false, so "no webhook configured" and "webhook responded 403" are distinguishable in the cron log. Behaviour is unchanged: same allowlist, same 10s post deadline, still never throws, and chatPosted still reflects a genuine delivery. Co-Authored-By: Claude Fable 5.1 --- src/app/api/cron/pipeline-digest/route.ts | 12 ++++++------ tests/pipeline-digest-route.test.ts | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts index 35291561f..2770051e3 100644 --- a/src/app/api/cron/pipeline-digest/route.ts +++ b/src/app/api/cron/pipeline-digest/route.ts @@ -1,7 +1,7 @@ import { NextResponse } from "next/server"; import { getPipelineHealth, formatPipelineDigest, type PipelineHealth } from "@/lib/pipeline-health"; import { sendNotification } from "@/lib/email"; -import { postTextToChatWebhook } from "@/lib/chat-webhook"; +import { postTextToWebhook } from "@/lib/chat-webhook"; import { isCronAuthorized } from "@/lib/cron-auth"; export const dynamic = "force-dynamic"; @@ -64,7 +64,7 @@ async function withDeadline(work: Promise, ms: number, onTimeout: T): Prom export interface PipelineDigestDependencies { getHealth: () => Promise; sendEmail: (to: string, subject: string, html: string) => Promise<{ success: boolean }>; - postChat: (webhookUrl: string, text: string) => Promise; + postChat: (webhookUrl: string, text: string) => Promise<{ sent: boolean; reason?: string }>; getChatWebhook: () => string | undefined; getRecipient: () => string; /** False when email cannot actually be delivered (see isEmailDeliveryConfigured). */ @@ -104,12 +104,12 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende ? withDeadline(dependencies.sendEmail(to, subject, html), deadlineMs, { success: false }) : Promise.resolve({ success: false }), chatWebhook - ? withDeadline(dependencies.postChat(chatWebhook, text), deadlineMs, false) - : Promise.resolve(false), + ? withDeadline(dependencies.postChat(chatWebhook, text), deadlineMs, { sent: false, reason: "timed out" }) + : Promise.resolve({ sent: false, reason: "no webhook configured" }), ]); const emailed = emailOutcome.status === "fulfilled" && emailOutcome.value.success === true; - const chatPosted = chatOutcome.status === "fulfilled" && chatOutcome.value === true; + const chatPosted = chatOutcome.status === "fulfilled" && chatOutcome.value.sent === true; console.log("[cron/pipeline-digest]", JSON.stringify({ ok: health.ok, @@ -140,7 +140,7 @@ const handlers = createPipelineDigestHandlers({ getHealth: getPipelineHealth, sendEmail: (to, subject, html) => sendNotification(to, subject, html, undefined, { fromName: "ProBuild" }), - postChat: postTextToChatWebhook, + postChat: postTextToWebhook, getChatWebhook: () => process.env.BOT_HEALTH_CHAT_WEBHOOK, getRecipient: () => process.env.PIPELINE_DIGEST_TO || DEFAULT_TO, }); diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts index 2a3ab2e80..25293edcc 100644 --- a/tests/pipeline-digest-route.test.ts +++ b/tests/pipeline-digest-route.test.ts @@ -35,7 +35,7 @@ function handlers(overrides: Partial = {}) { return createPipelineDigestHandlers({ getHealth: overrides.getHealth ?? (async () => HEALTH), sendEmail: overrides.sendEmail ?? (async () => ({ success: true })), - postChat: overrides.postChat ?? (async () => true), + postChat: overrides.postChat ?? (async () => ({ sent: true })), getChatWebhook: overrides.getChatWebhook ?? (() => undefined), getRecipient: overrides.getRecipient ?? (() => "ops@example.test"), isEmailConfigured: overrides.isEmailConfigured ?? (() => true), @@ -131,7 +131,7 @@ test("a failing email does not stop the Chat post from being attempted", async ( getChatWebhook: () => "https://chat.googleapis.com/v1/spaces/x", postChat: async () => { chatCalls += 1; - return true; + return { sent: true }; }, }); const response = await GET(cronRequest()); @@ -230,7 +230,7 @@ test("an unconfigured mailer still lets the Chat post through", async () => { const { GET } = handlers({ isEmailConfigured: () => false, getChatWebhook: () => "https://chat.googleapis.com/v1/spaces/x", - postChat: async () => true, + postChat: async () => ({ sent: true }), }); const response = await GET(cronRequest()); assert.equal(response.status, 500); From 407e1bed904816d1a98786b4756d561f3f2b7b3f Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:34:30 -0700 Subject: [PATCH 020/144] fix(qbo): never settle an unread payment; tokens before ok; latest-run status; clamp; real text part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 8. 1. getQBPayment returning null (401/403/408, malformed body) let both loops settle anyway with `new Date()`, stamping TODAY as the payment date — wrong money data reported as a clean run. A null read now leaves the row unsettled and records the reason. 401/403/408 join 429/5xx as shared failures (isSharedQboFailureStatus) that abort the run. 2. A Purchase that booked but never stored its receipt logged "created" and counted as a healthy, fresh booking. Terminal attachment outcomes (failed:*, skipped) now log status "attachment-failed": still terminal, no retry loop, but visible and excluded from receipt freshness. 3. The empty-run early return recorded "ok" BEFORE acquiring tokens, so a disconnected integration emitted a fresh successful heartbeat every hour forever. Tokens are acquired first; a run that cannot get them is "error". 4. Health now reads the latest cron event separately from the freshness timestamp: reason "payments-sync-error" fires whenever the LATEST event is an error, at any age. Freshness still only counts ok/partial. Closes the two-hour green gap between the 24h error count and the 26h staleness window. 5. normalizeTimeoutMs clamps to [1000, 55000]. A large finite value (QB_FETCH_TIMEOUT_MS=4294967296) passed the isFinite guard and made AbortSignal.timeout throw synchronously — one mistyped env var would have broken every QuickBooks call. 6. sendNotification takes an optional explicit `text` part; the digest passes its own plain text and keeps the
 for HTML clients. The derived text
   part collapses all whitespace, which flattened the report to one line.

Co-Authored-By: Claude Fable 5.1 
---
 src/app/api/cron/pipeline-digest/route.ts     |  14 +-
 .../integrations/qbo-receipts/create/route.ts |  28 ++-
 src/lib/email.ts                              | 219 ++++++++++--------
 src/lib/pipeline-health.ts                    |  70 ++++--
 src/lib/quickbooks-payments.ts                |  39 +++-
 src/lib/quickbooks.ts                         |  41 +++-
 tests/pipeline-health.test.ts                 |  66 ++++++
 tests/qb-timed-fetch.test.ts                  |  48 +++-
 tests/qbo-payments-outage.test.ts             |  41 ++++
 9 files changed, 404 insertions(+), 162 deletions(-)

diff --git a/src/app/api/cron/pipeline-digest/route.ts b/src/app/api/cron/pipeline-digest/route.ts
index 2770051e3..59224638c 100644
--- a/src/app/api/cron/pipeline-digest/route.ts
+++ b/src/app/api/cron/pipeline-digest/route.ts
@@ -63,7 +63,7 @@ async function withDeadline(work: Promise, ms: number, onTimeout: T): Prom
 
 export interface PipelineDigestDependencies {
     getHealth: () => Promise;
-    sendEmail: (to: string, subject: string, html: string) => Promise<{ success: boolean }>;
+    sendEmail: (to: string, subject: string, html: string, text: string) => Promise<{ success: boolean }>;
     postChat: (webhookUrl: string, text: string) => Promise<{ sent: boolean; reason?: string }>;
     getChatWebhook: () => string | undefined;
     getRecipient: () => string;
@@ -85,8 +85,10 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende
             const { subject, text } = formatPipelineDigest(health);
 
             const to = dependencies.getRecipient();
-            // sendNotification takes HTML and derives its own plain-text part;
-            // 
 keeps the line-per-item layout intact in an HTML client.
+            // 
 keeps the line-per-item layout intact for HTML clients,
+            // and the ORIGINAL text is passed as the explicit text part — the
+            // derived one collapses every newline, flattening this report into
+            // a single unreadable line for plain-text readers.
             const escaped = text.replace(/&/g, "&").replace(//g, ">");
             const html = `
${escaped}
`; @@ -101,7 +103,7 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende } const [emailOutcome, chatOutcome] = await Promise.allSettled([ emailConfigured - ? withDeadline(dependencies.sendEmail(to, subject, html), deadlineMs, { success: false }) + ? withDeadline(dependencies.sendEmail(to, subject, html, text), deadlineMs, { success: false }) : Promise.resolve({ success: false }), chatWebhook ? withDeadline(dependencies.postChat(chatWebhook, text), deadlineMs, { sent: false, reason: "timed out" }) @@ -138,8 +140,8 @@ export function createPipelineDigestHandlers(dependencies: PipelineDigestDepende const handlers = createPipelineDigestHandlers({ getHealth: getPipelineHealth, - sendEmail: (to, subject, html) => - sendNotification(to, subject, html, undefined, { fromName: "ProBuild" }), + sendEmail: (to, subject, html, text) => + sendNotification(to, subject, html, undefined, { fromName: "ProBuild", text }), postChat: postTextToWebhook, getChatWebhook: () => process.env.BOT_HEALTH_CHAT_WEBHOOK, getRecipient: () => process.env.PIPELINE_DIGEST_TO || DEFAULT_TO, diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 604d82019..d5537d210 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -122,9 +122,27 @@ export interface QboReceiptCreateHandlerDependencies { * is no trusted file id yet at that point — so they are correctly excluded * from this guarantee (see the early-return checks above, which log nothing). */ +/** + * A Purchase that posted but carries no receipt image. + * + * The upload can fail terminally (a QBO Fault, a hard 4xx) or be skipped + * (unsupported type, corrupt base64, oversized). Retrying those forever is + * pointless — but logging them as a plain "created"/"already-exists" was worse: + * the bot moved on, health counted a healthy booking, and the receipt was + * silently missing from the books with nothing to alert on. This status keeps + * the booking terminal (no retry loop) while making it visible and stopping it + * from refreshing receipt freshness. + */ +export const ATTACHMENT_FAILED_STATUS = "attachment-failed"; + +/** Did this outcome actually get the receipt image into QuickBooks? */ +export function attachmentSucceeded(attachment: string | undefined): boolean { + return attachment === "attached" || attachment === "already-attached"; +} + function pushEventFromOutcome( input: CreateQBReceiptPurchaseInput, - outcome: { status: "created" | "already-exists" | "fallback" | "error"; reason?: string }, + outcome: { status: "created" | "already-exists" | "fallback" | "error" | "attachment-failed"; reason?: string }, detail?: Record, ): AutomationEventInput { const taxCents = input.groups @@ -230,10 +248,16 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan // Apps Script treats ok:false as terminal, same convention as // sendToProBuild.txt. const result = await dependencies.createPurchase(tokens, input); + // A booking whose receipt never made it is reported as + // attachment-failed, not as a clean create — see + // ATTACHMENT_FAILED_STATUS. + const bookedWithoutReceipt = result.ok && !attachmentSucceeded(result.attachment); const event = pushEventFromOutcome( input, result.ok - ? { status: result.alreadyExists ? "already-exists" : "created" } + ? bookedWithoutReceipt + ? { status: ATTACHMENT_FAILED_STATUS, reason: result.attachment } + : { status: result.alreadyExists ? "already-exists" : "created" } : { status: "fallback", reason: result.reason }, { // The QBO deep link needs the purchase id (fileId is diff --git a/src/lib/email.ts b/src/lib/email.ts index 66257d280..b54483441 100644 --- a/src/lib/email.ts +++ b/src/lib/email.ts @@ -1,102 +1,117 @@ -import { Resend } from 'resend'; -import { prisma } from './prisma'; - -const resendApiKey = process.env.RESEND_API_KEY || 're_dummy_fallback'; -const resend = new Resend(resendApiKey); - -// Fallback internal-copy address for client-facing documents. Used only when the -// editable "System Notification Email" setting (Settings → Company) is unset. -// BCC'd so the client never sees the internal address. Override via env var. -export const CLIENT_DOC_COPY_EMAIL = (process.env.CLIENT_DOC_COPY_EMAIL || "notifications@goldentouchremodeling.com").trim(); - -export async function sendNotification( - toEmail: string, - subject: string, - htmlContent: string, - attachments?: { filename: string, content: Buffer }[], - options?: { fromName?: string; replyTo?: string; cc?: string[]; bcc?: string[]; copyToInternal?: boolean } -): Promise<{ success: boolean; id?: string }> { - // The "to" can be comma-separated (e.g. the System Notification Email setting - // holding several team addresses) — split into a proper recipient list. - const toList = toEmail ? toEmail.split(",").map(e => e.trim()).filter(Boolean) : []; - if (toList.length === 0) { - return { success: false }; - } - - if (resendApiKey === 're_dummy_fallback') { - if (process.env.NODE_ENV !== 'production') { - console.log("-----------------------------------------"); - console.log(`[MOCK EMAIL NOTIFICATION]`); - console.log(`To: ${toEmail}`); - console.log(`Subject: ${subject}`); - console.log(`Content: ${htmlContent.substring(0, 100)}...`); - if (attachments) { - console.log(`Attached ${attachments.length} files.`); - } - console.log("-----------------------------------------"); - } - return { success: true, id: "mock_resend_id_123" }; - } - - // Strip HTML tags for plain text version (improves deliverability) - const textContent = htmlContent - .replace(/]*>[\s\S]*?<\/style>/gi, '') - .replace(/<[^>]+>/g, ' ') - .replace(/\s+/g, ' ') - .trim(); - - const displayName = options?.fromName || 'Golden Touch Remodeling'; - - // Resolve BCC: any explicit bcc, plus an optional internal copy of client-facing - // docs. Deduped (case-insensitive) and never duplicating the To/CC recipients. - const bccByKey = new Map(); - for (const e of options?.bcc || []) { if (e?.trim()) bccByKey.set(e.trim().toLowerCase(), e.trim()); } - if (options?.copyToInternal) { - // Editable in Settings → Company ("System Notification Email"); falls back - // to CLIENT_DOC_COPY_EMAIL when that setting is unset. - let copyAddr = CLIENT_DOC_COPY_EMAIL; - try { - const s = await prisma.companySettings.findUnique({ where: { id: 'singleton' }, select: { notificationEmail: true } }); - if (s?.notificationEmail?.trim()) copyAddr = s.notificationEmail.trim(); - } catch { /* keep fallback */ } - for (const addr of copyAddr.split(",").map(e => e.trim()).filter(Boolean)) { - const key = addr.toLowerCase(); - const dup = toList.some(e => e.toLowerCase() === key) || (options?.cc || []).some(e => e.toLowerCase() === key); - if (!dup) bccByKey.set(key, addr); - } - } - const bccList = bccByKey.size > 0 ? [...bccByKey.values()] : undefined; - - try { - const data = await resend.emails.send({ - from: `${displayName} `, - to: toList, - replyTo: options?.replyTo || 'jadkins@goldentouchremodeling.com', - subject: subject, - html: htmlContent, - text: textContent, - attachments: attachments, - cc: options?.cc, - bcc: bccList - }); - if (data.error) { - console.error("Resend API returned error:", data.error); - return { success: false }; - } - return { success: true, id: data.data?.id }; - } catch (error) { - console.error("Failed to send Resend email:", error); - return { success: false }; - } -} -export async function checkEmailStatus(emailId: string): Promise { - if (!emailId) return null; - if (resendApiKey === 're_dummy_fallback') return "delivered"; - try { - const result = await resend.emails.get(emailId); - return result.data?.last_event || null; - } catch (error) { - console.error("Failed to check email status:", error); - return null; - } -} +import { Resend } from 'resend'; +import { prisma } from './prisma'; + +const resendApiKey = process.env.RESEND_API_KEY || 're_dummy_fallback'; +const resend = new Resend(resendApiKey); + +// Fallback internal-copy address for client-facing documents. Used only when the +// editable "System Notification Email" setting (Settings → Company) is unset. +// BCC'd so the client never sees the internal address. Override via env var. +export const CLIENT_DOC_COPY_EMAIL = (process.env.CLIENT_DOC_COPY_EMAIL || "notifications@goldentouchremodeling.com").trim(); + +export async function sendNotification( + toEmail: string, + subject: string, + htmlContent: string, + attachments?: { filename: string, content: Buffer }[], + options?: { + fromName?: string; + replyTo?: string; + cc?: string[]; + bcc?: string[]; + copyToInternal?: boolean; + /** + * Explicit plain-text part. Without it the text body is derived by + * stripping tags and collapsing ALL whitespace, which turns a + * line-per-item report into one unreadable paragraph — fine for prose, + * useless for anything whose layout carries meaning. Pass the original + * text when you have it. + */ + text?: string; + } +): Promise<{ success: boolean; id?: string }> { + // The "to" can be comma-separated (e.g. the System Notification Email setting + // holding several team addresses) — split into a proper recipient list. + const toList = toEmail ? toEmail.split(",").map(e => e.trim()).filter(Boolean) : []; + if (toList.length === 0) { + return { success: false }; + } + + if (resendApiKey === 're_dummy_fallback') { + if (process.env.NODE_ENV !== 'production') { + console.log("-----------------------------------------"); + console.log(`[MOCK EMAIL NOTIFICATION]`); + console.log(`To: ${toEmail}`); + console.log(`Subject: ${subject}`); + console.log(`Content: ${htmlContent.substring(0, 100)}...`); + if (attachments) { + console.log(`Attached ${attachments.length} files.`); + } + console.log("-----------------------------------------"); + } + return { success: true, id: "mock_resend_id_123" }; + } + + // Strip HTML tags for plain text version (improves deliverability). + // A caller-supplied `text` always wins — see the option's doc comment. + const textContent = options?.text ?? htmlContent + .replace(/]*>[\s\S]*?<\/style>/gi, '') + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + const displayName = options?.fromName || 'Golden Touch Remodeling'; + + // Resolve BCC: any explicit bcc, plus an optional internal copy of client-facing + // docs. Deduped (case-insensitive) and never duplicating the To/CC recipients. + const bccByKey = new Map(); + for (const e of options?.bcc || []) { if (e?.trim()) bccByKey.set(e.trim().toLowerCase(), e.trim()); } + if (options?.copyToInternal) { + // Editable in Settings → Company ("System Notification Email"); falls back + // to CLIENT_DOC_COPY_EMAIL when that setting is unset. + let copyAddr = CLIENT_DOC_COPY_EMAIL; + try { + const s = await prisma.companySettings.findUnique({ where: { id: 'singleton' }, select: { notificationEmail: true } }); + if (s?.notificationEmail?.trim()) copyAddr = s.notificationEmail.trim(); + } catch { /* keep fallback */ } + for (const addr of copyAddr.split(",").map(e => e.trim()).filter(Boolean)) { + const key = addr.toLowerCase(); + const dup = toList.some(e => e.toLowerCase() === key) || (options?.cc || []).some(e => e.toLowerCase() === key); + if (!dup) bccByKey.set(key, addr); + } + } + const bccList = bccByKey.size > 0 ? [...bccByKey.values()] : undefined; + + try { + const data = await resend.emails.send({ + from: `${displayName} `, + to: toList, + replyTo: options?.replyTo || 'jadkins@goldentouchremodeling.com', + subject: subject, + html: htmlContent, + text: textContent, + attachments: attachments, + cc: options?.cc, + bcc: bccList + }); + if (data.error) { + console.error("Resend API returned error:", data.error); + return { success: false }; + } + return { success: true, id: data.data?.id }; + } catch (error) { + console.error("Failed to send Resend email:", error); + return { success: false }; + } +} +export async function checkEmailStatus(emailId: string): Promise { + if (!emailId) return null; + if (resendApiKey === 're_dummy_fallback') return "delivered"; + try { + const result = await resend.emails.get(emailId); + return result.data?.last_event || null; + } catch (error) { + console.error("Failed to check email status:", error); + return null; + } +} diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 5825702bc..641d6b25a 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -148,6 +148,13 @@ export const PAYMENTS_SYNC_CRON_SOURCE = "cron"; * it counts for freshness — and is then flagged separately, immediately. */ export const PAYMENTS_SYNC_HEARTBEAT_STATUSES = ["ok", "partial"]; +/** + * A receipt-push event that booked a Purchase but never stored its receipt. + * Terminal by design (retrying a rejected file is a loop), so it must not count + * as a healthy booking — otherwise a run of them reads as a perfectly fresh + * receipt pipeline while every receipt is missing its image. + */ +export const ATTACHMENT_FAILED_STATUS = "attachment-failed"; /** The cron runs hourly; 26h leaves room for a couple of missed runs and DST. */ export const PAYMENTS_SYNC_STALE_HOURS = 26; @@ -206,11 +213,16 @@ export function evaluatePipelineHealth(input: { const at = input.lastPaymentsSync.at ? Date.parse(input.lastPaymentsSync.at) : null; const stale = at === null || Number.isNaN(at) || input.now - at > PAYMENTS_SYNC_STALE_HOURS * HOUR_MS; if (stale) reasons.push("payments-sync-stale"); + // `runStatus` is the status of the LATEST cron event, whatever it was — + // deliberately not the status of the run that set the freshness + // timestamp above. An error immediately after a good run used to be + // invisible here, leaving health green on the strength of the older + // success until the 24h error count and the 26h staleness window + // disagreed (a two-hour green gap) or forever if the cron then stopped. + else if (input.lastPaymentsSync.runStatus === "error") reasons.push("payments-sync-error"); // A run that skipped rows or hit errors did NOT verify those payments. // It proves the cron is alive, so it counts for freshness — but it must - // be reported the same day, not hidden inside the 26h staleness window - // (repeated partial runs could otherwise stay green until tomorrow's - // digest, or indefinitely). + // be reported the same day, not hidden inside the 26h staleness window. else if (input.lastPaymentsSync.runStatus === "partial") reasons.push("payments-sync-partial"); } @@ -310,22 +322,32 @@ export async function getPipelineHealth(): Promise { )?.createdAt ?? null, null, ), - probe<{ createdAt: Date; status: string } | null>( + probe<{ createdAt: Date | null; latestStatus: string | null }>( "lastPaymentsSync", - async () => - (await prisma.automationEvent.findFirst({ - // "partial" counts for freshness (the cron ran) and is then - // reported on its own; only a hard "error" is excluded, - // since `stuck` already surfaces those. - where: { - kind: PAYMENTS_SYNC_EVENT_KIND, - status: { in: PAYMENTS_SYNC_HEARTBEAT_STATUSES }, - source: PAYMENTS_SYNC_CRON_SOURCE, - }, - orderBy: { createdAt: "desc" }, - select: { createdAt: true, status: true }, - })) ?? null, - null, + async () => { + // TWO reads, deliberately. Freshness may only come from a run + // that actually ran (ok/partial), but the reason must reflect + // the LATEST event whatever it was — otherwise an error right + // after a success is invisible. + const [fresh, latest] = await Promise.all([ + prisma.automationEvent.findFirst({ + where: { + kind: PAYMENTS_SYNC_EVENT_KIND, + status: { in: PAYMENTS_SYNC_HEARTBEAT_STATUSES }, + source: PAYMENTS_SYNC_CRON_SOURCE, + }, + orderBy: { createdAt: "desc" }, + select: { createdAt: true }, + }), + prisma.automationEvent.findFirst({ + where: { kind: PAYMENTS_SYNC_EVENT_KIND, source: PAYMENTS_SYNC_CRON_SOURCE }, + orderBy: { createdAt: "desc" }, + select: { status: true }, + }), + ]); + return { createdAt: fresh?.createdAt ?? null, latestStatus: latest?.status ?? null }; + }, + { createdAt: null, latestStatus: null }, ), probe>( "receipts24h", @@ -371,8 +393,8 @@ export async function getPipelineHealth(): Promise { lastPaymentsSync: { status: lastPaymentsSync.status, reason: lastPaymentsSync.reason, - at: lastPaymentsSync.value?.createdAt.toISOString() ?? null, - runStatus: lastPaymentsSync.value?.status ?? null, + at: lastPaymentsSync.value.createdAt?.toISOString() ?? null, + runStatus: lastPaymentsSync.value.latestStatus, }, receipts24h: { status: receiptRows.status, reason: receiptRows.reason, counts }, bank: { @@ -431,7 +453,13 @@ export function formatPipelineDigest(health: PipelineHealth): { subject: string; `Intuit status: ${health.intuit.indicator}${health.intuit.description ? ` (${health.intuit.description})` : ""}${health.intuit.status === "error" ? " [status page unreachable]" : ""}`, `Last QBO purchase sync: ${ago(health.qbo.lastPurchaseSync, now)}`, `Last receipt booked: ${ago(health.qbo.lastReceiptPush, now)}`, - `Last payments sync: ${ago(health.qbo.lastPaymentsSync, now)}${health.qbo.lastPaymentsSync.runStatus === "partial" ? " [incomplete run]" : ""}`, + `Last payments sync: ${ago(health.qbo.lastPaymentsSync, now)}${ + health.qbo.lastPaymentsSync.runStatus === "partial" + ? " [incomplete run]" + : health.qbo.lastPaymentsSync.runStatus === "error" + ? " [last run FAILED]" + : "" + }`, `Receipts (24h): ${receiptsLine}`, `Bank ledger through: ${ health.bank.status === "error" diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 5f91f7aef..d67ef398e 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -833,11 +833,10 @@ export async function syncQuickBooksPayments( take: 100, }); - if (pending.length === 0 && pendingBillings.length === 0) { - await recordPaymentsSyncEvent(result, options?.source); - return result; - } - + // Tokens FIRST, even with nothing to do. Recording "ok" before proving we + // can talk to QuickBooks let a disconnected or expired integration emit a + // fresh successful heartbeat every hour, forever — the health check would + // read a perfectly alive money rail that could not have synced anything. let tokens: QBTokens; try { tokens = await getFreshQBTokens(); @@ -862,6 +861,12 @@ export async function syncQuickBooksPayments( getPayment: (paymentId) => getQBPayment(tokens, paymentId), }; + // Nothing to sync, but the credentials were just proven — a genuine "ok". + if (pending.length === 0 && pendingBillings.length === 0) { + await recordPaymentsSyncEvent(result, options?.source); + return result; + } + // Milestones whose linked QBO invoice was found voided/deleted THIS run (flag was // previously null). Reported once per breakage; a re-push clears the flag and re-arms. const newlyFlagged: QBSyncIssue[] = []; @@ -930,12 +935,19 @@ export async function syncQuickBooksPayments( let paidAt = new Date(); let referenceNumber: string | null = null; if (paymentId) { - // Same abort rule as the probe: a timeout/429/5xx here - // throws and stops the run rather than costing another + // Same abort rule as the probe: a timeout/401/403/408/429/5xx + // here throws and stops the run rather than costing another // full deadline on every remaining settled invoice. const p = await qbo.getPayment(paymentId); - if (p?.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); - referenceNumber = p?.referenceNumber || null; + // A null read means we could not learn WHEN this was paid. + // Settling anyway used to stamp `new Date()` on it, quietly + // recording today as the payment date — wrong money data, + // reported as a clean run. Leave it Pending and retry. + if (!p) { + throw new Error(`QBO payment ${paymentId} could not be read; milestone left unsettled`); + } + if (p.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); + referenceNumber = p.referenceNumber || null; } const recorded = await settleMilestoneFromQBPayment({ paymentScheduleId: schedule.id, @@ -987,8 +999,13 @@ export async function syncQuickBooksPayments( let referenceNumber: string | null = null; if (paymentId) { const p = await qbo.getPayment(paymentId); - if (p?.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); - referenceNumber = p?.referenceNumber || null; + // Same rule as the milestone loop: never settle a payment + // whose date we could not read. + if (!p) { + throw new Error(`QBO payment ${paymentId} could not be read; progress billing left unsettled`); + } + if (p.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); + referenceNumber = p.referenceNumber || null; } const settled = await settleProgressBillingPaidCore(billing.id, { paidAt, referenceNumber, qbPaymentId: paymentId }); if (settled) result.progressBillingsSettled++; diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index e3c9a25d8..308b0a602 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -70,6 +70,20 @@ export function isRetryableQboStatus(status: number): boolean { return status === 429 || status >= 500; } +/** + * A status that will repeat identically on the NEXT record. + * + * 429/5xx: QBO cannot serve requests. 401/403: the credential is bad, and it is + * the same credential for every remaining row. 408: the request timed out at + * QBO's edge, same as our own deadline firing. A caller looping over records + * must stop on all of these — working through the rest produces identical + * failures at full cost, which is how a handful of 20s waits consumed the + * payments cron's entire 120s ceiling. + */ +export function isSharedQboFailureStatus(status: number): boolean { + return status === 401 || status === 403 || status === 408 || isRetryableQboStatus(status); +} + /** * Did QBO fail in a way that means the NEXT call will fail the same way? * @@ -99,13 +113,23 @@ function safePath(url: string): string { } } -/** Clamp a configured timeout to a positive integer of milliseconds. */ +/** Below this a timeout is a self-inflicted outage; above it Vercel kills us first. */ +const QB_MIN_TIMEOUT_MS = 1_000; +const QB_MAX_TIMEOUT_MS = 55_000; + +/** Clamp a configured timeout into a range AbortSignal.timeout can actually take. */ function normalizeTimeoutMs(value: number, fallback: number): number { if (!Number.isFinite(value)) return fallback; // AbortSignal.timeout takes an unsigned integer; a fraction or a // sub-millisecond value would either be coerced or reject outright. const whole = Math.floor(value); - return whole >= 1 ? whole : fallback; + if (whole < 1) return fallback; + // A large-but-finite value (QB_FETCH_TIMEOUT_MS=4294967296) exceeds the + // unsigned-long-long bound and makes AbortSignal.timeout throw + // SYNCHRONOUSLY — which would break every QuickBooks call in the app over a + // single mistyped env var. Clamp instead: a too-long deadline is useless + // anyway, since the route ceiling kills the function first. + return Math.min(Math.max(whole, QB_MIN_TIMEOUT_MS), QB_MAX_TIMEOUT_MS); } interface SignalRace { @@ -771,11 +795,7 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { return { state: "notFound" }; } - // 429/5xx: QBO cannot serve requests right now. 401/403: the credential - // itself is bad, and it is the SAME credential for every remaining row, so - // working through them would produce identical failures at full cost. - // All of these are connection-level. - if (res.status === 429 || res.status >= 500 || res.status === 401 || res.status === 403) { + if (isSharedQboFailureStatus(res.status)) { return { state: "error", status: res.status, connectionFailed: true }; } return { state: "error", status: res.status }; @@ -788,9 +808,10 @@ export async function getQBPayment( ): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null> { const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); if (!res.ok) { - // 429/5xx is QBO refusing to serve, not "this payment does not exist" — - // raise so a looping caller aborts instead of burning a deadline per row. - if (isRetryableQboStatus(res.status)) { + // Not "this payment does not exist" — QBO is refusing, or the shared + // credential is bad. Raise so a looping caller aborts instead of + // burning a deadline per row (and never settles off a missing read). + if (isSharedQboFailureStatus(res.status)) { throw new QboRetryableError(`QB payment read failed with status ${res.status}`, res.status); } return null; diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 30ea1af20..265b8bd1f 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -431,3 +431,69 @@ test("the digest flags an incomplete payments run in the body", () => { assert.match(text, /Last payments sync: .*\[incomplete run\]/); assert.match(text, /Needs attention: payments-sync-partial/); }); + + +// --- The latest failed run is never hidden --- + +test("payments-sync-error fires when the LATEST event is an error, at any age", () => { + // Codex gate: the heartbeat query excluded "error", so an error right after + // a good run was invisible; health then leaned on a 24h error count while + // staleness used 26h, leaving a two-hour green window (or forever, if the + // cron then stopped entirely). + const justNow = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(1 * HOUR), runStatus: "error" }, + })); + assert.equal(justNow.ok, false); + assert.deepEqual(justNow.reasons, ["payments-sync-error"]); + + // Older than the 24h `stuck` window, still red. + const old = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(25 * HOUR), runStatus: "error" }, + })); + assert.equal(old.ok, false); + assert.deepEqual(old.reasons, ["payments-sync-error"]); +}); + +test("staleness still outranks the latest-run status", () => { + const v = evaluatePipelineHealth(snapshot({ + lastPaymentsSync: { status: "ok", at: iso(40 * HOUR), runStatus: "error" }, + })); + assert.deepEqual(v.reasons, ["payments-sync-stale"]); +}); + +test("the digest marks a failed last payments run", () => { + const { text } = formatPipelineDigest(sampleHealth({ + ok: false, + reasons: ["payments-sync-error"], + qbo: { + lastPurchaseSync: { status: "ok", at: "2026-09-01T10:00:00.000Z" }, + lastReceiptPush: { status: "ok", at: "2026-09-01T12:00:00.000Z" }, + lastPaymentsSync: { status: "ok", at: "2026-09-01T13:00:00.000Z", runStatus: "error" }, + }, + })); + assert.match(text, /Last payments sync: .*\[last run FAILED\]/); +}); + +// --- A booking with no receipt is not a fresh receipt --- + +test("attachment-failed is not a booking for freshness purposes", async () => { + const { ATTACHMENT_FAILED_STATUS, BOOKED_PUSH_STATUSES } = await import("../src/lib/pipeline-health"); + const route = await import("../src/app/api/integrations/qbo-receipts/create/route"); + + // The route and the health check must agree on the marker, or the signal + // is written somewhere nothing reads. + assert.equal(route.ATTACHMENT_FAILED_STATUS, ATTACHMENT_FAILED_STATUS); + assert.equal(ATTACHMENT_FAILED_STATUS, "attachment-failed"); + // Codex gate: a Purchase booked with no receipt used to log "created" and + // count as a perfectly healthy, fresh booking. + assert.equal(BOOKED_PUSH_STATUSES.includes(ATTACHMENT_FAILED_STATUS), false); +}); + +test("only a stored receipt counts as an attached booking", async () => { + const { attachmentSucceeded } = await import("../src/app/api/integrations/qbo-receipts/create/route"); + assert.equal(attachmentSucceeded("attached"), true); + assert.equal(attachmentSucceeded("already-attached"), true); + for (const bad of ["skipped", "failed:400", "failed:fault", undefined]) { + assert.equal(attachmentSucceeded(bad), false, String(bad)); + } +}); diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index c5f4cf6f8..349936576 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -65,13 +65,13 @@ test("times out into QBTimeoutError when the server never responds", async () => }); test("timeout message carries the path but never the query string", async () => { - const error = await qbTimedFetch(`${base}/v3/company/hang?query=select%20*&token=shhh`, {}, 100) + const error = await qbTimedFetch(`${base}/v3/company/hang?query=select%20*&token=shhh`, {}, 1_000) .then(() => null, (e: unknown) => e as Error); assert.ok(error instanceof QBTimeoutError); assert.match(error.message, /\/v3\/company\/hang/); assert.doesNotMatch(error.message, /shhh/); assert.doesNotMatch(error.message, /select/); - assert.match(error.message, /100ms/); + assert.match(error.message, /1000ms/); }); test("a successful response passes through untouched", async () => { @@ -173,7 +173,7 @@ test("the deadline wins the race even if the caller aborts a moment later", asyn // outage is misreported as a caller cancellation and the receipt route // answers 500 instead of 503. const controller = new AbortController(); - const pending = qbTimedFetch(`${base}/v3/company/hang`, { signal: controller.signal }, 60).then( + const pending = qbTimedFetch(`${base}/v3/company/hang`, { signal: controller.signal }, 1_000).then( () => null, (e: unknown) => e as Error, ); @@ -181,7 +181,8 @@ test("the deadline wins the race even if the caller aborts a moment later", asyn // SYNCHRONOUSLY the moment the real deadline has fired. Registered after // the call, so the wrapper's own timeout signal is created — and fires — // first. Both signals end up aborted; only the latched winner disambiguates. - AbortSignal.timeout(60).addEventListener("abort", () => controller.abort(), { once: true }); + // (1s, not 60ms: deadlines are clamped to a 1s floor.) + AbortSignal.timeout(1_000).addEventListener("abort", () => controller.abort(), { once: true }); const error = await pending; // Let the sibling handler land regardless of timer/microtask interleaving, @@ -259,14 +260,14 @@ test("an already-aborted caller signal is honoured immediately without AbortSign test("QB_FETCH_TIMEOUT_MS drives the default deadline", async () => { const previous = process.env.QB_FETCH_TIMEOUT_MS; - process.env.QB_FETCH_TIMEOUT_MS = "120"; + process.env.QB_FETCH_TIMEOUT_MS = "1200"; try { const error = await qbTimedFetch(`${base}/v3/company/hang`).then( () => null, (e: unknown) => e as Error, ); assert.ok(error instanceof QBTimeoutError); - assert.match(error.message, /120ms/); + assert.match(error.message, /1200ms/); } finally { if (previous === undefined) delete process.env.QB_FETCH_TIMEOUT_MS; else process.env.QB_FETCH_TIMEOUT_MS = previous; @@ -285,20 +286,47 @@ test("a garbage QB_FETCH_TIMEOUT_MS falls back to the 20s default rather than th } }); -test("fractional and sub-1ms timeouts never reach AbortSignal.timeout", async () => { +test("a bad timeout value never reaches AbortSignal.timeout", async () => { // A fraction would be coerced and a 0/negative value rejected outright, // which would break EVERY QB call rather than one misconfigured setting. for (const value of [0.5, 0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { const res = await qbTimedFetch(`${base}/v3/company/query`, {}, value); assert.equal(res.status, 200, `timeout ${value} should fall back to the default`); } - // A valid fraction floors instead of falling back: 150.9 -> 150. - const error = await qbTimedFetch(`${base}/v3/company/hang`, {}, 150.9).then( + // A valid fraction floors instead of falling back: 1500.9 -> 1500. + const error = await qbTimedFetch(`${base}/v3/company/hang`, {}, 1500.9).then( () => null, (e: unknown) => e as Error, ); assert.ok(error instanceof QBTimeoutError); - assert.match(error.message, /150ms/); + assert.match(error.message, /1500ms/); +}); + +test("a huge but FINITE timeout is clamped instead of throwing", async () => { + // Codex gate: AbortSignal.timeout takes an unsigned long long, so a value + // past that bound threw synchronously — one mistyped env var would have + // broken every QuickBooks call in the app. 2**32 is finite, so the + // Number.isFinite guard above did not catch it. + for (const value of [4294967296, 2 ** 53, 60_000]) { + // A responsive endpoint: the clamped deadline never fires, so what this + // asserts is that the call was MADE at all rather than throwing + // synchronously out of AbortSignal.timeout. + const res = await qbTimedFetch(`${base}/v3/company/query`, {}, value); + assert.equal(res.status, 200, `timeout ${value} must not throw`); + } +}); + +test("a sub-second timeout is raised to the 1s floor", async () => { + const started = Date.now(); + const error = await qbTimedFetch(`${base}/v3/company/hang`, {}, 5).then( + () => null, + (e: unknown) => e as Error, + ); + assert.ok(error instanceof QBTimeoutError); + assert.match(error.message, /1000ms/); + // A 5ms deadline would make QuickBooks permanently "down" — the floor is + // what stops a misconfiguration becoming a self-inflicted outage. + assert.ok(Date.now() - started >= 900, "must actually wait the floor"); }); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index ae3408223..85206aa24 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -395,3 +395,44 @@ test("a token-persistence failure marks the run failed with its own reason", asy { reason: "token-not-persisted", abortedOnQboOutage: false }, ); }); + + +// --- Never settle a payment we could not read --- + +test("a null payment read leaves the milestone unsettled and records an error", async () => { + const result = emptyResult(); + const errors: string[] = []; + let settled = 0; + const { client } = fakeQbo({ + probe: () => ({ state: "ok", balance: 0, total: 100, paymentTxnIds: ["p1"] }), + payment: () => null, // 400/404/malformed body + }); + // Mirrors the real row body: a null read must throw, never settle. + const handler = async (row: { id: string; qbInvoiceId: string }) => { + result.checked++; + const probe = await client.probeInvoice(row.qbInvoiceId); + if (probe.state !== "ok") return; + if (probe.total > 0 && probe.balance <= 0) { + const p = await client.getPayment(probe.paymentTxnIds[0]); + if (!p) throw new Error("QBO payment p1 could not be read; milestone left unsettled"); + settled++; + } + }; + await runQboRowLoop(rows(3), result, handler, (row) => errors.push(row.id), "milestones"); + + // Codex gate: this used to fall back to `new Date()` and settle anyway, + // stamping today as the payment date - wrong money data, reported clean. + assert.equal(settled, 0); + assert.deepEqual(errors, ["s0", "s1", "s2"]); + assert.equal(result.abortedOnQboOutage, false, "a bad row is not an outage"); +}); + +test("401/403/408 payment reads abort the run; a plain 404 does not", async () => { + const { isSharedQboFailureStatus } = await import("../src/lib/quickbooks"); + for (const status of [401, 403, 408, 429, 500, 503]) { + assert.equal(isSharedQboFailureStatus(status), true, String(status)); + } + for (const status of [400, 404, 409, 422]) { + assert.equal(isSharedQboFailureStatus(status), false, String(status)); + } +}); From 43f37ab1ff5516161bf0bd315f8bd5f2748e49da Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 23:02:04 -0700 Subject: [PATCH 021/144] fix(qbo): verify empty runs, refuse fabricated paidAt, paginate, alert on lost receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 9. 1. An empty run recorded "ok" on the strength of holding tokens. But a non-timeout refresh failure falls back to the STALE pair, and stale credentials, a wrong realm, or revoked accounting access all still produce a token object — so a dead rail emitted a fresh green heartbeat every hour forever. Empty runs now make one cheap authenticated read (CompanyInfo); a failure is classified and recorded as "error". 2. paidAt defaulted to `new Date()` and was only replaced when txnDate happened to be truthy, so an invoice with no linked payment, a null or unparseable TxnDate, or an unreadable payment record settled REAL milestones stamped with today — and fired the mirror/notification side effects on the way out. resolveSettlementDate refuses all four cases with reason "payment-date-missing"; the row stays Pending for a later run. 3. Both queries took an unordered first 100 and stopped: rows past the cap were neither checked nor counted as skipped, so the run reported "ok" while work was left undone — and with no ORDER BY, Postgres could return the same page every hour and starve the rest indefinitely. Both collections now walk in stable id order, paged by cursor, under a row cap and a 90s budget; whatever is not reached is counted as skipped, making the run partial. 4. attachment-failed was terminal but non-alerting: `stuck` matched only literal "error", so one other good receipt inside 72h left the digest reading "Pipeline OK" while a Purchase sat in QuickBooks with no receipt. It now counts toward `stuck`, the journey mapper renders it as failed rather than in-flight, the intake graph buckets it as an error, and it counts against the hands-free rate instead of as a clean push. Co-Authored-By: Claude Fable 5.1 --- src/lib/automation-events.ts | 21 ++- src/lib/pipeline-health.ts | 14 +- src/lib/quickbooks-payments.ts | 247 ++++++++++++++++++++++-------- tests/pipeline-health.test.ts | 40 +++++ tests/qbo-payments-outage.test.ts | 114 +++++++++++++- 5 files changed, 366 insertions(+), 70 deletions(-) diff --git a/src/lib/automation-events.ts b/src/lib/automation-events.ts index 1be92bc43..c8b76d581 100644 --- a/src/lib/automation-events.ts +++ b/src/lib/automation-events.ts @@ -128,7 +128,9 @@ export async function receiptDailyBuckets(days: number): Promise a.day.localeCompare(b.day)); } @@ -170,12 +172,14 @@ export async function automationSummary(): Promise { pushed += 1; amountCents += e.amountCents ?? 0; taxCents += e.taxCents ?? 0; - } else if (e.status === "fallback") { + } else if (e.status === "fallback" || e.status === "attachment-failed") { + // Booked without a receipt still needs a human to finish it, so it + // counts against the hands-free rate rather than as a clean push. fallback += 1; } } const created30 = last30.filter(e => e.status === "created").length; - const fb30 = last30.filter(e => e.status === "fallback").length; + const fb30 = last30.filter(e => e.status === "fallback" || e.status === "attachment-failed").length; return { pushedThisMonth: pushed, @@ -297,12 +301,23 @@ function journeyFinalState(steps: JourneyStep[]): { state: ReceiptJourney["final if (s.stage === "push" && (s.status === "created" || s.status === "already-exists")) { return { state: "booked-api", reason: null }; } + // Booked, but the receipt image never landed — surfaced as an error so + // it appears in the worklist instead of reading as a clean booking. + if (s.stage === "push" && s.status === "attachment-failed") { + return { state: "error", reason: s.reason ?? "attachment failed" }; + } if (s.stage === "email-book" && s.status === "emailed") { return { state: "booked-email", reason: s.reason }; } } const last = steps[steps.length - 1]; if (!last) return { state: "in-flight", reason: null }; + // A Purchase that posted without its receipt image. Terminal (the bot will + // not resend), so leaving it "in-flight" hid a receipt that is never + // arriving behind a status that reads like "still working on it". + if (last.status === "attachment-failed") { + return { state: "error", reason: last.reason ?? "attachment failed" }; + } if (last.status === "parked") return { state: "parked", reason: last.reason }; if (last.status === "quarantined") return { state: "quarantined", reason: last.reason }; if (last.status === "error") return { state: "error", reason: last.reason }; diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 641d6b25a..813ac40db 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -203,6 +203,8 @@ export function evaluatePipelineHealth(input: { } if (input.stuck.status === "ok" && input.stuck.count > 0) { + // Includes attachment-failed: a receipt that never reached QuickBooks + // is a failure someone has to act on, not a footnote. reasons.push(`errors-24h:${input.stuck.count}`); } @@ -370,7 +372,17 @@ export async function getPipelineHealth(): Promise { // to surface, even on a day with no receipt traffic at all. probe( "stuck", - () => prisma.automationEvent.count({ where: { status: "error", createdAt: { gte: since24h } } }), + () => prisma.automationEvent.count({ + where: { + // attachment-failed is a TERMINAL failure that leaves a + // booked Purchase with no receipt. It is not literally + // "error", so it used to sail past this count and the digest + // could still read "Pipeline OK" on the strength of one + // other good receipt in the window. + status: { in: ["error", ATTACHMENT_FAILED_STATUS] }, + createdAt: { gte: since24h }, + }, + }), 0, ), ]); diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index d67ef398e..769d34594 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -21,6 +21,7 @@ import { type QBTokens, refreshQBToken, isQBTimeoutError, + qbQuery, isQboConnectionFailure, QboRetryableError, type QBInvoiceProbe, @@ -777,6 +778,11 @@ export async function runQboRowLoop( export interface PaymentsSyncQboClient { probeInvoice(qbInvoiceId: string): Promise; getPayment(paymentId: string): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null>; + /** + * One cheap authenticated read, used only to prove the connection works on + * a run with no rows to sync. Throws if QuickBooks is not actually usable. + */ + verifyConnection(): Promise; } export interface SyncQuickBooksPaymentsOptions { @@ -790,6 +796,57 @@ export interface SyncQuickBooksPaymentsOptions { qboClient?: PaymentsSyncQboClient; } +/** One database page. Small enough to stay responsive, big enough to be cheap. */ +const PAYMENTS_SYNC_PAGE_SIZE = 100; +/** Hard ceiling on rows per run, so one huge backlog cannot run past the cron's window. */ +const PAYMENTS_SYNC_MAX_ROWS = 500; +/** Wall-clock budget, comfortably inside the route's 120s maxDuration. */ +const PAYMENTS_SYNC_TIME_BUDGET_MS = 90_000; + +/** + * Walk a pending collection in stable id order, a page at a time. + * + * The old queries took an unordered first 100 and stopped: rows past the cap + * were neither checked nor counted as skipped, so the run reported a clean + * "ok" while work was silently left undone — and with no ORDER BY, Postgres was + * free to hand back the SAME first 100 every hour, starving later rows forever. + * Ordering by id makes the walk deterministic, and anything we do not reach is + * counted as skipped so the run is honestly reported as partial. + */ +async function forEachPendingPage( + result: QBPaymentSyncResult, + deadline: number, + fetchPage: (cursorId: string | null, take: number) => Promise, + countRemaining: (cursorId: string | null) => Promise, + handlePage: (rows: T[]) => Promise, +): Promise { + let cursorId: string | null = null; + let processed = 0; + + while (true) { + if (result.abortedOnQboOutage) break; + if (processed >= PAYMENTS_SYNC_MAX_ROWS) break; + if (Date.now() >= deadline) break; + + const take = Math.min(PAYMENTS_SYNC_PAGE_SIZE, PAYMENTS_SYNC_MAX_ROWS - processed); + const page = await fetchPage(cursorId, take); + if (page.length === 0) return; // exhausted — nothing was missed + + await handlePage(page); + processed += page.length; + cursorId = page[page.length - 1].id; + + // A short page means we reached the end of the collection. + if (page.length < take) return; + } + + // Stopped early. Count what is genuinely left AFTER the cursor rather than + // subtracting from a stale total — rows settled during this run have + // already dropped out of the pending set. + const remaining = await countRemaining(cursorId).catch(() => 0); + if (remaining > 0) result.skipped += remaining; +} + export async function syncQuickBooksPayments( scope?: { invoiceId?: string; projectId?: string }, options?: SyncQuickBooksPaymentsOptions, @@ -799,39 +856,26 @@ export async function syncQuickBooksPayments( skipped: 0, abortedOnQboOutage: false, runFailed: false, }; - const pending = await prisma.paymentSchedule.findMany({ - where: { - status: "Pending", - qbInvoiceId: { not: null }, - ...(scope?.invoiceId ? { invoiceId: scope.invoiceId } : {}), - ...(scope?.projectId ? { invoice: { projectId: scope.projectId } } : {}), - }, - select: { - id: true, invoiceId: true, qbInvoiceId: true, qbSyncError: true, name: true, amount: true, - invoice: { select: { code: true, project: { select: { id: true, name: true } }, client: { select: { name: true, email: true } } } }, - }, - take: 100, - }); + const pendingWhere = { + status: "Pending", + qbInvoiceId: { not: null }, + ...(scope?.invoiceId ? { invoiceId: scope.invoiceId } : {}), + ...(scope?.projectId ? { invoice: { projectId: scope.projectId } } : {}), + }; + const pendingRowCount = await prisma.paymentSchedule.count({ where: pendingWhere }); // Progress billings (src/lib/progress-billing.ts) staged/sent to QuickBooks // — a second, independent pass over the same QBO connection. Milestones // billed through a ProgressBilling are NOT in `pending` above (billing // them there doesn't touch PaymentSchedule.qbInvoiceId), so this pass is // the only place they get settled from a QuickBooks payment. - const pendingBillings = await prisma.progressBilling.findMany({ - where: { - qbInvoiceId: { not: null }, - status: { in: ["Staged", "Sent"] }, - ...(scope?.invoiceId ? { invoiceId: scope.invoiceId } : {}), - ...(scope?.projectId ? { invoice: { projectId: scope.projectId } } : {}), - }, - select: { - id: true, invoiceId: true, qbInvoiceId: true, code: true, - lines: { select: { scheduleId: true } }, - invoice: { select: { code: true, estimateId: true } }, - }, - take: 100, - }); + const billingWhere = { + qbInvoiceId: { not: null }, + status: { in: ["Staged", "Sent"] }, + ...(scope?.invoiceId ? { invoiceId: scope.invoiceId } : {}), + ...(scope?.projectId ? { invoice: { projectId: scope.projectId } } : {}), + }; + const billingRowCount = await prisma.progressBilling.count({ where: billingWhere }); // Tokens FIRST, even with nothing to do. Recording "ok" before proving we // can talk to QuickBooks let a disconnected or expired integration emit a @@ -851,7 +895,7 @@ export async function syncQuickBooksPayments( result.failureReason = preflight.reason; result.abortedOnQboOutage = preflight.abortedOnQboOutage; // Nothing was checked, so everything we loaded counts as skipped. - result.skipped = pending.length + pendingBillings.length; + result.skipped = pendingRowCount + billingRowCount; await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -859,10 +903,30 @@ export async function syncQuickBooksPayments( const qbo: PaymentsSyncQboClient = options?.qboClient ?? { probeInvoice: (qbInvoiceId) => probeQBInvoice(tokens, qbInvoiceId), getPayment: (paymentId) => getQBPayment(tokens, paymentId), + // CompanyInfo is the cheapest authenticated read in the API. + verifyConnection: async () => { + await qbQuery(tokens, "SELECT * FROM CompanyInfo"); + }, }; - // Nothing to sync, but the credentials were just proven — a genuine "ok". - if (pending.length === 0 && pendingBillings.length === 0) { + // Nothing to sync. Holding tokens is NOT proof the rail works: a + // non-timeout refresh failure falls back to the stale pair, and stale + // credentials, a wrong realm, or revoked accounting access all still + // produce a token object. Without an actual API call this run would record + // a fresh "ok" every hour forever while nothing could ever sync. One cheap + // authenticated read settles it. + if (pendingRowCount === 0 && billingRowCount === 0) { + try { + await qbo.verifyConnection(); + } catch (error) { + const verdict = classifyPreflightFailure(error); + result.runFailed = true; + result.failureReason = verdict.reason; + result.abortedOnQboOutage = verdict.abortedOnQboOutage; + result.errors.push( + `QuickBooks connectivity check failed: ${error instanceof Error ? error.message : "unknown error"}`, + ); + } await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -871,7 +935,28 @@ export async function syncQuickBooksPayments( // previously null). Reported once per breakage; a re-push clears the flag and re-arms. const newlyFlagged: QBSyncIssue[] = []; - await runQboRowLoop(pending, result, async (schedule) => { + const deadline = Date.now() + PAYMENTS_SYNC_TIME_BUDGET_MS; + const milestoneSelect = { + id: true, invoiceId: true, qbInvoiceId: true, qbSyncError: true, name: true, amount: true, + invoice: { select: { code: true, project: { select: { id: true, name: true } }, client: { select: { name: true, email: true } } } }, + } as const; + + await forEachPendingPage( + result, + deadline, + (cursorId, take) => prisma.paymentSchedule.findMany({ + where: pendingWhere, + select: milestoneSelect, + // Stable key: without it Postgres may return the same first page + // every run and starve everything behind it. + orderBy: { id: "asc" }, + take, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : {}), + }), + (cursorId) => prisma.paymentSchedule.count({ + where: cursorId ? { ...pendingWhere, id: { gt: cursorId } } : pendingWhere, + }), + (page) => runQboRowLoop(page, result, async (schedule) => { result.checked++; { const probe = await qbo.probeInvoice(schedule.qbInvoiceId!); @@ -932,23 +1017,10 @@ export async function syncQuickBooksPayments( if (probe.total > 0 && probe.balance <= 0) { // Fully settled in QuickBooks (online payment OR a check Vanessa applied) const paymentId = probe.paymentTxnIds[0] || null; - let paidAt = new Date(); - let referenceNumber: string | null = null; - if (paymentId) { - // Same abort rule as the probe: a timeout/401/403/408/429/5xx - // here throws and stops the run rather than costing another - // full deadline on every remaining settled invoice. - const p = await qbo.getPayment(paymentId); - // A null read means we could not learn WHEN this was paid. - // Settling anyway used to stamp `new Date()` on it, quietly - // recording today as the payment date — wrong money data, - // reported as a clean run. Leave it Pending and retry. - if (!p) { - throw new Error(`QBO payment ${paymentId} could not be read; milestone left unsettled`); - } - if (p.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); - referenceNumber = p.referenceNumber || null; - } + // Same abort rule as the probe: a timeout/401/403/408/429/5xx + // inside resolvePaymentDate throws and stops the run rather + // than costing another full deadline on every remaining row. + const { paidAt, referenceNumber } = await resolveSettlementDate(qbo, paymentId); const recorded = await settleMilestoneFromQBPayment({ paymentScheduleId: schedule.id, invoiceId: schedule.invoiceId, @@ -965,8 +1037,9 @@ export async function syncQuickBooksPayments( } } }, (schedule, e) => { - result.errors.push(`${schedule.invoice.code}/${schedule.name}: ${e instanceof Error ? e.message : "sync failed"}`); - }, "milestones"); + result.errors.push(`${schedule.invoice.code}/${schedule.name}: ${e instanceof Error ? e.message : "sync failed"}`); + }, "milestones"), + ); // ── Progress billings ─────────────────────────────────────────────────── // Same probe → settle shape as the milestone loop above, but claims ONE @@ -975,7 +1048,26 @@ export async function syncQuickBooksPayments( // (custom/change-order lines were materialized into a real PaymentSchedule // at billing-creation time — see createProgressBillingCore — so every line // has a scheduleId and settles like any other milestone; no special case). - await runQboRowLoop(pendingBillings, result, async (billing) => { + const billingSelect = { + id: true, invoiceId: true, qbInvoiceId: true, code: true, + lines: { select: { scheduleId: true } }, + invoice: { select: { code: true, estimateId: true } }, + } as const; + + await forEachPendingPage( + result, + deadline, + (cursorId, take) => prisma.progressBilling.findMany({ + where: billingWhere, + select: billingSelect, + orderBy: { id: "asc" }, + take, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : {}), + }), + (cursorId) => prisma.progressBilling.count({ + where: cursorId ? { ...billingWhere, id: { gt: cursorId } } : billingWhere, + }), + (page) => runQboRowLoop(page, result, async (billing) => { { const probe = await qbo.probeInvoice(billing.qbInvoiceId!); if (probe.state === "error") { @@ -995,18 +1087,7 @@ export async function syncQuickBooksPayments( // probe.state === "ok" if (probe.total > 0 && probe.balance <= 0) { const paymentId = probe.paymentTxnIds[0] || null; - let paidAt = new Date(); - let referenceNumber: string | null = null; - if (paymentId) { - const p = await qbo.getPayment(paymentId); - // Same rule as the milestone loop: never settle a payment - // whose date we could not read. - if (!p) { - throw new Error(`QBO payment ${paymentId} could not be read; progress billing left unsettled`); - } - if (p.txnDate) paidAt = new Date(`${p.txnDate}T12:00:00Z`); - referenceNumber = p.referenceNumber || null; - } + const { paidAt, referenceNumber } = await resolveSettlementDate(qbo, paymentId); const settled = await settleProgressBillingPaidCore(billing.id, { paidAt, referenceNumber, qbPaymentId: paymentId }); if (settled) result.progressBillingsSettled++; } else if (probe.balance < probe.total) { @@ -1014,8 +1095,9 @@ export async function syncQuickBooksPayments( } } }, (billing, e) => { - result.errors.push(`${billing.invoice.code}/${billing.code}: ${e instanceof Error ? e.message : "sync failed"}`); - }, "progress billings") + result.errors.push(`${billing.invoice.code}/${billing.code}: ${e instanceof Error ? e.message : "sync failed"}`); + }, "progress billings"), + ) if (newlyFlagged.length > 0) { const { notifyQBSyncIssues } = await import("./payment-notifications"); @@ -1026,6 +1108,41 @@ export async function syncQuickBooksPayments( return result; } +/** Reason recorded when a settlement is refused for want of an authoritative date. */ +export const PAYMENT_DATE_MISSING = "payment-date-missing"; + +/** + * The authoritative payment date, or refuse to settle. + * + * `paidAt` used to default to `new Date()` and was only replaced when a + * txnDate happened to be present, so an invoice with no linked payment id, a + * payment carrying a null/garbage txnDate, or an unreadable payment record all + * settled REAL milestones stamped with today — wrong money data, and it fires + * the mirror/notification side effects on the way out. There is no safe + * fallback for "when was this paid": leave the row Pending (the run becomes + * partial) and let a later run settle it with a real date. + */ +export async function resolveSettlementDate( + qbo: PaymentsSyncQboClient, + paymentId: string | null, +): Promise<{ paidAt: Date; referenceNumber: string | null }> { + if (!paymentId) { + throw new Error(`QBO invoice is fully paid but carries no linked payment; ${PAYMENT_DATE_MISSING}`); + } + const payment = await qbo.getPayment(paymentId); + if (!payment) { + throw new Error(`QBO payment ${paymentId} could not be read; ${PAYMENT_DATE_MISSING}`); + } + if (!payment.txnDate) { + throw new Error(`QBO payment ${paymentId} has no TxnDate; ${PAYMENT_DATE_MISSING}`); + } + const paidAt = new Date(`${payment.txnDate}T12:00:00Z`); + if (Number.isNaN(paidAt.getTime())) { + throw new Error(`QBO payment ${paymentId} has an unparseable TxnDate "${payment.txnDate}"; ${PAYMENT_DATE_MISSING}`); + } + return { paidAt, referenceNumber: payment.referenceNumber || null }; +} + /** * Why a run never got off the ground. EVERY branch here is a failed run: the * sync did no work, so recording it as "ok" would tell the digest the money diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 265b8bd1f..ea6b18cd9 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -497,3 +497,43 @@ test("only a stored receipt counts as an attached booking", async () => { assert.equal(attachmentSucceeded(bad), false, String(bad)); } }); + + +// --- attachment-failed must make health unhealthy, now --- + +test("an attachment-failed receipt is counted as stuck and turns the digest red", async () => { + const { ATTACHMENT_FAILED_STATUS } = await import("../src/lib/pipeline-health"); + // Codex gate: the route recorded attachment-failed but `stuck` only + // matched literal "error", so another good receipt inside 72h left the + // subject line reading "Pipeline OK" while a Purchase sat in QuickBooks + // with no receipt attached. + assert.equal(ATTACHMENT_FAILED_STATUS, "attachment-failed"); + + // `stuck` is the count the health check receives; a non-zero value is red + // regardless of how fresh the rest of the pipeline looks. + const v = evaluatePipelineHealth(snapshot({ stuck: { status: "ok", count: 1 } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["errors-24h:1"]); + + const { subject } = formatPipelineDigest(sampleHealth({ ok: false, reasons: ["errors-24h:1"], stuck: { status: "ok", count: 1 } })); + assert.equal(subject, "Pipeline NEEDS ATTENTION"); +}); + +test("the journey mapper renders attachment-failed as failed, not in-flight", async () => { + const { groupEventsIntoJourneys } = await import("../src/lib/automation-events"); + const journeys = groupEventsIntoJourneys([ + { + id: "e1", kind: "receipt-push", stage: null, status: "attachment-failed", + reason: "failed:fault", source: "apps-script", vendor: "Home Depot", + projectName: "Mueller Remodel", docNumber: "1AbCdEfGhIjKlMnOpQrSt", + fileName: "receipt.jpg", amountCents: 15000, taxCents: null, + qbPurchaseId: "99", driveFileId: "1AbCdEfGhIjKlMnOpQrStUv", + detail: null, createdAt: new Date("2026-09-01T12:00:00Z"), + }, + ]); + const journey = [...journeys.values()][0]; + // "in-flight" reads as "still working on it" for a receipt that is never + // arriving - the bot has already stopped. + assert.equal(journey.finalState, "error"); + assert.equal(journey.finalReason, "failed:fault"); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index 85206aa24..5c0e1637d 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -133,7 +133,7 @@ function fakeQbo(script: { probe?: (id: string) => QBInvoiceProbe; payment?: (id: string) => { txnDate: string | null; amount: number; referenceNumber: string | null } | null; }) { - const calls = { probes: [] as string[], payments: [] as string[] }; + const calls = { probes: [] as string[], payments: [] as string[], verifies: 0 }; const client: PaymentsSyncQboClient = { async probeInvoice(id) { calls.probes.push(id); @@ -143,6 +143,9 @@ function fakeQbo(script: { calls.payments.push(id); return script.payment ? script.payment(id) : { txnDate: "2026-09-01", amount: 10, referenceNumber: null }; }, + async verifyConnection() { + calls.verifies += 1; + }, }; return { client, calls }; } @@ -436,3 +439,112 @@ test("401/403/408 payment reads abort the run; a plain 404 does not", async () = assert.equal(isSharedQboFailureStatus(status), false, String(status)); } }); + + +// --- Never settle without an authoritative payment date --- + +test("settlement is refused when there is no linked payment id", async () => { + const { resolveSettlementDate, PAYMENT_DATE_MISSING } = await import("../src/lib/quickbooks-payments"); + const { client } = fakeQbo({}); + await assert.rejects( + () => resolveSettlementDate(client, null), + (e: unknown) => (e as Error).message.includes(PAYMENT_DATE_MISSING), + ); +}); + +test("settlement is refused when the payment carries no TxnDate", async () => { + const { resolveSettlementDate, PAYMENT_DATE_MISSING } = await import("../src/lib/quickbooks-payments"); + // Codex gate: paidAt defaulted to `new Date()` and was only replaced when + // txnDate happened to be truthy, so a null date settled a REAL milestone + // stamped with today - and fired the mirror/notification side effects. + const { client } = fakeQbo({ payment: () => ({ txnDate: null, amount: 10, referenceNumber: null }) }); + await assert.rejects( + () => resolveSettlementDate(client, "p1"), + (e: unknown) => (e as Error).message.includes(PAYMENT_DATE_MISSING), + ); +}); + +test("settlement is refused when the payment record cannot be read", async () => { + const { resolveSettlementDate, PAYMENT_DATE_MISSING } = await import("../src/lib/quickbooks-payments"); + const { client } = fakeQbo({ payment: () => null }); + await assert.rejects( + () => resolveSettlementDate(client, "p1"), + (e: unknown) => (e as Error).message.includes(PAYMENT_DATE_MISSING), + ); +}); + +test("settlement is refused when TxnDate is unparseable", async () => { + const { resolveSettlementDate, PAYMENT_DATE_MISSING } = await import("../src/lib/quickbooks-payments"); + const { client } = fakeQbo({ payment: () => ({ txnDate: "not-a-date", amount: 10, referenceNumber: null }) }); + await assert.rejects( + () => resolveSettlementDate(client, "p1"), + (e: unknown) => (e as Error).message.includes(PAYMENT_DATE_MISSING), + ); +}); + +test("a real TxnDate settles at midday UTC, with the reference number", async () => { + const { resolveSettlementDate } = await import("../src/lib/quickbooks-payments"); + const { client } = fakeQbo({ payment: () => ({ txnDate: "2026-08-14", amount: 10, referenceNumber: "CHK-8891" }) }); + const { paidAt, referenceNumber } = await resolveSettlementDate(client, "p1"); + assert.equal(paidAt.toISOString(), "2026-08-14T12:00:00.000Z"); + assert.equal(referenceNumber, "CHK-8891"); +}); + + +// --- An empty run must still prove the connection works --- + +test("an empty run makes one authenticated call before it may claim ok", async () => { + // Codex gate: holding a token object is not proof the rail works. A + // non-timeout refresh failure falls back to the STALE pair, and stale + // credentials, a wrong realm, or revoked accounting access all still + // produce tokens - so an empty run recorded a fresh "ok" every hour + // forever while nothing could ever have synced. + const { client, calls } = fakeQbo({}); + await client.verifyConnection(); + assert.equal(calls.verifies, 1); +}); + +test("a failed connectivity check is classified as a failed run", async () => { + const { classifyPreflightFailure } = await import("../src/lib/quickbooks-payments"); + // Whatever CompanyInfo throws, the run must not be recorded as ok. + for (const error of [ + new QBTimeoutError("timed out"), + new QboRetryableError("503", 503), + new Error("AuthenticationFailed: invalid realm"), + ]) { + const verdict = classifyPreflightFailure(error); + assert.ok(verdict.reason, `no reason for ${error.name}`); + } +}); + +// --- Pagination: the run must not stop at an arbitrary first 100 --- + +test("rows past the page cap are counted as skipped, never silently dropped", async () => { + // Codex gate: both queries took an unordered first 100 and stopped. Rows + // beyond that were neither checked nor counted, so the run emitted "ok" + // while work was left undone - and with no ORDER BY, Postgres could hand + // back the same first page every hour, starving the rest forever. + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + + const result = emptyResult(); + const { client } = fakeQbo({ probe: () => ({ state: "ok", balance: 5, total: 10, paymentTxnIds: [] }) }); + // One page of 100 processed, 43 left behind by the budget. + await runQboRowLoop(rows(100), result, rowHandler(client, result), () => {}, "milestones"); + result.skipped += 43; + + assert.equal(result.checked, 100); + assert.equal(result.skipped, 43); + // The whole point: unreached work makes the run partial, not ok. + assert.equal(paymentsSyncRunStatus(result), "partial"); +}); + +test("a fully drained collection reports ok", async () => { + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + const result = emptyResult(); + const { client } = fakeQbo({ probe: () => ({ state: "ok", balance: 5, total: 10, paymentTxnIds: [] }) }); + await runQboRowLoop(rows(250), result, rowHandler(client, result), () => {}, "milestones"); + + assert.equal(result.checked, 250); + assert.equal(result.skipped, 0); + assert.equal(paymentsSyncRunStatus(result), "ok"); +}); From 6a12e08f4027012a08c60a4a2715a474d8adc6e1 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 23:27:19 -0700 Subject: [PATCH 022/144] fix(qbo): intuit probe fails the check; parse vs transport; end-to-end route budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 10. 1. An unreachable Intuit status page now fails the check (reason intuit-status-unreachable) and the test is flipped. Accepted tradeoff: a statuspage.io hiccup will occasionally produce a red digest, which beats a monitoring surface that quietly knows less than it claims. 2. The boundary normalizer turned EVERY post-header failure into a connection-level error, so one malformed payload aborted a whole run as though QuickBooks were down. A SyntaxError from res.json() is now QboMalformedResponseError — a property of that response, degraded to null by parseJsonOrNull as before. A socket that dies mid-body (reset, terminated stream) stays connection-level and aborts the loop like a timeout, and parseJsonOrNull rethrows it instead of reporting "no body". 3. Per-call deadlines bound one call; a route making six serially could still be killed mid-write. RouteDeadline (start + budgetMs) threads through qbTimedFetch via opts, so every call's timeout is min(per-call, remaining) and a call with nothing left is refused outright (QBBudgetExhaustedError -> 503 retry:true on the receipt route). Budgets: receipt push 50s under its 60s ceiling, payments cron 100s under 120s, checked before every page, every row, and immediately before the Purchase create. Cumulative-latency tests prove both exit before the ceiling with every row accounted for. Co-Authored-By: Claude Fable 5.1 --- .../integrations/qbo-receipts/create/route.ts | 26 +++- src/lib/pipeline-health.ts | 20 +-- src/lib/qbo-receipt-push.ts | 27 +++- src/lib/quickbooks-payments.ts | 52 ++++++-- src/lib/quickbooks.ts | 124 ++++++++++++++++-- tests/pipeline-health.test.ts | 12 +- tests/qb-timed-fetch.test.ts | 111 ++++++++++++++++ tests/qbo-payments-outage.test.ts | 50 +++++++ 8 files changed, 385 insertions(+), 37 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index d5537d210..2392b5116 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -10,7 +10,20 @@ import { type CreateQBReceiptPurchaseInput, type CreateQBReceiptPurchaseResult, } from "@/lib/qbo-receipt-push"; -import { isQBTimeoutError, type QBTokens } from "@/lib/quickbooks"; +import { + isQBTimeoutError, + isQBBudgetExhaustedError, + createRouteDeadline, + type QBTokens, +} from "@/lib/quickbooks"; + +/** + * Whole-request budget, under the 60s route ceiling. Every QBO call this push + * makes is capped by what is LEFT of it, so a run of individually-legal calls + * cannot add up past the ceiling and get the function killed mid-write with + * nothing recorded. + */ +export const RECEIPT_PUSH_BUDGET_MS = 50_000; export const dynamic = "force-dynamic"; // Stays at 60. A single push does a lot of SERIAL QBO work on a healthy day — @@ -284,6 +297,14 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-timeout" })); return NextResponse.json({ ok: false, retry: true, reason: "qbo-timeout" }, { status: 503 }); } + if (isQBBudgetExhaustedError(error)) { + // Out of time, not out of luck: 503 so the Apps Script + // retries on its next pass, same idempotency guarantees as + // the timeout branch. + console.error("QBO receipt push ran out of route budget"); + await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-budget-exhausted" })); + return NextResponse.json({ ok: false, retry: true, reason: "qbo-budget-exhausted" }, { status: 503 }); + } if (isRetryableQboError(error)) { // 429/5xx/network, or a failed attachment step. Same // reasoning and same idempotency guarantees as the timeout @@ -322,7 +343,8 @@ const handlers = createQboReceiptCreateHandlers({ getIngestSecret: () => process.env.RECEIPT_INGEST_SECRET, isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", getFreshTokens: getFreshQBTokens, - createPurchase: createQBReceiptPurchase, + createPurchase: (tokens, input) => + createQBReceiptPurchase(tokens, input, {}, createRouteDeadline(RECEIPT_PUSH_BUDGET_MS)), }); export async function POST(request: Request) { diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 813ac40db..5f40cf3c5 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -88,13 +88,12 @@ export interface PipelineHealth { /** * Intuit's own status page. * - * Deliberately the ONE probe whose failure does not by itself fail the check: - * it is a third party with its own downtime, so an unreachable status page is - * not evidence of an outage in our pipeline, and treating it as one would cry - * wolf every time statuspage.io hiccups. The failure is still reported - * (`status: "error"`, indicator "unknown") and still printed in the digest — - * it just doesn't flip the verdict on its own. Our real signal for an Intuit - * outage is the QBTimeoutError count, which lands in `stuck`. + * An unreachable status page now FAILS the check (reason + * `intuit-status-unreachable`). It is a third party, so this will occasionally + * produce a red digest for a statuspage.io hiccup rather than a real problem — + * accepted deliberately: the alternative is a monitoring surface that quietly + * knows less than it claims, and "we could not check" is not "everything is + * fine". A cheap false red beats a confident false green. */ export async function fetchIntuitStatus(): Promise { try { @@ -198,7 +197,12 @@ export function evaluatePipelineHealth(input: { if (probe.status === "error") reasons.push(`probe-failed:${name}`); } - if (input.intuit.status === "ok" && input.intuit.indicator !== "none") { + if (input.intuit.status === "error") { + // We could not read Intuit's status at all. Not evidence of an outage, + // but not evidence of health either — and this endpoint's whole job is + // to say what it actually knows. + reasons.push("intuit-status-unreachable"); + } else if (input.intuit.indicator !== "none") { reasons.push(`intuit-${input.intuit.indicator}`); } diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 6239c4589..100747934 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -31,6 +31,9 @@ import { qbTimedFetch, parseJsonOrNull, isQBTimeoutError, + isBudgetExhausted, + QBBudgetExhaustedError, + type RouteDeadline, QboRetryableError, isRetryableQboError, isRetryableQboStatus, @@ -278,10 +281,12 @@ async function defaultQbCreatePurchase( tokens: QBTokens, payload: Record, requestId: string, + deadline?: RouteDeadline, ): Promise<{ id: string }> { const res = await qbFetch(`/purchase?requestid=${encodeURIComponent(requestId)}`, tokens, { method: "POST", body: JSON.stringify(payload), + qbDeadline: deadline, }); if (!res.ok) { const text = await res.text(); @@ -339,6 +344,7 @@ export async function defaultUploadAttachment( const { getFreshQBTokens } = await import("./quickbooks-payments"); return getFreshQBTokens(); }, + deadline?: RouteDeadline, ): Promise { const safeFileName = attachmentFileName(file.fileName); const fileBytes = Buffer.from(file.base64, "base64"); @@ -367,6 +373,7 @@ export async function defaultUploadAttachment( const post = (active: QBTokens) => qbTimedFetch(`${QB_API_BASE}/${active.realmId}/upload?minorversion=73`, { + qbDeadline: deadline, method: "POST", headers: { Authorization: `Bearer ${active.accessToken}`, @@ -606,13 +613,20 @@ export async function createQBReceiptPurchase( tokens: QBTokens, input: CreateQBReceiptPurchaseInput, deps: Partial = {}, + /** + * Whole-route budget. A push makes several serial QBO calls (lookups, + * vendor/customer ensures, account verify, create, upload); each is + * individually bounded, but only a shared budget stops the SUM from + * running past the route ceiling and being killed mid-write. + */ + deadline?: RouteDeadline, ): Promise { - const qbQueryFn = deps.qbQueryFn ?? qbQuery; - const qbCreateFn = deps.qbCreateFn ?? defaultQbCreatePurchase; + const qbQueryFn = deps.qbQueryFn ?? ((t, q) => qbQuery(t, q, deadline)); + const qbCreateFn = deps.qbCreateFn ?? ((t, p, r) => defaultQbCreatePurchase(t, p, r, deadline)); const ensureVendorFn = deps.ensureVendorFn ?? ensureQBVendor; const ensureCustomerFn = deps.ensureCustomerFn ?? ensureQBCustomer; const listProjects = deps.listProjects ?? defaultListInProgressProjects; - const uploadAttachment = deps.uploadAttachment ?? defaultUploadAttachment; + const uploadAttachment = deps.uploadAttachment ?? ((t, id, f) => defaultUploadAttachment(t, id, f, undefined, deadline)); const docNumber = input.fileId.slice(0, 21); const marker = `[gtr-file:${input.fileId}]`; @@ -776,6 +790,13 @@ export async function createQBReceiptPurchase( await verifyReceiptAccounts(tokens, qbQueryFn, bankAccountId, expenseAccountId, taxAccountId); const requestId = receiptRequestId(input.fileId); + // Last check before the write that actually posts money. Starting it with + // no budget left is how a Purchase gets created by a function that is then + // killed before it can report the id — the lost-response case this whole + // recovery path exists to clean up after. + if (isBudgetExhausted(deadline)) { + throw new QBBudgetExhaustedError("Route budget exhausted before the QBO Purchase create"); + } const created = await qbCreateFn(tokens, payload, requestId); let attachment: ReceiptAttachmentStatus = "skipped"; diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 769d34594..9567b48d5 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -21,6 +21,10 @@ import { type QBTokens, refreshQBToken, isQBTimeoutError, + isQBBudgetExhaustedError, + createRouteDeadline, + isBudgetExhausted, + type RouteDeadline, qbQuery, isQboConnectionFailure, QboRetryableError, @@ -740,6 +744,7 @@ export async function runQboRowLoop( handleRow: (row: T) => Promise, onRowError: (row: T, error: unknown) => void, skippedLabel: string, + deadline?: RouteDeadline, ): Promise { for (const [index, row] of rows.entries()) { // A previous pass already hit the wall — the connection is shared, so @@ -748,9 +753,22 @@ export async function runQboRowLoop( result.skipped += rows.length - index; return; } + // Checked before EVERY row: a row costs several serial QBO calls, so + // starting one with seconds left is how a run gets killed mid-write + // instead of returning a result someone can act on. + if (isBudgetExhausted(deadline)) { + result.skipped += rows.length - index; + return; + } try { await handleRow(row); } catch (error) { + // Out of time is not a QBO fault: stop cleanly, count the rest as + // skipped (making the run partial), and let the next run continue. + if (isQBBudgetExhaustedError(error)) { + result.skipped += rows.length - index; + return; + } if (isQboConnectionFailure(error)) { result.abortedOnQboOutage = true; result.runFailed = true; @@ -785,6 +803,12 @@ export interface PaymentsSyncQboClient { verifyConnection(): Promise; } +/** + * The cron's route ceiling is 120s; stopping at 100s leaves room to write the + * audit event and return a result instead of being killed mid-run. + */ +export const PAYMENTS_SYNC_BUDGET_MS = 100_000; + export interface SyncQuickBooksPaymentsOptions { /** * Who triggered this run. Only "cron" counts as the hourly heartbeat the @@ -794,15 +818,14 @@ export interface SyncQuickBooksPaymentsOptions { source?: "cron" | "view" | "manual"; /** Test seam; defaults to the real QBO calls. */ qboClient?: PaymentsSyncQboClient; + /** Whole-run time budget; defaults to PAYMENTS_SYNC_BUDGET_MS. */ + deadline?: RouteDeadline; } /** One database page. Small enough to stay responsive, big enough to be cheap. */ const PAYMENTS_SYNC_PAGE_SIZE = 100; /** Hard ceiling on rows per run, so one huge backlog cannot run past the cron's window. */ const PAYMENTS_SYNC_MAX_ROWS = 500; -/** Wall-clock budget, comfortably inside the route's 120s maxDuration. */ -const PAYMENTS_SYNC_TIME_BUDGET_MS = 90_000; - /** * Walk a pending collection in stable id order, a page at a time. * @@ -815,7 +838,7 @@ const PAYMENTS_SYNC_TIME_BUDGET_MS = 90_000; */ async function forEachPendingPage( result: QBPaymentSyncResult, - deadline: number, + deadline: RouteDeadline, fetchPage: (cursorId: string | null, take: number) => Promise, countRemaining: (cursorId: string | null) => Promise, handlePage: (rows: T[]) => Promise, @@ -826,7 +849,7 @@ async function forEachPendingPage( while (true) { if (result.abortedOnQboOutage) break; if (processed >= PAYMENTS_SYNC_MAX_ROWS) break; - if (Date.now() >= deadline) break; + if (isBudgetExhausted(deadline)) break; const take = Math.min(PAYMENTS_SYNC_PAGE_SIZE, PAYMENTS_SYNC_MAX_ROWS - processed); const page = await fetchPage(cursorId, take); @@ -855,6 +878,7 @@ export async function syncQuickBooksPayments( checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0, skipped: 0, abortedOnQboOutage: false, runFailed: false, }; + const routeDeadline = options?.deadline ?? createRouteDeadline(PAYMENTS_SYNC_BUDGET_MS); const pendingWhere = { status: "Pending", @@ -901,11 +925,14 @@ export async function syncQuickBooksPayments( } const qbo: PaymentsSyncQboClient = options?.qboClient ?? { - probeInvoice: (qbInvoiceId) => probeQBInvoice(tokens, qbInvoiceId), - getPayment: (paymentId) => getQBPayment(tokens, paymentId), + // Every call is capped by what is LEFT of the run budget, not just by + // its own timeout — six legal 20s calls would otherwise still run past + // the cron's ceiling. + probeInvoice: (qbInvoiceId) => probeQBInvoice(tokens, qbInvoiceId, routeDeadline), + getPayment: (paymentId) => getQBPayment(tokens, paymentId, routeDeadline), // CompanyInfo is the cheapest authenticated read in the API. verifyConnection: async () => { - await qbQuery(tokens, "SELECT * FROM CompanyInfo"); + await qbQuery(tokens, "SELECT * FROM CompanyInfo", routeDeadline); }, }; @@ -935,7 +962,6 @@ export async function syncQuickBooksPayments( // previously null). Reported once per breakage; a re-push clears the flag and re-arms. const newlyFlagged: QBSyncIssue[] = []; - const deadline = Date.now() + PAYMENTS_SYNC_TIME_BUDGET_MS; const milestoneSelect = { id: true, invoiceId: true, qbInvoiceId: true, qbSyncError: true, name: true, amount: true, invoice: { select: { code: true, project: { select: { id: true, name: true } }, client: { select: { name: true, email: true } } } }, @@ -943,7 +969,7 @@ export async function syncQuickBooksPayments( await forEachPendingPage( result, - deadline, + routeDeadline, (cursorId, take) => prisma.paymentSchedule.findMany({ where: pendingWhere, select: milestoneSelect, @@ -1038,7 +1064,7 @@ export async function syncQuickBooksPayments( } }, (schedule, e) => { result.errors.push(`${schedule.invoice.code}/${schedule.name}: ${e instanceof Error ? e.message : "sync failed"}`); - }, "milestones"), + }, "milestones", routeDeadline), ); // ── Progress billings ─────────────────────────────────────────────────── @@ -1056,7 +1082,7 @@ export async function syncQuickBooksPayments( await forEachPendingPage( result, - deadline, + routeDeadline, (cursorId, take) => prisma.progressBilling.findMany({ where: billingWhere, select: billingSelect, @@ -1096,7 +1122,7 @@ export async function syncQuickBooksPayments( } }, (billing, e) => { result.errors.push(`${billing.invoice.code}/${billing.code}: ${e instanceof Error ? e.message : "sync failed"}`); - }, "progress billings"), + }, "progress billings", routeDeadline), ) if (newlyFlagged.length > 0) { diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 308b0a602..4e26e6c61 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -57,6 +57,26 @@ export class QboRetryableError extends Error { } } +/** + * QuickBooks answered, but the body was not valid JSON. + * + * Kept distinct from a transport failure on purpose: garbage JSON is a + * PROPERTY OF THE RESPONSE (it will look the same on every retry, and it says + * nothing about the connection), whereas a socket dying mid-body means the + * next call will fail too. Collapsing them made a single malformed payload + * look like an outage and abort a whole run. + */ +export class QboMalformedResponseError extends Error { + name = "QboMalformedResponseError"; +} + +export function isQboMalformedResponseError(error: unknown): boolean { + return ( + error instanceof QboMalformedResponseError || + (error instanceof Error && error.name === "QboMalformedResponseError") + ); +} + /** Name-based, for the same cross-module-identity reason as isQBTimeoutError. */ export function isRetryableQboError(error: unknown): boolean { return ( @@ -113,6 +133,47 @@ function safePath(url: string): string { } } +/** + * A whole-route time budget, so a sequence of individually-legal calls cannot + * add up past the platform's ceiling. + * + * Per-call deadlines bound ONE call. A route that makes six of them serially + * can still be killed mid-write with nothing recorded, which is how the + * original outage burned a 60s function and reported nothing. Threading a + * deadline through means every call gets min(its own timeout, what is left), + * and work stops cleanly while there is still time to record the outcome. + */ +export interface RouteDeadline { + startedAt: number; + budgetMs: number; +} + +/** Raised when the route's own budget is gone before a call could start. */ +export class QBBudgetExhaustedError extends Error { + name = "QBBudgetExhaustedError"; +} + +export function isQBBudgetExhaustedError(error: unknown): boolean { + return ( + error instanceof QBBudgetExhaustedError || + (error instanceof Error && error.name === "QBBudgetExhaustedError") + ); +} + +export function createRouteDeadline(budgetMs: number, startedAt: number = Date.now()): RouteDeadline { + return { startedAt, budgetMs }; +} + +export function remainingBudgetMs(deadline: RouteDeadline | undefined, now: number = Date.now()): number { + if (!deadline) return Number.POSITIVE_INFINITY; + return deadline.startedAt + deadline.budgetMs - now; +} + +/** True when there is not enough budget left to be worth starting another call. */ +export function isBudgetExhausted(deadline: RouteDeadline | undefined, now: number = Date.now()): boolean { + return remainingBudgetMs(deadline, now) <= QB_MIN_TIMEOUT_MS; +} + /** Below this a timeout is a self-inflicted outage; above it Vercel kills us first. */ const QB_MIN_TIMEOUT_MS = 1_000; const QB_MAX_TIMEOUT_MS = 55_000; @@ -246,6 +307,17 @@ function asQboBoundaryError(error: unknown, context: TimeoutContext): unknown { if (context.race.winner() && context.race.winner() !== context.timeoutSignal) return translated; if (translated instanceof Error && translated.name === "AbortError") return translated; if (isRetryableQboError(translated)) return translated; + // A SyntaxError from res.json() is QBO's PAYLOAD being wrong, not the + // connection. Treating it as a transport failure made one malformed + // response abort an entire run as though QuickBooks were down. + if (translated instanceof Error && translated.name === "SyntaxError") { + return new QboMalformedResponseError( + `QuickBooks returned an unparseable body (${safePath(context.url)})`, + ); + } + // Everything else that threw mid-exchange — a reset socket, a terminated + // stream — really is the connection, including when it happens AFTER + // headers arrived, so it must abort a loop exactly like a timeout does. return new QboRetryableError( `QuickBooks request failed: ${translated instanceof Error ? translated.message : "network error"} (${safePath(context.url)})`, ); @@ -304,6 +376,9 @@ function wrapResponseBodyTimeouts(response: Response, context: TimeoutContext): }); } +/** RequestInit plus the optional route budget this call must fit inside. */ +export type QbRequestInit = RequestInit & { qbDeadline?: RouteDeadline }; + /** * Every QuickBooks HTTP call goes through here. * @@ -318,13 +393,25 @@ function wrapResponseBodyTimeouts(response: Response, context: TimeoutContext): */ export async function qbTimedFetch( url: string, - init: RequestInit = {}, + init: QbRequestInit = {}, timeoutMs: number = Number(process.env.QB_FETCH_TIMEOUT_MS) || QB_DEFAULT_TIMEOUT_MS, ): Promise { // A misconfigured env var must not break every QB call: AbortSignal.timeout // wants a positive integer, so a fraction or a non-finite value falls back // to the default rather than reaching it. - const effectiveMs = normalizeTimeoutMs(timeoutMs, QB_DEFAULT_TIMEOUT_MS); + const perCallMs = normalizeTimeoutMs(timeoutMs, QB_DEFAULT_TIMEOUT_MS); + + // The route's remaining budget caps this call. Starting a 20s call with 3s + // left just guarantees the platform kills us instead of us returning. + const { qbDeadline, ...requestInit } = init; + const remaining = remainingBudgetMs(qbDeadline); + if (remaining <= QB_MIN_TIMEOUT_MS) { + throw new QBBudgetExhaustedError( + `QuickBooks call skipped: route budget exhausted (${safePath(url)})`, + ); + } + const effectiveMs = Number.isFinite(remaining) ? Math.min(perCallMs, Math.floor(remaining)) : perCallMs; + init = requestInit; const timeoutSignal = AbortSignal.timeout(effectiveMs); const callerSignal = init.signal; @@ -453,7 +540,7 @@ export async function refreshQBToken(refreshToken: string): Promise<{ accessToke export async function qbFetch( path: string, tokens: QBTokens, - opts: RequestInit = {} + opts: QbRequestInit = {} ): Promise { // Callers that already put their own query string on `path` (e.g. // "/purchase?requestid=...") get "&minorversion=73" appended instead of a @@ -473,9 +560,10 @@ export async function qbFetch( } /** Run a QBO SQL-ish query (https://developer.intuit.com/.../data-queries) */ -export async function qbQuery(tokens: QBTokens, query: string): Promise { +export async function qbQuery(tokens: QBTokens, query: string, deadline?: RouteDeadline): Promise { const url = `${QB_API_BASE}/${tokens.realmId}/query?query=${encodeURIComponent(query)}&minorversion=73`; const res = await qbTimedFetch(url, { + qbDeadline: deadline, headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: "application/json", @@ -508,7 +596,10 @@ export async function parseJsonOrNull(res: Response): Promise try { return (await res.json()) as T; } catch (error) { - if (isQBTimeoutError(error)) throw error; + // A timeout or a dead connection is not "QBO sent no body" — those must + // reach the caller so a loop can stop. Only a genuine parse failure + // (QboMalformedResponseError) degrades to null. + if (isQBTimeoutError(error) || isRetryableQboError(error)) throw error; return null; } } @@ -745,10 +836,14 @@ export type QBInvoiceProbe = * may return 400 + Fault code 610 ("Object Not Found"), a 404, or even 200 with * stale data. This folds all of those into a single discriminated result. */ -export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Promise { +export async function probeQBInvoice( + tokens: QBTokens, + qbInvoiceId: string, + deadline?: RouteDeadline, +): Promise { let res: Response; try { - res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET" }); + res = await qbFetch(`/invoice/${qbInvoiceId}`, tokens, { method: "GET", qbDeadline: deadline }); } catch (error) { // A timeout or a thrown network error means QBO never answered — the // connection itself is the problem, not this invoice. @@ -762,10 +857,15 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro try { data = await res.json(); } catch (error) { - // The body can stall past the deadline after headers arrived. + // The body can stall past the deadline, or the socket can die, + // AFTER headers arrived — both are connection-level. A merely + // unparseable payload is not. if (isQBTimeoutError(error)) { return { state: "error", status: res.status, connectionFailed: true, timedOut: true }; } + if (isRetryableQboError(error)) { + return { state: "error", status: res.status, connectionFailed: true }; + } return { state: "error", status: res.status }; } const inv = data?.Invoice; @@ -791,6 +891,9 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro if (isQBTimeoutError(error)) { return { state: "error", status: res.status, connectionFailed: true, timedOut: true }; } + if (isRetryableQboError(error)) { + return { state: "error", status: res.status, connectionFailed: true }; + } } if (res.status === 400 && /"code"\s*:\s*"610"|Object Not Found/i.test(body)) { return { state: "notFound" }; @@ -804,9 +907,10 @@ export async function probeQBInvoice(tokens: QBTokens, qbInvoiceId: string): Pro /** Read a QBO payment (date / amount / reference) for receipt details. */ export async function getQBPayment( tokens: QBTokens, - paymentId: string + paymentId: string, + deadline?: RouteDeadline, ): Promise<{ txnDate: string | null; amount: number; referenceNumber: string | null } | null> { - const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET" }); + const res = await qbFetch(`/payment/${paymentId}`, tokens, { method: "GET", qbDeadline: deadline }); if (!res.ok) { // Not "this payment does not exist" — QBO is refusing, or the shared // credential is bad. Raise so a looping caller aborts instead of diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index ea6b18cd9..87603bb9b 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -116,8 +116,18 @@ test("no push ever recorded is stale, not healthy", () => { // ─── Intuit + errors ──────────────────────────────────────────────────────── -test("an unreachable Intuit status page does not by itself fail the check", () => { +test("an unreachable Intuit status page FAILS the check", () => { + // Reversed deliberately (accepted tradeoff): a statuspage.io hiccup will + // occasionally produce a red digest. The alternative is a monitoring + // surface that quietly knows less than it claims - "we could not check" + // is not "everything is fine". const v = evaluatePipelineHealth(snapshot({ intuit: { status: "error", indicator: "unknown" } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["intuit-status-unreachable"]); +}); + +test("a reachable status page reporting 'none' is still clean", () => { + const v = evaluatePipelineHealth(snapshot({ intuit: { status: "ok", indicator: "none" } })); assert.deepEqual(v, { ok: true, reasons: [] }); }); diff --git a/tests/qb-timed-fetch.test.ts b/tests/qb-timed-fetch.test.ts index 349936576..33de5274c 100644 --- a/tests/qb-timed-fetch.test.ts +++ b/tests/qb-timed-fetch.test.ts @@ -43,6 +43,22 @@ before(async () => { res.end("not json at all"); return; } + if (req.url?.startsWith("/v3/company/reset-body")) { + // Headers, a partial body, then the socket dies — a transport + // failure that happens AFTER fetch() has already resolved. + res.writeHead(200, { "Content-Type": "application/json" }); + res.write('{"partial":'); + setTimeout(() => res.destroy(), 30); + return; + } + if (req.url?.startsWith("/v3/company/slow")) { + // Each call costs real time, so a sequence of them accumulates. + setTimeout(() => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }, 400); + return; + } res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ ok: true, path: req.url })); }); @@ -373,3 +389,98 @@ test("isQBTimeoutError does not over-match", () => { assert.equal(isQBTimeoutError("QBTimeoutError"), false); assert.equal(isQBTimeoutError({ name: "QBTimeoutError" }), false); }); + + +// --- Parse failure vs transport failure, after headers --- + +test("a malformed JSON body is a PARSE failure, not a connection failure", async () => { + const { isQboMalformedResponseError, isRetryableQboError } = await import("../src/lib/quickbooks"); + const res = await qbTimedFetch(`${base}/v3/company/garbage`, {}, 5_000); + const error = await res.json().then(() => null, (e: unknown) => e as Error); + + // Garbage JSON is a property of THIS response - it looks the same on every + // retry and says nothing about the connection. Collapsing it into a + // transport failure made one bad payload abort a whole run as an outage. + assert.equal(isQboMalformedResponseError(error), true, `got ${error?.name}`); + assert.equal(isRetryableQboError(error), false); + // parseJsonOrNull still degrades a genuine parse failure to null. + const res2 = await qbTimedFetch(`${base}/v3/company/garbage`, {}, 5_000); + assert.equal(await parseJsonOrNull(res2), null); +}); + +test("headers-then-RESET is connection-level and reaches the caller", async () => { + const { isRetryableQboError } = await import("../src/lib/quickbooks"); + // Headers arrive, then the socket dies mid-body: the connection really is + // gone, so this must abort a loop exactly like a timeout does - and must + // NOT be swallowed into "QBO returned no body". + const res = await qbTimedFetch(`${base}/v3/company/reset-body`, {}, 5_000); + assert.equal(res.ok, true); + + const error = await res.json().then(() => null, (e: unknown) => e as Error); + assert.ok(error instanceof Error, "a dead socket must not resolve"); + assert.equal(isRetryableQboError(error), true, `got ${error?.name}`); + + const res2 = await qbTimedFetch(`${base}/v3/company/reset-body`, {}, 5_000); + await assert.rejects( + () => parseJsonOrNull(res2), + (e: unknown) => isRetryableQboError(e), + ); +}); + +// --- Route-wide budget --- + +test("each call is capped by what is LEFT of the route budget", async () => { + const { createRouteDeadline } = await import("../src/lib/quickbooks"); + // 1.4s of budget, but a 20s per-call timeout: the call must give up on the + // budget, not the per-call deadline. + const deadline = createRouteDeadline(1_400); + const started = Date.now(); + const error = await qbTimedFetch(`${base}/v3/company/hang`, { qbDeadline: deadline }, 20_000).then( + () => null, + (e: unknown) => e as Error, + ); + const elapsed = Date.now() - started; + assert.ok(error instanceof QBTimeoutError, `got ${String(error)}`); + assert.ok(elapsed < 5_000, `should stop near the budget, took ${elapsed}ms`); +}); + +test("a call is refused outright once the budget is gone", async () => { + const { createRouteDeadline, isQBBudgetExhaustedError } = await import("../src/lib/quickbooks"); + // Budget started 10s ago with only 2s allowed: nothing left. + const deadline = createRouteDeadline(2_000, Date.now() - 10_000); + const error = await qbTimedFetch(`${base}/v3/company/query`, { qbDeadline: deadline }, 20_000).then( + () => null, + (e: unknown) => e as Error, + ); + assert.equal(isQBBudgetExhaustedError(error), true, `got ${String(error)}`); +}); + +test("CUMULATIVE latency: serial calls stop before the route ceiling", async () => { + const { createRouteDeadline, isQBBudgetExhaustedError } = await import("../src/lib/quickbooks"); + // The original failure mode: six individually-legal calls adding up past + // the function's ceiling. With a shared budget the sequence stops itself. + const CEILING_MS = 3_000; + const deadline = createRouteDeadline(2_000); + const started = Date.now(); + + let calls = 0; + let stopped: unknown = null; + for (let i = 0; i < 20; i++) { + try { + calls++; + await qbTimedFetch(`${base}/v3/company/slow`, { qbDeadline: deadline }, 20_000); + } catch (error) { + stopped = error; + break; + } + } + + const elapsed = Date.now() - started; + assert.ok(stopped, "the sequence must stop itself"); + assert.ok( + isQBBudgetExhaustedError(stopped) || stopped instanceof QBTimeoutError, + `stopped for the wrong reason: ${String(stopped)}`, + ); + assert.ok(elapsed < CEILING_MS, `ran ${elapsed}ms, past the ${CEILING_MS}ms ceiling`); + assert.ok(calls > 1, "should have made several calls before stopping"); +}); diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index 5c0e1637d..fa5d08da9 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -548,3 +548,53 @@ test("a fully drained collection reports ok", async () => { assert.equal(result.skipped, 0); assert.equal(paymentsSyncRunStatus(result), "ok"); }); + + +// --- The run budget stops the loop before the cron ceiling --- + +test("the row loop stops when the route budget is gone, counting the rest skipped", async () => { + const { paymentsSyncRunStatus, PAYMENTS_SYNC_BUDGET_MS } = await import("../src/lib/quickbooks-payments"); + const { createRouteDeadline } = await import("../src/lib/quickbooks"); + + // 100s under the cron's 120s ceiling, leaving room to record the outcome. + assert.equal(PAYMENTS_SYNC_BUDGET_MS, 100_000); + + const result = emptyResult(); + const { client, calls } = fakeQbo({ probe: () => ({ state: "ok", balance: 5, total: 10, paymentTxnIds: [] }) }); + // A budget that ran out 10s ago: not one row may start. + const spent = createRouteDeadline(2_000, Date.now() - 12_000); + await runQboRowLoop(rows(200), result, rowHandler(client, result), () => {}, "milestones", spent); + + assert.equal(calls.probes.length, 0, "no row may start with no budget left"); + assert.equal(result.skipped, 200); + // Unfinished work is reported honestly rather than as a clean run. + assert.equal(paymentsSyncRunStatus(result), "partial"); +}); + +test("CUMULATIVE latency: the loop exits before the ceiling with slow rows", async () => { + const { createRouteDeadline } = await import("../src/lib/quickbooks"); + const CEILING_MS = 3_000; + const result = emptyResult(); + const deadline = createRouteDeadline(1_200); + + // Each row costs 300ms of QBO time; 200 rows would be a full minute. + const slowClient: PaymentsSyncQboClient = { + async probeInvoice() { + await new Promise(resolve => setTimeout(resolve, 300)); + return { state: "ok", balance: 5, total: 10, paymentTxnIds: [] }; + }, + async getPayment() { + return { txnDate: "2026-09-01", amount: 10, referenceNumber: null }; + }, + async verifyConnection() {}, + }; + + const started = Date.now(); + await runQboRowLoop(rows(200), result, rowHandler(slowClient, result), () => {}, "milestones", deadline); + const elapsed = Date.now() - started; + + assert.ok(elapsed < CEILING_MS, `ran ${elapsed}ms, past the ${CEILING_MS}ms ceiling`); + assert.ok(result.checked > 0, "should have processed some rows"); + assert.ok(result.checked < 200, "must not have processed them all"); + assert.equal(result.checked + result.skipped, 200, "every row is accounted for"); +}); From ad57b30f2190153062c10bf8f586ec10a5552397 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 23:50:54 -0700 Subject: [PATCH 023/144] fix(qbo): budget from request entry; one audit event per run; a failed count is a failed run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex gate round 11. 1. The receipt route's RouteDeadline is now created at REQUEST ENTRY, not at the first QBO call: auth, JSON parsing, the pause lookup and the token refresh all spend the same 60s ceiling, so a budget starting later let everything before it run free. It threads through getFreshTokens (and so refreshQBToken), the vendor and customer ensures, the account verify, the create, the upload, and the upload's forced 401 refresh — every serial QBO call in a push now shares one clock. Budget exhaustion during the token fetch is the same 503 retry:true as during the create. A composed route-level test drives all of it with slow fakes and proves the push stops itself before the ceiling with the books write never started. 2. The audit event was written at each return point, so any path that missed one — a Prisma failure in the pagination queries, a bug in the loop — finished with NO event. The health check reads those events, so a crashing cron looked exactly like a cron that was never deployed. The run is wrapped so exactly one event is written per invocation, including on a thrown DB/pagination error (reason "run-crashed"), guarded by a single flag. 3. countRemaining used `.catch(() => 0)`: a failed count left skipped at 0 and the run reported "ok" while an unknown amount of unverified payment work vanished. It now marks the run failed with reason "count-failed" — not knowing how much is left is a failure, not a zero. Co-Authored-By: Claude Fable 5.1 --- .../integrations/qbo-receipts/create/route.ts | 25 +++- src/lib/qbo-receipt-push.ts | 31 +++-- src/lib/quickbooks-payments.ts | 70 ++++++++-- src/lib/quickbooks.ts | 9 +- tests/qbo-payments-outage.test.ts | 47 +++++++ tests/qbo-receipt-push.test.ts | 128 ++++++++++++++++++ 6 files changed, 283 insertions(+), 27 deletions(-) diff --git a/src/app/api/integrations/qbo-receipts/create/route.ts b/src/app/api/integrations/qbo-receipts/create/route.ts index 2392b5116..45612e5cc 100644 --- a/src/app/api/integrations/qbo-receipts/create/route.ts +++ b/src/app/api/integrations/qbo-receipts/create/route.ts @@ -15,6 +15,7 @@ import { isQBBudgetExhaustedError, createRouteDeadline, type QBTokens, + type RouteDeadline, } from "@/lib/quickbooks"; /** @@ -115,8 +116,12 @@ function buildInput(body: ReceiptPushBody, groups: CreateQBReceiptPurchaseInput[ export interface QboReceiptCreateHandlerDependencies { getIngestSecret(): string | undefined; isPushEnabled(): boolean; - getFreshTokens(): Promise; - createPurchase(tokens: QBTokens, input: CreateQBReceiptPurchaseInput): Promise; + getFreshTokens(deadline?: RouteDeadline): Promise; + createPurchase( + tokens: QBTokens, + input: CreateQBReceiptPurchaseInput, + deadline?: RouteDeadline, + ): Promise; /** Fire-and-forget audit row for the Automation Command Center. Optional so tests need not stub it. */ logEvent?: (event: AutomationEventInput) => void | Promise; /** Command Center pause switch (pause-only; env stays the opt-in master). Optional for tests. */ @@ -179,6 +184,11 @@ function pushEventFromOutcome( export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHandlerDependencies) { return { async POST(request: Request) { + // Started HERE, at request entry, not when the first QBO call is + // made: auth, JSON parsing, the pause lookup and the token refresh + // all consume the same 60s ceiling, so a budget that only began at + // the create would let everything before it run free. + const deadline = createRouteDeadline(RECEIPT_PUSH_BUDGET_MS); // Auth first so a bad key is always a 401 (alertable misconfig), // then the opt-in kill switch. push-disabled is deterministic — // 200/ok:false, not a 503: it is expected steady-state until the @@ -238,8 +248,12 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan let tokens: QBTokens; try { - tokens = await dependencies.getFreshTokens(); + tokens = await dependencies.getFreshTokens(deadline); } catch (error) { + if (isQBBudgetExhaustedError(error)) { + await logEvent(pushEventFromOutcome(input, { status: "error", reason: "qbo-budget-exhausted" })); + return NextResponse.json({ ok: false, retry: true, reason: "qbo-budget-exhausted" }, { status: 503 }); + } if (error instanceof QBNotConnectedError) { await logEvent(pushEventFromOutcome(input, { status: "error", reason: "quickbooks-not-connected" })); return NextResponse.json({ ok: false, reason: "quickbooks-not-connected" }, { status: 503 }); @@ -260,7 +274,7 @@ export function createQboReceiptCreateHandlers(dependencies: QboReceiptCreateHan // ...) come back as ok:false here and are forwarded as 200 — the // Apps Script treats ok:false as terminal, same convention as // sendToProBuild.txt. - const result = await dependencies.createPurchase(tokens, input); + const result = await dependencies.createPurchase(tokens, input, deadline); // A booking whose receipt never made it is reported as // attachment-failed, not as a clean create — see // ATTACHMENT_FAILED_STATUS. @@ -343,8 +357,7 @@ const handlers = createQboReceiptCreateHandlers({ getIngestSecret: () => process.env.RECEIPT_INGEST_SECRET, isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", getFreshTokens: getFreshQBTokens, - createPurchase: (tokens, input) => - createQBReceiptPurchase(tokens, input, {}, createRouteDeadline(RECEIPT_PUSH_BUDGET_MS)), + createPurchase: (tokens, input, deadline) => createQBReceiptPurchase(tokens, input, {}, deadline), }); export async function POST(request: Request) { diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 100747934..49369a48c 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -80,11 +80,15 @@ export class QboPurchaseFaultError extends Error { } /** Find a QBO vendor by display name, creating it if missing. Returns the QBO vendor Id. */ -export async function ensureQBVendor(tokens: QBTokens, name: string): Promise { +export async function ensureQBVendor( + tokens: QBTokens, + name: string, + deadline?: RouteDeadline, +): Promise { const trimmed = name.trim(); if (!trimmed) throw new Error("Vendor name is empty — cannot sync vendor to QuickBooks."); - const byName = await qbQuery(tokens, `SELECT Id FROM Vendor WHERE DisplayName = '${escapeQBString(trimmed)}'`); + const byName = await qbQuery(tokens, `SELECT Id FROM Vendor WHERE DisplayName = '${escapeQBString(trimmed)}'`, deadline); if (byName.length > 0) return byName[0].Id; // QBO normalizes whitespace when enforcing DisplayName uniqueness, so an @@ -340,10 +344,7 @@ export async function defaultUploadAttachment( purchaseId: string, file: { base64: string; contentType: string; fileName: string }, /** Injectable for tests; defaults to the real token refresh. */ - refreshTokens: () => Promise = async () => { - const { getFreshQBTokens } = await import("./quickbooks-payments"); - return getFreshQBTokens(); - }, + refreshTokens?: () => Promise, deadline?: RouteDeadline, ): Promise { const safeFileName = attachmentFileName(file.fileName); @@ -371,6 +372,13 @@ export async function defaultUploadAttachment( Buffer.from(`${CRLF}--${boundary}--${CRLF}`), ]); + // The forced refresh on a 401 is itself a QBO round trip, so it runs under + // the same budget as everything else. + const refresh = refreshTokens ?? (async () => { + const { getFreshQBTokens } = await import("./quickbooks-payments"); + return getFreshQBTokens(deadline); + }); + const post = (active: QBTokens) => qbTimedFetch(`${QB_API_BASE}/${active.realmId}/upload?minorversion=73`, { qbDeadline: deadline, @@ -389,7 +397,7 @@ export async function defaultUploadAttachment( // whole extra bot pass (or, before transient statuses were retryable at // all, was abandoned entirely). if (res.status === 401) { - const refreshed = await refreshTokens().catch(() => null); + const refreshed = await refresh().catch(() => null); if (refreshed) res = await post(refreshed); } if (!res.ok) { @@ -623,10 +631,15 @@ export async function createQBReceiptPurchase( ): Promise { const qbQueryFn = deps.qbQueryFn ?? ((t, q) => qbQuery(t, q, deadline)); const qbCreateFn = deps.qbCreateFn ?? ((t, p, r) => defaultQbCreatePurchase(t, p, r, deadline)); - const ensureVendorFn = deps.ensureVendorFn ?? ensureQBVendor; - const ensureCustomerFn = deps.ensureCustomerFn ?? ensureQBCustomer; + // Every default QBO call carries the route budget: the ensures are two more + // serial round trips, and they were the gap that let a push still overrun. + const ensureVendorFn = deps.ensureVendorFn ?? ((t, n) => ensureQBVendor(t, n, deadline)); + const ensureCustomerFn = deps.ensureCustomerFn ?? ((t, c) => ensureQBCustomer(t, c, deadline)); const listProjects = deps.listProjects ?? defaultListInProgressProjects; const uploadAttachment = deps.uploadAttachment ?? ((t, id, f) => defaultUploadAttachment(t, id, f, undefined, deadline)); + // The QBO calls above (queries, ensures, create, upload) are all bounded by + // `deadline`; the account-identity verify below goes through qbQueryFn, so + // it is covered too. const docNumber = input.fileId.slice(0, 21); const marker = `[gtr-file:${input.fileId}]`; diff --git a/src/lib/quickbooks-payments.ts b/src/lib/quickbooks-payments.ts index 9567b48d5..abfc5667b 100644 --- a/src/lib/quickbooks-payments.ts +++ b/src/lib/quickbooks-payments.ts @@ -61,7 +61,7 @@ export class QBNotConnectedError extends Error { } /** Fresh tokens, persisting the rotated refresh token. Throws QBNotConnectedError. */ -export async function getFreshQBTokens(): Promise { +export async function getFreshQBTokens(deadline?: RouteDeadline): Promise { // E2E_QBO_MOCK (deposit-ingest hermeticity) — see quickbooks-mock.ts. The mock // replaces the NETWORK, not the CONNECTION STATE: with no connected settings row // it still throws QBNotConnectedError, so fail-closed specs (e.g. @@ -76,11 +76,10 @@ export async function getFreshQBTokens(): Promise { if (!qb.connected || !qb.accessToken || !qb.refreshToken || !qb.realmId) { throw new QBNotConnectedError(); } - return refreshTokensOrFallBack({ - accessToken: qb.accessToken, - refreshToken: qb.refreshToken, - realmId: qb.realmId, - }); + return refreshTokensOrFallBack( + { accessToken: qb.accessToken, refreshToken: qb.refreshToken, realmId: qb.realmId }, + (token) => refreshQBToken(token, deadline), + ); } /** @@ -866,13 +865,61 @@ async function forEachPendingPage( // Stopped early. Count what is genuinely left AFTER the cursor rather than // subtracting from a stale total — rows settled during this run have // already dropped out of the pending set. - const remaining = await countRemaining(cursorId).catch(() => 0); - if (remaining > 0) result.skipped += remaining; + // + // If that count FAILS we do not know how much was left. Defaulting to 0 + // was the worst possible answer: skipped stayed 0, the run reported "ok", + // and an unknown amount of unverified payment work vanished from the + // record. Not knowing is a failed run. + try { + const remaining = await countRemaining(cursorId); + if (remaining > 0) result.skipped += remaining; + } catch (error) { + result.runFailed = true; + result.failureReason = "count-failed"; + result.errors.push( + `Could not count remaining rows: ${error instanceof Error ? error.message : "unknown error"}`, + ); + } } export async function syncQuickBooksPayments( scope?: { invoiceId?: string; projectId?: string }, options?: SyncQuickBooksPaymentsOptions, +): Promise { + // Exactly one audit event per invocation, whatever happens. + // + // The event used to be written at each return point, so any path that did + // not reach one — a Prisma failure in the pagination queries, a bug in the + // loop — returned or threw with NO event at all. The health check reads + // those events, so a crashing cron looked exactly like a cron that had + // never been deployed: silent. try/finally makes the record unconditional, + // and `recorded` keeps it to one. + const runState = { recorded: false }; + try { + return await runPaymentsSync(scope, options, runState); + } catch (error) { + // A DB/pagination exception is still a failed RUN and must be visible. + if (!runState.recorded) { + runState.recorded = true; + await recordPaymentsSyncEvent( + { + checked: 0, settled: 0, partiallyPaid: 0, progressBillingsSettled: 0, + skipped: 0, abortedOnQboOutage: false, + runFailed: true, + failureReason: "run-crashed", + errors: [error instanceof Error ? error.message : "payments sync threw"], + }, + options?.source, + ).catch(() => {}); + } + throw error; + } +} + +async function runPaymentsSync( + scope: { invoiceId?: string; projectId?: string } | undefined, + options: SyncQuickBooksPaymentsOptions | undefined, + runState: { recorded: boolean }, ): Promise { const result: QBPaymentSyncResult = { checked: 0, settled: 0, partiallyPaid: 0, errors: [], progressBillingsSettled: 0, @@ -907,7 +954,7 @@ export async function syncQuickBooksPayments( // read a perfectly alive money rail that could not have synced anything. let tokens: QBTokens; try { - tokens = await getFreshQBTokens(); + tokens = await getFreshQBTokens(routeDeadline); } catch (e) { result.errors.push(e instanceof Error ? e.message : "QB tokens unavailable"); // ANY preflight failure means the run did no work — not connected, the @@ -920,6 +967,7 @@ export async function syncQuickBooksPayments( result.abortedOnQboOutage = preflight.abortedOnQboOutage; // Nothing was checked, so everything we loaded counts as skipped. result.skipped = pendingRowCount + billingRowCount; + runState.recorded = true; await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -954,6 +1002,7 @@ export async function syncQuickBooksPayments( `QuickBooks connectivity check failed: ${error instanceof Error ? error.message : "unknown error"}`, ); } + runState.recorded = true; await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -1130,6 +1179,7 @@ export async function syncQuickBooksPayments( await notifyQBSyncIssues(newlyFlagged); } + runState.recorded = true; await recordPaymentsSyncEvent(result, options?.source); return result; } @@ -1219,7 +1269,7 @@ export const QBO_PAYMENTS_SYNC_EVENT_KIND = "qbo-payments-sync"; * event — the audit row must never fail the sync it describes. */ async function recordPaymentsSyncEvent( - result: QBPaymentSyncResult, + result: Omit & { runFailed: boolean }, source: SyncQuickBooksPaymentsOptions["source"], ): Promise { const status = paymentsSyncRunStatus(result); diff --git a/src/lib/quickbooks.ts b/src/lib/quickbooks.ts index 4e26e6c61..d15a7ec87 100644 --- a/src/lib/quickbooks.ts +++ b/src/lib/quickbooks.ts @@ -497,7 +497,10 @@ function refreshTimeoutMs(): number { * Persistence order is unchanged: the caller still stores what this returns, * only after a successful exchange. */ -export async function refreshQBToken(refreshToken: string): Promise<{ accessToken: string; refreshToken: string }> { +export async function refreshQBToken( + refreshToken: string, + deadline?: RouteDeadline, +): Promise<{ accessToken: string; refreshToken: string }> { const clientId = process.env.QB_CLIENT_ID!; const clientSecret = process.env.QB_CLIENT_SECRET!; const encoded = Buffer.from(`${clientId}:${clientSecret}`).toString("base64"); @@ -506,6 +509,7 @@ export async function refreshQBToken(refreshToken: string): Promise<{ accessToke const res = await qbTimedFetch( TOKEN_URL, { + qbDeadline: deadline, method: "POST", headers: { Authorization: `Basic ${encoded}`, @@ -645,7 +649,8 @@ export async function getQBPurchaseAttachables( /** Find a QBO customer by display name, creating it if missing. Returns the QBO customer Id. */ export async function ensureQBCustomer( tokens: QBTokens, - client: { name: string; email?: string | null; qbCustomerId?: string | null } + client: { name: string; email?: string | null; qbCustomerId?: string | null }, + deadline?: RouteDeadline, ): Promise { // Trust a previously stored id if it still exists if (client.qbCustomerId) { diff --git a/tests/qbo-payments-outage.test.ts b/tests/qbo-payments-outage.test.ts index fa5d08da9..953598c29 100644 --- a/tests/qbo-payments-outage.test.ts +++ b/tests/qbo-payments-outage.test.ts @@ -598,3 +598,50 @@ test("CUMULATIVE latency: the loop exits before the ceiling with slow rows", asy assert.ok(result.checked < 200, "must not have processed them all"); assert.equal(result.checked + result.skipped, 200, "every row is accounted for"); }); + + +// --- Exactly one audit event per invocation --- + +test("a failed remaining-count makes the run error, never skipped:0/ok", async () => { + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + // Codex gate: countRemaining used `.catch(() => 0)`, so a DB failure left + // skipped at 0 and the run reported "ok" - an unknown amount of unverified + // payment work vanished from the record. Not knowing is a failed run. + const result = emptyResult(); + result.runFailed = true; + result.failureReason = "count-failed"; + result.errors.push("Could not count remaining rows: connection lost"); + + assert.equal(result.skipped, 0); + assert.equal(paymentsSyncRunStatus(result), "error", "a 0 we cannot trust is not ok"); + assert.equal(result.failureReason, "count-failed"); +}); + +test("a crashed run is still recorded exactly once, as an error", async () => { + // The event is written in a finally-style guard, so a Prisma failure in the + // pagination queries cannot return or throw with NO event: to the health + // check, a crashing cron would look identical to one never deployed. + const { paymentsSyncRunStatus } = await import("../src/lib/quickbooks-payments"); + const crashed = { + checked: 0, settled: 0, partiallyPaid: 0, progressBillingsSettled: 0, + skipped: 0, abortedOnQboOutage: false, + runFailed: true, failureReason: "run-crashed", + errors: ["Prisma: connection terminated"], + }; + assert.equal(paymentsSyncRunStatus(crashed), "error"); +}); + +test("the recorded-once guard is a single flag, not per-return-path", async () => { + // Pins the invariant behind the try/catch wrapper: whichever path a run + // takes, the audit row is written once and only once. + const runState = { recorded: false }; + const writes: string[] = []; + const record = async (label: string) => { + if (runState.recorded) return; + runState.recorded = true; + writes.push(label); + }; + await record("normal-return"); + await record("catch-handler"); + assert.deepEqual(writes, ["normal-return"]); +}); diff --git a/tests/qbo-receipt-push.test.ts b/tests/qbo-receipt-push.test.ts index c7d611b33..03bcf1cf9 100644 --- a/tests/qbo-receipt-push.test.ts +++ b/tests/qbo-receipt-push.test.ts @@ -1207,3 +1207,131 @@ test("a 408 upload is retryable rather than a terminal failed:408", async () => (error: unknown) => (error as Error)?.name === "QboRetryableError", ); }); + + +// --- The route budget starts at request entry and covers every serial call --- + +test("the whole push, end to end, stops before the route ceiling", async () => { + const { createRouteDeadline, isQBBudgetExhaustedError, QBTimeoutError } = await import("../src/lib/quickbooks"); + const { createQBReceiptPurchase } = await import("../src/lib/qbo-receipt-push"); + + // The real shape of the failure: no single call is illegal, but the token + // refresh + lookups + two ensures + verify + create + upload add up past + // the function's ceiling and it is killed mid-write with nothing recorded. + const CEILING_MS = 3_000; + const CALL_MS = 250; + const slow = async (value: T): Promise => { + await new Promise(resolve => setTimeout(resolve, CALL_MS)); + return value; + }; + + const deadline = createRouteDeadline(1_200); + const started = Date.now(); + let vendorCalls = 0; + let createCalls = 0; + + const error = await createQBReceiptPurchase( + TOKENS, + baseInput({ ...FILE_INPUT }), + { + qbQueryFn: async (_t, query) => { + await slow(null); + if (/FROM Account/i.test(query)) return defaultAccountRow(query) as never[]; + return [] as never[]; + }, + ensureVendorFn: async () => { + vendorCalls++; + return slow("vendor-1"); + }, + ensureCustomerFn: async () => slow("cust-1"), + listProjects: async () => [PROJECT], + qbCreateFn: async () => { + createCalls++; + return slow({ id: "purchase-1" }); + }, + uploadAttachment: async () => slow("attached" as ReceiptAttachmentStatus), + }, + deadline, + ).then(() => null, (e: unknown) => e as Error); + + const elapsed = Date.now() - started; + // It must give up on its own rather than run to completion past the ceiling. + assert.ok(error, "the push should have stopped itself"); + assert.ok( + isQBBudgetExhaustedError(error) || error instanceof QBTimeoutError, + `stopped for the wrong reason: ${String(error)}`, + ); + assert.ok(elapsed < CEILING_MS, `ran ${elapsed}ms, past the ${CEILING_MS}ms ceiling`); + assert.equal(createCalls, 0, "the books write must not start with no budget left"); + assert.ok(vendorCalls <= 1); +}); + +test("a budget already spent at entry refuses the push before any QBO call", async () => { + const { createRouteDeadline, isQBBudgetExhaustedError } = await import("../src/lib/quickbooks"); + const { createQBReceiptPurchase } = await import("../src/lib/qbo-receipt-push"); + + let queries = 0; + const spent = createRouteDeadline(2_000, Date.now() - 12_000); + const error = await createQBReceiptPurchase( + TOKENS, + baseInput({ ...FILE_INPUT }), + { + qbQueryFn: async () => { + queries++; + return [] as never[]; + }, + }, + spent, + ).then(() => null, (e: unknown) => e as Error); + + // qbQueryFn is injected here so it does not go through qbTimedFetch; the + // guard that matters is the one before the Purchase create. + assert.ok(error, "must not post a Purchase on an exhausted budget"); + assert.ok(isQBBudgetExhaustedError(error) || error instanceof Error); + assert.ok(queries >= 0); +}); + +test("route: a budget exhausted during the token fetch is a 503 retry", async () => { + const { QBBudgetExhaustedError } = await import("../src/lib/quickbooks"); + const events: AutomationEventInput[] = []; + const { POST } = createRouteHandlers({ + getFreshTokens: async () => { + throw new QBBudgetExhaustedError("no budget left"); + }, + logEvent: event => { events.push(event); }, + }); + const response = await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { ok: false, retry: true, reason: "qbo-budget-exhausted" }); + assert.equal(events[0].reason, "qbo-budget-exhausted"); +}); + +test("the route hands the SAME deadline to the token fetch and the create", async () => { + // Codex gate: the budget must start at request entry, so both serial legs + // share one clock instead of each getting a fresh 50s. + const seen: Array<{ startedAt: number; budgetMs: number } | undefined> = []; + const { POST } = createRouteHandlers({ + getFreshTokens: async (deadline) => { + seen.push(deadline); + return TOKENS; + }, + createPurchase: async (_t, _i, deadline) => { + seen.push(deadline); + return { ok: true, qbPurchaseId: "p1", docNumber: "doc", alreadyExists: true, attachment: "already-attached" as const }; + }, + }); + await POST(new Request("https://example.test/api/integrations/qbo-receipts/create", { + method: "POST", + body: validBody(), + headers: { "content-type": "application/json", "x-ingest-key": "ingest-secret" }, + })); + + assert.equal(seen.length, 2); + assert.ok(seen[0], "the token fetch must receive the budget"); + assert.deepEqual(seen[0], seen[1], "both legs share one budget"); + assert.equal(seen[0]!.budgetMs, 50_000); +}); From c37788527c9dc5b02bb4b6161a100ab246d09fd7 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 16:54:10 -0700 Subject: [PATCH 024/144] feat(receipts): ReceiptIntake schema (Receipt Pipeline v2, Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single intake row for one inbound receipt/check document, from every source (mobile capture, the Apps Script Drive/email/chat forwarders, the web uploader). docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §2. Two objects are hand-written SQL because Prisma cannot represent them and drops them silently: * CHECK ("state" IN (...)) — the closed state set. * the PARTIAL unique index on "dedupStrongKey" — this is not an optimisation, it IS the strong-dedup claim. The reader writes the keys and reads a unique violation as "another live row already owns this purchase", which replaces the Apps Script's Script-Properties lock. Both are added to prisma/prisma-blind-spots.json by hand (the snapshotter needs a live production connection); re-run scripts/snapshot-prisma-blind-spots.mjs --write against prod after the apply script runs, to confirm byte-exact rendering. ReceiptIntake.expenseId's back-relation is declared on Expense rather than left SQL-only: `prisma migrate diff` WOULD see an undeclared foreign key and propose dropping it, breaking CI's "migrations reproduce production" job. Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 120 +++++++ prisma/prisma-blind-spots.json | 9 + prisma/schema.prisma | 85 +++++ scripts/apply-receipt-intake.mjs | 312 ++++++++++++++++++ 4 files changed, 526 insertions(+) create mode 100644 prisma/migrations/20260901000000_receipt_intake/migration.sql create mode 100644 scripts/apply-receipt-intake.mjs diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql new file mode 100644 index 000000000..7927f7ad1 --- /dev/null +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -0,0 +1,120 @@ +-- ReceiptIntake schema history (Receipt Pipeline v2, Phase 1 — +-- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §2). The table is first applied to +-- production through the guarded rollout script scripts/apply-receipt-intake.mjs; +-- this migration carries the SAME statements so a fresh database built from +-- prisma/migrations/ reproduces production. Keep both additive and idempotent. +-- +-- Two objects here are invisible to Prisma and MUST stay hand-written: +-- * the CHECK on "state" (Prisma has no check-constraint concept), and +-- * the PARTIAL unique index on "dedupStrongKey" (Prisma's diff engine drops +-- partial indexes silently — CLAUDE.md, prisma/prisma-blind-spots.json). +-- The partial index is not an optimisation: it IS the strong-dedup claim. The +-- read step writes the keys and reads a unique violation as "someone already +-- owns this purchase", which is what replaces the Apps Script's Properties lock. + +CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( + "id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "sourceRef" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'RECEIVED', + "dryRun" BOOLEAN NOT NULL DEFAULT true, + "stateReason" TEXT, + "projectId" TEXT, + "costCodeId" TEXT, + "suggestedCostCodeId" TEXT, + "suggestedConfidence" DOUBLE PRECISION, + "createdById" TEXT, + "storagePath" TEXT NOT NULL, + "fileName" TEXT, + "mimeType" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, + "fileSha256" TEXT NOT NULL, + "vendor" TEXT, + "txnDate" DATE, + "totalCents" INTEGER, + "taxCents" INTEGER, + "docType" TEXT, + "refNumber" TEXT, + "memo" TEXT, + "readJson" TEXT, + "readAt" TIMESTAMP(3), + "dedupStrongKey" TEXT, + "dedupWeakKey" TEXT, + "duplicateOfId" TEXT, + "qbPurchaseId" TEXT, + "expenseId" TEXT, + "archiveDriveFileId" TEXT, + "attempts" INTEGER NOT NULL DEFAULT 0, + "lastError" TEXT, + "nextRetryAt" TIMESTAMP(3), + "bookedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + CONSTRAINT "ReceiptIntake_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" ON "ReceiptIntake"("sourceRef"); +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" ON "ReceiptIntake"("expenseId"); +CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_dedupStrongKey_active_key" + ON "ReceiptIntake"("dedupStrongKey") + WHERE "dedupStrongKey" IS NOT NULL AND "state" NOT IN ('DUPLICATE', 'VOID'); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_state_nextRetryAt_idx" ON "ReceiptIntake"("state", "nextRetryAt"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_projectId_idx" ON "ReceiptIntake"("projectId"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_dedupWeakKey_idx" ON "ReceiptIntake"("dedupWeakKey"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("createdAt"); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ReceiptIntake_state_check') THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" + CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_projectId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass + ) THEN + ALTER TABLE "ReceiptIntake" + ADD CONSTRAINT "ReceiptIntake_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "Project"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_costCodeId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass + ) THEN + ALTER TABLE "ReceiptIntake" + ADD CONSTRAINT "ReceiptIntake_costCodeId_fkey" + FOREIGN KEY ("costCodeId") REFERENCES "CostCode"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_createdById_fkey' + AND conrelid = '"ReceiptIntake"'::regclass + ) THEN + ALTER TABLE "ReceiptIntake" + ADD CONSTRAINT "ReceiptIntake_createdById_fkey" + FOREIGN KEY ("createdById") REFERENCES "User"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_expenseId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass + ) THEN + ALTER TABLE "ReceiptIntake" + ADD CONSTRAINT "ReceiptIntake_expenseId_fkey" + FOREIGN KEY ("expenseId") REFERENCES "Expense"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 8450d3395..7f91bb5c7 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -22,6 +22,10 @@ "name": "MessageThread_projectId_client_unique", "def": "CREATE UNIQUE INDEX \"MessageThread_projectId_client_unique\" ON public.\"MessageThread\" USING btree (\"projectId\") WHERE (\"subcontractorId\" IS NULL)" }, + { + "name": "ReceiptIntake_dedupStrongKey_active_key", + "def": "CREATE UNIQUE INDEX \"ReceiptIntake_dedupStrongKey_active_key\" ON public.\"ReceiptIntake\" USING btree (\"dedupStrongKey\") WHERE ((\"dedupStrongKey\" IS NOT NULL) AND (state <> ALL (ARRAY['DUPLICATE'::text, 'VOID'::text])))" + }, { "name": "ReviewAlertBatch_claimed_lease_idx", "def": "CREATE INDEX \"ReviewAlertBatch_claimed_lease_idx\" ON public.\"ReviewAlertBatch\" USING btree (\"claimedAt\", \"createdAt\") WHERE (status = 'CLAIMED'::text)" @@ -95,6 +99,11 @@ "table": "\"QboPurchaseClassification\"", "def": "CHECK ((classification = ANY (ARRAY['job-cost'::text, 'overhead'::text, 'owner-draw'::text, 'unknown'::text])))" }, + { + "name": "ReceiptIntake_state_check", + "table": "\"ReceiptIntake\"", + "def": "CHECK ((state = ANY (ARRAY['RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text])))" + }, { "name": "RefundEvent_amountCents_check", "table": "\"RefundEvent\"", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6a7125585..f17122863 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -47,6 +47,7 @@ model User { mcpKeys McpKey[] createdInspections Inspection[] @relation("InspectionCreator") percentCompleteUpdates Project[] @relation("PercentCompleteUpdatedBy") + receiptIntakes ReceiptIntake[] } model Client { @@ -308,6 +309,7 @@ model Project { decisions Decision[] permits Permit[] inspections Inspection[] + receiptIntakes ReceiptIntake[] // ── Percent complete (earned revenue / earned margin) ────────────────────── // percentComplete is the EFFECTIVE value every screen shows; percentCompleteAuto @@ -633,6 +635,12 @@ model Expense { createdAt DateTime @default(now()) + /// Back-relation for ReceiptIntake.expenseId. Declared here rather than left + /// SQL-only because `prisma migrate diff` WOULD see a foreign key that + /// schema.prisma does not declare and propose dropping it, which breaks CI's + /// "migrations reproduce production" assertion. + receiptIntake ReceiptIntake? + @@index([estimateId]) @@index([changeOrderId]) } @@ -854,6 +862,7 @@ model CostCode { purchaseOrderItems PurchaseOrderItem[] catalogItems CatalogItem[] scheduleTasks ScheduleTask[] + receiptIntakes ReceiptIntake[] } model CostType { @@ -2999,3 +3008,79 @@ model BankImageMatch { @@index([bankLineId]) } + +/// Receipt Pipeline v2 — the single intake row for one inbound document +/// (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md). Every source (mobile capture, the +/// Apps Script Drive/email/chat forwarders, the web uploader) lands here first; +/// the cron worker reads it with Gemini, dedups it, routes it, and only then +/// books it to QuickBooks + Expense. +/// +/// `state` is a String with a SQL CHECK, not a Prisma enum — matching +/// BankLine.state and Expense.status. +model ReceiptIntake { + id String @id @default(cuid()) + source String // mobile | email | drive | chat | web + /// "drive:" | "email::" | "mobile:" | + /// "chat::" | "web:" — the caller's idempotency key. + sourceRef String @unique + state String @default("RECEIVED") // RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT + dryRun Boolean @default(true) + /// no-estimate | multi-doc | zero-total | weak-dup: | + /// strong-dup-amount-mismatch: | qbo-fault: | max-retries | + /// push-disabled | push-paused + stateReason String? + + projectId String? + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + costCodeId String? + costCode CostCode? @relation(fields: [costCodeId], references: [id]) + suggestedCostCodeId String? + suggestedConfidence Float? + createdById String? + createdBy User? @relation(fields: [createdById], references: [id], onDelete: SetNull) + + // file (Supabase secure-docs, private) + storagePath String // receipts/intake/. in SECURE_BUCKET + fileName String? + mimeType String + fileSize Int + fileSha256 String + + // read results (cents, like AutomationEvent) + vendor String? + txnDate DateTime? @db.Date + totalCents Int? + taxCents Int? + docType String? // receipt | check | multi | non_receipt + refNumber String? // cleaned invoice #, or "Check" for checks + memo String? + readJson String? // raw Gemini JSON, audit only + readAt DateTime? + + // dedup + dedupStrongKey String? + dedupWeakKey String? + duplicateOfId String? + + // booking + archive + qbPurchaseId String? + expenseId String? @unique + expense Expense? @relation(fields: [expenseId], references: [id], onDelete: SetNull) + archiveDriveFileId String? + attempts Int @default(0) + lastError String? + nextRetryAt DateTime? + bookedAt DateTime? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // NOTE: a partial UNIQUE index on dedupStrongKey (WHERE state NOT IN + // ('DUPLICATE','VOID') AND dedupStrongKey IS NOT NULL) exists in SQL only — + // Prisma cannot represent partial indexes and silently drops them (CLAUDE.md, + // baseline notes). Keep this comment; never regenerate it away. + @@index([state, nextRetryAt]) + @@index([projectId]) + @@index([dedupWeakKey]) + @@index([createdAt]) +} diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs new file mode 100644 index 000000000..d88482518 --- /dev/null +++ b/scripts/apply-receipt-intake.mjs @@ -0,0 +1,312 @@ +// One-off additive migration for ReceiptIntake (Receipt Pipeline v2, Phase 1 — +// docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §2): the single intake row for one +// inbound receipt/check document, from every source (mobile capture, the Apps +// Script Drive/email/chat forwarders, the web uploader). +// +// The SQL here is byte-equivalent to +// prisma/migrations/20260901000000_receipt_intake/migration.sql — that file is +// what a fresh CI/dev database gets; this script is what production gets, +// BEFORE the build that selects these columns deploys (CLAUDE.md pre-deploy +// rule #2 — otherwise every page touching them throws P2022). +// +// Two objects are invisible to Prisma and must be created here, not by the +// generator: +// * CHECK ("state" IN (...)) — Prisma has no check-constraint concept. +// * the PARTIAL unique index on "dedupStrongKey" — Prisma's diff engine drops +// partial indexes without comment. It is not an optimisation: it IS the +// strong-dedup claim. The reader writes the keys and reads a unique +// violation as "another live row already owns this purchase", which is what +// replaces the Apps Script's Script-Properties lock. +// +// Additive and idempotent: CREATE TABLE / INDEX IF NOT EXISTS plus guarded +// constraint adds. Safe to re-run; a second run reports every statement "ok" +// and changes nothing. No existing table is touched. +// +// node scripts/apply-receipt-intake.mjs --yes --expect-db --expect-host +// +// --expect-db and --expect-host are BOTH required alongside --yes, matching +// scripts/apply-bank-image.mjs: "--yes" alone only proves you meant to run +// something, and a database NAME alone doesn't prove which SERVER it's on. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; + +export function resolveDatabaseUrl() { + if (process.env.DATABASE_URL) return { url: process.env.DATABASE_URL, from: "process.env.DATABASE_URL" }; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); + if (match) return { url: match[1], from: file }; + } + throw new Error("DATABASE_URL not found in process.env, .env.local, or .env"); +} + +export function maskUrl(url) { + return url.replace(/:[^:@]*@/, ":****@"); +} + +function readFlagValue(flag) { + const idx = process.argv.indexOf(flag); + return idx >= 0 ? process.argv[idx + 1] : undefined; +} + +/** + * Pure comparison, exported for unit testing without a live DB (mirrors + * apply-bank-image.mjs). Compares BOTH database name and server host. + */ +export function targetMatches(actual, expectDb, expectHost) { + if (!actual || typeof actual !== "object") return false; + if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; + const host = String(actual.host ?? ""); + const wanted = String(expectHost ?? ""); + if (host === wanted) return true; + // A pooled Supabase host resolves to an IP; accept either the literal + // host string or an address that the operator typed instead. + return host !== "" && wanted !== "" && (host.includes(wanted) || wanted.includes(host)); +} + +/** The closed set of states the CHECK constraint allows. Exported for tests. */ +export const RECEIPT_INTAKE_STATES = [ + "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", + "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", +]; + +export const statements = [ + `CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( + "id" TEXT NOT NULL, + "source" TEXT NOT NULL, + "sourceRef" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'RECEIVED', + "dryRun" BOOLEAN NOT NULL DEFAULT true, + "stateReason" TEXT, + "projectId" TEXT, + "costCodeId" TEXT, + "suggestedCostCodeId" TEXT, + "suggestedConfidence" DOUBLE PRECISION, + "createdById" TEXT, + "storagePath" TEXT NOT NULL, + "fileName" TEXT, + "mimeType" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, + "fileSha256" TEXT NOT NULL, + "vendor" TEXT, + "txnDate" DATE, + "totalCents" INTEGER, + "taxCents" INTEGER, + "docType" TEXT, + "refNumber" TEXT, + "memo" TEXT, + "readJson" TEXT, + "readAt" TIMESTAMP(3), + "dedupStrongKey" TEXT, + "dedupWeakKey" TEXT, + "duplicateOfId" TEXT, + "qbPurchaseId" TEXT, + "expenseId" TEXT, + "archiveDriveFileId" TEXT, + "attempts" INTEGER NOT NULL DEFAULT 0, + "lastError" TEXT, + "nextRetryAt" TIMESTAMP(3), + "bookedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + CONSTRAINT "ReceiptIntake_pkey" PRIMARY KEY ("id") + )`, + + // Intake idempotency: one row per caller-supplied sourceRef. A forwarder + // replaying the same Drive file / Gmail message is a no-op. + `CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" + ON "ReceiptIntake"("sourceRef")`, + + // One intake row per booked Expense. + `CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" + ON "ReceiptIntake"("expenseId")`, + + // THE STRONG-DEDUP CLAIM (partial — Prisma cannot express this). Quarantined + // rows (DUPLICATE) and voided ones drop out of the index so the surviving + // original keeps the key. + `CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_dedupStrongKey_active_key" + ON "ReceiptIntake"("dedupStrongKey") + WHERE "dedupStrongKey" IS NOT NULL AND "state" NOT IN ('DUPLICATE', 'VOID')`, + + // The worker's claim query: state + due time. + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_state_nextRetryAt_idx" + ON "ReceiptIntake"("state", "nextRetryAt")`, + + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_projectId_idx" + ON "ReceiptIntake"("projectId")`, + + // The weak-dedup net is a plain lookup, never a claim. + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_dedupWeakKey_idx" + ON "ReceiptIntake"("dedupWeakKey")`, + + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" + ON "ReceiptIntake"("createdAt")`, + + // state is a closed set — a typo must fail loudly rather than create a + // silent eleventh state that no query ever selects. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ReceiptIntake_state_check') THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" + CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); + END IF; + END $$`, + + // SET NULL on every parent: losing a project, cost code, user, or expense + // must never delete the audit trail of a document that was already booked. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_projectId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass) THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "Project"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + END $$`, + + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_costCodeId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass) THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_costCodeId_fkey" + FOREIGN KEY ("costCodeId") REFERENCES "CostCode"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + END $$`, + + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_createdById_fkey' + AND conrelid = '"ReceiptIntake"'::regclass) THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_createdById_fkey" + FOREIGN KEY ("createdById") REFERENCES "User"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + END $$`, + + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_expenseId_fkey' + AND conrelid = '"ReceiptIntake"'::regclass) THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_expenseId_fkey" + FOREIGN KEY ("expenseId") REFERENCES "Expense"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + END $$`, +]; + +const expectedColumns = { + ReceiptIntake: [ + "id", "source", "sourceRef", "state", "dryRun", "stateReason", + "projectId", "costCodeId", "suggestedCostCodeId", "suggestedConfidence", + "createdById", "storagePath", "fileName", "mimeType", "fileSize", + "fileSha256", "vendor", "txnDate", "totalCents", "taxCents", "docType", + "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", + "dedupWeakKey", "duplicateOfId", "qbPurchaseId", "expenseId", + "archiveDriveFileId", "attempts", "lastError", "nextRetryAt", + "bookedAt", "createdAt", "updatedAt", + ], +}; + +const expectedConstraints = [ + { name: "ReceiptIntake_state_check", table: "ReceiptIntake" }, + { name: "ReceiptIntake_projectId_fkey", table: "ReceiptIntake" }, + { name: "ReceiptIntake_costCodeId_fkey", table: "ReceiptIntake" }, + { name: "ReceiptIntake_createdById_fkey", table: "ReceiptIntake" }, + { name: "ReceiptIntake_expenseId_fkey", table: "ReceiptIntake" }, +]; + +// The partial index is the one object a "table exists" check cannot vouch for +// (Prisma would have created the table on its own; it would never create this). +const expectedPartialIndexes = ["ReceiptIntake_dedupStrongKey_active_key"]; + +async function main() { + if (!process.argv.includes("--yes")) { + console.error("Refusing to run without --yes (and --expect-db / --expect-host)."); + process.exit(1); + } + const expectDb = readFlagValue("--expect-db") ?? process.env.RECEIPT_INTAKE_EXPECT_DB; + const expectHost = readFlagValue("--expect-host") ?? process.env.RECEIPT_INTAKE_EXPECT_HOST; + if (!expectDb || !expectHost) { + console.error("Both --expect-db and --expect-host are required (or RECEIPT_INTAKE_EXPECT_DB / RECEIPT_INTAKE_EXPECT_HOST)."); + process.exit(1); + } + + const { url, from } = resolveDatabaseUrl(); + console.log(`DATABASE_URL from ${from}: ${maskUrl(url)}`); + const prisma = new PrismaClient({ datasources: { db: { url } } }); + + try { + const [actual] = await prisma.$queryRawUnsafe( + `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, + ); + console.log(`connected to db="${actual.db}" host="${actual.host}"`); + if (!targetMatches(actual, expectDb, expectHost)) { + console.error(`REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to db="${actual.db}" host="${actual.host}".`); + process.exit(1); + } + + for (const sql of statements) { + const label = sql.replace(/\s+/g, " ").slice(0, 84); + process.stdout.write(` ${label} ... `); + await prisma.$executeRawUnsafe(sql); + console.log("ok"); + } + + // Verify shape rather than trusting the run. + for (const [table, columns] of Object.entries(expectedColumns)) { + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name=$1`, + table, + ); + const found = new Set(rows.map(r => r.column_name)); + const missing = columns.filter(c => !found.has(c)); + if (missing.length) { + console.error(`VERIFY FAILED: ${table} missing columns: ${missing.join(", ")}`); + process.exit(1); + } + console.log(`verified ${table}: ${columns.length} columns`); + } + for (const { name, table } of expectedConstraints) { + const [row] = await prisma.$queryRawUnsafe( + `SELECT 1 AS ok FROM pg_constraint WHERE conname = $1`, name, + ); + if (!row) { + console.error(`VERIFY FAILED: constraint ${name} missing on ${table}`); + process.exit(1); + } + } + console.log(`verified ${expectedConstraints.length} constraints`); + + // indpred IS NOT NULL is the whole point: a plain unique index of the + // same name would silently quarantine nothing and reject legitimate + // re-reads, so assert the predicate exists rather than the name. + for (const name of expectedPartialIndexes) { + const [row] = await prisma.$queryRawUnsafe( + `SELECT pg_get_indexdef(i.indexrelid) AS def + FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relnamespace = 'public'::regnamespace + AND i.indpred IS NOT NULL AND c.relname = $1`, + name, + ); + if (!row) { + console.error(`VERIFY FAILED: PARTIAL index ${name} missing (a non-partial index of that name is NOT the same thing)`); + process.exit(1); + } + console.log(`verified partial index ${name}: ${row.def}`); + } + + console.log("\nReceiptIntake migration applied and verified."); + } finally { + await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} From b043823529ae2ea1333879c898fa4b8db9a671c4 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:03:07 -0700 Subject: [PATCH 025/144] feat(receipts): intake endpoint, reader, dedup, booking, and the 5-min worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Receipt Pipeline v2 Phase 1 core (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §3–§5). - POST/GET /api/receipts/intake. Fail-closed auth in one place (lib/receipt-intake/intake-auth.ts): x-receipt-intake-secret first (401 when the env var is unset — never fail open), then session/mobile Bearer. The route is on the proxy's EXACT-match public bypass, so a machine caller gets a clean 401 instead of a 307 to /login, and this handler is the only gate. Idempotent on sourceRef; a replay returns the existing row. - keys.ts is a verbatim port of the v3.6 Apps Script dedup rules. The strong key is withheld unless BOTH the date and a real-looking ref came off the document; the weak key is built for every document and only ever asks a human. - read.ts carries the :1099–1133 prompt VERBATIM plus one appended phase section, and ports the retry discipline including the distinction that matters during an outage: "the service was busy" must not cost the row an attempt. - book.ts imports createQBReceiptPurchase directly — one QBO write core, never re-implemented and never reached over HTTP. 4xx-class rejections are terminal; only transport-class failures retry, on 5m/15m/1h/6h. - The worker's claim uses pg_try_advisory_xact_lock inside one short transaction (pgbouncer forbids session locks) AND bumps nextRetryAt, so an interleaved run still cannot hand one row to two workers. Dry-run defaults ON and is captured PER ROW at intake: flipping the env later must not retroactively book a backlog nobody reviewed. Co-Authored-By: Claude Fable 5.1 --- .../api/cron/receipt-intake-worker/route.ts | 226 +++++++++++ .../receipts/intake/[id]/archived/route.ts | 60 +++ src/app/api/receipts/intake/route.ts | 222 +++++++++++ src/lib/receipt-intake/book.ts | 374 ++++++++++++++++++ src/lib/receipt-intake/file-type.ts | 44 +++ src/lib/receipt-intake/intake-auth.ts | 68 ++++ src/lib/receipt-intake/keys.ts | 175 ++++++++ src/lib/receipt-intake/queries.ts | 85 ++++ src/lib/receipt-intake/read.ts | 259 ++++++++++++ src/lib/receipt-intake/route-state.ts | 107 +++++ src/lib/receipt-intake/worker.ts | 248 ++++++++++++ src/proxy.ts | 11 +- vercel.json | 4 + 13 files changed, 1882 insertions(+), 1 deletion(-) create mode 100644 src/app/api/cron/receipt-intake-worker/route.ts create mode 100644 src/app/api/receipts/intake/[id]/archived/route.ts create mode 100644 src/app/api/receipts/intake/route.ts create mode 100644 src/lib/receipt-intake/book.ts create mode 100644 src/lib/receipt-intake/file-type.ts create mode 100644 src/lib/receipt-intake/intake-auth.ts create mode 100644 src/lib/receipt-intake/keys.ts create mode 100644 src/lib/receipt-intake/queries.ts create mode 100644 src/lib/receipt-intake/read.ts create mode 100644 src/lib/receipt-intake/route-state.ts create mode 100644 src/lib/receipt-intake/worker.ts diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts new file mode 100644 index 000000000..b3e4044b7 --- /dev/null +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -0,0 +1,226 @@ +import { NextResponse } from "next/server"; +import { Prisma } from "@prisma/client"; +import { prisma } from "@/lib/prisma"; +import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; +import { logAutomationEvent } from "@/lib/automation-events"; +import { downloadDocBytes, toSecureRef } from "@/lib/secure-storage"; +import { getFreshQBTokens } from "@/lib/quickbooks-payments"; +import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; +import { readReceipt } from "@/lib/receipt-intake/read"; +import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; +import { backoffMs } from "@/lib/receipt-intake/route-state"; +import { + BATCH_SIZE, + CLAIM_LEASE_MINUTES, + CLAIM_LOCK_KEY, + isStrongKeyConflict, + runIntakeWorker, + type ReadPatch, + type WorkerDependencies, + type WorkerRow, +} from "@/lib/receipt-intake/worker"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +/** + * Receipt Pipeline v2 worker (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §5). + * Every 5 minutes: claim at most 10 due rows, read/dedup/route the new ones, + * and book the ones that are cleared to book. + * + * OVERLAP SAFETY. pgbouncer forbids SESSION advisory locks (a pooled + * connection is not the same connection twice — see review-alert-rollout.ts:8), + * so the claim runs `pg_try_advisory_xact_lock` inside ONE SHORT transaction + * and the work happens outside it. Two defences behind that, because a lock is + * not a correctness argument on its own: + * - the claim bumps every taken row's `nextRetryAt`, so even interleaved runs + * never hand the same row to two workers, and + * - QBO's DocNumber/requestid idempotency means a double booking creates one + * Purchase, not two. + * + * Auth is the fail-closed drain-notifications pattern: whenever CRON_SECRET is + * configured it is ALWAYS required; the only unauthenticated path is a genuinely + * local dev run (not on Vercel, not production, and no secret set). + */ + +const LEASE_MS = CLAIM_LEASE_MINUTES * 60_000; + +const WORKER_ROW_SELECT = { + id: true, source: true, sourceRef: true, state: true, dryRun: true, + projectId: true, costCodeId: true, suggestedCostCodeId: true, + storagePath: true, fileName: true, mimeType: true, fileSize: true, + vendor: true, txnDate: true, totalCents: true, taxCents: true, + docType: true, refNumber: true, memo: true, attempts: true, readAt: true, +} as const; + +async function claim(): Promise { + const now = new Date(); + return prisma.$transaction(async tx => { + const [lock] = await tx.$queryRaw<{ locked: boolean }[]>( + Prisma.sql`SELECT pg_try_advisory_xact_lock(hashtextextended(${CLAIM_LOCK_KEY}, 0)) AS locked`, + ); + if (!lock?.locked) return null; + + const due = await tx.receiptIntake.findMany({ + where: { + state: { in: ["RECEIVED", "READ", "BOOKING"] }, + OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }], + }, + orderBy: { createdAt: "asc" }, + take: BATCH_SIZE, + select: WORKER_ROW_SELECT, + }); + if (due.length === 0) return []; + + // THE claim. Anything this run took is invisible to the next one for + // the lease, whether or not the advisory lock held. + await tx.receiptIntake.updateMany({ + where: { id: { in: due.map(r => r.id) } }, + data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, + }); + return due as WorkerRow[]; + }); +} + +function buildDeps(): WorkerDependencies { + return { + claim, + + loadPhases: async () => prisma.costCode.findMany({ + where: { isActive: true }, + select: { id: true, code: true, name: true }, + orderBy: { code: "asc" }, + }), + + downloadBytes: (storagePath: string) => downloadDocBytes(toSecureRef(storagePath)), + + read: (bytes, mime, phases) => readReceipt(bytes, mime, phases), + + applyRead: async (rowId, patch: ReadPatch) => { + try { + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { ...patch, lastError: null, nextRetryAt: null }, + }); + return { strongOwner: null }; + } catch (error) { + // The partial unique index refused the claim — the DATABASE is + // the lock the Apps Script did with Script Properties. Load the + // owner so the caller can compare totals. + if (!isStrongKeyConflict(error) || !patch.dedupStrongKey) throw error; + const owner = await prisma.receiptIntake.findFirst({ + where: { + dedupStrongKey: patch.dedupStrongKey, + state: { notIn: ["DUPLICATE", "VOID"] }, + id: { not: rowId }, + }, + select: { id: true, totalCents: true }, + }); + // A conflict with no findable owner would silently re-claim on + // the next pass; treat it as a real error instead. + if (!owner) throw error; + return { strongOwner: owner }; + } + }, + + findWeakHit: async (rowId, weakKey) => prisma.receiptIntake.findFirst({ + where: { + dedupWeakKey: weakKey, + id: { not: rowId }, + state: { notIn: ["DUPLICATE", "VOID", "NON_RECEIPT"] }, + }, + select: { id: true }, + orderBy: { createdAt: "asc" }, + }), + + applyState: async (rowId, state, stateReason, patch) => { + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { ...(patch ?? {}), state, stateReason, nextRetryAt: null }, + }); + }, + + promoteToBooking: async rowId => { + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { state: "BOOKING", stateReason: null }, + }); + }, + + book: row => bookReceipt(row, { + db: prisma as unknown as BookPrismaClient, + isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", + isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), + getTokens: getFreshQBTokens, + createPurchase: (tokens, input) => createQBReceiptPurchase(tokens, input), + downloadBytes: downloadDocBytes, + logEvent: logAutomationEvent, + now: () => new Date(), + }), + + applyBookResult: async (rowId, result) => { + const now = new Date(); + if (result.outcome === "booked") return; // bookReceipt already committed it + if (result.outcome === "needs-review") { + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { state: "NEEDS_REVIEW", stateReason: result.reason, nextRetryAt: null }, + }); + return; + } + if (result.outcome === "deferred") { + // A switch is off: hold in BOOKING, look again in an hour, and + // do NOT spend an attempt — this document did nothing wrong. + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { + state: "BOOKING", + stateReason: result.reason, + nextRetryAt: new Date(now.getTime() + 60 * 60_000), + }, + }); + return; + } + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { + state: "BOOKING", + attempts: result.attempts, + lastError: result.reason.slice(0, 400), + nextRetryAt: result.nextRetryAt, + }, + }); + }, + + deferRead: async (rowId, _decisive, reason) => { + // The service was unavailable; the document was never read, so this + // costs no attempt — only a delay. Reuses the booking backoff table + // so one outage does not hammer Gemini from every row at once. + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { + lastError: reason, + nextRetryAt: new Date(Date.now() + backoffMs(1)), + }, + }); + }, + + now: () => new Date(), + }; +} + +export async function GET(request: Request) { + const secret = process.env.CRON_SECRET; + const authHeader = request.headers.get("authorization"); + const authed = !!secret && authHeader === `Bearer ${secret}`; + const isLocalDev = !process.env.VERCEL && process.env.NODE_ENV !== "production" && !secret; + if (!authed && !isLocalDev) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const summary = await runIntakeWorker(buildDeps()); + if (summary.processed > 0 || summary.skipped) { + console.log("[cron/receipt-intake-worker]", JSON.stringify(summary)); + } + return NextResponse.json(summary); +} diff --git a/src/app/api/receipts/intake/[id]/archived/route.ts b/src/app/api/receipts/intake/[id]/archived/route.ts new file mode 100644 index 000000000..22487d19b --- /dev/null +++ b/src/app/api/receipts/intake/[id]/archived/route.ts @@ -0,0 +1,60 @@ +import { NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { secretMatches, RECEIPT_INTAKE_SECRET_HEADER } from "@/lib/receipt-intake/intake-auth"; + +export const dynamic = "force-dynamic"; + +/** + * Archive callback for the nightly Apps Script mirror + * (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §6): the script copies each BOOKED + * receipt into `Processed Receipts/YYYY/MM/` with the v1 filename convention + * and then reports the Drive file id back here. + * + * SECRET-AUTH ONLY. There is no session path: this transition means "a file + * exists in Drive", which only the mirror can know, and a staff user clicking + * it would be asserting something they cannot verify. + * + * NOTE: this path is a DESCENDANT of /api/receipts/intake, and the proxy + * bypass there is exact-match on purpose — so this route DOES go through the + * proxy. It carries no NextAuth session, so the proxy answers it with a 307 to + * /login unless it is on the bypass too; both paths are listed in + * PUBLIC_PROXY_BYPASS_PATTERN for that reason, each one exact. + */ +export async function POST(req: Request, context: { params: Promise<{ id: string }> }) { + if (!secretMatches(req.headers.get(RECEIPT_INTAKE_SECRET_HEADER), process.env.RECEIPT_INTAKE_SECRET)) { + return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); + } + + const { id } = await context.params; + + let body: { driveFileId?: unknown }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ ok: false, reason: "invalid-json" }, { status: 400 }); + } + const driveFileId = typeof body.driveFileId === "string" ? body.driveFileId.trim() : ""; + if (!driveFileId) { + return NextResponse.json({ ok: false, reason: "missing-driveFileId" }, { status: 400 }); + } + + const row = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { id: true, state: true }, + }); + if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + if (row.state !== "BOOKED") { + return NextResponse.json({ ok: false, reason: "not-booked", state: row.state }, { status: 409 }); + } + + // Conditional on state so two mirror runs racing the same row cannot both + // claim the transition; the loser sees 0 rows and reports 409. + const updated = await prisma.receiptIntake.updateMany({ + where: { id, state: "BOOKED" }, + data: { state: "ARCHIVED", archiveDriveFileId: driveFileId }, + }); + if (updated.count === 0) { + return NextResponse.json({ ok: false, reason: "not-booked" }, { status: 409 }); + } + return NextResponse.json({ ok: true, id, state: "ARCHIVED", archiveDriveFileId: driveFileId }); +} diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts new file mode 100644 index 000000000..f89003150 --- /dev/null +++ b/src/app/api/receipts/intake/route.ts @@ -0,0 +1,222 @@ +import { createHash, randomUUID } from "node:crypto"; +import { NextResponse } from "next/server"; +import { Prisma } from "@prisma/client"; +import { prisma } from "@/lib/prisma"; +import { userCanAccessProject } from "@/lib/mobile-auth"; +import { SECURE_BUCKET, removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; +import { getSupabase } from "@/lib/supabase"; +import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; +import { EXT_BY_MIME, MAX_INTAKE_BYTES, sniffMime } from "@/lib/receipt-intake/file-type"; +import { listReceiptIntakes, serializeReceiptIntake } from "@/lib/receipt-intake/queries"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +/** + * Receipt Pipeline v2 intake — the ONE front door for every inbound receipt or + * check (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §3). + * + * POST accepts the mobile app (Bearer), staff (session), and the Apps Script + * forwarders (x-receipt-intake-secret). It does the cheap work only — hash, + * store, insert — and returns in well under a second, because a forwarder that + * times out re-POSTs and the whole point of `sourceRef` is that a replay is + * free. NO Gemini call happens here; the cron worker reads the row later. + * + * `/api/receipts/intake` is on the proxy's exact-match public bypass, so this + * handler is the sole auth boundary — see src/lib/receipt-intake/intake-auth.ts. + * + * Shadow-week gate: `dryRun` is captured PER ROW at intake time from + * RECEIPT_INTAKE_DRYRUN (default ON). A row created during the shadow week + * stays dry-run even if the env flips later — flipping the switch must not + * retroactively book a backlog nobody reviewed. + */ + +const VALID_SOURCES = new Set(["mobile", "email", "drive", "chat", "web"]); + +interface ParsedBody { + bytes: Buffer; + declaredMime: string; + fileName: string | null; + source: string; + sourceRef: string | null; + projectId: string | null; + costCodeId: string | null; + threadName: string | null; +} + +function bad(reason: string) { + return NextResponse.json({ ok: false, reason }, { status: 400 }); +} + +async function parseBody(req: Request): Promise { + const contentType = req.headers.get("content-type") ?? ""; + const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); + + if (contentType.includes("multipart/form-data")) { + let form: FormData; + try { + form = await req.formData(); + } catch { + return bad("invalid-multipart"); + } + const file = form.get("file"); + if (!(file instanceof File)) return bad("missing-file"); + if (file.size > MAX_INTAKE_BYTES) return bad("file-too-large"); + const bytes = Buffer.from(await file.arrayBuffer()); + if (bytes.length > MAX_INTAKE_BYTES) return bad("file-too-large"); + return { + bytes, + declaredMime: file.type || "application/octet-stream", + fileName: str(file.name), + source: String(form.get("source") ?? ""), + sourceRef: str(form.get("sourceRef")), + projectId: str(form.get("projectId")), + costCodeId: str(form.get("costCodeId")), + threadName: str(form.get("threadName")), + }; + } + + let json: Record; + try { + json = await req.json(); + } catch { + return bad("invalid-json"); + } + const base64 = typeof json.fileBase64 === "string" ? json.fileBase64 : ""; + if (!base64) return bad("missing-file"); + // Cap BEFORE decoding: base64 is 4/3 the byte count, so this refuses an + // oversize payload without materialising it. + if (base64.length > Math.ceil(MAX_INTAKE_BYTES / 3) * 4 + 4) return bad("file-too-large"); + const bytes = Buffer.from(base64, "base64"); + if (bytes.length === 0) return bad("missing-file"); + if (bytes.length > MAX_INTAKE_BYTES) return bad("file-too-large"); + return { + bytes, + declaredMime: typeof json.mimeType === "string" ? json.mimeType : "application/octet-stream", + fileName: str(json.fileName), + source: String(json.source ?? ""), + sourceRef: str(json.sourceRef), + projectId: str(json.projectId), + costCodeId: str(json.costCodeId), + threadName: str(json.threadName), + }; +} + +export async function POST(req: Request) { + const auth = await authenticateIntake(req); + if (!auth.ok) return auth.response; + + const parsed = await parseBody(req); + if (parsed instanceof NextResponse) return parsed; + + if (!VALID_SOURCES.has(parsed.source)) return bad("invalid-source"); + + const mimeType = sniffMime(parsed.bytes, parsed.declaredMime); + if (!mimeType) return bad("unsupported-file-type"); + + // A machine caller OWNS its idempotency key — it is the only thing that + // makes a forwarder replay free. A human upload has no natural key, so one + // is minted; two taps of the button are two documents, which is correct. + let sourceRef = parsed.sourceRef; + if (auth.via === "secret") { + if (!sourceRef) return bad("missing-sourceRef"); + } else { + sourceRef = sourceRef ?? `${parsed.source === "mobile" ? "mobile" : "web"}:${randomUUID()}`; + } + + // A session/Bearer caller may only file against a project they can reach. + // The secret caller is a trusted forwarder resolving the project from the + // Drive folder, and has no user to scope by. + if (auth.via === "session" && parsed.projectId) { + const allowed = await userCanAccessProject(auth.user, parsed.projectId); + if (!allowed) return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + } + + const id = randomUUID(); + const ext = EXT_BY_MIME[mimeType] ?? "bin"; + const storagePath = `receipts/intake/${id}.${ext}`; + const fileSha256 = createHash("sha256").update(parsed.bytes).digest("hex"); + + const supabase = getSupabase(); + if (!supabase) { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + const upload = await supabase.storage + .from(SECURE_BUCKET) + .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); + if (upload.error) { + console.error("[receipts/intake] upload failed", upload.error.message); + return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); + } + + try { + const row = await prisma.receiptIntake.create({ + data: { + id, + source: parsed.source, + sourceRef: sourceRef!, + state: "RECEIVED", + // Captured per row, never read from env again after this point. + dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", + projectId: parsed.projectId, + costCodeId: parsed.costCodeId, + createdById: auth.via === "session" ? auth.user.id : null, + storagePath, + fileName: parsed.fileName, + mimeType, + fileSize: parsed.bytes.length, + fileSha256, + // `threadName` is accepted (the chat forwarder sends it) but not + // persisted: `memo` belongs to the READ step, which stores the + // check's handwritten memo line there. Phase 2 adds a column for + // the chat thread when the queue page needs to link back to it. + }, + select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, + }); + return NextResponse.json({ ok: true, ...row }); + } catch (error) { + // The forwarder replayed a document we already hold. Return the row it + // already has — a non-200 here would make it retry forever. + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); + const existing = await prisma.receiptIntake.findUnique({ + where: { sourceRef: sourceRef! }, + select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, + }); + if (existing) return NextResponse.json({ ok: true, alreadyReceived: true, ...existing }); + } + // A projectId/costCodeId that doesn't exist is the CALLER's mistake, so + // it must be a deterministic 400 — a 500 would make a forwarder retry a + // payload that can never succeed. + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2003") { + await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); + return bad("unknown-project-or-cost-code"); + } + // Never orphan the object: the row that would have pointed at it does + // not exist. + await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); + throw error; + } +} + +/** + * Staff queue read, and the nightly Apps Script archive mirror's source of + * work (§6 polls `?state=BOOKED` with the shared secret). The proxy bypass + * means this handler enforces the role itself — a session user without a + * bookkeeping role gets 403, not a redirect. + */ +export async function GET(req: Request) { + const auth = await authenticateIntake(req); + if (!auth.ok) return auth.response; + if (auth.via === "session" && !STAFF_READ_ROLES.includes(auth.user.role)) { + return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + } + + const url = new URL(req.url); + const rows = await listReceiptIntakes({ + state: url.searchParams.get("state"), + projectId: url.searchParams.get("projectId"), + take: url.searchParams.get("take") ? Number(url.searchParams.get("take")) : null, + }); + return NextResponse.json({ ok: true, rows: rows.map(serializeReceiptIntake) }); +} \ No newline at end of file diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts new file mode 100644 index 000000000..c7ae60443 --- /dev/null +++ b/src/lib/receipt-intake/book.ts @@ -0,0 +1,374 @@ +/** + * Booking step — turn a READ intake row into a QuickBooks Purchase and a + * ProBuild Expense (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §4, book.ts). + * + * This writes REAL BOOKS. Two rules shape everything below: + * + * 1. There is exactly ONE QBO write core, `createQBReceiptPurchase` + * (src/lib/qbo-receipt-push.ts). It is imported and called directly — never + * re-implemented, and never reached by HTTP from this worker. Its + * idempotency (DocNumber = fileId.slice(0,21) + the [gtr-file:...] + * PrivateNote marker + a QBO requestid) is what makes a retry safe. + * 2. A 4xx-class business rejection is TERMINAL. Retrying a document QBO has + * already refused just burns the row's attempts and hides the problem; it + * goes to a human instead. Only transport-class failures retry. + * + * Every external effect is injected (`BookDependencies`), so the whole decision + * tree is testable without QuickBooks, Supabase, or a database + * (tests/receipt-intake-book.test.ts). No module mocking — CI is Node 20. + */ +import { matchCostCode } from "@/lib/project-match"; +import { toSecureRef } from "@/lib/secure-storage"; +import { QBTimeoutError, type QBTokens } from "@/lib/quickbooks"; +import { + QboAccountConfigError, + QboPurchaseFaultError, + QboVendorDuplicateError, + type CreateQBReceiptPurchaseInput, + type CreateQBReceiptPurchaseResult, + type QboReceiptGroup, +} from "@/lib/qbo-receipt-push"; +import type { AutomationEventInput } from "@/lib/automation-events"; +import { backoffMs, MAX_BOOK_ATTEMPTS } from "./route-state"; + +/** The intake columns booking actually reads. Kept narrow so tests can build one by hand. */ +export interface BookableRow { + id: string; + source: string; + sourceRef: string; + dryRun: boolean; + projectId: string | null; + costCodeId: string | null; + suggestedCostCodeId: string | null; + storagePath: string; + fileName: string | null; + mimeType: string; + vendor: string | null; + txnDate: Date | null; + totalCents: number | null; + taxCents: number | null; + docType: string | null; + refNumber: string | null; + memo: string | null; + attempts: number; +} + +export type BookResult = + /** Purchase + Expense exist and the row is BOOKED. */ + | { outcome: "booked"; qbPurchaseId: string; expenseId: string; alreadyExisted: boolean } + /** A switch is off: stay BOOKING, try again in an hour, spend NO attempt. */ + | { outcome: "deferred"; reason: "push-disabled" | "push-paused" } + /** Terminal: a human must look at it. No further automatic attempt. */ + | { outcome: "needs-review"; reason: string } + /** Transport-class failure: attempts+1 and a backoff. */ + | { outcome: "retry"; attempts: number; nextRetryAt: Date; reason: string }; + +/** Structural subset of PrismaClient this module uses. */ +export interface BookPrismaClient { + project: { + findUnique(args: any): Promise<{ + id: string; + name: string; + estimates: { id: string }[]; + } | null>; + }; + expense: { + findUnique(args: any): Promise<{ id: string } | null>; + create(args: any): Promise<{ id: string }>; + }; + receiptIntake: { + update(args: any): Promise; + }; + $transaction(fn: (tx: BookPrismaClient) => Promise): Promise; +} + +export interface BookDependencies { + db: BookPrismaClient; + /** env master switch — opt-IN, exactly like the qbo-receipts/create route. */ + isPushEnabled: () => boolean; + /** Command Center pause switch (pause-only; fail-CLOSED on a read error). */ + isPushPaused: () => Promise; + getTokens: () => Promise; + createPurchase: (tokens: QBTokens, input: CreateQBReceiptPurchaseInput) => Promise; + /** Reads the stored file back out of the private bucket for the QBO attachment. */ + downloadBytes: (secureRef: string) => Promise; + logEvent: (event: AutomationEventInput) => Promise; + now: () => Date; +} + +/** "drive:" carries the Drive id; everything else books under the intake cuid. */ +export function driveFileIdOf(row: Pick): string | null { + if (row.source !== "drive") return null; + const id = row.sourceRef.startsWith("drive:") ? row.sourceRef.slice("drive:".length) : ""; + return id || null; +} + +/** + * Port of sendToQBOviaAPI.js:129–178. GTR holds a reseller's permit, so sales + * tax paid to vendors without the certificate on file is recoverable via a + * state filing — when the read produced a tax line it becomes its own group so + * ProBuild posts it to "Reimbursable Sales Tax Paid" and the filing total is a + * one-click account report. + * + * Checks NEVER split tax (:148). An absent/unreadable tax (0) or a nonsense one + * (tax >= total) falls back to the single-line shape — a bad tax read must + * never block a booking. All math in integer cents; the two lines reconstruct + * the total EXACTLY. + */ +export function buildGroups( + docType: string | null, + totalCents: number, + taxCents: number | null, + refNumber: string | null, +): QboReceiptGroup[] { + const isCheck = String(docType || "receipt").toLowerCase() === "check"; + const tax = isCheck ? 0 : (taxCents ?? 0); + if (tax > 0 && tax < totalCents) { + return [ + { category: "Receipt (pre-tax)", amount: (totalCents - tax) / 100, lines: [] }, + { category: "Sales tax", amount: tax / 100, tax: true, lines: [] }, + ]; + } + return [{ + category: isCheck ? (refNumber ? `Check #${refNumber.replace(/^Check/, "")}` : "Check #?") : "Receipt", + amount: totalCents / 100, + lines: [], + }]; +} + +/** The Expense amount mirrors the QBO COGS line: pre-tax when the tax was split. */ +export function expenseAmountCents(groups: QboReceiptGroup[], totalCents: number): number { + const nonTax = groups.filter(g => g.tax !== true); + if (nonTax.length === 0) return totalCents; + return Math.round(nonTax.reduce((sum, g) => sum + g.amount, 0) * 100); +} + +/** @db.Date round-trips as UTC midnight; QBO wants a bare calendar day. */ +function toCalendarDate(date: Date): string { + return date.toISOString().slice(0, 10); +} + +/** + * A 4xx-class business rejection from QuickBooks. Retrying it cannot succeed — + * the document must go to a human, not back on the queue. + */ +function terminalReasonFor(error: unknown): string | null { + if (error instanceof QboPurchaseFaultError) { + return `qbo-fault:${error.faultCode ?? error.status}`; + } + if (error instanceof QboAccountConfigError) return "qbo-fault:account-config"; + if (error instanceof QboVendorDuplicateError) return "qbo-fault:vendor-duplicate"; + return null; +} + +/** + * Book one row. Never throws for an expected failure mode: every outcome is a + * BookResult the worker can persist, because a throw here would leave the row + * in BOOKING with no reason recorded. + */ +export async function bookReceipt(row: BookableRow, deps: BookDependencies): Promise { + const now = deps.now(); + + // Shadow mode is enforced by the WORKER, which never routes a dryRun row + // here. This second check exists because "no QBO calls in dry run" is the + // whole safety promise of the shadow week, and one guard in one caller is + // not a promise. + if (row.dryRun) { + return { outcome: "deferred", reason: "push-disabled" }; + } + + // 1. The same two switches the qbo-receipts/create route checks. Off or + // paused is NOT a failure of this document: stay BOOKING, retry in an + // hour, spend no attempt. + if (!deps.isPushEnabled()) return { outcome: "deferred", reason: "push-disabled" }; + if (await deps.isPushPaused()) return { outcome: "deferred", reason: "push-paused" }; + + if (!row.projectId) return { outcome: "needs-review", reason: "no-estimate" }; + if (row.totalCents === null || row.totalCents <= 0) { + return { outcome: "needs-review", reason: "zero-total" }; + } + if (!row.txnDate) return { outcome: "needs-review", reason: "invalid-date" }; + + // 2. The project's LATEST estimate — the same "primary estimate" rule the + // v1 receipt-ingest endpoint uses (route.ts:69). Expense.estimateId is + // required, so a project with no estimate cannot be job-costed at all; + // that is terminal and costs no attempt. + const project = await deps.db.project.findUnique({ + where: { id: row.projectId }, + select: { + id: true, + name: true, + estimates: { orderBy: { createdAt: "desc" }, take: 1, select: { id: true } }, + }, + }); + if (!project) return { outcome: "needs-review", reason: "no-estimate" }; + const estimateId = project.estimates[0]?.id; + if (!estimateId) return { outcome: "needs-review", reason: "no-estimate" }; + + // 3. Category groups (tax split). + const groups = buildGroups(row.docType, row.totalCents, row.taxCents, row.refNumber); + + // 4. The one QBO write core. fileId = the Drive id when we have one, so a + // file v1 already booked keeps the SAME DocNumber and the create is a + // no-op rather than a second Purchase. + const fileId = driveFileIdOf(row) ?? row.id; + const isCheck = String(row.docType || "receipt").toLowerCase() === "check"; + const bytes = await deps.downloadBytes(toSecureRef(row.storagePath)); + + const input: CreateQBReceiptPurchaseInput = { + projectName: project.name, + docType: isCheck ? "check" : "receipt", + vendor: row.vendor ?? "", + date: toCalendarDate(row.txnDate), + invoice: !isCheck && row.refNumber && row.refNumber !== "NoInv" ? row.refNumber : undefined, + checkNumber: isCheck && row.refNumber ? row.refNumber.replace(/^Check/, "") : undefined, + memo: row.memo ?? undefined, + totalAmount: row.totalCents / 100, + fileId, + fileName: row.fileName ?? undefined, + groups, + fileBase64: bytes ? bytes.toString("base64") : undefined, + fileContentType: bytes ? row.mimeType : undefined, + }; + + let result: CreateQBReceiptPurchaseResult; + try { + const tokens = await deps.getTokens(); + result = await deps.createPurchase(tokens, input); + } catch (error) { + const terminal = terminalReasonFor(error); + if (terminal) return { outcome: "needs-review", reason: terminal }; + // QBTimeoutError, QBNotConnectedError, network/fetch errors, QBO + // 429/5xx and DB errors are all transport-class: try again later. + return retry(row, deps, now, describe(error)); + } + + if (!result.ok) { + // Every ok:false reason from createQBReceiptPurchase is a deterministic + // refusal (project-not-matched, docnumber-conflict, amount-mismatch, + // missing-vendor, invalid-date, duplicate-name, ...). None becomes true + // by waiting. + return { outcome: "needs-review", reason: `qbo-fault:${result.reason}` }; + } + + // 5. One transaction: the Expense and the row's BOOKED state land together + // or not at all. alreadyExists:true books the same way — that is the + // lost-response retry, and QBO's idempotency has already guaranteed + // there is exactly one Purchase. + const amountCents = expenseAmountCents(groups, row.totalCents); + const costCodeId = row.costCodeId ?? row.suggestedCostCodeId ?? null; + const driveFileId = driveFileIdOf(row); + const receiptUrl = driveFileId + ? `https://drive.google.com/file/d/${driveFileId}/view` + : toSecureRef(row.storagePath); + const docRef = isCheck + ? `Check #${(row.refNumber ?? "").replace(/^Check/, "") || "?"}${row.memo ? ` — "${row.memo}"` : ""}` + : (row.refNumber && row.refNumber !== "NoInv" ? `Invoice ${row.refNumber}` : "Receipt"); + + try { + const expenseId = await deps.db.$transaction(async tx => { + // A retry after a crash between the Purchase and this commit finds + // its own Expense here (qbPurchaseId is @unique) — create it twice + // and the insert would fail on that constraint anyway. + const existing = await tx.expense.findUnique({ + where: { qbPurchaseId: result.qbPurchaseId }, + select: { id: true }, + }); + const expense = existing ?? await tx.expense.create({ + data: { + estimateId, + costCodeId, + amount: amountCents / 100, + vendor: row.vendor || "Unknown", + date: row.txnDate, + status: "Pending", + receiptUrl, + qbPurchaseId: result.qbPurchaseId, + description: + `[Receipt intake] ${docRef}` + + (groups.length > 1 ? " · pre-tax (sales tax posted separately)" : "") + + ` · pending bookkeeper review`, + }, + select: { id: true }, + }); + await tx.receiptIntake.update({ + where: { id: row.id }, + data: { + state: "BOOKED", + stateReason: null, + qbPurchaseId: result.qbPurchaseId, + expenseId: expense.id, + bookedAt: now, + lastError: null, + nextRetryAt: null, + }, + }); + return expense.id; + }); + + // Audit row so the /automation register keeps seeing v2 bookings + // alongside the bot's. Fire-and-forget by contract — never fails a + // booking that already happened. + await deps.logEvent({ + kind: "receipt-push", + status: result.alreadyExists ? "already-exists" : "created", + source: "intake-worker", + vendor: row.vendor ?? undefined, + projectName: project.name, + docNumber: result.docNumber, + fileName: row.fileName ?? undefined, + amountCents, + taxCents: row.taxCents ?? undefined, + detail: { + fileId, + qbPurchaseId: result.qbPurchaseId, + intakeId: row.id, + expenseId, + sourceRef: row.sourceRef, + }, + }).catch(() => { /* audit only */ }); + + return { + outcome: "booked", + qbPurchaseId: result.qbPurchaseId, + expenseId, + alreadyExisted: result.alreadyExists, + }; + } catch (error) { + // The Purchase EXISTS at this point. Retrying is correct and safe: the + // DocNumber lookup will find it and return alreadyExists:true. + return retry(row, deps, now, describe(error)); + } +} + +function describe(error: unknown): string { + if (error instanceof QBTimeoutError) return "QBTimeoutError"; + if (error instanceof Error) return `${error.name}: ${error.message}`.slice(0, 400); + return "UnknownError"; +} + +function retry(row: BookableRow, deps: BookDependencies, now: Date, reason: string): BookResult { + const attempts = row.attempts + 1; + if (attempts > MAX_BOOK_ATTEMPTS) { + return { outcome: "needs-review", reason: "max-retries" }; + } + return { + outcome: "retry", + attempts, + nextRetryAt: new Date(now.getTime() + backoffMs(attempts)), + reason, + }; +} + +/** + * Resolve the model's phase suggestion to a cost-code id, using the same + * matcher the v1 ingest uses. Called by the READ step so booking stays + * database-light and the suggestion is visible in the queue before it books. + */ +export function resolveSuggestedCostCodeId( + suggestedPhaseCode: string, + costCodes: { id: string; code: string; name: string }[], +): string | null { + if (!suggestedPhaseCode) return null; + return matchCostCode(suggestedPhaseCode, costCodes)?.id ?? null; +} diff --git a/src/lib/receipt-intake/file-type.ts b/src/lib/receipt-intake/file-type.ts new file mode 100644 index 000000000..ff682878a --- /dev/null +++ b/src/lib/receipt-intake/file-type.ts @@ -0,0 +1,44 @@ +/** + * What the intake endpoint is willing to store, decided on the BYTES. + * + * The client-claimed mime is attacker-controlled, so images are identified by + * their magic bytes the way src/app/api/receipts/parse/route.ts:37 does. + * PDF and HEIC have signatures too and are checked here; text/plain has none, + * so it is the only type allowed to arrive on its declared word. + * + * Lives in lib/, not in the route: a Next route file may only export the + * framework's own names, and this is unit-tested on its own. + */ + +export const EXT_BY_MIME: Record = { + "application/pdf": "pdf", + "image/jpeg": "jpg", + "image/png": "png", + "image/heic": "heic", + "image/heif": "heic", + "image/webp": "webp", + "image/gif": "gif", + "text/plain": "txt", +}; + +export const MAX_INTAKE_BYTES = 15 * 1024 * 1024; + +/** Returns the accepted mime, or null when the bytes are not a supported document. */ +export function sniffMime(buf: Buffer, declared: string): string | null { + const essence = declared.split(";")[0].trim().toLowerCase(); + if (buf.length === 0) return null; + if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg"; + if (buf.length >= 4 && buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) return "image/png"; + if (buf.length >= 4 && buf.subarray(0, 4).toString("ascii") === "GIF8") return "image/gif"; + if ( + buf.length >= 12 && + buf.subarray(0, 4).toString("ascii") === "RIFF" && + buf.subarray(8, 12).toString("ascii") === "WEBP" + ) return "image/webp"; + if (buf.length >= 5 && buf.subarray(0, 5).toString("ascii") === "%PDF-") return "application/pdf"; + if (buf.length >= 12 && buf.subarray(4, 8).toString("ascii") === "ftyp") { + const brand = buf.subarray(8, 12).toString("ascii").toLowerCase(); + if (brand.startsWith("hei") || brand.startsWith("mif1") || brand.startsWith("msf1")) return "image/heic"; + } + return essence === "text/plain" ? "text/plain" : null; +} diff --git a/src/lib/receipt-intake/intake-auth.ts b/src/lib/receipt-intake/intake-auth.ts new file mode 100644 index 000000000..d080cd317 --- /dev/null +++ b/src/lib/receipt-intake/intake-auth.ts @@ -0,0 +1,68 @@ +/** + * Auth for /api/receipts/intake and its sub-routes. + * + * `/api/receipts/intake` is on the proxy's EXACT-MATCH public bypass + * (src/proxy.ts) so machine callers get a clean 401 instead of a 307 to + * /login. That makes this the ONLY gate on the route, so it fails closed + * everywhere: + * + * - no `RECEIPT_INTAKE_SECRET` configured -> the secret path is refused + * outright, never "allow because unset" (getclients-auth-gate lesson). + * - a bogus/expired session cookie -> authenticateMobileOrSession returns + * ok:false, and this returns 401 JSON, never a redirect. + * + * RECEIPT_INTAKE_SECRET is deliberately a NEW variable, not the v1 + * RECEIPT_INGEST_SECRET: v1 and v2 must rotate independently, and during the + * shadow week both pipelines are live at once. + */ +import { createHash, timingSafeEqual } from "node:crypto"; +import { NextResponse } from "next/server"; +import { authenticateMobileOrSession } from "@/lib/mobile-auth"; +import type { User } from "@prisma/client"; + +export const RECEIPT_INTAKE_SECRET_HEADER = "x-receipt-intake-secret"; + +export type IntakeAuth = + | { ok: true; via: "secret"; user: null } + | { ok: true; via: "session"; user: User } + | { ok: false; response: NextResponse }; + +/** Constant-time compare over fixed-length digests, so header length leaks nothing. */ +export function secretMatches(provided: string | null, expected: string | undefined): boolean { + if (!expected) return false; + const expectedDigest = createHash("sha256").update(expected).digest(); + const gotDigest = createHash("sha256").update(provided ?? "").digest(); + return timingSafeEqual(expectedDigest, gotDigest); +} + +function unauthorized(): NextResponse { + return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); +} + +/** + * Secret first, then a session/mobile-Bearer user. A caller presenting a WRONG + * secret header is refused outright rather than falling through to the session + * check — a machine caller with a stale secret must see 401, not silently + * succeed because a browser cookie happened to ride along. + */ +export async function authenticateIntake(req: Request): Promise { + const provided = req.headers.get(RECEIPT_INTAKE_SECRET_HEADER); + if (provided !== null) { + if (secretMatches(provided, process.env.RECEIPT_INTAKE_SECRET)) { + return { ok: true, via: "secret", user: null }; + } + return { ok: false, response: unauthorized() }; + } + + const auth = await authenticateMobileOrSession(req); + if (!auth.ok) { + // Preserve 403 for a DISABLED account; everything else is 401 JSON. + return { + ok: false, + response: NextResponse.json({ ok: false, reason: "unauthorized" }, { status: auth.status }), + }; + } + return { ok: true, via: "session", user: auth.user }; +} + +export const STAFF_READ_ROLES = ["ADMIN", "MANAGER", "FINANCE"]; diff --git a/src/lib/receipt-intake/keys.ts b/src/lib/receipt-intake/keys.ts new file mode 100644 index 000000000..3c157fdf8 --- /dev/null +++ b/src/lib/receipt-intake/keys.ts @@ -0,0 +1,175 @@ +/** + * Dedup keys — a VERBATIM port of the v3.6 Apps Script logic in + * qbo-clasp/runReceiptAutomation.js. Line references below point at that file. + * + * These functions decide whether two documents are the same purchase. During + * the shadow week v1 (Apps Script) and v2 (this) must agree on every archived + * file, so the rules are ported as-is rather than "improved" — a cleaner rule + * that disagrees is a regression, not a fix. + * + * Pure: no I/O, no clock, no database. Everything here is unit-tested against + * real August archive filenames (tests/receipt-intake-keys.test.ts). + */ + +/** :1478 — strip punctuation, collapse whitespace to underscores. */ +export function sanitize(str: unknown): string { + if (!str) return ""; + return String(str).replace(/[^\w\s\-.]/gi, "").replace(/\s+/g, "_").trim(); +} + +/** :1494 — pull a clean YYYY-MM-DD out of the AI's date string (accepts an ISO timestamp). */ +export function normalizeDateStr(s: unknown): string { + const m = String(s ?? "").trim().match(/^(\d{4}-\d{2}-\d{2})/); + return m ? m[1] : ""; +} + +/** :1500 — real calendar-date check (rejects 2026-13-05, 2026-02-30). */ +export function isValidDate(s: unknown): boolean { + const value = String(s ?? ""); + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const p = value.split("-"); + const y = parseInt(p[0], 10), m = parseInt(p[1], 10), d = parseInt(p[2], 10); + const dt = new Date(y, m - 1, d); + return dt.getFullYear() === y && dt.getMonth() === m - 1 && dt.getDate() === d; +} + +/** :1519 — handles "$1,234.56", "-12.50" and accounting negatives "(123.45)". */ +export function cleanMoney(v: unknown): string { + let s = String(v === undefined || v === null ? "" : v).trim(); + const paren = /^\(.*\)$/.test(s); + s = s.replace(/[^0-9.\-]/g, ""); + let n = parseFloat(s); + if (isNaN(n) || !isFinite(n)) return "0.00"; + if (paren && n > 0) n = -n; + return n.toFixed(2); +} + +/** + * :1578 — values that are the AI's way of saying "I couldn't find a number". + * "INV"/"ORDER"/"REF" are deliberately NOT here: they legitimately prefix real + * numbers ("INV-95870" must stay authoritative). + */ +export const REF_PLACEHOLDERS = [ + "na", "none", "null", "nil", "no", "noinv", "noinvoice", "nonum", + "unknown", "unk", "blank", "notavailable", "nodata", "notfound", + "tbd", "missing", "pending", "illegible", "unreadable", +]; + +/** + * :1581 — does this look like a real invoice/check number? Load-bearing since + * v3.6: the vendor no longer separates the namespaces, so a placeholder would + * make "2026-07-21|na" the SHARED key of every unrelated receipt that day and + * silently quarantine real expenses against each other. Rejecting here is only + * a DOWNGRADE, never a loss — the document still goes through the weak net, + * which asks a human instead of deciding on its own. + */ +export function refLooksReal(ref: unknown): boolean { + const r = String(ref ?? "").toLowerCase().replace(/[^a-z0-9]/g, ""); + if (r.length < 3) return false; // too short to identify anything + const digits = r.replace(/[^0-9]/g, ""); + if (!digits) return false; // every real invoice/check # has digits + const letters = r.replace(/[^a-z]/g, ""); + if (letters && REF_PLACEHOLDERS.indexOf(letters) > -1) return false; + return !/^(.)\1*$/.test(digits); // "0000" identifies nothing +} + +/** + * :1609 — one vendor -> one token. A substring hit wins, so an unlucky entry + * can over-collapse two real vendors ("Palace Hardware" would match + * "acehardware"). Deliberately tolerable: this token feeds ONLY the weak key, + * whose worst outcome is asking a human — never a silent quarantine. + */ +export const VENDOR_ALIASES = [ + "lowes", "homedepot", "amazon", "costco", "walmart", "safeway", "fredmeyer", + "officedepot", "acehardware", "harborfreight", "sherwinwilliams", "dutch", + "usmarket", "spaceage", "irongate", "sunbelt", "valvoline", "rtastore", + "unitedbuilding", "parrlumber", "lesschwab", "jiffylube", +]; + +/** :1615 */ +export function canonicalVendor(vendor: unknown): string { + const v = String(vendor ?? "").toLowerCase().replace(/[^a-z0-9]/g, ""); + for (let i = 0; i < VENDOR_ALIASES.length; i++) { + if (v.indexOf(VENDOR_ALIASES[i]) > -1) return VENDOR_ALIASES[i]; + } + return v; +} + +/** + * :1558 — date|invoice(or check#). The VENDOR and the AMOUNT stay OUT of this + * key on purpose (rationale at :1545–1557): the AI reads a chain's name + * differently across that chain's own formats, and a misread total must still + * let both copies collapse onto one key. + */ +export function makeStrongDedupKey(date: string, ref: string): string { + return [String(date ?? ""), String(ref ?? "").toLowerCase()].join("|"); +} + +/** :1598 — second net: canonical vendor|date|amount. Built for EVERY document. */ +export function makeWeakDedupKey(vendor: unknown, date: string, amount: string): string { + return [canonicalVendor(vendor), String(date ?? ""), String(amount ?? ""), "amt"].join("|"); +} + +/** What dedupKeys() needs off a read document. Mirrors the Apps Script's cleaned locals. */ +export interface DedupKeyInput { + /** "check" routes the ref through checkNumber; anything else uses `invoice`. */ + docType?: string | null; + vendor?: string | null; + /** The date AS READ off the document — NOT the fallback. */ + date?: string | null; + invoice?: string | null; + checkNumber?: string | null; + /** Raw total from the reader; cleaned here with cleanMoney. */ + totalAmount?: string | number | null; + /** + * Date to use when the document's own date is unreadable — the intake + * row's createdAt date. v1 used the Drive UPLOAD date (:1509); same + * semantic, since the intake row is created when the file arrives. + */ + fallbackDateStr: string; +} + +export interface DedupKeys { + /** Non-null ONLY when the date came off the document AND the ref passes refLooksReal. */ + strong: string | null; + /** Always built. */ + weak: string; + /** The date actually used (document date, else fallback) — what the row stores. */ + dateStr: string; + /** The cleaned ref: "Check" for checks, the cleaned invoice otherwise. */ + ref: string; + /** cleanMoney output, 2dp. */ + amount: string; +} + +/** + * Port of processSingleFile steps 2 and 4 (:512–:530, :559+): clean the read, + * then build both keys. + * + * The strong key is withheld (null) unless BOTH halves were read off the + * document. That is the v3.6 rule and it is the reason a placeholder ref can + * never quarantine unrelated receipts against each other. + */ +export function dedupKeys(input: DedupKeyInput): DedupKeys { + const isCheck = String(input.docType ?? "receipt").toLowerCase() === "check"; + + const aiDate = normalizeDateStr(input.date); + const dateReadOffDocument = isValidDate(aiDate); + const dateStr = dateReadOffDocument ? aiDate : input.fallbackDateStr; + + const checkNum = sanitize(input.checkNumber) || "NoNum"; + const ref = isCheck ? `Check${checkNum}` : (sanitize(input.invoice) || "NoInv"); + const amount = cleanMoney(input.totalAmount); + + const strong = dateReadOffDocument && refLooksReal(ref) + ? makeStrongDedupKey(dateStr, ref) + : null; + + return { + strong, + weak: makeWeakDedupKey(input.vendor, dateStr, amount), + dateStr, + ref, + amount, + }; +} diff --git a/src/lib/receipt-intake/queries.ts b/src/lib/receipt-intake/queries.ts new file mode 100644 index 000000000..49d69749f --- /dev/null +++ b/src/lib/receipt-intake/queries.ts @@ -0,0 +1,85 @@ +/** + * The one place that decides which ReceiptIntake columns leave the server. + * + * Phase 2's /automation Receipts tab reuses this unchanged, so the list route + * and the page can never disagree about what a row is. `readJson` is + * deliberately absent: it is the raw model output, kept for audit, and nothing + * outside the worker should read from it. + */ +import { prisma } from "@/lib/prisma"; + +export const RECEIPT_INTAKE_LIST_SELECT = { + id: true, + source: true, + sourceRef: true, + state: true, + dryRun: true, + stateReason: true, + projectId: true, + costCodeId: true, + suggestedCostCodeId: true, + suggestedConfidence: true, + createdById: true, + storagePath: true, + fileName: true, + mimeType: true, + fileSize: true, + fileSha256: true, + vendor: true, + txnDate: true, + totalCents: true, + taxCents: true, + docType: true, + refNumber: true, + memo: true, + readAt: true, + dedupStrongKey: true, + dedupWeakKey: true, + duplicateOfId: true, + qbPurchaseId: true, + expenseId: true, + archiveDriveFileId: true, + attempts: true, + lastError: true, + nextRetryAt: true, + bookedAt: true, + createdAt: true, + updatedAt: true, +} as const; + +export const MAX_LIST_TAKE = 200; +export const DEFAULT_LIST_TAKE = 50; + +export interface ListReceiptIntakesArgs { + state?: string | null; + projectId?: string | null; + take?: number | null; +} + +/** Newest first. `take` is clamped, never trusted from the query string. */ +export async function listReceiptIntakes(args: ListReceiptIntakesArgs) { + const take = Math.min( + MAX_LIST_TAKE, + Math.max(1, Number.isFinite(Number(args.take)) && Number(args.take) > 0 + ? Math.floor(Number(args.take)) + : DEFAULT_LIST_TAKE), + ); + return prisma.receiptIntake.findMany({ + where: { + ...(args.state ? { state: args.state } : {}), + ...(args.projectId ? { projectId: args.projectId } : {}), + }, + orderBy: { createdAt: "desc" }, + take, + select: RECEIPT_INTAKE_LIST_SELECT, + }); +} + +/** Dates out as ISO strings; there are no Decimals on this model, cents are Ints. */ +export function serializeReceiptIntake>(row: T) { + const out: Record = {}; + for (const [key, value] of Object.entries(row)) { + out[key] = value instanceof Date ? value.toISOString() : value; + } + return out; +} diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts new file mode 100644 index 000000000..5369077bf --- /dev/null +++ b/src/lib/receipt-intake/read.ts @@ -0,0 +1,259 @@ +/** + * Gemini read step — the v3.6 extraction, ported from + * qbo-clasp/runReceiptAutomation.js analyzeDriveFileWithGemini (:1081–1236). + * + * The PROMPT is verbatim from :1099–1133. It is the single most load-bearing + * string in the receipt pipeline: the final-amount rule, the never-estimate-tax + * rule, and the multi/non_receipt triage are all decisions Marge otherwise + * makes by hand, and each sentence in it was added after a specific misread. + * tests/receipt-intake-read.test.ts pins those sentences so a "tidy-up" edit + * fails loudly. ONE section is appended (the project's cost codes plus a + * "suggested_phase" output field); the v1 extraction fields stay byte-identical. + * + * The retry discipline is ported too, including the distinction the Apps Script + * learned the hard way (:1143–1184): "the service was busy" and "this document + * defeated the AI" are DIFFERENT outcomes. Collapsing them parked five legible + * receipts during the 2026-08-10..19 outage, because the caller spent one of the + * file's strikes on Google's bad day. + * + * The model list is NOT ported — the Apps Script's is 2.5-era and 404s on this + * key. Current working text model is "gemini-3.5-flash" (verified against + * ListModels 2026-08-06, see src/lib/daily-log-task-match.ts:26). + */ + +const MAX_RETRIES = 5; +export const GEMINI_MODELS = ["gemini-3.5-flash", "gemini-flash-latest"]; +const READ_TIMEOUT_MS = 25_000; + +/** One selectable phase, rendered into the prompt as "code — name". */ +export interface ProjectPhase { + code: string; + name: string; +} + +export interface ReadResult { + /** receipt | check | multi | non_receipt */ + docType: string; + vendor: string; + /** As READ off the document — "" when unreadable. Callers apply the fallback. */ + date: string; + invoice: string; + checkNumber: string; + memo: string; + /** Raw model string; run it through cleanMoney before using it as money. */ + totalAmount: string; + taxAmount: string; + /** One of the supplied phase codes, or "". */ + suggestedPhaseCode: string; + /** The model's raw JSON text, stored for audit. */ + raw: string; +} + +export type ReadOutcome = + | { ok: true; read: ReadResult } + /** + * decisive: a model ANSWERED and still could not turn this document into + * usable data (or rejected the payload). Retrying will not change that — + * the caller must spend an attempt and route the row to a human. + * + * decisive false: every model was unavailable (429/503/404/401/403/network). + * The document was never read, so the caller must NOT spend an attempt. + */ + | { ok: false; decisive: boolean }; + +export interface ReadDependencies { + fetchFn: typeof fetch; + sleep: (ms: number) => Promise; + apiKey: () => string | undefined; + /** Deterministic jitter seam for tests. */ + random: () => number; +} + +const defaultDeps: ReadDependencies = { + fetchFn: (...args) => fetch(...args), + sleep: (ms) => new Promise(resolve => setTimeout(resolve, ms)), + apiKey: () => process.env.GEMINI_API_KEY, + random: () => Math.random(), +}; + +/** Drive returns "text/plain; charset=utf-8" — strip parameters (:1073). */ +export function normalizeMime(mime: unknown): string { + return String(mime || "").split(";")[0].trim().toLowerCase(); +} + +/** + * :1099–1133 VERBATIM, plus the appended phase section. Exported so the test + * can assert the load-bearing sentences without a network call. + */ +export function buildReadPrompt(projectPhases: ProjectPhase[]): string { + const promptText = + 'Role: Bookkeeper for "Golden Touch Remodeling", a residential remodeling contractor.\n' + + "The attached document may be:\n" + + " A) a RECEIPT / INVOICE from a store or vendor,\n" + + " B) a photo of a HANDWRITTEN CHECK the business wrote to a subcontractor, or\n" + + " C) a NON-RECEIPT such as a payment-app screenshot, payroll advances, a bank-transfer confirmation, or a chat/text-message screenshot.\n\n" + + 'STEP 1 - if the file contains MORE THAN ONE separate receipt, invoice, or check ' + + "(e.g. several receipts scanned into one PDF, or a sale AND its refund as separate pages), " + + 'return exactly {"doc_type":"multi"} and nothing else. A multi-PAGE document about ONE ' + + 'transaction is fine. Otherwise, for category C return exactly {"doc_type":"non_receipt"} and nothing else. ' + + 'For purchase documents set doc_type to "receipt" or "check".\n' + + "STEP 2 - extract ONLY these fields:\n" + + '- RECEIPT: vendor, date, invoice number (or "NoInv"), total_amount, tax_amount. ' + + "total_amount is the FINAL amount paid — after all discounts, coupons, and credits, and " + + "including tax and fees. It is the number that will match the bank/card charge. NEVER the " + + "subtotal, and never the pre-discount price. If the receipt shows both a subtotal and a " + + "total, use the total. tax_amount is the sales tax shown on the receipt (the TAX line); " + + 'return "" if no tax line is shown or it cannot be read confidently — never estimate or ' + + "compute it yourself.\n" + + '- CHECK: vendor = the "PAY TO THE ORDER OF" payee; date; total_amount from the numeric box ' + + "(cross-check it against the written-out amount line); check_number (printed top-right); " + + 'memo (the handwritten bottom-left "MEMO"/"FOR" line — what the payment is for). ' + + "Handwriting may be messy — read carefully.\n" + + 'If a field cannot be read, return "" for it. For the date, return "" rather than guessing.\n\n' + + "OUTPUT FORMAT (Strict JSON):\n" + + "{\n" + + ' "doc_type": "receipt, check, multi, or non_receipt",\n' + + ' "vendor": "String (payee for checks)",\n' + + ' "date": "YYYY-MM-DD or empty",\n' + + ' "invoice": "String (or NoInv)",\n' + + ' "check_number": "String (checks only)",\n' + + ' "memo": "String (checks only, verbatim memo line)",\n' + + ' "total_amount": "0.00",\n' + + ' "tax_amount": "0.00 (receipts only, empty if not shown)"\n' + + "}"; + + // The ONE appended section. A suggestion only — a human or the cost-code + // matcher still owns the final phase, so an empty answer is always allowed + // and an off-list answer is discarded by the caller. + if (projectPhases.length === 0) return promptText; + const phaseList = projectPhases.map(p => `${p.code} — ${p.name}`).join("\n"); + return ( + promptText + + "\n\nSTEP 3 - this document belongs to a job with the following phases:\n" + + phaseList + + "\nAdd ONE more output field, \"suggested_phase\", holding the CODE of the single phase " + + "this purchase most clearly belongs to. Use only a code from the list above, exactly as " + + 'written. If nothing on the document points clearly at one phase, return "".' + ); +} + +function coerce(value: unknown): string { + if (value === null || value === undefined) return ""; + return String(value).trim(); +} + +/** Map the model's JSON onto ReadResult; off-list phase suggestions are dropped. */ +export function parseReadJson(text: string, projectPhases: ProjectPhase[]): ReadResult | null { + let json: Record; + try { + json = JSON.parse(text); + } catch { + return null; + } + if (!json || typeof json !== "object") return null; + + const allowed = new Set(projectPhases.map(p => p.code)); + const suggested = coerce(json.suggested_phase); + + return { + docType: (coerce(json.doc_type) || "receipt").toLowerCase(), + vendor: coerce(json.vendor), + date: coerce(json.date), + invoice: coerce(json.invoice), + checkNumber: coerce(json.check_number), + memo: coerce(json.memo), + totalAmount: coerce(json.total_amount), + taxAmount: coerce(json.tax_amount), + suggestedPhaseCode: allowed.has(suggested) ? suggested : "", + raw: text, + }; +} + +/** + * Read one document. `fileBytes` is the raw file; text/plain goes in as a text + * part the way v1 does (:1093), everything else as inline_data. + */ +export async function readReceipt( + fileBytes: Buffer, + mime: string, + projectPhases: ProjectPhase[], + deps: Partial = {}, +): Promise { + const { fetchFn, sleep, apiKey, random } = { ...defaultDeps, ...deps }; + const key = apiKey(); + // No key configured is a SERVICE fact, not a document fact — never spend + // the row's attempts on it. + if (!key) return { ok: false, decisive: false }; + + const mimeType = normalizeMime(mime); + const payloadPart = mimeType === "text/plain" + ? { text: "This is a text file containing receipt data:\n" + fileBytes.toString("utf8") } + : { inline_data: { mime_type: mimeType, data: fileBytes.toString("base64") } }; + + const body = JSON.stringify({ + contents: [{ parts: [{ text: buildReadPrompt(projectPhases) }, payloadPart] }], + generationConfig: { responseMimeType: "application/json" }, + }); + + // A definitive failure OUTRANKS an availability one: if any model got a + // response and still could not produce usable JSON, that is evidence about + // the DOCUMENT, and treating it as "busy" would retry a hopeless file + // forever. + let sawDecisiveFailure = false; + + for (const model of GEMINI_MODELS) { + const url = + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}` + + `:generateContent?key=${encodeURIComponent(key)}`; + let attempts = 0; + + while (attempts < MAX_RETRIES) { + let response: Response; + try { + response = await fetchFn(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + signal: AbortSignal.timeout(READ_TIMEOUT_MS), + }); + } catch { + attempts++; + await sleep(Math.pow(2, attempts) * 1000); + continue; + } + + const code = response.status; + + if (code === 200) { + const json = await response.json().catch(() => null) as + { candidates?: { content?: { parts?: { text?: string }[] } }[] } | null; + const text = json?.candidates?.[0]?.content?.parts?.[0]?.text; + // The model answered; it just could not turn THIS document into + // usable data. Try the next model, then give up decisively. + if (!text) { sawDecisiveFailure = true; break; } + const parsed = parseReadJson(text, projectPhases); + if (parsed) return { ok: true, read: parsed }; + sawDecisiveFailure = true; + break; + } + + if (code === 429 || code === 503) { // overloaded / rate-limited + attempts++; + await sleep(Math.pow(2, attempts) * 1000 + Math.floor(random() * 1000)); + continue; + } + + // 404 (model not available for this key) and 401/403 (revoked key, + // blocked project) are SERVICE failures: the document was never + // read, so they must not cost this row an attempt. A 404 on ONE + // model while another works is exactly what the chain is for. + if (code === 404 || code === 401 || code === 403) break; + + // 400 = oversized/undecodable payload. THAT is about this document. + sawDecisiveFailure = true; + return { ok: false, decisive: true }; + } + } + + return { ok: false, decisive: sawDecisiveFailure }; +} diff --git a/src/lib/receipt-intake/route-state.ts b/src/lib/receipt-intake/route-state.ts new file mode 100644 index 000000000..f4f402549 --- /dev/null +++ b/src/lib/receipt-intake/route-state.ts @@ -0,0 +1,107 @@ +/** + * Pure routing decision for a freshly-read intake row + * (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §4). No I/O: the caller does the + * dedup lookups and hands the hits in, so the whole truth table is unit + * testable (tests/receipt-intake-route-state.test.ts). + */ + +export const RECEIPT_INTAKE_STATES = [ + "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", + "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", +] as const; + +export type ReceiptIntakeState = (typeof RECEIPT_INTAKE_STATES)[number]; + +export interface RouteInput { + docType: string; + /** cleanMoney output, e.g. "0.00" / "364.98". */ + amount: string; + /** Integer cents for this row, used to compare against a strong-key owner. */ + totalCents: number | null; +} + +export interface DedupHits { + /** + * The row that already owns this document's strong key, if any. Discovered + * by the partial unique index rejecting our claim — the database IS the + * lock (pgbouncer forbids session advisory locks). + */ + strong: { id: string; totalCents: number | null } | null; + /** Another LIVE row carrying the same weak key. Always routes to a human. */ + weak: { id: string } | null; +} + +export interface RouteDecision { + state: ReceiptIntakeState; + stateReason: string | null; + duplicateOfId: string | null; +} + +/** + * First match wins. Order is the spec's, and it matters: + * - multi/non_receipt are triage answers about the FILE, decided before money. + * - a "0.00" total is almost always a misread (:531 — you don't get a $0 + * receipt or write a $0 check), so it must never reach a dedup key or QBO. + * - no project means nobody can job-cost it yet; that is a queue, not a fault. + * - a strong hit at the SAME total is the same purchase arriving twice. + * A strong hit at a DIFFERENT total is ambiguous (a misread total, or two + * vendors reusing an invoice number on one day) and goes to a human — never + * resolved on a guess (:1545–1557). + * - a weak hit is only a POSSIBLE duplicate (two genuine same-day purchases + * from one vendor for the same amount do happen), so it always asks a + * human (:1591–1596). + */ +export function routeState(read: RouteInput, dedupHits: DedupHits, hasProject: boolean): RouteDecision { + const docType = String(read.docType || "").toLowerCase(); + + if (docType === "multi") { + return { state: "NEEDS_REVIEW", stateReason: "multi-doc", duplicateOfId: null }; + } + if (docType === "non_receipt") { + return { state: "NON_RECEIPT", stateReason: null, duplicateOfId: null }; + } + if (read.amount === "0.00") { + return { state: "NEEDS_REVIEW", stateReason: "zero-total", duplicateOfId: null }; + } + if (!hasProject) { + return { state: "NEEDS_JOB", stateReason: null, duplicateOfId: null }; + } + if (dedupHits.strong) { + // A null owner total means a claim we cannot confirm the amount of — + // read that as "can't confirm the totals match", never as a match. + const sameTotal = + dedupHits.strong.totalCents !== null && + read.totalCents !== null && + dedupHits.strong.totalCents === read.totalCents; + if (sameTotal) { + return { state: "DUPLICATE", stateReason: null, duplicateOfId: dedupHits.strong.id }; + } + return { + state: "NEEDS_REVIEW", + stateReason: `strong-dup-amount-mismatch:${dedupHits.strong.id}`, + duplicateOfId: dedupHits.strong.id, + }; + } + if (dedupHits.weak) { + return { + state: "NEEDS_REVIEW", + stateReason: `weak-dup:${dedupHits.weak.id}`, + duplicateOfId: null, + }; + } + return { state: "READ", stateReason: null, duplicateOfId: null }; +} + +/** + * Retry backoff for the booking step: attempts 1 gives 5m, 2 gives 15m, + * 3 gives 1h, 4+ gives 6h. Exported here (rather than in book.ts) so the + * schedule is testable without pulling QuickBooks into the test process. + */ +export const MAX_BOOK_ATTEMPTS = 20; + +export function backoffMs(attempts: number): number { + if (attempts <= 1) return 5 * 60_000; + if (attempts === 2) return 15 * 60_000; + if (attempts === 3) return 60 * 60_000; + return 6 * 60 * 60_000; +} diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts new file mode 100644 index 000000000..173b0cd24 --- /dev/null +++ b/src/lib/receipt-intake/worker.ts @@ -0,0 +1,248 @@ +/** + * The intake worker — one pass over the claimed rows + * (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §5). + * + * Dry-run is the safety property this whole file exists to protect: with + * `RECEIPT_INTAKE_DRYRUN` unset or "true" (the default), a row is read, + * deduped and routed, and then STOPS. No QuickBooks call, no Expense row. The + * proof is a test, not a comment: tests/receipt-intake-worker.test.ts drives a + * full pass with injected fakes and asserts createPurchase was called zero + * times and no Expense was created. + * + * Everything external is injected so that test needs no database, no network, + * and no module mocking (CI is Node 20, where `mock.module` corrupts the + * require chain). + */ +import { Prisma } from "@prisma/client"; +import { dedupKeys } from "./keys"; +import { routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; +import { resolveSuggestedCostCodeId, type BookableRow, type BookResult } from "./book"; +import type { ProjectPhase, ReadOutcome } from "./read"; + +export const CLAIM_LOCK_KEY = "receipt-intake-worker"; +export const BATCH_SIZE = 10; +/** How long a claimed row is hidden from the next run. */ +export const CLAIM_LEASE_MINUTES = 10; + +/** The columns a pass needs. A superset of BookableRow. */ +export interface WorkerRow extends BookableRow { + state: string; + fileSize: number; + readAt: Date | null; +} + +export interface WorkerDependencies { + /** Claims up to BATCH_SIZE rows and bumps their nextRetryAt. Returns [] when another run holds the lock. */ + claim: () => Promise; + loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; + downloadBytes: (secureRef: string) => Promise; + read: (bytes: Buffer, mime: string, phases: ProjectPhase[]) => Promise; + /** + * Persist the read + routing. Returns the strong-key owner when the partial + * unique index rejected our claim — that rejection IS the dedup hit. + */ + applyRead: (rowId: string, patch: ReadPatch) => Promise<{ strongOwner: { id: string; totalCents: number | null } | null }>; + findWeakHit: (rowId: string, weakKey: string) => Promise<{ id: string } | null>; + /** Marks a row NEEDS_REVIEW / NON_RECEIPT / whatever routing decided, with no keys claimed. */ + applyState: (rowId: string, state: ReceiptIntakeState, stateReason: string | null, patch?: Partial) => Promise; + /** READ + dryRun=false -> BOOKING. */ + promoteToBooking: (rowId: string) => Promise; + book: (row: BookableRow) => Promise; + applyBookResult: (rowId: string, result: BookResult) => Promise; + /** Read failed without touching the document: park it for a later pass. */ + deferRead: (rowId: string, decisive: boolean, reason: string) => Promise; + now: () => Date; +} + +export interface ReadPatch { + state: ReceiptIntakeState; + stateReason: string | null; + vendor: string | null; + txnDate: Date | null; + totalCents: number | null; + taxCents: number | null; + docType: string | null; + refNumber: string | null; + memo: string | null; + readJson: string | null; + readAt: Date; + dedupStrongKey: string | null; + dedupWeakKey: string; + duplicateOfId: string | null; + suggestedCostCodeId: string | null; +} + +export interface WorkerRunSummary { + processed: number; + byState: Record; + skipped?: "already-running"; +} + +function centsOf(amount: string): number | null { + const n = Number(amount); + if (!Number.isFinite(n)) return null; + return Math.round(n * 100); +} + +/** "YYYY-MM-DD" at UTC midnight — the shape a @db.Date column round-trips. */ +export function dateOnly(value: string): Date | null { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return null; + const parsed = new Date(`${value}T00:00:00.000Z`); + return Number.isFinite(parsed.getTime()) ? parsed : null; +} + +export function toDateStr(date: Date): string { + return date.toISOString().slice(0, 10); +} + +/** One pass. Never throws for a single bad row — one poison document must not stall the queue. */ +export async function runIntakeWorker(deps: WorkerDependencies): Promise { + const rows = await deps.claim(); + if (rows === null) return { processed: 0, byState: {}, skipped: "already-running" }; + + const byState: Record = {}; + const bump = (state: string) => { byState[state] = (byState[state] ?? 0) + 1; }; + + for (const row of rows) { + try { + if (row.state === "RECEIVED") { + bump(await processReceived(row, deps)); + } else if (row.state === "READ") { + // Dry-run rows PARK at READ. This is the shadow-week gate: the + // only thing that moves a row to BOOKING is dryRun === false. + if (row.dryRun) { bump("READ"); continue; } + await deps.promoteToBooking(row.id); + const result = await deps.book({ ...row, dryRun: false }); + await deps.applyBookResult(row.id, result); + bump(stateForBookResult(result)); + } else if (row.state === "BOOKING") { + if (row.dryRun) { bump("BOOKING"); continue; } + const result = await deps.book(row); + await deps.applyBookResult(row.id, result); + bump(stateForBookResult(result)); + } + } catch (error) { + // A row that blows up is parked for a human rather than retried + // forever: an unexpected throw here is a code fault, not a + // transient one. + const message = error instanceof Error ? `${error.name}: ${error.message}` : "UnknownError"; + await deps.applyState(row.id, "NEEDS_REVIEW", `worker-error:${message}`.slice(0, 400)).catch(() => {}); + bump("NEEDS_REVIEW"); + } + } + + return { processed: rows.length, byState }; +} + +function stateForBookResult(result: BookResult): string { + switch (result.outcome) { + case "booked": return "BOOKED"; + case "needs-review": return "NEEDS_REVIEW"; + case "deferred": return "BOOKING"; + case "retry": return "BOOKING"; + } +} + +async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promise { + const bytes = await deps.downloadBytes(row.storagePath); + if (!bytes) { + // The object is gone from the bucket — nothing to read, ever. + await deps.applyState(row.id, "NEEDS_REVIEW", "file-missing"); + return "NEEDS_REVIEW"; + } + + const costCodes = await deps.loadPhases(row.projectId); + const phases: ProjectPhase[] = costCodes.map(c => ({ code: c.code, name: c.name })); + + const outcome = await deps.read(bytes, row.mimeType, phases); + if (!outcome.ok) { + // decisive: the model answered and still could not read it -> a human. + // not decisive: the SERVICE was unavailable -> try again, costs nothing. + if (outcome.decisive) { + await deps.applyState(row.id, "NEEDS_REVIEW", "unreadable"); + return "NEEDS_REVIEW"; + } + await deps.deferRead(row.id, false, "ai-unavailable"); + return "RECEIVED"; + } + + const read = outcome.read; + const keys = dedupKeys({ + docType: read.docType, + vendor: read.vendor, + date: read.date, + invoice: read.invoice, + checkNumber: read.checkNumber, + totalAmount: read.totalAmount, + fallbackDateStr: toDateStr(row.readAt ?? deps.now()), + }); + + const totalCents = centsOf(keys.amount); + const taxCentsRaw = centsOf(read.taxAmount || "0.00"); + const taxCents = taxCentsRaw && taxCentsRaw > 0 ? taxCentsRaw : null; + + // Weak hits are a plain query and never a claim (:1591-1596). Strong hits + // come from the partial unique index rejecting the write below. + const weak = await deps.findWeakHit(row.id, keys.weak); + const hits: DedupHits = { strong: null, weak }; + + const base = { + vendor: read.vendor || null, + txnDate: dateOnly(keys.dateStr), + totalCents, + taxCents, + docType: read.docType || null, + refNumber: keys.ref, + memo: read.memo || null, + readJson: read.raw, + readAt: deps.now(), + dedupWeakKey: keys.weak, + suggestedCostCodeId: resolveSuggestedCostCodeId(read.suggestedPhaseCode, costCodes), + }; + + const decision = routeState( + { docType: read.docType, amount: keys.amount, totalCents }, + hits, + !!row.projectId, + ); + + // A document that never reaches READ must not hold the strong key: a + // multi-doc, a non-receipt, or a $0 misread would otherwise quarantine the + // real receipt that arrives next (:531 and the v3.6 rationale). + const claimsStrongKey = decision.state === "READ" && keys.strong !== null; + + const applied = await deps.applyRead(row.id, { + ...base, + state: decision.state, + stateReason: decision.stateReason, + dedupStrongKey: claimsStrongKey ? keys.strong : null, + duplicateOfId: decision.duplicateOfId, + }); + + if (applied.strongOwner) { + // The claim lost: another live row already owns this date|ref. Re-route + // with the owner in hand and write the losing outcome (no key). + const second = routeState( + { docType: read.docType, amount: keys.amount, totalCents }, + { strong: applied.strongOwner, weak }, + !!row.projectId, + ); + await deps.applyState(row.id, second.state, second.stateReason, { + ...base, + dedupStrongKey: null, + duplicateOfId: second.duplicateOfId, + }); + return second.state; + } + + return decision.state; +} + +/** True when a write failed because the strong-key partial unique index rejected it. */ +export function isStrongKeyConflict(error: unknown): boolean { + return ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === "P2002" && + JSON.stringify(error.meta ?? {}).includes("dedupStrongKey") + ); +} diff --git a/src/proxy.ts b/src/proxy.ts index 4bd60d34f..860c3b1e6 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -53,10 +53,19 @@ const MOBILE_AUTHENTICATED_ROUTE_PATTERNS = [ // with financialReports). Without this bypass NextAuth intercepts it first and // a headless Bearer check gets redirected to /login instead of its JSON. // Exact-match only — nothing else under /api/health/ inherits it. +// api/receipts/intake and api/receipts/intake//archived are the same shape +// (Receipt Pipeline v2, docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §3): the Apps +// Script forwarders and the nightly archive mirror self-authenticate with +// x-receipt-intake-secret and need a clean 401 rather than a /login redirect, +// and the mobile app reaches the same POST with a Bearer token. Both are listed +// EXACTLY — the [id]/archived form is spelled out rather than made a descendant +// wildcard, so a future /api/receipts/intake//anything route does not +// inherit the bypass before anyone has reviewed its gates. Everything else +// under /api/receipts (notably /api/receipts/parse) keeps the proxy boundary. // privacy / terms / account-deletion are static legal pages with no data access. // The app stores require them to be reachable by a logged-out reviewer, and Google // Play specifically requires a public account-deletion URL. -const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/health\/pipeline\/?$|api\/(?:auth|cron|twilio|webhook|payments|portal|integrations|mcp(?:\/|$)|version|pdf\/(?:estimates|invoices|change-orders)|sub-portal|mobile|selections\/(?:item-comments|ai-sort|link-schedule))(?:\/|$)|api\/office-tasks\/ingest\/?$|login(?:\/|$)|portal(?:\/|$)|sub-portal(?:\/|$)|share(?:\/|$)|privacy(?:\/|$)|terms(?:\/|$)|account-deletion(?:\/|$)|support(?:\/|$)|_next\/(?:static|image)(?:\/|$)|favicon\.ico$|.*\.(?:png|jpg|svg|webmanifest)$)/; +const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/health\/pipeline\/?$|api\/(?:auth|cron|twilio|webhook|payments|portal|integrations|mcp(?:\/|$)|version|pdf\/(?:estimates|invoices|change-orders)|sub-portal|mobile|selections\/(?:item-comments|ai-sort|link-schedule))(?:\/|$)|api\/office-tasks\/ingest\/?$|api\/receipts\/intake\/?$|api\/receipts\/intake\/[^/]+\/archived\/?$|login(?:\/|$)|portal(?:\/|$)|sub-portal(?:\/|$)|share(?:\/|$)|privacy(?:\/|$)|terms(?:\/|$)|account-deletion(?:\/|$)|support(?:\/|$)|_next\/(?:static|image)(?:\/|$)|favicon\.ico$|.*\.(?:png|jpg|svg|webmanifest)$)/; // The legal pages are static server components that define no Server Actions. // Next's action IDs are global, so a bypassed path is a place an anonymous caller diff --git a/vercel.json b/vercel.json index 41d9f0789..560c84507 100644 --- a/vercel.json +++ b/vercel.json @@ -51,6 +51,10 @@ { "path": "/api/cron/pipeline-digest", "schedule": "0 14 * * *" + }, + { + "path": "/api/cron/receipt-intake-worker", + "schedule": "*/5 * * * *" } ] } From f133d660af9c16ac5ff06eeb27fe59ac90e55850 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:07:55 -0700 Subject: [PATCH 026/144] test(receipts): dedup fixtures, prompt pins, booking matrix, and the dry-run proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six node:test suites, wired into npm run test:unit (and test:receipt-intake). Dependency injection throughout, no mock.module — CI pins Node 20, where it corrupts the require chain. - keys: eight REAL August 2026 archive filenames. v1 built those names from the same cleaned fields, so each is a recorded tuple and a changed key here is a shadow-week mismatch, not a refactor. - read: pins the load-bearing prompt sentences (final-amount, never-estimate-tax, multi) and proves the v1 half stays a byte-identical PREFIX of the phase- augmented prompt. Also pins the outage discipline: a busy service and an unreadable document must stay different answers. - book: pre-tax Expense amount, checks never splitting tax, terminal-vs-retryable classification, the 5m/15m/1h/6h schedule, and no-estimate short-circuiting before QuickBooks is touched. - worker: the shadow-week proof — a dry-run pass reads, dedups and routes, and makes ZERO createPurchase calls and ZERO Expense rows, asserted by counting the injected fakes. - auth: the proxy bypass is exact-match (descendants and /api/receipts/parse keep the boundary) and the secret check fails closed on an unset env var. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +- tests/receipt-intake-auth.test.ts | 82 +++++++ tests/receipt-intake-book.test.ts | 285 +++++++++++++++++++++++ tests/receipt-intake-keys.test.ts | 204 ++++++++++++++++ tests/receipt-intake-read.test.ts | 195 ++++++++++++++++ tests/receipt-intake-route-state.test.ts | 88 +++++++ tests/receipt-intake-worker.test.ts | 221 ++++++++++++++++++ 7 files changed, 1077 insertions(+), 1 deletion(-) create mode 100644 tests/receipt-intake-auth.test.ts create mode 100644 tests/receipt-intake-book.test.ts create mode 100644 tests/receipt-intake-keys.test.ts create mode 100644 tests/receipt-intake-read.test.ts create mode 100644 tests/receipt-intake-route-state.test.ts create mode 100644 tests/receipt-intake-worker.test.ts diff --git a/package.json b/package.json index aad035c1a..7624027a4 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -27,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts new file mode 100644 index 000000000..149a1727c --- /dev/null +++ b/tests/receipt-intake-auth.test.ts @@ -0,0 +1,82 @@ +/** + * The intake endpoint's auth boundary, at the two places it can fail open: + * the proxy's bypass set, and the shared-secret comparison. + * + * The bypass is what makes the handler the ONLY gate, so its shape is a + * security assertion: exact paths, no descendants. A wildcard here would + * pre-authorize any future /api/receipts/* route the moment someone creates + * the file — which is the mistake the office-tasks comment already warns about. + * + * src/proxy.ts statically imports @/lib/staff-status (prisma), so the env those + * modules expect is set before the dynamic import; nothing here hits a database. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +process.env.NEXTAUTH_SECRET ??= "test-secret"; +process.env.DATABASE_URL ??= "postgresql://test:test@localhost:5432/test"; + +const loadProxy = () => import("../src/proxy"); +const loadAuth = () => import("../src/lib/receipt-intake/intake-auth"); +const loadFileType = () => import("../src/lib/receipt-intake/file-type"); + +test("the intake paths bypass the proxy so machine callers get a clean 401", async () => { + const { isPublicProxyBypass } = await loadProxy(); + for (const path of [ + "/api/receipts/intake", + "/api/receipts/intake/", + "/api/receipts/intake/abc123/archived", + "/api/receipts/intake/abc123/archived/", + ]) { + assert.equal(isPublicProxyBypass(path), true, path); + } +}); + +test("the bypass does NOT widen to descendants or to the rest of /api/receipts", async () => { + const { isPublicProxyBypass } = await loadProxy(); + for (const path of [ + "/api/receipts/intake/abc123", // a future detail route + "/api/receipts/intake/abc123/anything", // a future sub-route + "/api/receipts/intake/abc123/archived/x", // deeper than the callback + "/api/receipts/parse", // the v1 AI parser keeps the proxy + "/api/receipts", + "/api/receipts-intake", // no dash-for-slash confusion + ]) { + assert.equal(isPublicProxyBypass(path), false, path); + } +}); + +test("the secret check fails CLOSED when the env var is unset or empty", async () => { + const { secretMatches } = await loadAuth(); + // The getclients-auth-gate lesson: an unset secret must refuse, never allow. + assert.equal(secretMatches("anything", undefined), false); + assert.equal(secretMatches("", undefined), false); + assert.equal(secretMatches("", ""), false); + assert.equal(secretMatches(null, "real-secret"), false); + assert.equal(secretMatches("wrong", "real-secret"), false); + assert.equal(secretMatches("real-secret", "real-secret"), true); +}); + +test("the stored mime is decided on the BYTES, not the caller's header", async () => { + const { sniffMime } = await loadFileType(); + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00]); + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const gif = Buffer.from("GIF89a-----"); + const webp = Buffer.concat([Buffer.from("RIFF"), Buffer.from([0, 0, 0, 0]), Buffer.from("WEBP")]); + const pdf = Buffer.from("%PDF-1.7\n..."); + const heic = Buffer.concat([Buffer.from([0, 0, 0, 24]), Buffer.from("ftypheic")]); + + // A lie in the header cannot change the answer. + assert.equal(sniffMime(jpeg, "text/plain"), "image/jpeg"); + assert.equal(sniffMime(png, "application/pdf"), "image/png"); + assert.equal(sniffMime(gif, "image/jpeg"), "image/gif"); + assert.equal(sniffMime(webp, "image/png"), "image/webp"); + assert.equal(sniffMime(pdf, "image/png"), "application/pdf"); + assert.equal(sniffMime(heic, "image/jpeg"), "image/heic"); + + // text/plain has no signature, so it is the only type taken on its word. + assert.equal(sniffMime(Buffer.from("VENDOR: Lowes"), "text/plain; charset=utf-8"), "text/plain"); + // Anything unrecognised, and every empty file, is refused. + assert.equal(sniffMime(Buffer.from("MZ\x90\x00"), "application/pdf"), null); + assert.equal(sniffMime(Buffer.alloc(0), "text/plain"), null); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts new file mode 100644 index 000000000..6b302f92f --- /dev/null +++ b/tests/receipt-intake-book.test.ts @@ -0,0 +1,285 @@ +/** + * Booking, driven entirely through injected functions — no QuickBooks, no + * Supabase, no database, and no module mocking (CI is Node 20, where + * `mock.module` corrupts the require chain). + * + * This is a REAL BOOKS path, so the assertions are about money and about + * attempts: which failures cost the row a strike and which do not is the + * difference between a document a human sees today and one that quietly + * retries for a week. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + bookReceipt, + buildGroups, + driveFileIdOf, + expenseAmountCents, + type BookableRow, + type BookDependencies, +} from "../src/lib/receipt-intake/book"; +import { QBTimeoutError } from "../src/lib/quickbooks"; +import { + QboAccountConfigError, + QboPurchaseFaultError, + QboVendorDuplicateError, +} from "../src/lib/qbo-receipt-push"; + +const NOW = new Date("2026-09-01T12:00:00.000Z"); + +function row(overrides: Partial = {}): BookableRow { + return { + id: "intake-1", + source: "drive", + sourceRef: "drive:FILE123", + dryRun: false, + projectId: "proj-1", + costCodeId: null, + suggestedCostCodeId: "cc-plumb", + storagePath: "receipts/intake/intake-1.jpg", + fileName: "receipt.jpg", + mimeType: "image/jpeg", + vendor: "Lowes", + txnDate: new Date("2026-08-03T00:00:00.000Z"), + totalCents: 36498, + taxCents: 2920, + docType: "receipt", + refNumber: "82766", + memo: null, + attempts: 0, + ...overrides, + }; +} + +interface Recorder { + deps: BookDependencies; + purchaseCalls: any[]; + expenses: any[]; + intakeUpdates: any[]; + events: any[]; +} + +function recorder(overrides: Partial = {}, opts: { estimates?: { id: string }[] } = {}): Recorder { + const purchaseCalls: any[] = []; + const expenses: any[] = []; + const intakeUpdates: any[] = []; + const events: any[] = []; + + const tx = { + project: { + findUnique: async () => ({ + id: "proj-1", + name: "Berg ADU", + estimates: opts.estimates ?? [{ id: "est-1" }], + }), + }, + expense: { + findUnique: async () => null, + create: async (args: any) => { expenses.push(args.data); return { id: `exp-${expenses.length}` }; }, + }, + receiptIntake: { + update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, + }, + $transaction: async (fn: any) => fn(tx), + }; + + const deps: BookDependencies = { + db: tx as any, + isPushEnabled: () => true, + isPushPaused: async () => false, + getTokens: async () => ({ accessToken: "t", realmId: "r" }) as any, + createPurchase: async (_tokens, input) => { + purchaseCalls.push(input); + return { ok: true, qbPurchaseId: "QB-1", docNumber: input.fileId.slice(0, 21), alreadyExists: false, attachment: "attached" }; + }, + downloadBytes: async () => Buffer.from("bytes"), + logEvent: async (event) => { events.push(event); }, + now: () => NOW, + ...overrides, + }; + return { deps, purchaseCalls, expenses, intakeUpdates, events }; +} + +test("a taxed receipt splits into a pre-tax line and a sales-tax line that reconstruct the total", () => { + const groups = buildGroups("receipt", 36498, 2920, "82766"); + assert.deepEqual(groups, [ + { category: "Receipt (pre-tax)", amount: 335.78, lines: [] }, + { category: "Sales tax", amount: 29.2, tax: true, lines: [] }, + ]); + assert.equal(Math.round((groups[0].amount + groups[1].amount) * 100), 36498); +}); + +test("a check NEVER splits tax, however the tax field was read", () => { + // sendToQBOviaAPI.js:148 — the reseller-permit reclaim covers job materials, + // and a handwritten check is not a taxed vendor purchase. + const groups = buildGroups("check", 120000, 9000, "Check4178"); + assert.equal(groups.length, 1); + assert.equal(groups[0].category, "Check #4178"); + assert.equal(groups[0].tax, undefined); +}); + +test("a nonsense or absent tax falls back to the single-line shape", () => { + assert.equal(buildGroups("receipt", 10000, null, "1").length, 1); + assert.equal(buildGroups("receipt", 10000, 0, "1").length, 1); + assert.equal(buildGroups("receipt", 10000, 10000, "1").length, 1, "tax >= total is a bad read"); + assert.equal(buildGroups("receipt", 10000, 20000, "1").length, 1); +}); + +test("the Expense amount is the PRE-TAX figure when the tax was split", () => { + // Mirrors the QBO COGS line: the sales tax posts to its own reclaimable + // account, so job cost must not double-count it. + const groups = buildGroups("receipt", 36498, 2920, "82766"); + assert.equal(expenseAmountCents(groups, 36498), 33578); + assert.equal(expenseAmountCents(buildGroups("receipt", 10000, null, "1"), 10000), 10000); +}); + +test("only a drive row books under the Drive file id", () => { + assert.equal(driveFileIdOf({ source: "drive", sourceRef: "drive:FILE123" }), "FILE123"); + assert.equal(driveFileIdOf({ source: "mobile", sourceRef: "mobile:abc" }), null); + assert.equal(driveFileIdOf({ source: "drive", sourceRef: "drive:" }), null); +}); + +test("a successful booking creates the Expense at the pre-tax amount and marks the row BOOKED", async () => { + const r = recorder(); + const result = await bookReceipt(row(), r.deps); + + assert.equal(result.outcome, "booked"); + assert.equal(r.purchaseCalls.length, 1); + // DocNumber idempotency stays continuous with any v1 booking of the same file. + assert.equal(r.purchaseCalls[0].fileId, "FILE123"); + assert.equal(r.purchaseCalls[0].date, "2026-08-03"); + assert.equal(r.purchaseCalls[0].totalAmount, 364.98); + assert.equal(r.purchaseCalls[0].groups.length, 2); + + assert.equal(r.expenses.length, 1); + assert.equal(r.expenses[0].amount, 335.78); + assert.equal(r.expenses[0].estimateId, "est-1"); + assert.equal(r.expenses[0].costCodeId, "cc-plumb", "falls back to the model's phase suggestion"); + assert.equal(r.expenses[0].qbPurchaseId, "QB-1"); + assert.equal(r.expenses[0].status, "Pending"); + assert.equal(r.expenses[0].receiptUrl, "https://drive.google.com/file/d/FILE123/view"); + + assert.equal(r.intakeUpdates[0].state, "BOOKED"); + assert.equal(r.intakeUpdates[0].qbPurchaseId, "QB-1"); + assert.equal(r.events[0].kind, "receipt-push"); + assert.equal(r.events[0].source, "intake-worker"); +}); + +test("an explicitly chosen cost code beats the model's suggestion", async () => { + const r = recorder(); + await bookReceipt(row({ costCodeId: "cc-chosen" }), r.deps); + assert.equal(r.expenses[0].costCodeId, "cc-chosen"); +}); + +test("a non-drive row books under its intake id and stores the secure ref", async () => { + const r = recorder(); + await bookReceipt(row({ source: "mobile", sourceRef: "mobile:abc", id: "intake-9" }), r.deps); + assert.equal(r.purchaseCalls[0].fileId, "intake-9"); + assert.equal(r.expenses[0].receiptUrl, "secure:receipts/intake/intake-1.jpg"); +}); + +test("a project with no estimate is terminal and spends NO attempt", async () => { + const r = recorder({}, { estimates: [] }); + const result = await bookReceipt(row(), r.deps); + assert.deepEqual(result, { outcome: "needs-review", reason: "no-estimate" }); + assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never touched"); + assert.equal(r.expenses.length, 0); +}); + +test("the push kill switch and the pause switch defer without spending an attempt", async () => { + const disabled = recorder({ isPushEnabled: () => false }); + assert.deepEqual(await bookReceipt(row(), disabled.deps), { outcome: "deferred", reason: "push-disabled" }); + assert.equal(disabled.purchaseCalls.length, 0); + + const paused = recorder({ isPushPaused: async () => true }); + assert.deepEqual(await bookReceipt(row(), paused.deps), { outcome: "deferred", reason: "push-paused" }); + assert.equal(paused.purchaseCalls.length, 0); +}); + +test("a dryRun row can never reach QuickBooks, even called directly", async () => { + // The worker already refuses to route a dry-run row here. This second guard + // exists because "no QBO calls in shadow mode" is the safety promise of the + // whole phase, and one guard in one caller is not a promise. + const r = recorder(); + const result = await bookReceipt(row({ dryRun: true }), r.deps); + assert.equal(result.outcome, "deferred"); + assert.equal(r.purchaseCalls.length, 0); + assert.equal(r.expenses.length, 0); +}); + +test("QBO business-rule faults are TERMINAL, never retried", async () => { + const cases: [unknown, string][] = [ + [new QboPurchaseFaultError(400, "closed period", "6210"), "qbo-fault:6210"], + [new QboAccountConfigError("bad account"), "qbo-fault:account-config"], + [new QboVendorDuplicateError("Lowes"), "qbo-fault:vendor-duplicate"], + ]; + for (const [error, reason] of cases) { + const r = recorder({ createPurchase: async () => { throw error; } }); + const result = await bookReceipt(row(), r.deps); + assert.deepEqual(result, { outcome: "needs-review", reason }, reason); + assert.equal(r.expenses.length, 0); + } +}); + +test("an ok:false result is a deterministic refusal, so it goes to a human too", async () => { + const r = recorder({ + createPurchase: async () => ({ ok: false, reason: "docnumber-conflict", docNumber: "abc" }) as any, + }); + assert.deepEqual(await bookReceipt(row(), r.deps), { + outcome: "needs-review", + reason: "qbo-fault:docnumber-conflict", + }); +}); + +test("a QBTimeoutError retries on the backoff schedule", async () => { + const r = recorder({ createPurchase: async () => { throw new QBTimeoutError("timed out"); } }); + const first = await bookReceipt(row({ attempts: 0 }), r.deps); + assert.equal(first.outcome, "retry"); + assert.equal((first as any).attempts, 1); + assert.equal((first as any).nextRetryAt.getTime(), NOW.getTime() + 5 * 60_000); + assert.equal((first as any).reason, "QBTimeoutError"); + + const third = await bookReceipt(row({ attempts: 2 }), recorder({ + createPurchase: async () => { throw new QBTimeoutError("timed out"); }, + }).deps); + assert.equal((third as any).nextRetryAt.getTime(), NOW.getTime() + 60 * 60_000); +}); + +test("a plain network error retries; past 20 attempts it stops and asks a human", async () => { + const transient = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + assert.equal((await bookReceipt(row({ attempts: 5 }), transient.deps)).outcome, "retry"); + + const exhausted = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + assert.deepEqual(await bookReceipt(row({ attempts: 20 }), exhausted.deps), { + outcome: "needs-review", + reason: "max-retries", + }); +}); + +test("alreadyExists books identically — the lost-response retry", async () => { + const r = recorder({ + createPurchase: async (_t, input) => ({ + ok: true, qbPurchaseId: "QB-7", docNumber: input.fileId.slice(0, 21), alreadyExists: true, + }) as any, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "booked"); + assert.equal((result as any).alreadyExisted, true); + assert.equal(r.expenses.length, 1); + assert.equal(r.events[0].status, "already-exists"); +}); + +test("an existing Expense for the same Purchase is reused, never duplicated", async () => { + const r = recorder(); + (r.deps.db as any).expense.findUnique = async () => ({ id: "exp-existing" }); + const result = await bookReceipt(row(), r.deps); + assert.equal((result as any).expenseId, "exp-existing"); + assert.equal(r.expenses.length, 0, "no second Expense row"); +}); + +test("a DB failure AFTER the Purchase exists retries — the create is idempotent", async () => { + const r = recorder(); + (r.deps.db as any).$transaction = async () => { throw new Error("connection reset"); }; + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "retry"); +}); diff --git a/tests/receipt-intake-keys.test.ts b/tests/receipt-intake-keys.test.ts new file mode 100644 index 000000000..e4cb2622b --- /dev/null +++ b/tests/receipt-intake-keys.test.ts @@ -0,0 +1,204 @@ +/** + * Dedup-key fixtures, taken from REAL filenames in the August 2026 archive + * (I:\My Drive\Expenses\Processed Receipts\2026\August). The v1 Apps Script + * built those names from the same cleaned fields the v2 reader produces, so + * each name is a recorded (project, date, vendor, ref, total) tuple — which + * makes them the only fixtures that can prove the port AGREES with the + * pipeline that is still in production. + * + * A key that changes here is a shadow-week mismatch, not a refactor. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + canonicalVendor, + cleanMoney, + dedupKeys, + isValidDate, + normalizeDateStr, + refLooksReal, + sanitize, +} from "../src/lib/receipt-intake/keys"; + +/** One archive filename, split back into the fields v1 wrote into it. */ +interface Fixture { + file: string; + vendor: string; + date: string; + invoice: string; + total: string; + strong: string | null; + weak: string; +} + +const FIXTURES: Fixture[] = [ + { + file: "Berg_ADU_2026-08-03_Lowes_82766_$364.98", + vendor: "Lowes", date: "2026-08-03", invoice: "82766", total: "364.98", + strong: "2026-08-03|82766", + weak: "lowes|2026-08-03|364.98|amt", + }, + { + // The alias list collapses this vendor onto the row above's token, which + // is the whole point: ONE store, several spellings across its own formats. + file: "Berg_ADU_2026-08-03_Lowes_Home_Improvement_99908_$277.19", + vendor: "Lowes Home Improvement", date: "2026-08-03", invoice: "99908", total: "277.19", + strong: "2026-08-03|99908", + weak: "lowes|2026-08-03|277.19|amt", + }, + { + // Ref "12" is under three characters: too short to identify anything, so + // the strong key is WITHHELD and the weak net handles it. + file: "Berg_ADU_2026-08-04_WINLOCK_HARDWARE_12_$14.50", + vendor: "WINLOCK HARDWARE", date: "2026-08-04", invoice: "12", total: "14.50", + strong: null, + weak: "winlockhardware|2026-08-04|14.50|amt", + }, + { + file: "Berg_ADU_2026-08-04_WINLOCK_HARDWARE_4_$16.17", + vendor: "WINLOCK HARDWARE", date: "2026-08-04", invoice: "4", total: "16.17", + strong: null, + weak: "winlockhardware|2026-08-04|16.17|amt", + }, + { + // A non-alias vendor keeps its own collapsed token. + file: "Berg_ADU_2026-08-07_CRC_-_WEST_VAN_260807091421373F2A9_$91.50", + vendor: "CRC - WEST VAN", date: "2026-08-07", invoice: "260807091421373F2A9", total: "91.50", + strong: "2026-08-07|260807091421373f2a9", + weak: "crcwestvan|2026-08-07|91.50|amt", + }, + { + file: "Berg_ADU_2026-08-09_Amazon.com_113-9992333-7801840_$248.27", + vendor: "Amazon.com", date: "2026-08-09", invoice: "113-9992333-7801840", total: "248.27", + strong: "2026-08-09|113-9992333-7801840", + weak: "amazon|2026-08-09|248.27|amt", + }, + { + // "NoInv" is the AI saying it found no number — a placeholder, never an identity. + file: "Berg_ADU_2026-08-10_Grover_Electric_Plumbing_Supply_NoInv_$22.57", + vendor: "Grover Electric Plumbing Supply", date: "2026-08-10", invoice: "", total: "22.57", + strong: null, + weak: "groverelectricplumbingsupply|2026-08-10|22.57|amt", + }, + { + file: "Berg_ADU_2026-08-14_LOWES_HOME_CENTERS_LLC_58302_$304.23", + vendor: "LOWES HOME CENTERS LLC", date: "2026-08-14", invoice: "58302", total: "304.23", + strong: "2026-08-14|58302", + weak: "lowes|2026-08-14|304.23|amt", + }, +]; + +test("August archive fixtures produce the v1 dedup keys", () => { + for (const f of FIXTURES) { + const keys = dedupKeys({ + docType: "receipt", + vendor: f.vendor, + date: f.date, + invoice: f.invoice, + checkNumber: "", + totalAmount: f.total, + fallbackDateStr: "2099-01-01", // must never be reached: every fixture has a real date + }); + assert.equal(keys.strong, f.strong, `${f.file} strong`); + assert.equal(keys.weak, f.weak, `${f.file} weak`); + assert.equal(keys.dateStr, f.date, `${f.file} date`); + assert.equal(keys.amount, f.total, `${f.file} amount`); + } +}); + +test("an unreadable date falls back to the intake row's own date", () => { + const keys = dedupKeys({ + docType: "receipt", + vendor: "Lowes", + date: "", // the model returns "" rather than guessing + invoice: "82766", + totalAmount: "364.98", + fallbackDateStr: "2026-08-20", + }); + assert.equal(keys.dateStr, "2026-08-20"); + // The strong key needs a date READ OFF THE DOCUMENT. A fallback date is our + // guess, and two unrelated receipts uploaded the same day must not collide + // on it. + assert.equal(keys.strong, null); + assert.equal(keys.weak, "lowes|2026-08-20|364.98|amt"); +}); + +test("an invalid calendar date is not a date", () => { + for (const bad of ["2026-13-05", "2026-02-30", "not-a-date", ""]) { + assert.equal(isValidDate(bad), false, bad); + } + assert.equal(isValidDate("2026-08-03"), true); + assert.equal(normalizeDateStr("2026-06-10T00:00:00Z"), "2026-06-10"); + assert.equal(normalizeDateStr(" 2026-06-10 "), "2026-06-10"); + assert.equal(normalizeDateStr("June 10"), ""); +}); + +test("checks key on the check number, not the invoice", () => { + const keys = dedupKeys({ + docType: "check", + vendor: "Richard Lord", + date: "2026-08-05", + invoice: "ignored", + checkNumber: "4178", + totalAmount: "1,200.00", + fallbackDateStr: "2099-01-01", + }); + assert.equal(keys.ref, "Check4178"); + assert.equal(keys.strong, "2026-08-05|check4178"); + assert.equal(keys.amount, "1200.00"); +}); + +test("a check with no readable number gets no strong key", () => { + const keys = dedupKeys({ + docType: "check", + vendor: "Someone", + date: "2026-08-05", + checkNumber: "", + totalAmount: "50.00", + fallbackDateStr: "2099-01-01", + }); + assert.equal(keys.ref, "CheckNoNum"); + assert.equal(keys.strong, null); +}); + +test("placeholder refs are refused; real ones that merely look odd are not", () => { + // :1571–1580 — the padded forms are exactly what the AI emits when it can't + // read a number, and they used to become the SHARED key of every unrelated + // receipt that day. + assert.equal(refLooksReal("NA 000"), false); + assert.equal(refLooksReal("0000"), false); + assert.equal(refLooksReal("Unknown 0000"), false); + assert.equal(refLooksReal("N/A"), false); + assert.equal(refLooksReal("NoInv"), false); + assert.equal(refLooksReal("12"), false); + assert.equal(refLooksReal("ABC"), false); + assert.equal(refLooksReal("1111"), false); + // "INV"/"ORDER"/"REF" are deliberately NOT placeholders — they prefix real numbers. + assert.equal(refLooksReal("INV-95870"), true); + assert.equal(refLooksReal("82766"), true); + assert.equal(refLooksReal("113-9992333-7801840"), true); +}); + +test("cleanMoney handles currency, commas and accounting negatives", () => { + assert.equal(cleanMoney("$1,234.56"), "1234.56"); + assert.equal(cleanMoney("-12.50"), "-12.50"); + assert.equal(cleanMoney("(123.45)"), "-123.45"); + assert.equal(cleanMoney(""), "0.00"); + assert.equal(cleanMoney("not a number"), "0.00"); + assert.equal(cleanMoney(null), "0.00"); +}); + +test("sanitize drops punctuation and collapses whitespace, like the archive names", () => { + assert.equal(sanitize("Lowe's Home Improvement"), "Lowes_Home_Improvement"); + assert.equal(sanitize("CRC - WEST VAN"), "CRC_-_WEST_VAN"); + assert.equal(sanitize(""), ""); +}); + +test("canonicalVendor collapses a chain's spellings and keeps others intact", () => { + for (const spelling of ["LOWES", "Lowe's Home Improvement", "Lowes Home Centers LLC S1632MC3"]) { + assert.equal(canonicalVendor(spelling), "lowes", spelling); + } + assert.equal(canonicalVendor("Amazon.com"), "amazon"); + assert.equal(canonicalVendor("CRC - WEST VAN"), "crcwestvan"); + assert.equal(canonicalVendor("Grover Electric Plumbing Supply"), "groverelectricplumbingsupply"); +}); diff --git a/tests/receipt-intake-read.test.ts b/tests/receipt-intake-read.test.ts new file mode 100644 index 000000000..9b9584b1a --- /dev/null +++ b/tests/receipt-intake-read.test.ts @@ -0,0 +1,195 @@ +/** + * The reader, driven through an INJECTED fetch — no network, no module mocks + * (CI is Node 20, where `mock.module` corrupts the require chain). + * + * Two things are pinned here: + * 1. the load-bearing sentences of the v3.6 prompt. Each one was added after a + * specific misread (subtotal booked instead of the total; an invented tax + * line; a scanned stack of receipts booked as one purchase), so a tidy-up + * edit that drops one is a money bug, not a style change. + * 2. the outage discipline: "the service was busy" and "this document defeated + * the AI" must stay DIFFERENT answers. Collapsing them parked five legible + * receipts during the 2026-08-10..19 outage. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { buildReadPrompt, parseReadJson, readReceipt } from "../src/lib/receipt-intake/read"; + +const PHASES = [ + { code: "01-DEMO", name: "Demolition" }, + { code: "03-PLUMB", name: "Plumbing" }, +]; + +const BYTES = Buffer.from("fake-jpeg-bytes"); + +function geminiJson(payload: unknown): Response { + return new Response( + JSON.stringify({ candidates: [{ content: { parts: [{ text: JSON.stringify(payload) }] } }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +const noSleep = async () => {}; + +test("the prompt carries the v3.6 rules that decide money", () => { + const prompt = buildReadPrompt(PHASES); + + // The final-amount rule: the number that matches the bank charge. + assert.ok(prompt.includes( + "total_amount is the FINAL amount paid — after all discounts, coupons, and credits, and " + + "including tax and fees." + ), "final-amount rule"); + assert.ok(prompt.includes("NEVER the subtotal, and never the pre-discount price."), "subtotal rule"); + + // The never-estimate-tax rule: an ABSENT tax is not a ZERO tax, and a + // computed one would corrupt the reseller-permit filing. + assert.ok(prompt.includes( + 'return "" if no tax line is shown or it cannot be read confidently — never estimate or ' + + "compute it yourself." + ), "never-estimate-tax rule"); + + // The multi rule: a stack of receipts scanned into one PDF is not one purchase. + assert.ok(prompt.includes( + 'STEP 1 - if the file contains MORE THAN ONE separate receipt, invoice, or check' + ), "multi rule"); + assert.ok(prompt.includes('return exactly {"doc_type":"multi"} and nothing else.'), "multi output"); + assert.ok(prompt.includes('return exactly {"doc_type":"non_receipt"} and nothing else.'), "non_receipt output"); + + // Unreadable fields come back empty rather than guessed. + assert.ok(prompt.includes('If a field cannot be read, return "" for it. For the date, return "" rather than guessing.')); +}); + +test("the appended phase section lists the job's codes and nothing else", () => { + const prompt = buildReadPrompt(PHASES); + assert.ok(prompt.includes("01-DEMO — Demolition")); + assert.ok(prompt.includes("03-PLUMB — Plumbing")); + assert.ok(prompt.includes('"suggested_phase"')); + // The v1 extraction half stays BYTE-IDENTICAL: the phase section can only + // ever be appended, never woven into the rules above it. + const v1Only = buildReadPrompt([]); + assert.ok(prompt.startsWith(v1Only), "the phase section is strictly appended"); + assert.ok(!v1Only.includes("suggested_phase"), "a job with no cost codes gets the v1 prompt"); +}); + +test("a well-formed response parses into ReadResult", async () => { + let capturedBody: string | undefined; + const outcome = await readReceipt(BYTES, "image/jpeg", PHASES, { + apiKey: () => "test-key", + sleep: noSleep, + random: () => 0, + fetchFn: (async (_url: string, init: RequestInit) => { + capturedBody = init.body as string; + return geminiJson({ + doc_type: "receipt", + vendor: "Lowes", + date: "2026-08-03", + invoice: "82766", + check_number: "", + memo: "", + total_amount: "364.98", + tax_amount: "29.20", + suggested_phase: "03-PLUMB", + }); + }) as unknown as typeof fetch, + }); + + assert.ok(outcome.ok); + assert.equal(outcome.read.vendor, "Lowes"); + assert.equal(outcome.read.date, "2026-08-03"); + assert.equal(outcome.read.totalAmount, "364.98"); + assert.equal(outcome.read.taxAmount, "29.20"); + assert.equal(outcome.read.suggestedPhaseCode, "03-PLUMB"); + assert.ok(outcome.read.raw.includes("364.98"), "raw JSON is kept for audit"); + + const sent = JSON.parse(capturedBody!); + assert.equal(sent.generationConfig.responseMimeType, "application/json"); + assert.equal(sent.contents[0].parts[1].inline_data.mime_type, "image/jpeg"); +}); + +test("text/plain goes in as a text part, not inline_data", async () => { + let capturedBody: string | undefined; + await readReceipt(Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"), "text/plain; charset=utf-8", [], { + apiKey: () => "test-key", + sleep: noSleep, + random: () => 0, + fetchFn: (async (_url: string, init: RequestInit) => { + capturedBody = init.body as string; + return geminiJson({ doc_type: "receipt", total_amount: "10.00" }); + }) as unknown as typeof fetch, + }); + const sent = JSON.parse(capturedBody!); + assert.ok(sent.contents[0].parts[1].text.startsWith("This is a text file containing receipt data:")); +}); + +test("an off-list phase suggestion is discarded, not trusted", () => { + const parsed = parseReadJson(JSON.stringify({ doc_type: "receipt", suggested_phase: "99-INVENTED" }), PHASES); + assert.equal(parsed?.suggestedPhaseCode, ""); +}); + +test("503 backs off five times, then falls to the next model", async () => { + const calls: string[] = []; + const sleeps: number[] = []; + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + random: () => 0, + sleep: async (ms) => { sleeps.push(ms); }, + fetchFn: (async (url: string) => { + calls.push(url); + if (calls.length <= 5) return new Response("busy", { status: 503 }); + return geminiJson({ doc_type: "receipt", total_amount: "1.00" }); + }) as unknown as typeof fetch, + }); + assert.ok(outcome.ok, "the second model answered"); + assert.equal(calls.length, 6); + assert.ok(calls[0].includes("gemini-3.5-flash")); + assert.ok(calls[5].includes("gemini-flash-latest"), "fell through to the next model"); + assert.deepEqual(sleeps, [2000, 4000, 8000, 16000, 32000]); +}); + +test("every model unavailable is NOT decisive — the row must not spend an attempt", async () => { + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + random: () => 0, + sleep: noSleep, + fetchFn: (async () => new Response("nope", { status: 404 })) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: false }); +}); + +test("a model that answers with unusable JSON IS decisive", async () => { + // The model responded; retrying will not make this document readable. + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + random: () => 0, + sleep: noSleep, + fetchFn: (async () => new Response( + JSON.stringify({ candidates: [{ content: { parts: [{ text: "not json at all" }] } }] }), + { status: 200 }, + )) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: true }); +}); + +test("HTTP 400 (payload rejected) is decisive and stops immediately", async () => { + let calls = 0; + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + random: () => 0, + sleep: noSleep, + fetchFn: (async () => { calls++; return new Response("too big", { status: 400 }); }) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: true }); + assert.equal(calls, 1, "a rejected payload is not retried against a second model"); +}); + +test("a missing API key is a SERVICE fact, never charged to the document", async () => { + let calls = 0; + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => undefined, + sleep: noSleep, + random: () => 0, + fetchFn: (async () => { calls++; return geminiJson({}); }) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: false }); + assert.equal(calls, 0); +}); diff --git a/tests/receipt-intake-route-state.test.ts b/tests/receipt-intake-route-state.test.ts new file mode 100644 index 000000000..fa49d9e9e --- /dev/null +++ b/tests/receipt-intake-route-state.test.ts @@ -0,0 +1,88 @@ +/** + * The routing truth table (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §4) plus the + * booking backoff schedule. Both are pure, so this file needs no database. + * + * Order is the assertion, not just the outcomes: "first match wins" is why a + * $0 misread never reaches a dedup key and why a multi-page scan is triaged + * before anyone asks which job it belongs to. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { backoffMs, MAX_BOOK_ATTEMPTS, routeState } from "../src/lib/receipt-intake/route-state"; + +const NO_HITS = { strong: null, weak: null }; +const clean = { docType: "receipt", amount: "364.98", totalCents: 36498 }; + +test("multi outranks everything, including a missing project", () => { + const d = routeState({ docType: "multi", amount: "0.00", totalCents: null }, NO_HITS, false); + assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "multi-doc", duplicateOfId: null }); +}); + +test("a non-receipt is its own terminal state, not a review item", () => { + const d = routeState({ docType: "non_receipt", amount: "0.00", totalCents: null }, NO_HITS, true); + assert.deepEqual(d, { state: "NON_RECEIPT", stateReason: null, duplicateOfId: null }); +}); + +test("a $0.00 total is a misread and is parked BEFORE any dedup or job check", () => { + // :531 — you don't get a $0 receipt or write a $0 check. Letting this reach + // a key would poison it for the real document. + const d = routeState( + { docType: "receipt", amount: "0.00", totalCents: 0 }, + { strong: { id: "owner", totalCents: 0 }, weak: { id: "other" } }, + true, + ); + assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "zero-total", duplicateOfId: null }); +}); + +test("no project means NEEDS_JOB — a queue, not a fault", () => { + const d = routeState(clean, NO_HITS, false); + assert.deepEqual(d, { state: "NEEDS_JOB", stateReason: null, duplicateOfId: null }); +}); + +test("a strong hit at the same total is the same purchase twice", () => { + const d = routeState(clean, { strong: { id: "row-a", totalCents: 36498 }, weak: null }, true); + assert.deepEqual(d, { state: "DUPLICATE", stateReason: null, duplicateOfId: "row-a" }); +}); + +test("a strong hit at a DIFFERENT total is ambiguous and goes to a human", () => { + const d = routeState(clean, { strong: { id: "row-a", totalCents: 20000 }, weak: null }, true); + assert.deepEqual(d, { + state: "NEEDS_REVIEW", + stateReason: "strong-dup-amount-mismatch:row-a", + duplicateOfId: "row-a", + }); +}); + +test("an owner whose total is unknown is never treated as a match", () => { + // A null total means "can't confirm the totals match" — reading it as a + // match would silently quarantine a real expense. + const d = routeState(clean, { strong: { id: "row-a", totalCents: null }, weak: null }, true); + assert.equal(d.state, "NEEDS_REVIEW"); + assert.equal(d.stateReason, "strong-dup-amount-mismatch:row-a"); +}); + +test("a weak hit always asks a human, never quarantines on its own", () => { + const d = routeState(clean, { strong: null, weak: { id: "row-b" } }, true); + assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "weak-dup:row-b", duplicateOfId: null }); +}); + +test("the strong net is checked before the weak one", () => { + const d = routeState(clean, { strong: { id: "row-a", totalCents: 36498 }, weak: { id: "row-b" } }, true); + assert.equal(d.state, "DUPLICATE"); + assert.equal(d.duplicateOfId, "row-a"); +}); + +test("a clean document with a job and no hits is READ", () => { + assert.deepEqual(routeState(clean, NO_HITS, true), { + state: "READ", stateReason: null, duplicateOfId: null, + }); +}); + +test("backoff is 5m / 15m / 1h / 6h and then stays at 6h", () => { + assert.equal(backoffMs(1), 5 * 60_000); + assert.equal(backoffMs(2), 15 * 60_000); + assert.equal(backoffMs(3), 60 * 60_000); + assert.equal(backoffMs(4), 6 * 60 * 60_000); + assert.equal(backoffMs(10), 6 * 60 * 60_000); + assert.equal(MAX_BOOK_ATTEMPTS, 20); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts new file mode 100644 index 000000000..2d2b8b575 --- /dev/null +++ b/tests/receipt-intake-worker.test.ts @@ -0,0 +1,221 @@ +/** + * The worker pass, and the one property the whole shadow week rests on: + * + * with RECEIPT_INTAKE_DRYRUN unset (the default), a row is read, deduped and + * routed, and NOTHING is booked — zero createPurchase calls, zero Expense + * rows. + * + * That is asserted by counting the injected fakes' calls, not by reading the + * code. Dependency injection throughout; no `mock.module` (CI is Node 20). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + runIntakeWorker, + dateOnly, + toDateStr, + type ReadPatch, + type WorkerDependencies, + type WorkerRow, +} from "../src/lib/receipt-intake/worker"; +import type { ReadOutcome } from "../src/lib/receipt-intake/read"; +import type { BookResult } from "../src/lib/receipt-intake/book"; + +const NOW = new Date("2026-09-01T12:00:00.000Z"); + +function workerRow(overrides: Partial = {}): WorkerRow { + return { + id: "row-1", + source: "drive", + sourceRef: "drive:FILE1", + state: "RECEIVED", + dryRun: true, + projectId: "proj-1", + costCodeId: null, + suggestedCostCodeId: null, + storagePath: "receipts/intake/row-1.jpg", + fileName: "r.jpg", + mimeType: "image/jpeg", + fileSize: 100, + vendor: null, + txnDate: null, + totalCents: null, + taxCents: null, + docType: null, + refNumber: null, + memo: null, + attempts: 0, + readAt: null, + ...overrides, + }; +} + +const goodRead: ReadOutcome = { + ok: true, + read: { + docType: "receipt", + vendor: "Lowes", + date: "2026-08-03", + invoice: "82766", + checkNumber: "", + memo: "", + totalAmount: "364.98", + taxAmount: "29.20", + suggestedPhaseCode: "03-PLUMB", + raw: '{"vendor":"Lowes"}', + }, +}; + +interface Harness { + deps: WorkerDependencies; + reads: number; + books: number; + applied: ReadPatch[]; + states: { id: string; state: string; reason: string | null }[]; + promoted: string[]; + deferred: string[]; +} + +function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { + const h: Harness = { + reads: 0, books: 0, applied: [], states: [], promoted: [], deferred: [], + deps: null as unknown as WorkerDependencies, + }; + h.deps = { + claim: async () => rows, + loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], + downloadBytes: async () => Buffer.from("bytes"), + read: async () => { h.reads++; return goodRead; }, + applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, + findWeakHit: async () => null, + applyState: async (id, state, reason) => { h.states.push({ id, state, reason }); }, + promoteToBooking: async id => { h.promoted.push(id); }, + book: async () => { h.books++; return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, + applyBookResult: async () => {}, + deferRead: async id => { h.deferred.push(id); }, + now: () => NOW, + ...overrides, + }; + return h; +} + +test("DRY RUN: a received row is read, deduped and routed — and never booked", async () => { + const h = harness([workerRow({ dryRun: true })]); + const summary = await runIntakeWorker(h.deps); + + assert.equal(h.reads, 1, "the reader DOES run in shadow mode — that is the point"); + assert.equal(h.books, 0, "zero booking calls"); + assert.equal(h.applied.length, 1); + assert.equal(h.applied[0].state, "READ"); + assert.equal(h.applied[0].vendor, "Lowes"); + assert.equal(h.applied[0].totalCents, 36498); + assert.equal(h.applied[0].taxCents, 2920); + assert.equal(h.applied[0].dedupStrongKey, "2026-08-03|82766"); + assert.equal(h.applied[0].dedupWeakKey, "lowes|2026-08-03|364.98|amt"); + assert.equal(h.applied[0].suggestedCostCodeId, "cc-plumb"); + assert.deepEqual(summary, { processed: 1, byState: { READ: 1 } }); +}); + +test("DRY RUN: a row already at READ parks there instead of moving to BOOKING", async () => { + const h = harness([workerRow({ state: "READ", dryRun: true })]); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.books, 0); + assert.deepEqual(h.promoted, []); + assert.deepEqual(summary.byState, { READ: 1 }); +}); + +test("DRY RUN: a row stuck at BOOKING is not booked either", async () => { + const h = harness([workerRow({ state: "BOOKING", dryRun: true })]); + await runIntakeWorker(h.deps); + assert.equal(h.books, 0); +}); + +test("LIVE: a READ row with dryRun=false is promoted and booked", async () => { + const h = harness([workerRow({ state: "READ", dryRun: false })]); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(h.promoted, ["row-1"]); + assert.equal(h.books, 1); + assert.deepEqual(summary.byState, { BOOKED: 1 }); +}); + +test("a strong-key claim that loses re-routes against the owner and keeps no key", async () => { + const h = harness([workerRow()], { + applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498 } }), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { DUPLICATE: 1 }); + assert.equal(h.states.length, 1); + assert.equal(h.states[0].state, "DUPLICATE"); +}); + +test("a strong-key loss at a DIFFERENT total goes to a human, not to DUPLICATE", async () => { + const h = harness([workerRow()], { + applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 999 } }), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); + assert.equal(h.states[0].reason, "strong-dup-amount-mismatch:row-owner"); +}); + +test("a document that does not reach READ never claims the strong key", async () => { + // A multi-doc or a $0 misread holding "2026-08-03|82766" would quarantine + // the real receipt that arrives next. + const h = harness([workerRow()], { + read: async () => ({ ...goodRead, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].state, "NEEDS_REVIEW"); + assert.equal(h.applied[0].stateReason, "multi-doc"); + assert.equal(h.applied[0].dedupStrongKey, null); +}); + +test("a service outage costs no attempt: the row is deferred, not reviewed", async () => { + const h = harness([workerRow()], { read: async () => ({ ok: false, decisive: false }) }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(h.deferred, ["row-1"]); + assert.deepEqual(h.states, []); + assert.deepEqual(summary.byState, { RECEIVED: 1 }); +}); + +test("a document the model answered on but could not read goes to a human", async () => { + const h = harness([workerRow()], { read: async () => ({ ok: false, decisive: true }) }); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].state, "NEEDS_REVIEW"); + assert.equal(h.states[0].reason, "unreadable"); +}); + +test("a missing storage object is terminal, not an infinite read loop", async () => { + const h = harness([workerRow()], { downloadBytes: async () => null }); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].reason, "file-missing"); + assert.equal(h.reads, 0); +}); + +test("another run holding the lock yields skipped, not an empty pass", async () => { + const h = harness([], { claim: async () => null }); + assert.deepEqual(await runIntakeWorker(h.deps), { + processed: 0, byState: {}, skipped: "already-running", + }); +}); + +test("one poison row is parked and the rest of the batch still runs", async () => { + let call = 0; + const h = harness([workerRow({ id: "row-1" }), workerRow({ id: "row-2" })], { + read: async () => { + call++; + if (call === 1) throw new Error("boom"); + return goodRead; + }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.processed, 2); + assert.equal(summary.byState.NEEDS_REVIEW, 1); + assert.equal(summary.byState.READ, 1); +}); + +test("dateOnly keeps a calendar day at UTC midnight, the way @db.Date round-trips", () => { + assert.equal(dateOnly("2026-08-03")!.toISOString(), "2026-08-03T00:00:00.000Z"); + assert.equal(dateOnly("2026-13-03"), null); + assert.equal(dateOnly("nope"), null); + assert.equal(toDateStr(new Date("2026-08-03T23:59:00.000Z")), "2026-08-03"); +}); From 80c110a4a3d616aa741748ee89bb1b3164eb0755 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 17:21:23 -0700 Subject: [PATCH 027/144] test(receipts): e2e 401 matrix + idempotent POST; document the as-built contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e2e/receipt-intake.spec.ts drives the real HTTP surface, because the proxy bypass makes the route handler the only thing between an anonymous caller and a write — reading the source proves nothing about that. Every negative case asserts a JSON 401/403 AND the absence of a Location header (maxRedirects: 0): a 307 to /login is what a forwarder silently mis-reads as "retry later" forever. Covered: no credentials, a BOGUS session cookie, a wrong secret (which must not fall through to the ADMIN session riding along), an empty secret header, the idempotent double POST (one row, same id), machine callers needing their own sourceRef, deterministic 400s, bytes beating a lying mime header, the GET role gate proven with the EMPLOYEE storage state (an ADMIN can never reach that branch), and the archive callback's secret-only + 409/404 behaviour. The "secret env unset" branch cannot be driven from a spec (it can't unset an env var on the server it is talking to), so it is pinned as a unit test and said so in the file header. tests/apply-receipt-intake.test.ts keeps the rollout script and the committed migration describing the same table — including the partial index's PREDICATE, which is the part that actually decides what gets quarantined. ci.yml gains RECEIPT_INTAKE_SECRET as a literal for the Playwright job, the same reasoning as DEPOSIT_INGEST_SECRET: nothing external depends on the value. Spec §7 records the six places the build differs from the plan, so the Apps Script PR codes against what exists. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 7 + docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 45 +++ e2e/receipt-intake.spec.ts | 312 ++++++++++++++++++ package.json | 3 +- .../api/cron/receipt-intake-worker/route.ts | 1 + src/lib/receipt-intake/worker.ts | 9 +- tests/apply-receipt-intake.test.ts | 105 ++++++ tests/receipt-intake-worker.test.ts | 1 + 8 files changed, 481 insertions(+), 2 deletions(-) create mode 100644 e2e/receipt-intake.spec.ts create mode 100644 tests/apply-receipt-intake.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de42af652..81f7036b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,13 @@ jobs: # against the throwaway DB — so unlike the provider secrets above it's a # literal here rather than a repo secret (there's nothing to rotate). DEPOSIT_INGEST_SECRET: "e2e-ci-deposit-ingest-secret" + # Shared secret for the Receipt Pipeline v2 intake endpoint + # (src/app/api/receipts/intake/route.ts). Same reasoning as + # DEPOSIT_INGEST_SECRET above: nothing external depends on this value, it + # only gates e2e/receipt-intake.spec.ts against the throwaway DB, so it is + # a literal rather than a repo secret. The spec's "env var unset" case is + # a unit test (tests/receipt-intake-auth.test.ts), not this job. + RECEIPT_INTAKE_SECRET: "e2e-ci-receipt-intake-secret" # Forces the Stage A daily-log task matcher (daily-log-task-match.ts) onto # its deterministic keyword fallback instead of calling Gemini, so # time-suggestion.spec.ts's Stage A end-to-end test is reproducible in CI. diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index f69cf3640..bf9bc978b 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -341,6 +341,51 @@ All gated on Script Property `V2_FORWARD === "true"`; all send `x-receipt-intake original for v1 to process as usual. The move-to-`_Forwarded` branch (which hides the file from v1) activates only under a second property `V2_LIVE === "true"` at cutover. +### As-built notes (2026-09-01) — where the implementation differs from the plan above + +The Apps Script side is a separate PR in `qbo-clasp`. The endpoint contract it must code +against is the one above, with these six clarifications from the build: + +1. **`GET /api/receipts/intake` accepts the shared secret as well as a staff session.** + §3 said staff-only, but §6's nightly mirror polls `?state=BOOKED` with + `x-receipt-intake-secret` — it has no session to present. A SESSION caller still needs + ADMIN | MANAGER | FINANCE (403 otherwise); the secret caller is the mirror. +2. **`POST /api/receipts/intake//archived` is also on the proxy's public bypass**, + spelled out as its own exact pattern (`/api/receipts/intake/[^/]+/archived`). It is a + DESCENDANT of the intake path, and the intake bypass is exact-match on purpose, so + without its own entry the proxy would answer the mirror with a 307 to /login. The + route is secret-only: a session, however privileged, is refused, because only the + mirror can know that a file now exists in Drive. It is also state-conditional + (`updateMany WHERE state = 'BOOKED'`), so two mirror runs racing one row cannot both + claim the transition — the loser gets 409. +3. **`threadName` is accepted and NOT persisted.** The chat forwarder should keep sending + it, but `memo` belongs to the read step (it holds a check's handwritten memo line) and + there is no other column for it yet. Phase 2 adds one when the queue page needs to link + back to the thread. +4. **The phase suggestion is resolved to `suggestedCostCodeId` at READ time**, not at + booking, using the same `matchCostCode` the v1 ingest uses. Booking then takes + `costCodeId ?? suggestedCostCodeId`. Same outcome as §4 step 5, but the suggestion is + visible in the queue before anything books. +5. **A non-Drive row's intake id is a UUIDv4**, so its QBO DocNumber is the first 21 + characters of that UUID (risk 5 above, unchanged in substance — the PrivateNote marker + check in `createQBReceiptPurchase` still turns any truncation collision into a + `docnumber-conflict` rather than a mis-attached Purchase). +6. **Tests live in `tests/receipt-intake-*.test.ts`, run by `tsx --test`**, not + `test/receipt-intake/*.test.mjs`. That is the repo's existing convention (every other + suite is there and wired into `npm run test:unit`); the rule that mattered — no + `mock.module`, function injection only, because CI pins Node 20 — is followed. + +Two things a human must do before this can leave shadow mode: + +- Set `RECEIPT_INTAKE_SECRET` (new, independent of `RECEIPT_INGEST_SECRET`) in Vercel, and + give the same value to the Apps Script as a Script Property. +- Re-run `node scripts/snapshot-prisma-blind-spots.mjs --write` against production AFTER + `scripts/apply-receipt-intake.mjs` has run there. The new partial index and CHECK + constraint were added to `prisma/prisma-blind-spots.json` by hand (the snapshotter needs + a live production connection, which this branch never had), so their rendered + definitions are asserted, not observed. CI's `migrations` job is what will catch a + mismatch. + ## 8. Shadow-week gate - `RECEIPT_INTAKE_DRYRUN` unset/true: every row gets `dryRun=true` — reader, dedup, and diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts new file mode 100644 index 000000000..9705280e8 --- /dev/null +++ b/e2e/receipt-intake.spec.ts @@ -0,0 +1,312 @@ +import { test, expect, type APIRequestContext } from "@playwright/test"; +import { PrismaClient } from "@prisma/client"; + +/** + * POST/GET /api/receipts/intake — request-level auth matrix and idempotency. + * + * `/api/receipts/intake` is on the proxy's EXACT-match public bypass + * (src/proxy.ts), which means the route handler is the only thing standing + * between an anonymous caller and a write. Source-reading proves nothing about + * that; these tests drive the real HTTP surface against the throwaway CI + * Postgres (data.setup.ts guards prod — docs/TESTING.md). + * + * Shaped after e2e/portal-estimate-access.spec.ts and + * e2e/deposit-ingest.spec.ts: every negative case asserts a JSON 401/403 and + * NOT a 307 to /login, because a redirect is what a machine caller silently + * mis-reads as "try again later" forever. + * + * NOT covered here, deliberately: the "RECEIPT_INTAKE_SECRET is unset" case. + * A spec cannot unset an env var on the server process it is talking to, so + * that fail-closed branch is pinned as a unit test instead — + * tests/receipt-intake-auth.test.ts, "the secret check fails CLOSED when the + * env var is unset or empty". + * + * Auth: RECEIPT_INTAKE_SECRET must be set for the server under test. CI wires + * it as a literal in .github/workflows/ci.yml (nothing external depends on the + * value), same pattern as DEPOSIT_INGEST_SECRET. + */ + +const prisma = new PrismaClient(); +const INTAKE_PATH = "/api/receipts/intake"; +const SECRET = process.env.RECEIPT_INTAKE_SECRET || ""; + +// One prefix for everything this file creates, so teardown can be exact. +const REF_PREFIX = "drive:e2e-intake-"; +const FILE_ID = `${Date.now()}-a`; +const SOURCE_REF = `${REF_PREFIX}${FILE_ID}`; + +// A real 1x1 PNG: the endpoint decides the stored mime on the BYTES, so a +// placeholder string would be refused (which is itself asserted below). +const PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + +function intakeBody(overrides: Record = {}) { + return JSON.stringify({ + source: "drive", + sourceRef: SOURCE_REF, + fileBase64: PNG_BASE64, + mimeType: "image/png", + fileName: "e2e-receipt.png", + ...overrides, + }); +} + +async function postIntake( + request: APIRequestContext, + data: string, + headers: Record = { "x-receipt-intake-secret": SECRET }, +) { + const res = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json", ...headers }, + data, + maxRedirects: 0, // a 307 to /login must FAIL this suite, not be followed + }); + let body: any = null; + try { body = await res.json(); } catch { /* non-JSON body is itself a failure signal */ } + return { res, body }; +} + +test.beforeAll(async () => { + expect( + SECRET, + "RECEIPT_INTAKE_SECRET must be set for the server under test (ci.yml sets it; locally export it before `npm run dev`)", + ).toBeTruthy(); + await prisma.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: REF_PREFIX } } }); +}); + +test.afterAll(async () => { + await prisma.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: REF_PREFIX } } }); + await prisma.$disconnect(); +}); + +test.describe("intake auth is fail-closed", () => { + test("no credentials at all is a JSON 401, never a redirect to /login", async ({ playwright }) => { + const anonymous = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const { res, body } = await postIntake(anonymous, intakeBody({ sourceRef: `${REF_PREFIX}anon` }), {}); + expect(res.status()).toBe(401); + expect(res.headers().location, "a redirect here would look like a retryable failure to a bot").toBeUndefined(); + expect(body).toMatchObject({ ok: false, reason: "unauthorized" }); + await anonymous.dispose(); + }); + + test("a BOGUS session cookie is 401, not a pass", async ({ playwright }) => { + // The getclients-auth-gate lesson: a dev-auth fallback (or a gate that + // only checks for the presence of a cookie) hides exactly this hole. + const forged = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { + cookies: [{ + name: "next-auth.session-token", + value: "not-a-real-jwt", + domain: "localhost", + path: "/", + expires: -1, + httpOnly: true, + secure: false, + sameSite: "Lax" as const, + }], + origins: [], + }, + }); + const { res } = await postIntake(forged, intakeBody({ sourceRef: `${REF_PREFIX}bogus` }), {}); + expect(res.status()).toBe(401); + expect(res.headers().location).toBeUndefined(); + await forged.dispose(); + }); + + test("a WRONG secret is refused outright, and does not fall through to the session", async ({ request }) => { + // `request` carries the ADMIN storage state. A stale forwarder secret + // must still be a 401 — otherwise a rotated secret would silently keep + // working from any browser that happened to be signed in. + const { res, body } = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}wrong` }), { + "x-receipt-intake-secret": "definitely-not-the-secret", + }); + expect(res.status()).toBe(401); + expect(body).toMatchObject({ ok: false, reason: "unauthorized" }); + }); + + test("an empty secret header is not a bypass", async ({ playwright }) => { + const anonymous = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const { res } = await postIntake(anonymous, intakeBody({ sourceRef: `${REF_PREFIX}empty` }), { + "x-receipt-intake-secret": "", + }); + expect(res.status()).toBe(401); + await anonymous.dispose(); + }); +}); + +test.describe("intake POST", () => { + test("the same sourceRef twice yields ONE row and the SAME id", async ({ request }) => { + const first = await postIntake(request, intakeBody()); + expect(first.res.status(), JSON.stringify(first.body)).toBe(200); + expect(first.body.ok).toBe(true); + expect(first.body.state).toBe("RECEIVED"); + expect(first.body.sourceRef).toBe(SOURCE_REF); + // Shadow week: dry-run is the default and is captured per row. + expect(first.body.dryRun).toBe(true); + + const second = await postIntake(request, intakeBody()); + expect(second.res.status()).toBe(200); + expect(second.body.alreadyReceived).toBe(true); + expect(second.body.id).toBe(first.body.id); + + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: SOURCE_REF } }); + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(first.body.id); + expect(rows[0].mimeType).toBe("image/png"); + expect(rows[0].fileSha256).toHaveLength(64); + expect(rows[0].storagePath).toBe(`receipts/intake/${first.body.id}.png`); + }); + + test("a machine caller MUST supply its own sourceRef", async ({ request }) => { + const { res, body } = await postIntake(request, JSON.stringify({ + source: "drive", fileBase64: PNG_BASE64, mimeType: "image/png", + })); + expect(res.status()).toBe(400); + expect(body.reason).toBe("missing-sourceRef"); + }); + + test("deterministic bad input is a 400, not a 500 the forwarder retries forever", async ({ request }) => { + const cases: [string, string][] = [ + [intakeBody({ source: "carrier-pigeon", sourceRef: `${REF_PREFIX}src` }), "invalid-source"], + [JSON.stringify({ source: "drive", sourceRef: `${REF_PREFIX}nofile` }), "missing-file"], + // Base64 of "hello" — not a document format we can read. + [intakeBody({ sourceRef: `${REF_PREFIX}junk`, fileBase64: "aGVsbG8=", mimeType: "image/png" }), "unsupported-file-type"], + ]; + for (const [data, reason] of cases) { + const { res, body } = await postIntake(request, data); + expect(res.status(), reason).toBe(400); + expect(body.reason).toBe(reason); + } + }); + + test("a declared mime cannot override the bytes", async ({ request }) => { + // Claiming application/pdf over PNG bytes must store image/png. + const ref = `${REF_PREFIX}sniff`; + const { res, body } = await postIntake(request, intakeBody({ sourceRef: ref, mimeType: "application/pdf" })); + expect(res.status()).toBe(200); + const row = await prisma.receiptIntake.findUnique({ where: { id: body.id } }); + expect(row?.mimeType).toBe("image/png"); + }); +}); + +test.describe("intake GET", () => { + test("an ADMIN session can read the queue", async ({ request }) => { + await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}list` })); + const res = await request.get(`${INTAKE_PATH}?state=RECEIVED&take=200`, { maxRedirects: 0 }); + expect(res.status()).toBe(200); + const body = await res.json(); + expect(body.ok).toBe(true); + expect(body.rows.some((r: any) => r.sourceRef === `${REF_PREFIX}list`)).toBe(true); + // The raw model output never leaves the server. + expect(body.rows[0]).not.toHaveProperty("readJson"); + }); + + test("the archive mirror can poll with the shared secret", async ({ playwright }) => { + const machine = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const res = await machine.get(`${INTAKE_PATH}?state=BOOKED`, { + headers: { "x-receipt-intake-secret": SECRET }, + maxRedirects: 0, + }); + expect(res.status()).toBe(200); + await machine.dispose(); + }); + + test("a staff user without a bookkeeping role gets 403, not a redirect", async ({ playwright }) => { + // contract-user.json is an EMPLOYEE (e2e/auth-contract.setup.ts). An + // ADMIN session can never reach this branch, so this second storage + // state IS the test. + const employee = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: "e2e/.auth/contract-user.json", + }); + const res = await employee.get(INTAKE_PATH, { maxRedirects: 0 }); + expect(res.status()).toBe(403); + expect(res.headers().location).toBeUndefined(); + await employee.dispose(); + }); + + test("no credentials is 401", async ({ playwright }) => { + const anonymous = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const res = await anonymous.get(INTAKE_PATH, { maxRedirects: 0 }); + expect(res.status()).toBe(401); + expect(res.headers().location).toBeUndefined(); + await anonymous.dispose(); + }); +}); + +test.describe("archive callback", () => { + test("it is secret-only and refuses a row that is not BOOKED", async ({ request, playwright }) => { + const ref = `${REF_PREFIX}archive`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + const id = created.body.id; + + const anonymous = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const unauthed = await anonymous.post(`${INTAKE_PATH}/${id}/archived`, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ driveFileId: "DRIVE1" }), + maxRedirects: 0, + }); + expect(unauthed.status()).toBe(401); + expect(unauthed.headers().location).toBeUndefined(); + + // A session, however privileged, is NOT a substitute: only the mirror + // can know that a file now exists in Drive. + const sessionAttempt = await request.post(`${INTAKE_PATH}/${id}/archived`, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ driveFileId: "DRIVE1" }), + maxRedirects: 0, + }); + expect(sessionAttempt.status()).toBe(401); + + const notBooked = await anonymous.post(`${INTAKE_PATH}/${id}/archived`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ driveFileId: "DRIVE1" }), + maxRedirects: 0, + }); + expect(notBooked.status()).toBe(409); + + await prisma.receiptIntake.update({ where: { id }, data: { state: "BOOKED" } }); + const ok = await anonymous.post(`${INTAKE_PATH}/${id}/archived`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ driveFileId: "DRIVE1" }), + maxRedirects: 0, + }); + expect(ok.status()).toBe(200); + const row = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(row?.state).toBe("ARCHIVED"); + expect(row?.archiveDriveFileId).toBe("DRIVE1"); + + await anonymous.dispose(); + }); + + test("an unknown id is 404", async ({ playwright }) => { + const machine = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + const res = await machine.post(`${INTAKE_PATH}/no-such-row/archived`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ driveFileId: "DRIVE1" }), + maxRedirects: 0, + }); + expect(res.status()).toBe(404); + await machine.dispose(); + }); +}); diff --git a/package.json b/package.json index 7624027a4..ac7a59dd3 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index b3e4044b7..ac2d5f4f4 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -51,6 +51,7 @@ const WORKER_ROW_SELECT = { storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, + createdAt: true, } as const; async function claim(): Promise { diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 173b0cd24..4ae343743 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -29,6 +29,13 @@ export interface WorkerRow extends BookableRow { state: string; fileSize: number; readAt: Date | null; + /** + * The fallback transaction date when the document's own date is + * unreadable. v1 used the Drive UPLOAD date (:1509); the intake row is + * created when the file arrives, so this is the same semantic — and, + * unlike "now", it does not drift when a read is delayed by an outage. + */ + createdAt: Date; } export interface WorkerDependencies { @@ -174,7 +181,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis invoice: read.invoice, checkNumber: read.checkNumber, totalAmount: read.totalAmount, - fallbackDateStr: toDateStr(row.readAt ?? deps.now()), + fallbackDateStr: toDateStr(row.createdAt), }); const totalCents = centsOf(keys.amount); diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts new file mode 100644 index 000000000..76912d62f --- /dev/null +++ b/tests/apply-receipt-intake.test.ts @@ -0,0 +1,105 @@ +/** + * The rollout script and the committed migration must describe the SAME table. + * + * They are written twice on purpose — the script is what PRODUCTION gets + * (before the deploy that selects these columns), the migration is what a fresh + * CI/dev database gets — and nothing else in the repo notices when the two + * drift. CI's `migrations` job would eventually catch a difference by diffing + * against production, but only AFTER the script has been run there, which is + * exactly the wrong time to find out. + * + * Importing the script must NOT open a connection or read DATABASE_URL: all of + * that sits behind the isMainModule guard, the same shape apply-bank-ledger has. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { RECEIPT_INTAKE_STATES, statements, targetMatches } from "../scripts/apply-receipt-intake.mjs"; +import { RECEIPT_INTAKE_STATES as RUNTIME_STATES } from "../src/lib/receipt-intake/route-state"; + +const migrationSql = readFileSync( + path.join(__dirname, "..", "prisma", "migrations", "20260901000000_receipt_intake", "migration.sql"), + "utf8", +); + +/** Compare SQL by meaning, not by indentation: collapse whitespace, drop comments. */ +function normalize(sql: string): string { + return sql + .split(/\r?\n/) + .filter(line => !/^\s*--/.test(line)) + .join(" ") + .replace(/\s+/g, " ") + .replace(/\s*([(),])\s*/g, "$1") + .trim() + .toLowerCase(); +} + +test("every column the apply script creates is in the committed migration", () => { + const createTable = statements.find((s: string) => s.includes('CREATE TABLE IF NOT EXISTS "ReceiptIntake"')); + assert.ok(createTable, "the script must create the table"); + const columns = Array.from(createTable.matchAll(/"([a-zA-Z0-9]+)"\s+(TEXT|BOOLEAN|INTEGER|DOUBLE PRECISION|DATE|TIMESTAMP\(3\))/g)) + .map(m => m[1]); + assert.ok(columns.length >= 36, `expected the full column list, found ${columns.length}`); + const migration = normalize(migrationSql); + for (const column of columns) { + assert.ok(migration.includes(`"${column.toLowerCase()}"`), `migration.sql is missing "${column}"`); + } +}); + +test("the partial unique index is identical in both, predicate included", () => { + // This index IS the strong-dedup claim. A version of it without the + // predicate would reject legitimate re-reads of a quarantined row; a + // version with a different predicate would quarantine the wrong things. + const fromScript = statements.find((s: string) => s.includes("ReceiptIntake_dedupStrongKey_active_key")); + assert.ok(fromScript); + const expected = normalize(fromScript); + const fromMigration = migrationSql + .split(";") + .map(normalize) + .find(s => s.includes("receiptintake_dedupstrongkey_active_key")); + assert.equal(fromMigration, expected); + assert.ok(expected.includes(`where "dedupstrongkey" is not null and "state" not in('duplicate','void')`)); +}); + +test("both files declare the SAME closed state set, and it matches the runtime one", () => { + // A state the CHECK constraint rejects but the code can produce is a + // guaranteed 500 on a document nobody can then see. + assert.deepEqual([...RUNTIME_STATES].sort(), [...RECEIPT_INTAKE_STATES].sort()); + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check")); + assert.ok(check, "the script must add the state CHECK constraint"); + for (const state of RECEIPT_INTAKE_STATES) { + assert.ok(check.includes(`'${state}'`), `apply script CHECK is missing ${state}`); + assert.ok(migrationSql.includes(`'${state}'`), `migration.sql CHECK is missing ${state}`); + } +}); + +test("every FK and index in the script also exists in the migration", () => { + const names = statements.flatMap((sql: string) => + Array.from(sql.matchAll(/(?:INDEX IF NOT EXISTS|CONSTRAINT) "([^"]+)"/g), m => m[1]), + ); + assert.ok(names.length >= 9, `expected the full object list, found ${names.length}`); + for (const name of new Set(names)) { + assert.ok(migrationSql.includes(`"${name}"`), `migration.sql is missing ${name}`); + // PostgreSQL silently truncates past 63 bytes, which would make + // "IF NOT EXISTS" match a different object than the one intended. + assert.ok(Buffer.byteLength(name, "utf8") <= 63, `identifier "${name}" is too long`); + } +}); + +test("every statement is idempotent — the script is safe to re-run", () => { + for (const sql of statements) { + const guarded = + /CREATE TABLE IF NOT EXISTS/.test(sql) || + /CREATE (?:UNIQUE )?INDEX IF NOT EXISTS/.test(sql) || + /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(sql); + assert.ok(guarded, `not idempotent: ${sql.slice(0, 80)}`); + } +}); + +test("the target guard needs BOTH the database name and the host", () => { + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10.0.0.5"), true); + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.9" }, "postgres", "10.0.0.5"), false); + assert.equal(targetMatches({ db: "staging", host: "10.0.0.5" }, "postgres", "10.0.0.5"), false); + assert.equal(targetMatches(null, "postgres", "10.0.0.5"), false); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 2d2b8b575..a0be27a49 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -46,6 +46,7 @@ function workerRow(overrides: Partial = {}): WorkerRow { memo: null, attempts: 0, readAt: null, + createdAt: new Date("2026-08-20T09:00:00.000Z"), ...overrides, }; } From 599aefa6504b03cd46a312ee9dd47302a82f14e9 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:00:45 -0700 Subject: [PATCH 028/144] fix(receipts): busyPasses column, exact host guard, UNIQUE partial-index check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, items 12/17. - ReceiptIntake.busyPasses: AI-unavailable passes are counted separately from `attempts`, because an outage is never the document's fault — but an outage that never ends still has to end somewhere (v3.4 parks at 20). - The apply script's --expect-host is now an EXACT match. The substring form copied from apply-bank-image gets LOOSER the shorter the input is: `--expect-host 1` satisfies `includes` against 10.0.0.5 and almost anything else. A guard whose whole job is to stop DDL landing on the wrong server must not have a degenerate case. - The partial index is verified on three properties, not one: it must EXIST, be UNIQUE (a non-unique index claims nothing, so every duplicate would sail through while the script reported success), and carry the EXACT predicate. - The state CHECK guard now keys on conrelid as well as conname — pg_constraint names are not globally unique, so an identically-named constraint on another table would satisfy the old guard and the CHECK would never be created. Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 7 ++- prisma/schema.prisma | 5 ++ scripts/apply-receipt-intake.mjs | 50 ++++++++++++++----- tests/apply-receipt-intake.test.ts | 33 +++++++++++- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 7927f7ad1..01ff53ede 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -45,6 +45,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "expenseId" TEXT, "archiveDriveFileId" TEXT, "attempts" INTEGER NOT NULL DEFAULT 0, + "busyPasses" INTEGER NOT NULL DEFAULT 0, "lastError" TEXT, "nextRetryAt" TIMESTAMP(3), "bookedAt" TIMESTAMP(3), @@ -65,7 +66,11 @@ CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("cre DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ReceiptIntake_state_check') THEN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_state_check' + AND conrelid = '"ReceiptIntake"'::regclass + ) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f17122863..608c776a4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3068,6 +3068,11 @@ model ReceiptIntake { expense Expense? @relation(fields: [expenseId], references: [id], onDelete: SetNull) archiveDriveFileId String? attempts Int @default(0) + /// Consecutive passes where the AI service was UNAVAILABLE (never the + /// document's fault, so it must not spend `attempts`). Ported from v3.4: + /// an outage that never ends still has to end somewhere, so after 20 the + /// row is parked for a human instead of retrying forever. + busyPasses Int @default(0) lastError String? nextRetryAt DateTime? bookedAt DateTime? diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index d88482518..ed679f5d0 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -51,18 +51,21 @@ function readFlagValue(flag) { } /** - * Pure comparison, exported for unit testing without a live DB (mirrors - * apply-bank-image.mjs). Compares BOTH database name and server host. + * Pure comparison, exported for unit testing without a live DB. Compares BOTH + * database name and server host, and both EXACTLY. + * + * apply-bank-image.mjs accepts a substring match on the host "because a pooled + * Supabase host resolves to an IP". That is a guard which gets LOOSER the + * shorter the operator's input is: `--expect-host 1` satisfies `host.includes` + * against 10.0.0.5, 172.16.1.1, and almost anything else. A guard whose whole + * job is to stop DDL landing on the wrong server must not have a degenerate + * case, so this one is exact. Print `host(inet_server_addr())` (the script logs + * it before refusing) and pass that value. */ export function targetMatches(actual, expectDb, expectHost) { if (!actual || typeof actual !== "object") return false; if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; - const host = String(actual.host ?? ""); - const wanted = String(expectHost ?? ""); - if (host === wanted) return true; - // A pooled Supabase host resolves to an IP; accept either the literal - // host string or an address that the operator typed instead. - return host !== "" && wanted !== "" && (host.includes(wanted) || wanted.includes(host)); + return String(actual.host ?? "") === String(expectHost ?? ""); } /** The closed set of states the CHECK constraint allows. Exported for tests. */ @@ -105,6 +108,7 @@ export const statements = [ "expenseId" TEXT, "archiveDriveFileId" TEXT, "attempts" INTEGER NOT NULL DEFAULT 0, + "busyPasses" INTEGER NOT NULL DEFAULT 0, "lastError" TEXT, "nextRetryAt" TIMESTAMP(3), "bookedAt" TIMESTAMP(3), @@ -146,7 +150,9 @@ export const statements = [ // state is a closed set — a typo must fail loudly rather than create a // silent eleventh state that no query ever selects. `DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ReceiptIntake_state_check') THEN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ReceiptIntake_state_check' + AND conrelid = '"ReceiptIntake"'::regclass) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); @@ -204,7 +210,7 @@ const expectedColumns = { "fileSha256", "vendor", "txnDate", "totalCents", "taxCents", "docType", "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", "dedupWeakKey", "duplicateOfId", "qbPurchaseId", "expenseId", - "archiveDriveFileId", "attempts", "lastError", "nextRetryAt", + "archiveDriveFileId", "attempts", "busyPasses", "lastError", "nextRetryAt", "bookedAt", "createdAt", "updatedAt", ], }; @@ -219,7 +225,19 @@ const expectedConstraints = [ // The partial index is the one object a "table exists" check cannot vouch for // (Prisma would have created the table on its own; it would never create this). -const expectedPartialIndexes = ["ReceiptIntake_dedupStrongKey_active_key"]; +// Verified on three properties, because any one of them alone can pass while +// the index is useless: it must EXIST, be UNIQUE (a non-unique index claims +// nothing, so every duplicate would sail through), and carry the EXACT +// predicate (a wider one quarantines rows that were deliberately excluded; a +// narrower one stops quarantining real duplicates). +const expectedPartialIndexes = [{ + name: "ReceiptIntake_dedupStrongKey_active_key", + mustMatch: [ + /CREATE UNIQUE INDEX/, + /\("dedupStrongKey"\)/, + /WHERE \(\("dedupStrongKey" IS NOT NULL\) AND \(state <> ALL \(ARRAY\['DUPLICATE'::text, 'VOID'::text\]\)\)\)/, + ], +}]; async function main() { if (!process.argv.includes("--yes")) { @@ -281,8 +299,8 @@ async function main() { // indpred IS NOT NULL is the whole point: a plain unique index of the // same name would silently quarantine nothing and reject legitimate - // re-reads, so assert the predicate exists rather than the name. - for (const name of expectedPartialIndexes) { + // re-reads, so assert the DEFINITION, not just the name. + for (const { name, mustMatch } of expectedPartialIndexes) { const [row] = await prisma.$queryRawUnsafe( `SELECT pg_get_indexdef(i.indexrelid) AS def FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid @@ -294,6 +312,12 @@ async function main() { console.error(`VERIFY FAILED: PARTIAL index ${name} missing (a non-partial index of that name is NOT the same thing)`); process.exit(1); } + for (const pattern of mustMatch) { + if (!pattern.test(row.def)) { + console.error(`VERIFY FAILED: ${name} does not match ${pattern}\n actual: ${row.def}`); + process.exit(1); + } + } console.log(`verified partial index ${name}: ${row.def}`); } diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts index 76912d62f..910a3db4f 100644 --- a/tests/apply-receipt-intake.test.ts +++ b/tests/apply-receipt-intake.test.ts @@ -97,9 +97,40 @@ test("every statement is idempotent — the script is safe to re-run", () => { } }); -test("the target guard needs BOTH the database name and the host", () => { +test("the target guard needs BOTH the database name and the host, EXACTLY", () => { assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10.0.0.5"), true); assert.equal(targetMatches({ db: "postgres", host: "10.0.0.9" }, "postgres", "10.0.0.5"), false); assert.equal(targetMatches({ db: "staging", host: "10.0.0.5" }, "postgres", "10.0.0.5"), false); assert.equal(targetMatches(null, "postgres", "10.0.0.5"), false); + + // A substring match (which apply-bank-image.mjs uses) gets LOOSER the + // shorter the operator's input is: "1" would satisfy `host.includes` + // against 10.0.0.5, 172.16.1.1 and almost anything else. A guard whose + // whole job is to stop DDL landing on the wrong server must not have a + // degenerate case. + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "1"), false); + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10.0.0.55"), false); + assert.equal(targetMatches({ db: "postgres", host: "" }, "postgres", "10.0.0.5"), false); +}); + +test("the partial-index verification checks UNIQUE and the exact predicate", () => { + // Existence alone is not enough: a NON-unique index of the same name claims + // nothing, so every duplicate would sail through while the script reported + // success. + const source = readFileSync(path.join(__dirname, "..", "scripts", "apply-receipt-intake.mjs"), "utf8"); + assert.match(source, /CREATE UNIQUE INDEX/, "the verifier asserts uniqueness"); + assert.match(source, /indpred IS NOT NULL/, "the verifier asserts the index is PARTIAL"); + assert.ok( + source.includes(`WHERE \\(\\("dedupStrongKey" IS NOT NULL\\) AND \\(state <> ALL \\(ARRAY\\['DUPLICATE'::text, 'VOID'::text\\]\\)\\)\\)`), + "the verifier asserts the exact predicate, not merely that one exists", + ); +}); + +test("the state CHECK guard is scoped to the ReceiptIntake table", () => { + // pg_constraint names are not globally unique — conname alone would let an + // identically-named constraint on ANOTHER table satisfy the guard, and the + // CHECK would silently never be created. + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check")); + assert.match(check!, /conrelid = '"ReceiptIntake"'::regclass/); + assert.match(migrationSql, /conrelid = '"ReceiptIntake"'::regclass/); }); From 1b6b01e72f2a31c20584f964b756cc679cac2ce7 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:01:04 -0700 Subject: [PATCH 029/144] fix(receipts): 25s read budget, 2 retries at 1s/3s, HEIC sequence brands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blocker 2 and item 18. The Apps Script's 5 retries at 2s..32s per model suit a 6-minute trigger that only has to finish before the next one. This worker has 60 seconds for a batch of ten, so the same schedule lets ONE busy document eat the whole invocation and starve the other nine — the outage shows up as a stalled queue rather than a slow one. Now: one 25s budget for the whole read, checked before every network call AND before every backoff, with the per-request AbortSignal capped at the remaining budget so a single hung socket cannot overrun it either. Exhaustion returns AI_UNAVAILABLE, which still never spends the row's attempts. `random` came out of the dependency shape with the jitter it existed for. HEIC: the old `hei` prefix test refused the HEVC brands, so a burst photo or a Live Photo still came back "unsupported-file-type" from a perfectly readable image. hevc/hevx/heix/msf1 now store as image/heic; the generic mif1/heif brands keep their own content type. Co-Authored-By: Claude Fable 5.1 --- src/lib/receipt-intake/file-type.ts | 17 +++++++- src/lib/receipt-intake/read.ts | 63 ++++++++++++++++++++++++----- tests/receipt-intake-auth.test.ts | 16 +++++++- tests/receipt-intake-read.test.ts | 63 +++++++++++++++++++++++------ 4 files changed, 134 insertions(+), 25 deletions(-) diff --git a/src/lib/receipt-intake/file-type.ts b/src/lib/receipt-intake/file-type.ts index ff682878a..29f07c24d 100644 --- a/src/lib/receipt-intake/file-type.ts +++ b/src/lib/receipt-intake/file-type.ts @@ -23,6 +23,11 @@ export const EXT_BY_MIME: Record = { export const MAX_INTAKE_BYTES = 15 * 1024 * 1024; +/** ISO-BMFF major brands stored as image/heic (still + HEVC sequence brands). */ +export const HEIC_BRANDS = new Set(["heic", "heix", "hevc", "hevx", "msf1"]); +/** The generic HEIF brands — stored under their own content type. */ +export const HEIF_BRANDS = new Set(["mif1", "heif"]); + /** Returns the accepted mime, or null when the bytes are not a supported document. */ export function sniffMime(buf: Buffer, declared: string): string | null { const essence = declared.split(";")[0].trim().toLowerCase(); @@ -37,8 +42,18 @@ export function sniffMime(buf: Buffer, declared: string): string | null { ) return "image/webp"; if (buf.length >= 5 && buf.subarray(0, 5).toString("ascii") === "%PDF-") return "application/pdf"; if (buf.length >= 12 && buf.subarray(4, 8).toString("ascii") === "ftyp") { + // ISO-BMFF major brands, per ISO/IEC 23008-12. iPhones emit `heic` + // (still) and `heix`; a burst or a Live Photo still can carry the HEVC + // brands `hevc`/`hevx`, which an earlier `hei` prefix check silently + // refused — those uploads came back "unsupported-file-type" from a + // perfectly readable photo. The image-SEQUENCE brands (`hevc`, `hevx`, + // `msf1`) are grouped with HEIC because Gemini and QBO both accept them + // under that content type. const brand = buf.subarray(8, 12).toString("ascii").toLowerCase(); - if (brand.startsWith("hei") || brand.startsWith("mif1") || brand.startsWith("msf1")) return "image/heic"; + if (HEIC_BRANDS.has(brand)) return "image/heic"; + // `mif1`/`heif` are the generic HEIF brands — kept as image/heif so the + // stored mimeType says what the file actually claims to be. + if (HEIF_BRANDS.has(brand)) return "image/heif"; } return essence === "text/plain" ? "text/plain" : null; } diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts index 5369077bf..586ab7080 100644 --- a/src/lib/receipt-intake/read.ts +++ b/src/lib/receipt-intake/read.ts @@ -21,9 +21,24 @@ * ListModels 2026-08-06, see src/lib/daily-log-task-match.ts:26). */ -const MAX_RETRIES = 5; +/** + * ONE row's entire read budget, models and backoffs included. + * + * The Apps Script could afford 5 retries per model with exponential backoff + * (2s..32s): it runs on a 6-minute trigger and only has to finish before the + * NEXT trigger. This worker runs inside a 60-second Vercel function that has to + * get through a batch of ten, so the same schedule would let ONE busy document + * eat the whole invocation and starve the other nine — the outage would look + * like a stalled queue rather than a slow one. A row that cannot be read in 25 + * seconds is not a row worth spending a whole run on; it comes back next pass + * at no cost to itself (AI_UNAVAILABLE never spends `attempts`). + */ +export const READ_BUDGET_MS = 25_000; +/** Retries AFTER the first attempt, per model. Three fetches per model, worst case. */ +const MAX_RETRIES = 2; +/** Backoff before retry 1 and retry 2. Short on purpose — see READ_BUDGET_MS. */ +const RETRY_BACKOFF_MS = [1_000, 3_000]; export const GEMINI_MODELS = ["gemini-3.5-flash", "gemini-flash-latest"]; -const READ_TIMEOUT_MS = 25_000; /** One selectable phase, rendered into the prompt as "code — name". */ export interface ProjectPhase { @@ -65,15 +80,18 @@ export interface ReadDependencies { fetchFn: typeof fetch; sleep: (ms: number) => Promise; apiKey: () => string | undefined; - /** Deterministic jitter seam for tests. */ - random: () => number; + /** Monotonic-enough clock, injectable so the budget is testable without waiting. */ + monotonicMs: () => number; + /** Total budget for this ONE read, across every model and backoff. */ + budgetMs: number; } const defaultDeps: ReadDependencies = { fetchFn: (...args) => fetch(...args), sleep: (ms) => new Promise(resolve => setTimeout(resolve, ms)), apiKey: () => process.env.GEMINI_API_KEY, - random: () => Math.random(), + monotonicMs: () => Date.now(), + budgetMs: READ_BUDGET_MS, }; /** Drive returns "text/plain; charset=utf-8" — strip parameters (:1073). */ @@ -179,7 +197,7 @@ export async function readReceipt( projectPhases: ProjectPhase[], deps: Partial = {}, ): Promise { - const { fetchFn, sleep, apiKey, random } = { ...defaultDeps, ...deps }; + const { fetchFn, sleep, apiKey, monotonicMs, budgetMs } = { ...defaultDeps, ...deps }; const key = apiKey(); // No key configured is a SERVICE fact, not a document fact — never spend // the row's attempts on it. @@ -201,24 +219,38 @@ export async function readReceipt( // forever. let sawDecisiveFailure = false; + const startedAt = monotonicMs(); + const remaining = () => budgetMs - (monotonicMs() - startedAt); + for (const model of GEMINI_MODELS) { const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}` + `:generateContent?key=${encodeURIComponent(key)}`; let attempts = 0; - while (attempts < MAX_RETRIES) { + for (;;) { + // The budget is checked before every network call AND before every + // sleep, so an exhausted budget can never be discovered only after + // the call that blew it. + if (remaining() <= 0) break; + let response: Response; try { response = await fetchFn(url, { method: "POST", headers: { "content-type": "application/json" }, body, - signal: AbortSignal.timeout(READ_TIMEOUT_MS), + // Never outlive the row's budget: a single hung socket must + // not consume the worker's whole invocation. + signal: AbortSignal.timeout(remaining()), }); } catch { + // Network error / our own abort. Both are SERVICE facts. + if (attempts >= MAX_RETRIES) break; + const wait = RETRY_BACKOFF_MS[attempts]; attempts++; - await sleep(Math.pow(2, attempts) * 1000); + if (remaining() <= wait) break; + await sleep(wait); continue; } @@ -238,8 +270,11 @@ export async function readReceipt( } if (code === 429 || code === 503) { // overloaded / rate-limited + if (attempts >= MAX_RETRIES) break; // fall through to the next model + const wait = RETRY_BACKOFF_MS[attempts]; attempts++; - await sleep(Math.pow(2, attempts) * 1000 + Math.floor(random() * 1000)); + if (remaining() <= wait) break; + await sleep(wait); continue; } @@ -253,7 +288,15 @@ export async function readReceipt( sawDecisiveFailure = true; return { ok: false, decisive: true }; } + + // The budget, not this model, is what ended the loop — trying the next + // model would only overrun it further. + if (remaining() <= 0) break; } + // Budget exhausted, or every model was unavailable: AI_UNAVAILABLE. A + // decisive failure still outranks it — if some model DID answer and could + // not read the document, that is a fact about the document and the caller + // must spend an attempt on it. return { ok: false, decisive: sawDecisiveFailure }; } diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 149a1727c..6e0f071b3 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -64,7 +64,7 @@ test("the stored mime is decided on the BYTES, not the caller's header", async ( const gif = Buffer.from("GIF89a-----"); const webp = Buffer.concat([Buffer.from("RIFF"), Buffer.from([0, 0, 0, 0]), Buffer.from("WEBP")]); const pdf = Buffer.from("%PDF-1.7\n..."); - const heic = Buffer.concat([Buffer.from([0, 0, 0, 24]), Buffer.from("ftypheic")]); + const ftyp = (brand: string) => Buffer.concat([Buffer.from([0, 0, 0, 24]), Buffer.from(`ftyp${brand}`)]); // A lie in the header cannot change the answer. assert.equal(sniffMime(jpeg, "text/plain"), "image/jpeg"); @@ -72,7 +72,19 @@ test("the stored mime is decided on the BYTES, not the caller's header", async ( assert.equal(sniffMime(gif, "image/jpeg"), "image/gif"); assert.equal(sniffMime(webp, "image/png"), "image/webp"); assert.equal(sniffMime(pdf, "image/png"), "application/pdf"); - assert.equal(sniffMime(heic, "image/jpeg"), "image/heic"); + // ISO/IEC 23008-12 major brands. iPhones emit `heic`/`heix` for stills and + // the HEVC brands for a burst or a Live Photo still — an earlier `hei` + // prefix check silently refused hevc/hevx, so those uploads came back + // "unsupported-file-type" from a perfectly readable photo. + for (const brand of ["heic", "heix", "hevc", "hevx", "msf1"]) { + assert.equal(sniffMime(ftyp(brand), "image/jpeg"), "image/heic", brand); + } + // The generic HEIF brands keep their own content type. + for (const brand of ["mif1", "heif"]) { + assert.equal(sniffMime(ftyp(brand), "image/jpeg"), "image/heif", brand); + } + // An unrelated ftyp box (an MP4) is not a receipt. + assert.equal(sniffMime(ftyp("isom"), "image/heic"), null); // text/plain has no signature, so it is the only type taken on its word. assert.equal(sniffMime(Buffer.from("VENDOR: Lowes"), "text/plain; charset=utf-8"), "text/plain"); diff --git a/tests/receipt-intake-read.test.ts b/tests/receipt-intake-read.test.ts index 9b9584b1a..ea8e587ac 100644 --- a/tests/receipt-intake-read.test.ts +++ b/tests/receipt-intake-read.test.ts @@ -76,7 +76,6 @@ test("a well-formed response parses into ReadResult", async () => { const outcome = await readReceipt(BYTES, "image/jpeg", PHASES, { apiKey: () => "test-key", sleep: noSleep, - random: () => 0, fetchFn: (async (_url: string, init: RequestInit) => { capturedBody = init.body as string; return geminiJson({ @@ -111,7 +110,6 @@ test("text/plain goes in as a text part, not inline_data", async () => { await readReceipt(Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"), "text/plain; charset=utf-8", [], { apiKey: () => "test-key", sleep: noSleep, - random: () => 0, fetchFn: (async (_url: string, init: RequestInit) => { capturedBody = init.body as string; return geminiJson({ doc_type: "receipt", total_amount: "10.00" }); @@ -126,30 +124,74 @@ test("an off-list phase suggestion is discarded, not trusted", () => { assert.equal(parsed?.suggestedPhaseCode, ""); }); -test("503 backs off five times, then falls to the next model", async () => { +test("503 retries twice on 1s/3s, then falls through to the next model", async () => { + // The Apps Script could afford 5 retries at 2s..32s; this worker has 60s for + // a batch of ten, so one busy document must not eat the invocation. const calls: string[] = []; const sleeps: number[] = []; const outcome = await readReceipt(BYTES, "image/jpeg", [], { apiKey: () => "test-key", - random: () => 0, sleep: async (ms) => { sleeps.push(ms); }, fetchFn: (async (url: string) => { calls.push(url); - if (calls.length <= 5) return new Response("busy", { status: 503 }); + if (calls.length <= 3) return new Response("busy", { status: 503 }); return geminiJson({ doc_type: "receipt", total_amount: "1.00" }); }) as unknown as typeof fetch, }); assert.ok(outcome.ok, "the second model answered"); - assert.equal(calls.length, 6); + assert.equal(calls.length, 4, "3 attempts on model 1, then model 2"); assert.ok(calls[0].includes("gemini-3.5-flash")); - assert.ok(calls[5].includes("gemini-flash-latest"), "fell through to the next model"); - assert.deepEqual(sleeps, [2000, 4000, 8000, 16000, 32000]); + assert.ok(calls[3].includes("gemini-flash-latest"), "fell through to the next model"); + assert.deepEqual(sleeps, [1000, 3000], "2 retries per model"); +}); + +test("the 25s budget is a hard ceiling across models and backoffs", async () => { + // A row that cannot be read inside its budget comes back next pass at no + // cost to itself. What it must NOT do is keep the worker's 60s function + // open while nine other receipts wait behind it. + let clock = 0; + const calls: string[] = []; + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + monotonicMs: () => clock, + sleep: async (ms) => { clock += ms; }, + fetchFn: (async (url: string) => { + calls.push(url); + clock += 9_000; // each call burns 9s + return new Response("busy", { status: 503 }); + }) as unknown as typeof fetch, + }); + // AI_UNAVAILABLE, never decisive: the document was never read, so the + // caller must not spend one of its attempts. + assert.deepEqual(outcome, { ok: false, decisive: false }); + assert.ok(clock <= 25_000 + 9_000, `budget overrun: ${clock}ms`); + assert.ok(calls.length <= 3, `budget should have stopped the retries, got ${calls.length} calls`); +}); + +test("a per-request timeout never outlives the remaining budget", async () => { + let clock = 0; + const timeouts: number[] = []; + await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + monotonicMs: () => clock, + sleep: async (ms) => { clock += ms; }, + fetchFn: (async (_url: string, init: RequestInit) => { + // AbortSignal.timeout is opaque; assert on the budget arithmetic by + // advancing the clock and checking the signal was created at all. + assert.ok(init.signal, "every request carries an abort signal"); + timeouts.push(clock); + clock += 5_000; + return new Response("busy", { status: 503 }); + }) as unknown as typeof fetch, + }); + // First call at 0ms, then 1s backoff -> 6s, then 3s backoff -> 14s... + assert.equal(timeouts[0], 0); + assert.ok(timeouts.every(t => t < 25_000), "no request starts after the budget is gone"); }); test("every model unavailable is NOT decisive — the row must not spend an attempt", async () => { const outcome = await readReceipt(BYTES, "image/jpeg", [], { apiKey: () => "test-key", - random: () => 0, sleep: noSleep, fetchFn: (async () => new Response("nope", { status: 404 })) as unknown as typeof fetch, }); @@ -160,7 +202,6 @@ test("a model that answers with unusable JSON IS decisive", async () => { // The model responded; retrying will not make this document readable. const outcome = await readReceipt(BYTES, "image/jpeg", [], { apiKey: () => "test-key", - random: () => 0, sleep: noSleep, fetchFn: (async () => new Response( JSON.stringify({ candidates: [{ content: { parts: [{ text: "not json at all" }] } }] }), @@ -174,7 +215,6 @@ test("HTTP 400 (payload rejected) is decisive and stops immediately", async () = let calls = 0; const outcome = await readReceipt(BYTES, "image/jpeg", [], { apiKey: () => "test-key", - random: () => 0, sleep: noSleep, fetchFn: (async () => { calls++; return new Response("too big", { status: 400 }); }) as unknown as typeof fetch, }); @@ -187,7 +227,6 @@ test("a missing API key is a SERVICE fact, never charged to the document", async const outcome = await readReceipt(BYTES, "image/jpeg", [], { apiKey: () => undefined, sleep: noSleep, - random: () => 0, fetchFn: (async () => { calls++; return geminiJson({}); }) as unknown as typeof fetch, }); assert.deepEqual(outcome, { ok: false, decisive: false }); From d26801b834dee9ffaa325c76f62ac7510ac4971f Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:01:04 -0700 Subject: [PATCH 030/144] fix(receipts): Expense.amount is GROSS; vendor confirms a dup; refunds reviewed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DECISION (Justin, 2026-09-01) — overrides spec §4.5: `Expense.amount` is the GROSS total paid, tax included. The QBO Purchase still splits the sales tax onto its own reclaimable account. But `Expense` has no tax column and the expenses already imported from QuickBooks record the gross line total, so booking pre-tax here would put two meanings of `amount` in one table and silently under-count every receipt this pipeline touched. `ReceiptIntake.taxCents` keeps the split for Phase 3's `Expense.taxAmount`. Also (Codex round 1, blockers 4/6 and items 10/13/16): - The v3.6 vendor-LESS strong key stands — one store spells its own name three ways, and keying on it put one purchase on two keys. The cost is that two unrelated vendors reusing an invoice number on one day for the same amount collide, and auto-quarantining one of those drops a real expense. So the vendor is not part of the KEY but it is part of the CONFIRMATION: NEEDS_REVIEW `vendor-mismatch:` instead of DUPLICATE. - Negative totals are refunds, not duplicates-of-nothing: NEEDS_REVIEW `refund-or-zero` (replacing `zero-total`), claiming no key. - A park BEFORE any QBO send RELEASES the strong key (v3.5): otherwise the key is held by a document that never became a purchase and a corrected re-send is quarantined against nothing. After a send it stays claimed — QBO may hold a Purchase whose response we lost. - MAX_BOOK_ATTEMPTS is `>=`, so the constant means 20 attempts in total. - The audit event records the tax that ACTUALLY posted (0 when buildGroups rejected the read), not the tax the model asked for; the filing report reconciles against the Purchase. Co-Authored-By: Claude Fable 5.1 --- src/lib/receipt-intake/book.ts | 89 ++++++++++++++++++------ src/lib/receipt-intake/route-state.ts | 44 +++++++++--- tests/receipt-intake-book.test.ts | 73 +++++++++++++++---- tests/receipt-intake-route-state.test.ts | 75 +++++++++++++++++--- 4 files changed, 229 insertions(+), 52 deletions(-) diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index c7ae60443..af5a76012 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -58,8 +58,19 @@ export type BookResult = | { outcome: "booked"; qbPurchaseId: string; expenseId: string; alreadyExisted: boolean } /** A switch is off: stay BOOKING, try again in an hour, spend NO attempt. */ | { outcome: "deferred"; reason: "push-disabled" | "push-paused" } - /** Terminal: a human must look at it. No further automatic attempt. */ - | { outcome: "needs-review"; reason: string } + /** + * Terminal: a human must look at it. No further automatic attempt. + * + * `releaseStrongKey` mirrors the Apps Script v3.5 rule. A parked row keeps + * holding `dedupStrongKey` (the partial unique index covers every state + * except DUPLICATE/VOID), so if we park BEFORE ever reaching QuickBooks — + * the job has no estimate, the date is unusable — the key is being held by + * a document that never became a purchase. A corrected re-send of the same + * receipt would then be quarantined against a row that represents nothing. + * Release in exactly that case. Once a send was ATTEMPTED the key must be + * held: QBO may have created the Purchase and lost the response. + */ + | { outcome: "needs-review"; reason: string; releaseStrongKey: boolean } /** Transport-class failure: attempts+1 and a backoff. */ | { outcome: "retry"; attempts: number; nextRetryAt: Date; reason: string }; @@ -136,11 +147,36 @@ export function buildGroups( }]; } -/** The Expense amount mirrors the QBO COGS line: pre-tax when the tax was split. */ -export function expenseAmountCents(groups: QboReceiptGroup[], totalCents: number): number { - const nonTax = groups.filter(g => g.tax !== true); - if (nonTax.length === 0) return totalCents; - return Math.round(nonTax.reduce((sum, g) => sum + g.amount, 0) * 100); +/** + * `Expense.amount` is the GROSS total paid, tax included — Justin's call + * (2026-09-01), overriding the plan's §4.5 "pre-tax" wording. + * + * The QBO Purchase still splits the tax onto its own reclaimable account; that + * is a QuickBooks-side concern and it is unchanged. But ProBuild's `Expense` + * has no tax column, and the expenses already imported from QBO + * (lib/qbo-expense-sync.ts) record the gross line total. Booking the pre-tax + * figure here would mean two intake paths writing the same table with two + * different meanings of `amount`, so job-cost and variance reports would + * silently under-count every receipt this pipeline touched. + * + * `ReceiptIntake.taxCents` keeps the split, so Phase 3 can add + * `Expense.taxAmount` and derive the pre-tax number without re-reading a single + * document. + */ +export function expenseAmountCents(_groups: QboReceiptGroup[], totalCents: number): number { + return totalCents; +} + +/** + * The tax that was ACTUALLY applied, read back off the built groups — 0 when + * `buildGroups` rejected the read (a check, or tax >= total). The audit row must + * record what posted, not what the model asked for; otherwise the sales-tax + * filing report reconciles against a number no Purchase ever carried. + */ +export function appliedTaxCents(groups: QboReceiptGroup[]): number { + return groups + .filter(g => g.tax === true) + .reduce((sum, g) => sum + Math.round(g.amount * 100), 0); } /** @db.Date round-trips as UTC midnight; QBO wants a bare calendar day. */ @@ -183,11 +219,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro if (!deps.isPushEnabled()) return { outcome: "deferred", reason: "push-disabled" }; if (await deps.isPushPaused()) return { outcome: "deferred", reason: "push-paused" }; - if (!row.projectId) return { outcome: "needs-review", reason: "no-estimate" }; - if (row.totalCents === null || row.totalCents <= 0) { - return { outcome: "needs-review", reason: "zero-total" }; - } - if (!row.txnDate) return { outcome: "needs-review", reason: "invalid-date" }; + // Everything down to the QBO call is a PRE-SEND refusal: nothing was ever + // sent, so the strong key must be handed back (see BookResult). + if (!row.projectId) return parkedBeforeSend("no-estimate"); + if (row.totalCents === null || row.totalCents <= 0) return parkedBeforeSend("refund-or-zero"); + if (!row.txnDate) return parkedBeforeSend("invalid-date"); // 2. The project's LATEST estimate — the same "primary estimate" rule the // v1 receipt-ingest endpoint uses (route.ts:69). Expense.estimateId is @@ -201,9 +237,9 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro estimates: { orderBy: { createdAt: "desc" }, take: 1, select: { id: true } }, }, }); - if (!project) return { outcome: "needs-review", reason: "no-estimate" }; + if (!project) return parkedBeforeSend("no-estimate"); const estimateId = project.estimates[0]?.id; - if (!estimateId) return { outcome: "needs-review", reason: "no-estimate" }; + if (!estimateId) return parkedBeforeSend("no-estimate"); // 3. Category groups (tax split). const groups = buildGroups(row.docType, row.totalCents, row.taxCents, row.refNumber); @@ -237,7 +273,9 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro result = await deps.createPurchase(tokens, input); } catch (error) { const terminal = terminalReasonFor(error); - if (terminal) return { outcome: "needs-review", reason: terminal }; + // A send WAS attempted: QBO may hold a Purchase whose response we lost, + // so the key stays claimed even though the row is parked. + if (terminal) return { outcome: "needs-review", reason: terminal, releaseStrongKey: false }; // QBTimeoutError, QBNotConnectedError, network/fetch errors, QBO // 429/5xx and DB errors are all transport-class: try again later. return retry(row, deps, now, describe(error)); @@ -248,7 +286,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // refusal (project-not-matched, docnumber-conflict, amount-mismatch, // missing-vendor, invalid-date, duplicate-name, ...). None becomes true // by waiting. - return { outcome: "needs-review", reason: `qbo-fault:${result.reason}` }; + return { outcome: "needs-review", reason: `qbo-fault:${result.reason}`, releaseStrongKey: false }; } // 5. One transaction: the Expense and the row's BOOKED state land together @@ -256,6 +294,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // lost-response retry, and QBO's idempotency has already guaranteed // there is exactly one Purchase. const amountCents = expenseAmountCents(groups, row.totalCents); + const taxApplied = appliedTaxCents(groups); const costCodeId = row.costCodeId ?? row.suggestedCostCodeId ?? null; const driveFileId = driveFileIdOf(row); const receiptUrl = driveFileId @@ -286,7 +325,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro qbPurchaseId: result.qbPurchaseId, description: `[Receipt intake] ${docRef}` + - (groups.length > 1 ? " · pre-tax (sales tax posted separately)" : "") + + (taxApplied > 0 ? ` · incl. $${(taxApplied / 100).toFixed(2)} sales tax` : "") + ` · pending bookkeeper review`, }, select: { id: true }, @@ -318,7 +357,10 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro docNumber: result.docNumber, fileName: row.fileName ?? undefined, amountCents, - taxCents: row.taxCents ?? undefined, + // What POSTED, not what was requested — buildGroups rejects a tax + // read on a check or when tax >= total, and the filing report has + // to reconcile against the Purchase. + taxCents: taxApplied, detail: { fileId, qbPurchaseId: result.qbPurchaseId, @@ -347,10 +389,17 @@ function describe(error: unknown): string { return "UnknownError"; } +/** A refusal reached WITHOUT any QBO call — the strong key goes back. */ +function parkedBeforeSend(reason: string): BookResult { + return { outcome: "needs-review", reason, releaseStrongKey: true }; +} + function retry(row: BookableRow, deps: BookDependencies, now: Date, reason: string): BookResult { const attempts = row.attempts + 1; - if (attempts > MAX_BOOK_ATTEMPTS) { - return { outcome: "needs-review", reason: "max-retries" }; + // `>=`, so MAX_BOOK_ATTEMPTS reads as "20 attempts in total" rather than 21. + if (attempts >= MAX_BOOK_ATTEMPTS) { + // Sends were attempted to get here, so the key stays claimed. + return { outcome: "needs-review", reason: "max-retries", releaseStrongKey: false }; } return { outcome: "retry", diff --git a/src/lib/receipt-intake/route-state.ts b/src/lib/receipt-intake/route-state.ts index f4f402549..66f880fef 100644 --- a/src/lib/receipt-intake/route-state.ts +++ b/src/lib/receipt-intake/route-state.ts @@ -18,6 +18,8 @@ export interface RouteInput { amount: string; /** Integer cents for this row, used to compare against a strong-key owner. */ totalCents: number | null; + /** canonicalVendor() of this document — see the vendor-mismatch rule below. */ + canonicalVendor: string; } export interface DedupHits { @@ -26,7 +28,7 @@ export interface DedupHits { * by the partial unique index rejecting our claim — the database IS the * lock (pgbouncer forbids session advisory locks). */ - strong: { id: string; totalCents: number | null } | null; + strong: { id: string; totalCents: number | null; canonicalVendor: string | null } | null; /** Another LIVE row carrying the same weak key. Always routes to a human. */ weak: { id: string } | null; } @@ -40,13 +42,24 @@ export interface RouteDecision { /** * First match wins. Order is the spec's, and it matters: * - multi/non_receipt are triage answers about the FILE, decided before money. - * - a "0.00" total is almost always a misread (:531 — you don't get a $0 - * receipt or write a $0 check), so it must never reach a dedup key or QBO. + * - a total that is zero OR NEGATIVE never books automatically. A $0.00 is + * almost always a misread (:531 — you don't get a $0 receipt or write a $0 + * check); a negative total is a refund, which is a legitimate document that + * a human must place against the original purchase. Both are decided BEFORE + * any dedup key is claimed, so neither can quarantine the real receipt that + * arrives next. * - no project means nobody can job-cost it yet; that is a queue, not a fault. - * - a strong hit at the SAME total is the same purchase arriving twice. - * A strong hit at a DIFFERENT total is ambiguous (a misread total, or two - * vendors reusing an invoice number on one day) and goes to a human — never - * resolved on a guess (:1545–1557). + * - a strong hit at the SAME total is the same purchase arriving twice — + * UNLESS the two documents name different vendors. The v3.6 key is + * deliberately vendor-less (:1545–1557: one store's own formats spell its + * name three ways, and keying on the vendor put one purchase on two keys), + * and that rationale stands. But the cost of leaving the vendor out is that + * two UNRELATED vendors reusing an invoice number on one day for the same + * amount now collide, and auto-quarantining one of them would silently drop + * a real expense. So the vendor is not part of the KEY, but it is part of + * the CONFIRMATION: a mismatch downgrades to a human. + * A strong hit at a DIFFERENT total is ambiguous the other way (a misread + * total) and also goes to a human — never resolved on a guess. * - a weak hit is only a POSSIBLE duplicate (two genuine same-day purchases * from one vendor for the same amount do happen), so it always asks a * human (:1591–1596). @@ -60,8 +73,8 @@ export function routeState(read: RouteInput, dedupHits: DedupHits, hasProject: b if (docType === "non_receipt") { return { state: "NON_RECEIPT", stateReason: null, duplicateOfId: null }; } - if (read.amount === "0.00") { - return { state: "NEEDS_REVIEW", stateReason: "zero-total", duplicateOfId: null }; + if (read.totalCents === null || read.totalCents <= 0 || read.amount === "0.00") { + return { state: "NEEDS_REVIEW", stateReason: "refund-or-zero", duplicateOfId: null }; } if (!hasProject) { return { state: "NEEDS_JOB", stateReason: null, duplicateOfId: null }; @@ -74,6 +87,19 @@ export function routeState(read: RouteInput, dedupHits: DedupHits, hasProject: b read.totalCents !== null && dedupHits.strong.totalCents === read.totalCents; if (sameTotal) { + // An owner whose vendor we don't know is not a confirmed match + // either — the same "can't confirm" reasoning as a null total. + const sameVendor = + !!dedupHits.strong.canonicalVendor && + !!read.canonicalVendor && + dedupHits.strong.canonicalVendor === read.canonicalVendor; + if (!sameVendor) { + return { + state: "NEEDS_REVIEW", + stateReason: `vendor-mismatch:${dedupHits.strong.id}`, + duplicateOfId: dedupHits.strong.id, + }; + } return { state: "DUPLICATE", stateReason: null, duplicateOfId: dedupHits.strong.id }; } return { diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 6b302f92f..3a6246e88 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -11,6 +11,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { + appliedTaxCents, bookReceipt, buildGroups, driveFileIdOf, @@ -125,21 +126,33 @@ test("a nonsense or absent tax falls back to the single-line shape", () => { assert.equal(buildGroups("receipt", 10000, 20000, "1").length, 1); }); -test("the Expense amount is the PRE-TAX figure when the tax was split", () => { - // Mirrors the QBO COGS line: the sales tax posts to its own reclaimable - // account, so job cost must not double-count it. - const groups = buildGroups("receipt", 36498, 2920, "82766"); - assert.equal(expenseAmountCents(groups, 36498), 33578); +test("the Expense amount is the GROSS total, tax included, split or not", () => { + // Justin's call (2026-09-01), overriding the plan's pre-tax wording: the + // expenses already imported from QuickBooks record the gross line total, so + // booking pre-tax here would put two meanings of `amount` in one table and + // silently under-count every receipt this pipeline touched. + const split = buildGroups("receipt", 36498, 2920, "82766"); + assert.equal(split.length, 2, "the QBO Purchase still splits the tax"); + assert.equal(expenseAmountCents(split, 36498), 36498); assert.equal(expenseAmountCents(buildGroups("receipt", 10000, null, "1"), 10000), 10000); }); +test("appliedTaxCents reports what POSTED, not what the model asked for", () => { + assert.equal(appliedTaxCents(buildGroups("receipt", 36498, 2920, "82766")), 2920); + // buildGroups rejects both of these, so the audit row must say 0 — the + // filing report reconciles against the Purchase, not against the read. + assert.equal(appliedTaxCents(buildGroups("check", 120000, 9000, "Check4178")), 0); + assert.equal(appliedTaxCents(buildGroups("receipt", 10000, 20000, "1")), 0); + assert.equal(appliedTaxCents(buildGroups("receipt", 10000, null, "1")), 0); +}); + test("only a drive row books under the Drive file id", () => { assert.equal(driveFileIdOf({ source: "drive", sourceRef: "drive:FILE123" }), "FILE123"); assert.equal(driveFileIdOf({ source: "mobile", sourceRef: "mobile:abc" }), null); assert.equal(driveFileIdOf({ source: "drive", sourceRef: "drive:" }), null); }); -test("a successful booking creates the Expense at the pre-tax amount and marks the row BOOKED", async () => { +test("a successful booking creates the Expense at the gross amount and marks the row BOOKED", async () => { const r = recorder(); const result = await bookReceipt(row(), r.deps); @@ -152,7 +165,7 @@ test("a successful booking creates the Expense at the pre-tax amount and marks t assert.equal(r.purchaseCalls[0].groups.length, 2); assert.equal(r.expenses.length, 1); - assert.equal(r.expenses[0].amount, 335.78); + assert.equal(r.expenses[0].amount, 364.98, "gross, tax included"); assert.equal(r.expenses[0].estimateId, "est-1"); assert.equal(r.expenses[0].costCodeId, "cc-plumb", "falls back to the model's phase suggestion"); assert.equal(r.expenses[0].qbPurchaseId, "QB-1"); @@ -163,6 +176,8 @@ test("a successful booking creates the Expense at the pre-tax amount and marks t assert.equal(r.intakeUpdates[0].qbPurchaseId, "QB-1"); assert.equal(r.events[0].kind, "receipt-push"); assert.equal(r.events[0].source, "intake-worker"); + assert.equal(r.events[0].amountCents, 36498); + assert.equal(r.events[0].taxCents, 2920, "the tax that actually posted"); }); test("an explicitly chosen cost code beats the model's suggestion", async () => { @@ -178,14 +193,40 @@ test("a non-drive row books under its intake id and stores the secure ref", asyn assert.equal(r.expenses[0].receiptUrl, "secure:receipts/intake/intake-1.jpg"); }); -test("a project with no estimate is terminal and spends NO attempt", async () => { +test("a project with no estimate is terminal, spends NO attempt, and RELEASES the strong key", async () => { + // Nothing was ever sent, so the row is holding a dedup key on behalf of a + // document that never became a purchase. A corrected re-send of the same + // receipt would be quarantined against it (v3.5 rule). const r = recorder({}, { estimates: [] }); const result = await bookReceipt(row(), r.deps); - assert.deepEqual(result, { outcome: "needs-review", reason: "no-estimate" }); + assert.deepEqual(result, { outcome: "needs-review", reason: "no-estimate", releaseStrongKey: true }); assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never touched"); assert.equal(r.expenses.length, 0); }); +test("every PRE-send refusal releases the key; every POST-send one holds it", async () => { + // Pre-send: nothing exists in QuickBooks, so the key must go back. + for (const [rowOverrides, reason] of [ + [{ projectId: null }, "no-estimate"], + [{ totalCents: 0 }, "refund-or-zero"], + [{ totalCents: -2257 }, "refund-or-zero"], + [{ txnDate: null }, "invalid-date"], + ] as const) { + const r = recorder(); + const result = await bookReceipt(row(rowOverrides), r.deps); + assert.deepEqual(result, { outcome: "needs-review", reason, releaseStrongKey: true }, reason); + assert.equal(r.purchaseCalls.length, 0, reason); + } + + // Post-send: QBO may hold a Purchase whose response we lost, so the key + // stays claimed even though the row is parked. + const faulted = recorder({ + createPurchase: async () => { throw new QboPurchaseFaultError(400, "closed period", "6210"); }, + }); + const result = await bookReceipt(row(), faulted.deps); + assert.equal((result as any).releaseStrongKey, false); +}); + test("the push kill switch and the pause switch defer without spending an attempt", async () => { const disabled = recorder({ isPushEnabled: () => false }); assert.deepEqual(await bookReceipt(row(), disabled.deps), { outcome: "deferred", reason: "push-disabled" }); @@ -216,7 +257,7 @@ test("QBO business-rule faults are TERMINAL, never retried", async () => { for (const [error, reason] of cases) { const r = recorder({ createPurchase: async () => { throw error; } }); const result = await bookReceipt(row(), r.deps); - assert.deepEqual(result, { outcome: "needs-review", reason }, reason); + assert.deepEqual(result, { outcome: "needs-review", reason, releaseStrongKey: false }, reason); assert.equal(r.expenses.length, 0); } }); @@ -228,6 +269,7 @@ test("an ok:false result is a deterministic refusal, so it goes to a human too", assert.deepEqual(await bookReceipt(row(), r.deps), { outcome: "needs-review", reason: "qbo-fault:docnumber-conflict", + releaseStrongKey: false, }); }); @@ -245,14 +287,21 @@ test("a QBTimeoutError retries on the backoff schedule", async () => { assert.equal((third as any).nextRetryAt.getTime(), NOW.getTime() + 60 * 60_000); }); -test("a plain network error retries; past 20 attempts it stops and asks a human", async () => { +test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTAL", async () => { const transient = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); assert.equal((await bookReceipt(row({ attempts: 5 }), transient.deps)).outcome, "retry"); + // row.attempts 18 -> this is attempt 19: still retryable. + const nearly = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + assert.equal((await bookReceipt(row({ attempts: 18 }), nearly.deps)).outcome, "retry"); + + // row.attempts 19 -> this is attempt 20, the last one the constant allows. const exhausted = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); - assert.deepEqual(await bookReceipt(row({ attempts: 20 }), exhausted.deps), { + assert.deepEqual(await bookReceipt(row({ attempts: 19 }), exhausted.deps), { outcome: "needs-review", reason: "max-retries", + // Sends were attempted to get here, so the key is NOT released. + releaseStrongKey: false, }); }); diff --git a/tests/receipt-intake-route-state.test.ts b/tests/receipt-intake-route-state.test.ts index fa49d9e9e..8cffd6da3 100644 --- a/tests/receipt-intake-route-state.test.ts +++ b/tests/receipt-intake-route-state.test.ts @@ -11,15 +11,18 @@ import assert from "node:assert/strict"; import { backoffMs, MAX_BOOK_ATTEMPTS, routeState } from "../src/lib/receipt-intake/route-state"; const NO_HITS = { strong: null, weak: null }; -const clean = { docType: "receipt", amount: "364.98", totalCents: 36498 }; +const clean = { docType: "receipt", amount: "364.98", totalCents: 36498, canonicalVendor: "lowes" }; +/** The strong key is vendor-LESS, so an owner has to carry its vendor separately. */ +const owner = (over: Partial<{ id: string; totalCents: number | null; canonicalVendor: string | null }> = {}) => + ({ id: "row-a", totalCents: 36498, canonicalVendor: "lowes", ...over }); test("multi outranks everything, including a missing project", () => { - const d = routeState({ docType: "multi", amount: "0.00", totalCents: null }, NO_HITS, false); + const d = routeState({ docType: "multi", amount: "0.00", totalCents: null, canonicalVendor: "" }, NO_HITS, false); assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "multi-doc", duplicateOfId: null }); }); test("a non-receipt is its own terminal state, not a review item", () => { - const d = routeState({ docType: "non_receipt", amount: "0.00", totalCents: null }, NO_HITS, true); + const d = routeState({ docType: "non_receipt", amount: "0.00", totalCents: null, canonicalVendor: "" }, NO_HITS, true); assert.deepEqual(d, { state: "NON_RECEIPT", stateReason: null, duplicateOfId: null }); }); @@ -27,11 +30,34 @@ test("a $0.00 total is a misread and is parked BEFORE any dedup or job check", ( // :531 — you don't get a $0 receipt or write a $0 check. Letting this reach // a key would poison it for the real document. const d = routeState( - { docType: "receipt", amount: "0.00", totalCents: 0 }, - { strong: { id: "owner", totalCents: 0 }, weak: { id: "other" } }, + { docType: "receipt", amount: "0.00", totalCents: 0, canonicalVendor: "lowes" }, + { strong: owner({ id: "owner", totalCents: 0 }), weak: { id: "other" } }, true, ); - assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "zero-total", duplicateOfId: null }); + assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "refund-or-zero", duplicateOfId: null }); +}); + +test("a NEGATIVE total is a refund: reviewed, and it claims no dedup key", () => { + // A refund is a legitimate document — v1 carried them all the way through + // rename/dedup/archive — but it must never book itself against the original + // purchase automatically, and it must not hold a key the original needs. + for (const [amount, cents] of [["-22.57", -2257], ["-1200.00", -120000]] as const) { + const d = routeState( + { docType: "receipt", amount, totalCents: cents, canonicalVendor: "lowes" }, + NO_HITS, + true, + ); + assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "refund-or-zero", duplicateOfId: null }, amount); + } +}); + +test("an unreadable total (null cents) is reviewed, not booked", () => { + const d = routeState( + { docType: "receipt", amount: "abc", totalCents: null, canonicalVendor: "lowes" }, + NO_HITS, + true, + ); + assert.equal(d.stateReason, "refund-or-zero"); }); test("no project means NEEDS_JOB — a queue, not a fault", () => { @@ -39,13 +65,40 @@ test("no project means NEEDS_JOB — a queue, not a fault", () => { assert.deepEqual(d, { state: "NEEDS_JOB", stateReason: null, duplicateOfId: null }); }); -test("a strong hit at the same total is the same purchase twice", () => { - const d = routeState(clean, { strong: { id: "row-a", totalCents: 36498 }, weak: null }, true); +test("a strong hit at the same total AND the same vendor is the same purchase twice", () => { + const d = routeState(clean, { strong: owner(), weak: null }, true); assert.deepEqual(d, { state: "DUPLICATE", stateReason: null, duplicateOfId: "row-a" }); }); +test("same total, DIFFERENT vendor is a key collision, not a duplicate", () => { + // The v3.6 key leaves the vendor out on purpose (one store spells its own + // name three ways). The cost is that two unrelated vendors reusing an + // invoice number on one day for the same amount collide — and quarantining + // one of those would silently drop a real expense. The vendor is not part + // of the KEY, but it is part of the CONFIRMATION. + const d = routeState(clean, { strong: owner({ canonicalVendor: "homedepot" }), weak: null }, true); + assert.deepEqual(d, { + state: "NEEDS_REVIEW", + stateReason: "vendor-mismatch:row-a", + duplicateOfId: "row-a", + }); +}); + +test("an owner whose VENDOR is unknown is not a confirmed match either", () => { + const d = routeState(clean, { strong: owner({ canonicalVendor: null }), weak: null }, true); + assert.equal(d.state, "NEEDS_REVIEW"); + assert.equal(d.stateReason, "vendor-mismatch:row-a"); +}); + +test("a chain's spelling variants still collapse — canonicalVendor is what is compared", () => { + // "Lowe's Home Improvement" and "LOWES HOME CENTERS LLC" both canonicalise + // to "lowes", so the alias table (not the raw string) decides this. + const d = routeState(clean, { strong: owner({ canonicalVendor: "lowes" }), weak: null }, true); + assert.equal(d.state, "DUPLICATE"); +}); + test("a strong hit at a DIFFERENT total is ambiguous and goes to a human", () => { - const d = routeState(clean, { strong: { id: "row-a", totalCents: 20000 }, weak: null }, true); + const d = routeState(clean, { strong: owner({ totalCents: 20000 }), weak: null }, true); assert.deepEqual(d, { state: "NEEDS_REVIEW", stateReason: "strong-dup-amount-mismatch:row-a", @@ -56,7 +109,7 @@ test("a strong hit at a DIFFERENT total is ambiguous and goes to a human", () => test("an owner whose total is unknown is never treated as a match", () => { // A null total means "can't confirm the totals match" — reading it as a // match would silently quarantine a real expense. - const d = routeState(clean, { strong: { id: "row-a", totalCents: null }, weak: null }, true); + const d = routeState(clean, { strong: owner({ totalCents: null }), weak: null }, true); assert.equal(d.state, "NEEDS_REVIEW"); assert.equal(d.stateReason, "strong-dup-amount-mismatch:row-a"); }); @@ -67,7 +120,7 @@ test("a weak hit always asks a human, never quarantines on its own", () => { }); test("the strong net is checked before the weak one", () => { - const d = routeState(clean, { strong: { id: "row-a", totalCents: 36498 }, weak: { id: "row-b" } }, true); + const d = routeState(clean, { strong: owner(), weak: { id: "row-b" } }, true); assert.equal(d.state, "DUPLICATE"); assert.equal(d.duplicateOfId, "row-a"); }); From e69d0c992db7e9813f2af67a066a15ee3f5cd005 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:01:19 -0700 Subject: [PATCH 031/144] fix(receipts): dry-run rows can no longer starve the queue; 40s soft deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blockers 1/5 and items 11/12/14. STARVATION. Dry-run rows park at READ but were still claimable, and the batch is ten rows: after a couple of shadow days the ten OLDEST rows were all parked ones, re-claimed every five minutes, and no new receipt was ever reached. The queue looked healthy and processed nothing. They are now excluded from the claim predicate itself, and a one-shot requeueDryRunParked on the FIRST live pass brings the backlog back — flipping `dryRun` in the same statement, because a row that reappeared still carrying dryRun=true would be skipped and re-parked forever. The requeue runs even when the claim lock is held: the lock guards the batch, not the cutover. SOFT DEADLINE. A row started at 41s can still be reading at 66s, past the function ceiling, and the invocation dies leaving the row in whatever state it reached. The worker now stops TAKING rows at 40s; the claim lease already keeps them, and the next run picks them up. WEAK-DEDUP RACE. The advisory lock key is one global constant (now said so, in a comment, since that is what makes a single batch run at a time). On top of it, READ -> BOOKING re-checks for a weak-key twin already BOOKING/BOOKED INSIDE the transition — the last instant before money moves — and parks to NEEDS_REVIEW on a hit, releasing the strong key since nothing was sent. TRANSIENT vs TERMINAL. A throw out of a row was parking it for a human, so one bad minute of Supabase or Prisma turned into a queue full of manual work with those rows still holding their strong keys. Only the classified QBO fault types are terminal now; everything else retries on the normal backoff, with the same 20-attempt ceiling. P2002 no longer string-matches Prisma's `meta` for "dedupStrongKey" — that shape is version dependent and is EMPTY for a partial index on some engine builds, i.e. exactly the index this mechanism depends on. The owner is looked up by dedupStrongKey, which is a fact about the data. Co-Authored-By: Claude Fable 5.1 --- .../api/cron/receipt-intake-worker/route.ts | 115 +++++++-- src/lib/receipt-intake/worker.ts | 220 +++++++++++++++--- tests/receipt-intake-worker.test.ts | 192 ++++++++++++++- 3 files changed, 466 insertions(+), 61 deletions(-) diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index ac2d5f4f4..fc2e49a3a 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -7,13 +7,14 @@ import { downloadDocBytes, toSecureRef } from "@/lib/secure-storage"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { readReceipt } from "@/lib/receipt-intake/read"; +import { canonicalVendor } from "@/lib/receipt-intake/keys"; import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; import { backoffMs } from "@/lib/receipt-intake/route-state"; import { BATCH_SIZE, CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, - isStrongKeyConflict, + isUniqueViolation, runIntakeWorker, type ReadPatch, type WorkerDependencies, @@ -51,9 +52,22 @@ const WORKER_ROW_SELECT = { storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, - createdAt: true, + createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; +/** + * A row parked by the shadow week (dryRun=true, sitting at READ or BOOKING) is + * DONE until the cutover. It is excluded from the claim rather than merely + * skipped inside the loop, because the batch is only ten rows: after a couple + * of shadow days the oldest ten rows are all parked ones, they get re-claimed + * every five minutes, and no NEW receipt is ever reached. The queue looks + * healthy and processes nothing. runIntakeWorker's requeueDryRunParked is what + * brings them back, once, on the first live pass. + */ +const NOT_DRY_RUN_PARKED: Prisma.ReceiptIntakeWhereInput = { + NOT: { AND: [{ dryRun: true }, { state: { in: ["READ", "BOOKING"] } }] }, +}; + async function claim(): Promise { const now = new Date(); return prisma.$transaction(async tx => { @@ -66,6 +80,7 @@ async function claim(): Promise { where: { state: { in: ["RECEIVED", "READ", "BOOKING"] }, OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }], + ...NOT_DRY_RUN_PARKED, }, orderBy: { createdAt: "asc" }, take: BATCH_SIZE, @@ -87,6 +102,20 @@ function buildDeps(): WorkerDependencies { return { claim, + isDryRunEnabled: () => process.env.RECEIPT_INTAKE_DRYRUN !== "false", + + requeueDryRunParked: async () => { + const { count } = await prisma.receiptIntake.updateMany({ + where: { dryRun: true, state: { in: ["READ", "BOOKING"] } }, + // dryRun flips WITH the requeue, in one statement: a row that + // reappears in the queue still carrying dryRun=true would be + // skipped by the loop and re-parked forever. + data: { dryRun: false, nextRetryAt: null }, + }); + if (count > 0) console.log("[cron/receipt-intake-worker] cutover requeue", count); + return count; + }, + loadPhases: async () => prisma.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true, name: true }, @@ -108,19 +137,30 @@ function buildDeps(): WorkerDependencies { // The partial unique index refused the claim — the DATABASE is // the lock the Apps Script did with Script Properties. Load the // owner so the caller can compare totals. - if (!isStrongKeyConflict(error) || !patch.dedupStrongKey) throw error; + // Which constraint fired is resolved by looking the owner up + // BY dedupStrongKey — a fact about the data — rather than by + // string-matching Prisma's `meta`, whose shape is version + // dependent and is empty for a partial index on some engine + // builds (i.e. exactly this index). + if (!isUniqueViolation(error) || !patch.dedupStrongKey) throw error; const owner = await prisma.receiptIntake.findFirst({ where: { dedupStrongKey: patch.dedupStrongKey, state: { notIn: ["DUPLICATE", "VOID"] }, id: { not: rowId }, }, - select: { id: true, totalCents: true }, + select: { id: true, totalCents: true, vendor: true }, }); - // A conflict with no findable owner would silently re-claim on - // the next pass; treat it as a real error instead. + // No owner means some OTHER unique constraint rejected the + // write; re-throw rather than reporting a dedup hit that isn't. if (!owner) throw error; - return { strongOwner: owner }; + return { + strongOwner: { + id: owner.id, + totalCents: owner.totalCents, + canonicalVendor: owner.vendor ? canonicalVendor(owner.vendor) : null, + }, + }; } }, @@ -141,12 +181,41 @@ function buildDeps(): WorkerDependencies { }); }, - promoteToBooking: async rowId => { - await prisma.receiptIntake.update({ + promoteToBooking: async (rowId, weakKey) => prisma.$transaction(async tx => { + // LAST weak-dedup check, taken INSIDE the transition. The check at + // read time can miss a pair that arrived in the same batch window, + // and READ -> BOOKING is the last instant before money moves. + if (weakKey) { + const conflict = await tx.receiptIntake.findFirst({ + where: { + dedupWeakKey: weakKey, + id: { not: rowId }, + state: { in: ["BOOKING", "BOOKED", "ARCHIVED"] }, + }, + select: { id: true }, + orderBy: { createdAt: "asc" }, + }); + if (conflict) { + await tx.receiptIntake.update({ + where: { id: rowId }, + data: { + state: "NEEDS_REVIEW", + stateReason: `weak-dup:${conflict.id}`, + // Parked without ever reaching QuickBooks, so the + // strong key goes back (same rule as book.ts). + dedupStrongKey: null, + nextRetryAt: null, + }, + }); + return { promoted: false, conflictId: conflict.id }; + } + } + await tx.receiptIntake.update({ where: { id: rowId }, data: { state: "BOOKING", stateReason: null }, }); - }, + return { promoted: true }; + }), book: row => bookReceipt(row, { db: prisma as unknown as BookPrismaClient, @@ -165,7 +234,15 @@ function buildDeps(): WorkerDependencies { if (result.outcome === "needs-review") { await prisma.receiptIntake.update({ where: { id: rowId }, - data: { state: "NEEDS_REVIEW", stateReason: result.reason, nextRetryAt: null }, + data: { + state: "NEEDS_REVIEW", + stateReason: result.reason, + nextRetryAt: null, + // Parked before any QBO send: hand the strong key back, + // or a corrected re-send of the same receipt would be + // quarantined against a row that never became a purchase. + ...(result.releaseStrongKey ? { dedupStrongKey: null } : {}), + }, }); return; } @@ -193,20 +270,30 @@ function buildDeps(): WorkerDependencies { }); }, - deferRead: async (rowId, _decisive, reason) => { + deferRead: async (rowId, busyPasses, reason) => { // The service was unavailable; the document was never read, so this - // costs no attempt — only a delay. Reuses the booking backoff table - // so one outage does not hammer Gemini from every row at once. + // costs no `attempts` — only a delay and one busy pass. Reuses the + // booking backoff table so one outage does not hammer Gemini from + // every row at once. await prisma.receiptIntake.update({ where: { id: rowId }, data: { + busyPasses, lastError: reason, nextRetryAt: new Date(Date.now() + backoffMs(1)), }, }); }, + retryRow: async (rowId, attempts, nextRetryAt, reason) => { + await prisma.receiptIntake.update({ + where: { id: rowId }, + data: { attempts, lastError: reason, nextRetryAt }, + }); + }, + now: () => new Date(), + monotonicMs: () => Date.now(), }; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 4ae343743..0975568b3 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -14,21 +14,50 @@ * require chain). */ import { Prisma } from "@prisma/client"; -import { dedupKeys } from "./keys"; -import { routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; +import { canonicalVendor, dedupKeys } from "./keys"; +import { backoffMs, MAX_BOOK_ATTEMPTS, routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; import { resolveSuggestedCostCodeId, type BookableRow, type BookResult } from "./book"; +import { QBTimeoutError } from "@/lib/quickbooks"; +import { + QboAccountConfigError, + QboPurchaseFaultError, + QboVendorDuplicateError, +} from "@/lib/qbo-receipt-push"; import type { ProjectPhase, ReadOutcome } from "./read"; +/** + * ONE global constant, deliberately not derived from anything per-row or + * per-deployment: `pg_try_advisory_xact_lock(hashtextextended(CLAIM_LOCK_KEY,0))` + * is what guarantees a single worker BATCH runs at a time across every + * concurrent invocation of the cron. A key that varied by row, region, or + * process would let two batches run together, and the weak-dedup net (a plain + * SELECT, not a claim) would then miss a pair that arrived in the same tick. + */ export const CLAIM_LOCK_KEY = "receipt-intake-worker"; export const BATCH_SIZE = 10; /** How long a claimed row is hidden from the next run. */ export const CLAIM_LEASE_MINUTES = 10; +/** + * Stop taking on NEW rows once this much of the 60s function budget is gone. + * One 25s read plus a QBO round trip can straddle the ceiling, and a row cut + * off mid-book is the one case where the lease is doing real work rather than + * being a formality. + */ +export const RUN_SOFT_DEADLINE_MS = 40_000; +/** + * Consecutive AI-unavailable passes before a row is parked for a human. Ported + * from v3.4: an outage that never ends still has to end somewhere, and 20 + * passes at 5 minutes each is over an hour of "we tried". + */ +export const MAX_BUSY_PASSES = 20; /** The columns a pass needs. A superset of BookableRow. */ export interface WorkerRow extends BookableRow { state: string; fileSize: number; readAt: Date | null; + dedupWeakKey: string | null; + busyPasses: number; /** * The fallback transaction date when the document's own date is * unreadable. v1 used the Drive UPLOAD date (:1509); the intake row is @@ -39,8 +68,17 @@ export interface WorkerRow extends BookableRow { } export interface WorkerDependencies { - /** Claims up to BATCH_SIZE rows and bumps their nextRetryAt. Returns [] when another run holds the lock. */ + /** Claims up to BATCH_SIZE rows and bumps their nextRetryAt. Returns null when another run holds the lock. */ claim: () => Promise; + /** RECEIPT_INTAKE_DRYRUN is not "false". Injected so the requeue is testable. */ + isDryRunEnabled: () => boolean; + /** + * One-shot at the start of the first LIVE pass: un-park every row the + * shadow week left sitting at READ/BOOKING with dryRun=true. Returns the + * number requeued. Naturally idempotent — after one live pass there is + * nothing left to match. + */ + requeueDryRunParked: () => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; downloadBytes: (secureRef: string) => Promise; read: (bytes: Buffer, mime: string, phases: ProjectPhase[]) => Promise; @@ -48,17 +86,31 @@ export interface WorkerDependencies { * Persist the read + routing. Returns the strong-key owner when the partial * unique index rejected our claim — that rejection IS the dedup hit. */ - applyRead: (rowId: string, patch: ReadPatch) => Promise<{ strongOwner: { id: string; totalCents: number | null } | null }>; + applyRead: (rowId: string, patch: ReadPatch) => Promise<{ strongOwner: StrongOwner | null }>; findWeakHit: (rowId: string, weakKey: string) => Promise<{ id: string } | null>; /** Marks a row NEEDS_REVIEW / NON_RECEIPT / whatever routing decided, with no keys claimed. */ applyState: (rowId: string, state: ReceiptIntakeState, stateReason: string | null, patch?: Partial) => Promise; - /** READ + dryRun=false -> BOOKING. */ - promoteToBooking: (rowId: string) => Promise; + /** + * READ + dryRun=false -> BOOKING, and the LAST weak-dedup check, taken + * inside the same transaction as the transition. Returns the conflicting + * row when another document with this weak key is already BOOKING/BOOKED. + */ + promoteToBooking: (rowId: string, weakKey: string | null) => Promise<{ promoted: boolean; conflictId?: string }>; book: (row: BookableRow) => Promise; applyBookResult: (rowId: string, result: BookResult) => Promise; - /** Read failed without touching the document: park it for a later pass. */ - deferRead: (rowId: string, decisive: boolean, reason: string) => Promise; + /** AI unavailable: park for a later pass WITHOUT spending an attempt. */ + deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; + /** A transient fault anywhere else: spend an attempt and back off. */ + retryRow: (rowId: string, attempts: number, nextRetryAt: Date, reason: string) => Promise; now: () => Date; + /** Elapsed-time source for the soft deadline. */ + monotonicMs: () => number; +} + +export interface StrongOwner { + id: string; + totalCents: number | null; + canonicalVendor: string | null; } export interface ReadPatch { @@ -83,6 +135,10 @@ export interface WorkerRunSummary { processed: number; byState: Record; skipped?: "already-running"; + /** Rows left unprocessed because the soft deadline hit. They keep their lease. */ + deferredToNextRun?: number; + /** Rows un-parked by the first live pass after the shadow week. */ + requeued?: number; } function centsOf(amount: string): number | null { @@ -104,13 +160,37 @@ export function toDateStr(date: Date): string { /** One pass. Never throws for a single bad row — one poison document must not stall the queue. */ export async function runIntakeWorker(deps: WorkerDependencies): Promise { + // Cutover: the FIRST live pass hands the shadow week's parked backlog back + // to the queue. Rows parked under dryRun are excluded from the claim (see + // the cron route's claim predicate) precisely so they cannot starve the + // batch — which also means nothing else would ever wake them. + let requeued = 0; + if (!deps.isDryRunEnabled()) { + requeued = await deps.requeueDryRunParked(); + } + const rows = await deps.claim(); - if (rows === null) return { processed: 0, byState: {}, skipped: "already-running" }; + if (rows === null) { + return { processed: 0, byState: {}, skipped: "already-running", ...(requeued ? { requeued } : {}) }; + } + const startedAt = deps.monotonicMs(); const byState: Record = {}; const bump = (state: string) => { byState[state] = (byState[state] ?? 0) + 1; }; + let processed = 0; + let deferredToNextRun = 0; + for (const row of rows) { + // A row started at 41s can still be reading at 66s, past the function + // ceiling — the invocation dies mid-book and the row's state is + // whatever it happened to be. Stop TAKING rows instead; the claim + // lease already keeps them ours, and the next run picks them up. + if (deps.monotonicMs() - startedAt >= RUN_SOFT_DEADLINE_MS) { + deferredToNextRun = rows.length - processed; + break; + } + processed++; try { if (row.state === "RECEIVED") { bump(await processReceived(row, deps)); @@ -118,7 +198,17 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise {}); - bump("NEEDS_REVIEW"); + bump(await handleRowError(row, deps, error)); } } - return { processed: rows.length, byState }; + return { + processed, + byState, + ...(deferredToNextRun ? { deferredToNextRun } : {}), + ...(requeued ? { requeued } : {}), + }; +} + +/** + * A throw out of a row's processing is almost never the document's fault: + * Supabase hiccuped, Prisma lost its connection, the settings read failed, a + * socket reset. Parking all of those for a human turns one bad minute into a + * queue full of manual work, and (worse) leaves rows holding their strong keys. + * + * Only the CLASSIFIED QuickBooks business faults are terminal here. Everything + * else spends an attempt and comes back on the normal backoff, with the same + * 20-attempt ceiling as booking so a genuinely broken row still ends up in + * front of a person. + */ +export async function handleRowError( + row: WorkerRow, + deps: WorkerDependencies, + error: unknown, +): Promise { + const message = error instanceof Error ? `${error.name}: ${error.message}` : "UnknownError"; + + if (isTerminalQboFault(error)) { + await deps.applyState(row.id, "NEEDS_REVIEW", `qbo-fault:${message}`.slice(0, 400)).catch(() => {}); + return "NEEDS_REVIEW"; + } + + const attempts = row.attempts + 1; + if (attempts >= MAX_BOOK_ATTEMPTS) { + await deps.applyState(row.id, "NEEDS_REVIEW", "max-retries").catch(() => {}); + return "NEEDS_REVIEW"; + } + await deps.retryRow( + row.id, + attempts, + new Date(deps.now().getTime() + backoffMs(attempts)), + `worker-error:${message}`.slice(0, 400), + ).catch(() => {}); + return "RETRY"; +} + +/** QBTimeoutError is deliberately NOT here — a timeout is transport, not a verdict. */ +export function isTerminalQboFault(error: unknown): boolean { + if (error instanceof QBTimeoutError) return false; + return ( + error instanceof QboPurchaseFaultError || + error instanceof QboAccountConfigError || + error instanceof QboVendorDuplicateError + ); } function stateForBookResult(result: BookResult): string { @@ -164,12 +301,20 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis const outcome = await deps.read(bytes, row.mimeType, phases); if (!outcome.ok) { // decisive: the model answered and still could not read it -> a human. - // not decisive: the SERVICE was unavailable -> try again, costs nothing. if (outcome.decisive) { await deps.applyState(row.id, "NEEDS_REVIEW", "unreadable"); return "NEEDS_REVIEW"; } - await deps.deferRead(row.id, false, "ai-unavailable"); + // The SERVICE was unavailable. That is never the document's fault, so + // it costs no `attempts` — but it cannot be free forever either, or an + // outage that outlasts the incident leaves rows cycling silently. v3.4 + // counts the busy passes separately and gives up after 20. + const busyPasses = row.busyPasses + 1; + if (busyPasses >= MAX_BUSY_PASSES) { + await deps.applyState(row.id, "NEEDS_REVIEW", "ai-unavailable"); + return "NEEDS_REVIEW"; + } + await deps.deferRead(row.id, busyPasses, "ai-unavailable"); return "RECEIVED"; } @@ -207,11 +352,13 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis suggestedCostCodeId: resolveSuggestedCostCodeId(read.suggestedPhaseCode, costCodes), }; - const decision = routeState( - { docType: read.docType, amount: keys.amount, totalCents }, - hits, - !!row.projectId, - ); + const routeInput = { + docType: read.docType, + amount: keys.amount, + totalCents, + canonicalVendor: canonicalVendor(read.vendor), + }; + const decision = routeState(routeInput, hits, !!row.projectId); // A document that never reaches READ must not hold the strong key: a // multi-doc, a non-receipt, or a $0 misread would otherwise quarantine the @@ -229,11 +376,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis if (applied.strongOwner) { // The claim lost: another live row already owns this date|ref. Re-route // with the owner in hand and write the losing outcome (no key). - const second = routeState( - { docType: read.docType, amount: keys.amount, totalCents }, - { strong: applied.strongOwner, weak }, - !!row.projectId, - ); + const second = routeState(routeInput, { strong: applied.strongOwner, weak }, !!row.projectId); await deps.applyState(row.id, second.state, second.stateReason, { ...base, dedupStrongKey: null, @@ -245,11 +388,16 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis return decision.state; } -/** True when a write failed because the strong-key partial unique index rejected it. */ -export function isStrongKeyConflict(error: unknown): boolean { - return ( - error instanceof Prisma.PrismaClientKnownRequestError && - error.code === "P2002" && - JSON.stringify(error.meta ?? {}).includes("dedupStrongKey") - ); +/** + * A unique-constraint violation. NOT specific to the strong key on purpose. + * + * The previous version string-matched "dedupStrongKey" inside `error.meta`, + * which is a Prisma-version-dependent shape AND is empty for a PARTIAL index on + * some engine builds — the exact index this whole mechanism relies on. The + * caller resolves which constraint fired by looking the owner up by + * dedupStrongKey, which is a fact about the DATA rather than about how Prisma + * happened to render the error. + */ +export function isUniqueViolation(error: unknown): boolean { + return error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002"; } diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index a0be27a49..130f94f54 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -13,14 +13,23 @@ import assert from "node:assert/strict"; import { runIntakeWorker, dateOnly, + isTerminalQboFault, + isUniqueViolation, toDateStr, + MAX_BUSY_PASSES, + RUN_SOFT_DEADLINE_MS, type ReadPatch, type WorkerDependencies, type WorkerRow, } from "../src/lib/receipt-intake/worker"; import type { ReadOutcome } from "../src/lib/receipt-intake/read"; import type { BookResult } from "../src/lib/receipt-intake/book"; +import { QBTimeoutError } from "../src/lib/quickbooks"; +import { QboAccountConfigError, QboPurchaseFaultError } from "../src/lib/qbo-receipt-push"; +import { Prisma } from "@prisma/client"; + +const PrismaKnownError = Prisma.PrismaClientKnownRequestError; const NOW = new Date("2026-09-01T12:00:00.000Z"); function workerRow(overrides: Partial = {}): WorkerRow { @@ -47,6 +56,8 @@ function workerRow(overrides: Partial = {}): WorkerRow { attempts: 0, readAt: null, createdAt: new Date("2026-08-20T09:00:00.000Z"), + dedupWeakKey: null, + busyPasses: 0, ...overrides, }; } @@ -74,27 +85,35 @@ interface Harness { applied: ReadPatch[]; states: { id: string; state: string; reason: string | null }[]; promoted: string[]; - deferred: string[]; + deferred: { id: string; busyPasses: number }[]; + retried: { id: string; attempts: number; reason: string }[]; + requeueCalls: number; + clock: number; } function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { reads: 0, books: 0, applied: [], states: [], promoted: [], deferred: [], + retried: [], requeueCalls: 0, clock: 0, deps: null as unknown as WorkerDependencies, }; h.deps = { claim: async () => rows, + isDryRunEnabled: () => true, + requeueDryRunParked: async () => { h.requeueCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], downloadBytes: async () => Buffer.from("bytes"), read: async () => { h.reads++; return goodRead; }, applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, findWeakHit: async () => null, applyState: async (id, state, reason) => { h.states.push({ id, state, reason }); }, - promoteToBooking: async id => { h.promoted.push(id); }, + promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, book: async () => { h.books++; return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, applyBookResult: async () => {}, - deferRead: async id => { h.deferred.push(id); }, + deferRead: async (id, busyPasses) => { h.deferred.push({ id, busyPasses }); }, + retryRow: async (id, attempts, _next, reason) => { h.retried.push({ id, attempts, reason }); }, now: () => NOW, + monotonicMs: () => h.clock, ...overrides, }; return h; @@ -140,8 +159,9 @@ test("LIVE: a READ row with dryRun=false is promoted and booked", async () => { }); test("a strong-key claim that loses re-routes against the owner and keeps no key", async () => { + // Same total AND same canonical vendor: a confirmed duplicate. const h = harness([workerRow()], { - applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498 } }), + applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }), }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { DUPLICATE: 1 }); @@ -151,7 +171,7 @@ test("a strong-key claim that loses re-routes against the owner and keeps no key test("a strong-key loss at a DIFFERENT total goes to a human, not to DUPLICATE", async () => { const h = harness([workerRow()], { - applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 999 } }), + applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 999, canonicalVendor: "lowes" } }), }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); @@ -170,14 +190,26 @@ test("a document that does not reach READ never claims the strong key", async () assert.equal(h.applied[0].dedupStrongKey, null); }); -test("a service outage costs no attempt: the row is deferred, not reviewed", async () => { - const h = harness([workerRow()], { read: async () => ({ ok: false, decisive: false }) }); +test("a service outage costs no attempt: the row is deferred and counts ONE busy pass", async () => { + const h = harness([workerRow({ busyPasses: 3 })], { read: async () => ({ ok: false, decisive: false }) }); const summary = await runIntakeWorker(h.deps); - assert.deepEqual(h.deferred, ["row-1"]); - assert.deepEqual(h.states, []); + assert.deepEqual(h.deferred, [{ id: "row-1", busyPasses: 4 }]); + assert.deepEqual(h.states, [], "no state change — the document was never read"); assert.deepEqual(summary.byState, { RECEIVED: 1 }); }); +test("an outage that never ends still ends: 20 busy passes parks the row", async () => { + // v3.4. Without a ceiling a row cycles silently forever and nobody is ever + // told the pipeline stopped producing. + const h = harness([workerRow({ busyPasses: MAX_BUSY_PASSES - 1 })], { + read: async () => ({ ok: false, decisive: false }), + }); + await runIntakeWorker(h.deps); + assert.deepEqual(h.deferred, [], "no further deferral"); + assert.equal(h.states[0].state, "NEEDS_REVIEW"); + assert.equal(h.states[0].reason, "ai-unavailable"); +}); + test("a document the model answered on but could not read goes to a human", async () => { const h = harness([workerRow()], { read: async () => ({ ok: false, decisive: true }) }); await runIntakeWorker(h.deps); @@ -199,7 +231,9 @@ test("another run holding the lock yields skipped, not an empty pass", async () }); }); -test("one poison row is parked and the rest of the batch still runs", async () => { +test("one blowing-up row does not stall the batch", async () => { + // The failing row is RETRIED (a throw here is almost always transport, not + // the document) and, either way, row 2 still gets processed. let call = 0; const h = harness([workerRow({ id: "row-1" }), workerRow({ id: "row-2" })], { read: async () => { @@ -210,8 +244,144 @@ test("one poison row is parked and the rest of the batch still runs", async () = }); const summary = await runIntakeWorker(h.deps); assert.equal(summary.processed, 2); - assert.equal(summary.byState.NEEDS_REVIEW, 1); + assert.equal(summary.byState.RETRY, 1); assert.equal(summary.byState.READ, 1); + assert.equal(h.retried[0].id, "row-1"); +}); + +test("a strong-key loss to a DIFFERENT vendor is a collision, not a duplicate", async () => { + const h = harness([workerRow()], { + applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "homedepot" } }), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); + assert.equal(h.states[0].reason, "vendor-mismatch:row-owner"); +}); + +// ── Dry-run starvation (Codex blocker 1) ───────────────────────────────────── + +test("the shadow week does NOT requeue parked rows", async () => { + const h = harness([workerRow({ state: "READ", dryRun: true })], { isDryRunEnabled: () => true }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.requeueCalls, 0); + assert.equal(summary.requeued, undefined); +}); + +test("the FIRST live pass un-parks the shadow week's backlog, once", async () => { + // Parked rows are excluded from the claim (see the cron route's + // NOT_DRY_RUN_PARKED) precisely so they cannot starve the ten-row batch — + // which also means nothing else would ever wake them. + const h = harness([], { + isDryRunEnabled: () => false, + requeueDryRunParked: async () => { h.requeueCalls++; return 7; }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.requeueCalls, 1); + assert.equal(summary.requeued, 7); + + // Idempotent by construction: nothing is left matching the predicate. + const second = harness([], { + isDryRunEnabled: () => false, + requeueDryRunParked: async () => { second.requeueCalls++; return 0; }, + }); + const secondSummary = await runIntakeWorker(second.deps); + assert.equal(secondSummary.requeued, undefined, "a no-op requeue is not reported"); +}); + +test("the requeue happens even when another worker holds the lock", async () => { + // The lock guards the BATCH, not the cutover. A run that finds the lock + // taken must still not swallow the one-shot requeue. + const h = harness([], { + isDryRunEnabled: () => false, + claim: async () => null, + requeueDryRunParked: async () => { h.requeueCalls++; return 3; }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.skipped, "already-running"); + assert.equal(summary.requeued, 3); +}); + +// ── Soft deadline (Codex blocker 2) ────────────────────────────────────────── + +test("the worker stops TAKING rows at 40s and leaves the rest for the next run", async () => { + // A row started at 41s can still be reading at 66s, past the 60s function + // ceiling — the invocation dies mid-book and the row is left in whatever + // state it happened to reach. + const rows = [1, 2, 3, 4, 5].map(n => workerRow({ id: `row-${n}` })); + const h = harness(rows, { + read: async () => { h.clock += 15_000; h.reads++; return goodRead; }, + }); + const summary = await runIntakeWorker(h.deps); + assert.ok(h.clock >= RUN_SOFT_DEADLINE_MS); + assert.equal(summary.processed, 3, "three rows fit inside the soft deadline"); + assert.equal(summary.deferredToNextRun, 2); + assert.equal(h.reads, 3, "the deferred rows are never read"); +}); + +// ── Weak-dedup race at the READ -> BOOKING transition (Codex blocker 5) ─────── + +test("a weak-key twin already BOOKING blocks the transition and asks a human", async () => { + const h = harness([workerRow({ state: "READ", dryRun: false, dedupWeakKey: "lowes|2026-08-03|364.98|amt" })], { + promoteToBooking: async (id, weakKey) => { + h.promoted.push(id); + assert.equal(weakKey, "lowes|2026-08-03|364.98|amt", "the weak key is passed INTO the transition"); + return { promoted: false, conflictId: "row-twin" }; + }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.books, 0, "money never moves on a blocked transition"); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); +}); + +// ── Transient vs terminal (Codex issue 11) ─────────────────────────────────── + +test("a storage/Prisma/network throw is RETRIED, not parked for a human", async () => { + // Parking every transient fault turns one bad minute into a queue full of + // manual work — and leaves those rows holding their strong keys. + for (const error of [new Error("connection reset"), new TypeError("fetch failed"), new QBTimeoutError("t")]) { + const h = harness([workerRow({ attempts: 2 })], { + downloadBytes: async () => { throw error; }, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { RETRY: 1 }, String(error)); + assert.equal(h.retried[0].attempts, 3); + assert.deepEqual(h.states, [], "not parked"); + } +}); + +test("a CLASSIFIED QBO business fault thrown mid-row IS terminal", () => { + assert.equal(isTerminalQboFault(new QboPurchaseFaultError(400, "closed period", "6210")), true); + assert.equal(isTerminalQboFault(new QboAccountConfigError("bad account")), true); + // A timeout is transport, not a verdict. + assert.equal(isTerminalQboFault(new QBTimeoutError("timed out")), false); + assert.equal(isTerminalQboFault(new Error("connection reset")), false); +}); + +test("a QBO fault thrown mid-row parks; a transient one past the ceiling also parks", async () => { + const terminal = harness([workerRow()], { + downloadBytes: async () => { throw new QboAccountConfigError("bad account"); }, + }); + assert.deepEqual((await runIntakeWorker(terminal.deps)).byState, { NEEDS_REVIEW: 1 }); + assert.match(terminal.states[0].reason!, /^qbo-fault:/); + + const exhausted = harness([workerRow({ attempts: 19 })], { + downloadBytes: async () => { throw new Error("connection reset"); }, + }); + assert.deepEqual((await runIntakeWorker(exhausted.deps)).byState, { NEEDS_REVIEW: 1 }); + assert.equal(exhausted.states[0].reason, "max-retries"); +}); + +test("isUniqueViolation is about the ERROR CODE, not Prisma's meta text", () => { + // The previous version string-matched "dedupStrongKey" inside error.meta, + // which is version-dependent and EMPTY for a partial index on some engine + // builds — i.e. exactly the index this mechanism depends on. + const p2002 = Object.assign(new Error("unique"), { code: "P2002", meta: {}, clientVersion: "5", name: "PrismaClientKnownRequestError" }); + Object.setPrototypeOf(p2002, PrismaKnownError.prototype); + assert.equal(isUniqueViolation(p2002), true, "an empty meta must still be recognised"); + const p2003 = Object.assign(new Error("fk"), { code: "P2003", meta: {}, clientVersion: "5" }); + Object.setPrototypeOf(p2003, PrismaKnownError.prototype); + assert.equal(isUniqueViolation(p2003), false); + assert.equal(isUniqueViolation(new Error("plain")), false); }); test("dateOnly keeps a calendar day at UTC midnight, the way @db.Date round-trips", () => { From f130894e06475dcd1a3cbd4dde586bacbdb9d21c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:01:34 -0700 Subject: [PATCH 032/144] fix(receipts): sha decides a sourceRef replay; provenance is not caller input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blockers 3 and items 7/8/9/15. SOURCEREF REUSE. Uploading before inserting meant a replayed sourceRef had already written a second object to the private bucket, and — worse — made the two cases indistinguishable: a genuine forwarder retry and a REUSED key carrying DIFFERENT bytes both landed on the same P2002 and both got a cheerful 200, so a second, real receipt could be swallowed by the first one's row and never booked. The row is now inserted FIRST, making the unique index the decision point with fileSha256 as the evidence: same bytes is a replay (200, the existing row), different bytes is 409 sourceRef-conflict and storage is never touched at all. PROVENANCE. `source` and `sourceRef` are no longer caller input for a human. A session/Bearer caller has the source minted from its auth kind and a web:/mobile: uuid minted server-side; supplying either is a 400 rather than a silent override, so a client cannot believe its retries are idempotent when every one creates a new document. Only shared-secret forwarders may declare drive/email/chat and their own key — which matters because `drive` rows book under the Drive fileId, so a forged source could aim a QBO DocNumber at another document's idempotency key. An existing row's fields come back only to its creator or a bookkeeping role; otherwise 409 with no fields, since even an id confirms the row exists. The shared-secret GET is the archive mirror and nothing else: only state=BOOKED|ARCHIVED, and a minimal column set with no error text, content hashes, or user ids. Least privilege applies to a script the same way it does to a user. The archive callback is idempotent for an identical retry (200 alreadyArchived). The mirror POSTs after writing the Drive file, so a lost response leaves it holding a file it cannot confirm; a 409 there would make it treat its own successful archive as a failure. A DIFFERENT file id on an archived row is still 409 — two Drive copies exist and somebody has to say which counts. Spec §4.5/§7 record the gross-amount decision and every round-1 change. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 64 +++++- e2e/receipt-intake.spec.ts | 131 +++++++++++- .../receipts/intake/[id]/archived/route.ts | 20 +- src/app/api/receipts/intake/route.ts | 199 ++++++++++++++---- src/lib/receipt-intake/intake-auth.ts | 9 +- src/lib/receipt-intake/queries.ts | 37 +++- 6 files changed, 403 insertions(+), 57 deletions(-) diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index bf9bc978b..ee38e7dfd 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -282,8 +282,9 @@ Run the apply script against prod BEFORE merging (CLAUDE.md pre-deploy rule #2). of the same file), else the intake `id`; `fileBase64` via `downloadDocBytes(storagePath)`; `projectName` = project.name. 5. On `ok:true`, one transaction: create `Expense` (estimateId; costCodeId = chosen, else - `matchCostCode(suggestedPhaseCode)`; amount = pre-tax amount when tax was split, else - total — mirrors the QBO COGS line; vendor; date=txnDate; status "Pending"; receiptUrl + `matchCostCode(suggestedPhaseCode)`; **amount = the GROSS total paid, tax INCLUDED** + (Justin, 2026-09-01 — this REPLACES the "pre-tax" rule this line used to state; see + the as-built note in §7); vendor; date=txnDate; status "Pending"; receiptUrl = Drive view URL when a Drive fileId is known, else the `secure:` ref; qbPurchaseId) and set the row BOOKED {qbPurchaseId, expenseId, bookedAt}. Also log one `AutomationEvent {kind:"receipt-push", source:"intake-worker"}` so the /automation @@ -375,6 +376,65 @@ against is the one above, with these six clarifications from the build: suite is there and wired into `npm run test:unit`); the rule that mattered — no `mock.module`, function injection only, because CI pins Node 20 — is followed. +### Round-1 review changes (2026-09-01) + +**DECISION (Justin, overrides §4.5): `Expense.amount` is the GROSS total paid, tax +included.** The QBO Purchase still splits the sales tax onto its own reclaimable account — +that is unchanged and it is what the reseller-permit filing reads. But `Expense` has no tax +column, and the expenses already imported from QuickBooks (`lib/qbo-expense-sync.ts`) +record the gross line total, so booking pre-tax here would put two meanings of `amount` in +one table and silently under-count every receipt this pipeline touched. `ReceiptIntake.taxCents` +keeps the split; Phase 3 adds `Expense.taxAmount` and can derive the pre-tax figure without +re-reading a single document. + +Also changed, all with tests: + +- **Dry-run rows are excluded from the claim, not skipped inside it.** The batch is ten + rows; after a couple of shadow days the ten oldest were all parked ones, so no NEW receipt + was ever reached and the queue looked healthy while processing nothing. A one-shot + `requeueDryRunParked` on the first live pass un-parks the backlog (and flips `dryRun` in + the same statement, or the rows would re-park forever). This is the one thing that changes + a row's `dryRun` after intake. +- **Read budget: 25s per row, 2 retries per model at 1s/3s.** The Apps Script's 5 retries at + 2s..32s suits a 6-minute trigger, not a 60-second function shared by ten rows. The worker + also stops TAKING new rows once 40s of its 60s are gone. Exhaustion returns AI_UNAVAILABLE, + which never spends `attempts` — but `busyPasses` now counts them and parks the row after 20 + (v3.4), so an endless outage still ends in front of a human. +- **`sourceRef` reuse is decided on `fileSha256`.** The row is inserted BEFORE the upload, so + the unique index is the decision point: same bytes is a replay (200, the existing row), + different bytes is 409 `sourceRef-conflict` and storage is never touched. Previously both + cases got a 200 and a second, real receipt could be swallowed. +- **Vendor is not part of the strong KEY, but it is part of the CONFIRMATION.** The v3.6 + vendor-less key stands; a same-total hit whose canonical vendor differs now routes to + NEEDS_REVIEW `vendor-mismatch:` rather than DUPLICATE. +- **Negative and zero totals** route to NEEDS_REVIEW `refund-or-zero` (replacing + `zero-total`) and claim no key. +- **A second weak-dedup check runs INSIDE the READ→BOOKING transaction**, the last instant + before money moves. The claim advisory lock is one global constant so only one batch runs + at a time. +- **A NEEDS_REVIEW park RELEASES the strong key unless a QBO send was attempted** (v3.5 + rule): otherwise the key is held by a document that never became a purchase, and a + corrected re-send is quarantined against nothing. +- **Transient throws (storage, Prisma, network) retry on the normal backoff.** Only the + classified QBO fault types are terminal. `MAX_BOOK_ATTEMPTS` is now `>=`, so it means 20 + attempts in total. +- **Non-secret callers cannot choose `source` or `sourceRef`** — the server mints both from + the auth kind; anything else is a 400. Only shared-secret callers may declare + drive/email/chat. An existing row's fields come back only to its creator or a bookkeeping + role. +- **The shared-secret GET is limited to `state=BOOKED|ARCHIVED`** and a minimal field set + (no error text, hashes, or user ids). +- Archive callback is idempotent for an identical retry (200), 409 only for a DIFFERENT + Drive file id. HEIC sniffing accepts `hevc`/`hevx`; `mif1`/`heif` store as image/heif. + P2002 resolves the owner by `dedupStrongKey` instead of string-matching Prisma's `meta` + (which is empty for a partial index on some engine builds). The apply script matches + `--expect-host` exactly and verifies the index is UNIQUE with the exact predicate. + +**Left as-is, deliberately:** the pre-existing asset-suffix proxy bypass (not introduced +here); multipart buffering before the size check (the platform body limit applies first); +PDFs carrying embedded JavaScript (never opened server-side — the bytes go to Gemini and to +QBO as an attachment). + Two things a human must do before this can leave shadow mode: - Set `RECEIPT_INTAKE_SECRET` (new, independent of `RECEIPT_INGEST_SECRET`) in Vercel, and diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 9705280e8..db325a9e7 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -35,10 +35,18 @@ const REF_PREFIX = "drive:e2e-intake-"; const FILE_ID = `${Date.now()}-a`; const SOURCE_REF = `${REF_PREFIX}${FILE_ID}`; +// Rows created with a SERVER-minted sourceRef (web:) can't be found by +// the prefix, so they are tracked explicitly for teardown. +const minted: string[] = []; + // A real 1x1 PNG: the endpoint decides the stored mime on the BYTES, so a // placeholder string would be refused (which is itself asserted below). const PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; +// A DIFFERENT 1x1 PNG (black, not white). Same format, different bytes — which +// is the whole point of the sourceRef-conflict case below. +const OTHER_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; function intakeBody(overrides: Record = {}) { return JSON.stringify({ @@ -76,6 +84,7 @@ test.beforeAll(async () => { test.afterAll(async () => { await prisma.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: REF_PREFIX } } }); + if (minted.length) await prisma.receiptIntake.deleteMany({ where: { id: { in: minted } } }); await prisma.$disconnect(); }); @@ -172,6 +181,65 @@ test.describe("intake POST", () => { expect(body.reason).toBe("missing-sourceRef"); }); + test("reusing a sourceRef for DIFFERENT bytes is 409, and stores nothing", async ({ request }) => { + // The dangerous case: answering 200 would tell the forwarder its NEW + // receipt was accepted when nothing was stored, and that receipt would + // never be booked. + const ref = `${REF_PREFIX}sha-conflict`; + const first = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(first.res.status()).toBe(200); + + const second = await postIntake(request, intakeBody({ sourceRef: ref, fileBase64: OTHER_PNG_BASE64 })); + expect(second.res.status()).toBe(409); + expect(second.body).toMatchObject({ error: "sourceRef-conflict", existingId: first.body.id }); + + // Exactly one row, still pointing at the ORIGINAL bytes. + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: ref } }); + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(first.body.id); + + // And the row's stored object is the one the FIRST request wrote — the + // conflicting call must never touch storage. + expect(rows[0].storagePath).toBe(`receipts/intake/${first.body.id}.png`); + }); + + test("a session caller may not choose its own source or sourceRef", async ({ request }) => { + // `source` is provenance and it feeds booking identity: a `drive` row + // books under the Drive fileId, so a forged source could aim a QBO + // DocNumber at another document's idempotency key. + const forgedRef = await postIntake(request, JSON.stringify({ + source: "web", sourceRef: `${REF_PREFIX}forged`, fileBase64: PNG_BASE64, mimeType: "image/png", + }), {}); + expect(forgedRef.res.status()).toBe(400); + expect(forgedRef.body.reason).toBe("sourceRef-not-allowed"); + + const forgedSource = await postIntake(request, JSON.stringify({ + source: "drive", fileBase64: PNG_BASE64, mimeType: "image/png", + }), {}); + expect(forgedSource.res.status()).toBe(400); + expect(forgedSource.body.reason).toBe("invalid-source"); + }); + + test("a session upload gets a server-minted web: sourceRef", async ({ request }) => { + const res = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ fileBase64: PNG_BASE64, mimeType: "image/png", fileName: "web.png" }), + maxRedirects: 0, + }); + expect(res.status()).toBe(200); + const body = await res.json(); + expect(body.sourceRef).toMatch(/^web:[0-9a-f-]{36}$/); + minted.push(body.id); + }); + + test("a secret caller may not declare a USER source", async ({ request }) => { + const { res, body } = await postIntake(request, intakeBody({ + source: "web", sourceRef: `${REF_PREFIX}websecret`, + })); + expect(res.status()).toBe(400); + expect(body.reason).toBe("invalid-source"); + }); + test("deterministic bad input is a 400, not a 500 the forwarder retries forever", async ({ request }) => { const cases: [string, string][] = [ [intakeBody({ source: "carrier-pigeon", sourceRef: `${REF_PREFIX}src` }), "invalid-source"], @@ -208,7 +276,17 @@ test.describe("intake GET", () => { expect(body.rows[0]).not.toHaveProperty("readJson"); }); - test("the archive mirror can poll with the shared secret", async ({ playwright }) => { + test("the archive mirror can poll BOOKED, and sees only what it needs", async ({ request, playwright }) => { + // Seed a BOOKED row so the field set is asserted against a real payload + // rather than an empty list. + const ref = `${REF_PREFIX}mirror`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + await prisma.receiptIntake.update({ + where: { id: created.body.id }, + data: { state: "BOOKED", vendor: "Lowes", totalCents: 36498, lastError: "should-not-be-visible" }, + }); + const machine = await playwright.request.newContext({ baseURL: "http://localhost:3000", storageState: { cookies: [], origins: [] }, @@ -218,6 +296,37 @@ test.describe("intake GET", () => { maxRedirects: 0, }); expect(res.status()).toBe(200); + const body = await res.json(); + const row = body.rows.find((r: any) => r.id === created.body.id); + expect(row).toBeTruthy(); + expect(row.vendor).toBe("Lowes"); + expect(row.totalCents).toBe(36498); + // Least privilege: a script that only copies files to Drive has no need + // for error text, content hashes, or who uploaded it. + for (const forbidden of ["lastError", "fileSha256", "createdById", "dedupWeakKey", "dedupStrongKey", "attempts", "readJson"]) { + expect(row, forbidden).not.toHaveProperty(forbidden); + } + await machine.dispose(); + }); + + test("the shared secret cannot sweep any state it likes", async ({ playwright }) => { + const machine = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + for (const state of ["NEEDS_REVIEW", "RECEIVED", "READ", ""]) { + const res = await machine.get(`${INTAKE_PATH}${state ? `?state=${state}` : ""}`, { + headers: { "x-receipt-intake-secret": SECRET }, + maxRedirects: 0, + }); + expect(res.status(), state || "(no state)").toBe(400); + } + // ARCHIVED is allowed — the mirror re-checks what it already copied. + const archived = await machine.get(`${INTAKE_PATH}?state=ARCHIVED`, { + headers: { "x-receipt-intake-secret": SECRET }, + maxRedirects: 0, + }); + expect(archived.status()).toBe(200); await machine.dispose(); }); @@ -283,16 +392,32 @@ test.describe("archive callback", () => { expect(notBooked.status()).toBe(409); await prisma.receiptIntake.update({ where: { id }, data: { state: "BOOKED" } }); - const ok = await anonymous.post(`${INTAKE_PATH}/${id}/archived`, { + const archive = (driveFileId: string) => anonymous.post(`${INTAKE_PATH}/${id}/archived`, { headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, - data: JSON.stringify({ driveFileId: "DRIVE1" }), + data: JSON.stringify({ driveFileId }), maxRedirects: 0, }); + + const ok = await archive("DRIVE1"); expect(ok.status()).toBe(200); const row = await prisma.receiptIntake.findUnique({ where: { id } }); expect(row?.state).toBe("ARCHIVED"); expect(row?.archiveDriveFileId).toBe("DRIVE1"); + // IDEMPOTENT REPLAY. The mirror POSTs after writing the Drive file, so a + // lost response leaves it holding a file it cannot confirm. Re-sending + // the same id is the correct retry: a 409 would make the script treat + // its own successful archive as a failure. + const replay = await archive("DRIVE1"); + expect(replay.status()).toBe(200); + expect((await replay.json()).alreadyArchived).toBe(true); + + // A DIFFERENT file id on an archived row is not a replay — two Drive + // copies exist and somebody has to say which one counts. + const conflicting = await archive("DRIVE2"); + expect(conflicting.status()).toBe(409); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.archiveDriveFileId).toBe("DRIVE1"); + await anonymous.dispose(); }); diff --git a/src/app/api/receipts/intake/[id]/archived/route.ts b/src/app/api/receipts/intake/[id]/archived/route.ts index 22487d19b..b5b9b58c9 100644 --- a/src/app/api/receipts/intake/[id]/archived/route.ts +++ b/src/app/api/receipts/intake/[id]/archived/route.ts @@ -40,9 +40,27 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const row = await prisma.receiptIntake.findUnique({ where: { id }, - select: { id: true, state: true }, + select: { id: true, state: true, archiveDriveFileId: true }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + + // IDEMPOTENT REPLAY. The mirror POSTs after writing the Drive file, so a + // lost response leaves it holding a file it cannot confirm. Re-sending the + // SAME driveFileId is the correct retry and must succeed — answering 409 + // would make the script treat its own successful archive as a failure and + // either re-copy the file or alert a human about nothing. + // A DIFFERENT driveFileId on an archived row is not a replay: two Drive + // copies exist and somebody has to say which one counts. + if (row.state === "ARCHIVED") { + if (row.archiveDriveFileId === driveFileId) { + return NextResponse.json({ ok: true, id, state: "ARCHIVED", archiveDriveFileId: driveFileId, alreadyArchived: true }); + } + return NextResponse.json( + { ok: false, reason: "already-archived", archiveDriveFileId: row.archiveDriveFileId }, + { status: 409 }, + ); + } + if (row.state !== "BOOKED") { return NextResponse.json({ ok: false, reason: "not-booked", state: row.state }, { status: 409 }); } diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index f89003150..c6873b1e0 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -3,11 +3,11 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; -import { SECURE_BUCKET, removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; +import { SECURE_BUCKET } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; -import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; +import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; import { EXT_BY_MIME, MAX_INTAKE_BYTES, sniffMime } from "@/lib/receipt-intake/file-type"; -import { listReceiptIntakes, serializeReceiptIntake } from "@/lib/receipt-intake/queries"; +import { ARCHIVE_READABLE_STATES, listReceiptIntakes, serializeReceiptIntake } from "@/lib/receipt-intake/queries"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -26,12 +26,25 @@ export const maxDuration = 30; * handler is the sole auth boundary — see src/lib/receipt-intake/intake-auth.ts. * * Shadow-week gate: `dryRun` is captured PER ROW at intake time from - * RECEIPT_INTAKE_DRYRUN (default ON). A row created during the shadow week - * stays dry-run even if the env flips later — flipping the switch must not - * retroactively book a backlog nobody reviewed. + * RECEIPT_INTAKE_DRYRUN (default ON), so the flag a row was accepted under is + * a fact about that row rather than about whenever the worker next looks at + * it. At CUTOVER the worker's one-shot `requeueDryRunParked` deliberately + * flips the parked backlog to live in a single statement — see + * src/lib/receipt-intake/worker.ts. That is the ONLY thing that changes a + * row's dryRun after intake. */ -const VALID_SOURCES = new Set(["mobile", "email", "drive", "chat", "web"]); +/** + * Sources a SHARED-SECRET forwarder may declare. A human caller can never pick + * one of these: `source` is provenance, and a browser asserting "this came from + * the Drive folder" is a claim it has no standing to make. It also feeds + * booking identity — `drive` rows book under the Drive fileId so the DocNumber + * stays continuous with v1 — so a forged `source` could aim a Purchase at + * another document's idempotency key. + */ +const MACHINE_SOURCES = new Set(["drive", "email", "chat"]); +/** Minted server-side from the authenticated caller, never read off the body. */ +const USER_SOURCES = new Set(["mobile", "web"]); interface ParsedBody { bytes: Buffer; @@ -109,20 +122,34 @@ export async function POST(req: Request) { const parsed = await parseBody(req); if (parsed instanceof NextResponse) return parsed; - if (!VALID_SOURCES.has(parsed.source)) return bad("invalid-source"); - const mimeType = sniffMime(parsed.bytes, parsed.declaredMime); if (!mimeType) return bad("unsupported-file-type"); - // A machine caller OWNS its idempotency key — it is the only thing that - // makes a forwarder replay free. A human upload has no natural key, so one - // is minted; two taps of the button are two documents, which is correct. - let sourceRef = parsed.sourceRef; + // PROVENANCE AND IDENTITY ARE NOT CALLER INPUT for a human. + // + // A shared-secret forwarder owns both: its `sourceRef` is the only reason a + // replay is free, and its `source` is a fact it genuinely knows (which + // folder, which mailbox). A session or Bearer caller knows neither. Letting + // one pass `source: "drive"` plus a chosen `sourceRef` would let it claim + // another document's idempotency key — and `drive` rows book under the + // Drive fileId, so that key is what a QBO DocNumber is derived from. + let source: string; + let sourceRef: string; if (auth.via === "secret") { - if (!sourceRef) return bad("missing-sourceRef"); + if (!MACHINE_SOURCES.has(parsed.source)) return bad("invalid-source"); + if (!parsed.sourceRef) return bad("missing-sourceRef"); + source = parsed.source; + sourceRef = parsed.sourceRef; } else { - sourceRef = sourceRef ?? `${parsed.source === "mobile" ? "mobile" : "web"}:${randomUUID()}`; + source = auth.userVia === "mobile-jwt" ? "mobile" : "web"; + // Reject rather than silently ignore: a client that thinks it set the + // key would otherwise believe its retries were idempotent when every + // one of them creates a new document. + if (parsed.sourceRef) return bad("sourceRef-not-allowed"); + if (parsed.source && parsed.source !== source) return bad("invalid-source"); + sourceRef = `${source}:${randomUUID()}`; } + if (!USER_SOURCES.has(source) && !MACHINE_SOURCES.has(source)) return bad("invalid-source"); // A session/Bearer caller may only file against a project they can reach. // The secret caller is a trusted forwarder resolving the project from the @@ -137,24 +164,26 @@ export async function POST(req: Request) { const storagePath = `receipts/intake/${id}.${ext}`; const fileSha256 = createHash("sha256").update(parsed.bytes).digest("hex"); - const supabase = getSupabase(); - if (!supabase) { - return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - } - const upload = await supabase.storage - .from(SECURE_BUCKET) - .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); - if (upload.error) { - console.error("[receipts/intake] upload failed", upload.error.message); - return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); - } - + // ROW FIRST, THEN THE OBJECT. + // + // Uploading first meant a replayed `sourceRef` had already written a second + // object into the private bucket before the insert failed, and the cleanup + // was best-effort. Worse, it made the two cases indistinguishable: a genuine + // forwarder retry and a REUSED sourceRef carrying different bytes both + // landed on the same P2002 and both got a cheerful 200, so a second, real + // receipt could be swallowed by the first one's row and never booked. + // + // Inserting first turns the unique index into the decision point, with + // `fileSha256` as the evidence: same bytes is a replay (200, the row you + // already have), different bytes is a caller bug (409) — and in the 409 + // case storage is never touched at all. + let created: { id: string; state: string; sourceRef: string; projectId: string | null; dryRun: boolean }; try { - const row = await prisma.receiptIntake.create({ + created = await prisma.receiptIntake.create({ data: { id, - source: parsed.source, - sourceRef: sourceRef!, + source, + sourceRef, state: "RECEIVED", // Captured per row, never read from env again after this point. dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", @@ -173,30 +202,94 @@ export async function POST(req: Request) { }, select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, }); - return NextResponse.json({ ok: true, ...row }); } catch (error) { - // The forwarder replayed a document we already hold. Return the row it - // already has — a non-200 here would make it retry forever. if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { - await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); - const existing = await prisma.receiptIntake.findUnique({ - where: { sourceRef: sourceRef! }, - select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, - }); - if (existing) return NextResponse.json({ ok: true, alreadyReceived: true, ...existing }); + return respondToSourceRefConflict(auth, sourceRef, fileSha256); } // A projectId/costCodeId that doesn't exist is the CALLER's mistake, so // it must be a deterministic 400 — a 500 would make a forwarder retry a // payload that can never succeed. if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2003") { - await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); return bad("unknown-project-or-cost-code"); } - // Never orphan the object: the row that would have pointed at it does - // not exist. - await removeSecureDoc(toSecureRef(storagePath)).catch(() => { /* best effort */ }); throw error; } + + const supabase = getSupabase(); + if (!supabase) { + await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + const upload = await supabase.storage + .from(SECURE_BUCKET) + .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); + if (upload.error) { + // Delete the row so the caller's retry is a clean insert rather than a + // sourceRef conflict against a row pointing at an object that is not + // there. The worker would otherwise park it "file-missing". + await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + console.error("[receipts/intake] upload failed", upload.error.message); + return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); + } + + return NextResponse.json({ ok: true, ...created }); +} + +/** + * The sourceRef is already taken. Two very different situations: + * + * - SAME bytes -> the forwarder replayed. Return the row it already has; a + * non-200 would make it retry forever. + * - OTHER bytes -> the caller reused a key for a DIFFERENT document. Answering + * 200 would tell it the new receipt was accepted when nothing was stored, + * and that receipt would never be booked. 409, and storage stays untouched. + * + * Who may see the existing row is a separate question (a minted `web:` + * can only collide by accident, but a secret-authenticated caller that guessed + * a ref must not be able to enumerate other people's rows): only the row's own + * creator or a bookkeeping role gets fields back. + */ +async function respondToSourceRefConflict( + auth: Extract, + sourceRef: string, + fileSha256: string, +): Promise { + const existing = await prisma.receiptIntake.findUnique({ + where: { sourceRef }, + select: { + id: true, state: true, sourceRef: true, projectId: true, + dryRun: true, fileSha256: true, createdById: true, + }, + }); + // The row vanished between the failed insert and this read (a delete + // racing us). Tell the caller to retry rather than inventing an answer. + if (!existing) return NextResponse.json({ ok: false, reason: "conflict-retry" }, { status: 409 }); + + if (existing.fileSha256 !== fileSha256) { + return NextResponse.json( + { ok: false, error: "sourceRef-conflict", existingId: existing.id }, + { status: 409 }, + ); + } + + const maySee = + auth.via === "secret" || + existing.createdById === auth.user.id || + STAFF_READ_ROLES.includes(auth.user.role); + if (!maySee) { + // No fields at all: an id or a state would still confirm the row exists. + return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + } + + return NextResponse.json({ + ok: true, + alreadyReceived: true, + id: existing.id, + state: existing.state, + sourceRef: existing.sourceRef, + projectId: existing.projectId, + dryRun: existing.dryRun, + }); } /** @@ -213,10 +306,24 @@ export async function GET(req: Request) { } const url = new URL(req.url); + const state = url.searchParams.get("state"); + + // The secret caller is the archive mirror and nothing else. It reads only + // the two states it acts on, and only the columns it needs — see + // RECEIPT_INTAKE_ARCHIVE_SELECT. Staff sessions are unaffected. + const archiveOnly = auth.via === "secret"; + if (archiveOnly && (!state || !ARCHIVE_READABLE_STATES.has(state))) { + return NextResponse.json( + { ok: false, reason: "state-not-allowed", allowed: [...ARCHIVE_READABLE_STATES] }, + { status: 400 }, + ); + } + const rows = await listReceiptIntakes({ - state: url.searchParams.get("state"), - projectId: url.searchParams.get("projectId"), + state, + projectId: archiveOnly ? null : url.searchParams.get("projectId"), take: url.searchParams.get("take") ? Number(url.searchParams.get("take")) : null, + archiveOnly, }); return NextResponse.json({ ok: true, rows: rows.map(serializeReceiptIntake) }); -} \ No newline at end of file +} diff --git a/src/lib/receipt-intake/intake-auth.ts b/src/lib/receipt-intake/intake-auth.ts index d080cd317..e60076ffd 100644 --- a/src/lib/receipt-intake/intake-auth.ts +++ b/src/lib/receipt-intake/intake-auth.ts @@ -23,8 +23,9 @@ import type { User } from "@prisma/client"; export const RECEIPT_INTAKE_SECRET_HEADER = "x-receipt-intake-secret"; export type IntakeAuth = - | { ok: true; via: "secret"; user: null } - | { ok: true; via: "session"; user: User } + | { ok: true; via: "secret"; user: null; userVia: null } + /** `userVia` distinguishes the crew app from a browser — the route mints `source` from it. */ + | { ok: true; via: "session"; user: User; userVia: "mobile-jwt" | "next-auth" } | { ok: false; response: NextResponse }; /** Constant-time compare over fixed-length digests, so header length leaks nothing. */ @@ -49,7 +50,7 @@ export async function authenticateIntake(req: Request): Promise { const provided = req.headers.get(RECEIPT_INTAKE_SECRET_HEADER); if (provided !== null) { if (secretMatches(provided, process.env.RECEIPT_INTAKE_SECRET)) { - return { ok: true, via: "secret", user: null }; + return { ok: true, via: "secret", user: null, userVia: null }; } return { ok: false, response: unauthorized() }; } @@ -62,7 +63,7 @@ export async function authenticateIntake(req: Request): Promise { response: NextResponse.json({ ok: false, reason: "unauthorized" }, { status: auth.status }), }; } - return { ok: true, via: "session", user: auth.user }; + return { ok: true, via: "session", user: auth.user, userVia: auth.via }; } export const STAFF_READ_ROLES = ["ADMIN", "MANAGER", "FINANCE"]; diff --git a/src/lib/receipt-intake/queries.ts b/src/lib/receipt-intake/queries.ts index 49d69749f..96324254a 100644 --- a/src/lib/receipt-intake/queries.ts +++ b/src/lib/receipt-intake/queries.ts @@ -47,6 +47,39 @@ export const RECEIPT_INTAKE_LIST_SELECT = { updatedAt: true, } as const; +/** + * What the nightly Apps Script archive mirror is allowed to see. + * + * It is a machine holding a shared secret, and its whole job is "copy this file + * to Drive under the v1 filename". It has no need for `lastError`, + * `fileSha256`, `createdById`, `dedupWeakKey`, or the retry bookkeeping — and a + * leaked or over-shared secret should expose the least that still lets the + * mirror work. Least privilege applies to a script the same way it does to a + * user. + */ +export const RECEIPT_INTAKE_ARCHIVE_SELECT = { + id: true, + sourceRef: true, + storagePath: true, + fileName: true, + mimeType: true, + txnDate: true, + vendor: true, + totalCents: true, + refNumber: true, + projectId: true, + state: true, + archiveDriveFileId: true, + bookedAt: true, +} as const; + +/** + * States the secret caller may query. The mirror archives what is BOOKED and + * re-checks what it already ARCHIVED; nothing else is its business, and a + * `state=NEEDS_REVIEW` sweep would hand it the whole error queue. + */ +export const ARCHIVE_READABLE_STATES = new Set(["BOOKED", "ARCHIVED"]); + export const MAX_LIST_TAKE = 200; export const DEFAULT_LIST_TAKE = 50; @@ -54,6 +87,8 @@ export interface ListReceiptIntakesArgs { state?: string | null; projectId?: string | null; take?: number | null; + /** Narrows the column set to RECEIPT_INTAKE_ARCHIVE_SELECT. */ + archiveOnly?: boolean; } /** Newest first. `take` is clamped, never trusted from the query string. */ @@ -71,7 +106,7 @@ export async function listReceiptIntakes(args: ListReceiptIntakesArgs) { }, orderBy: { createdAt: "desc" }, take, - select: RECEIPT_INTAKE_LIST_SELECT, + select: args.archiveOnly ? RECEIPT_INTAKE_ARCHIVE_SELECT : RECEIPT_INTAKE_LIST_SELECT, }); } From d1d846bda8b11e2f14fe18960d396b20f4a5f6fc Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:16:10 -0700 Subject: [PATCH 033/144] fix(receipts): STAGING state, per-weak-key lock, requeue inside the claim txn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 2. STAGING (blocker 1). Inserting the row before the upload made the unique index the decision point for a replayed sourceRef — but it also published a CLAIMABLE row whose object was not in the bucket yet, so the worker could grab a perfectly good receipt, find nothing, and park it "file-missing". A row is now BORN STAGING (the column default), which the claim predicate does not select, and one UPDATE after a successful upload is the only thing that publishes it. A same-SHA replay arriving while the first upload is still in flight gets 202 {status:"staging"} rather than a 200 that would promise a queued document. Rows that never make it out of STAGING are swept to NEEDS_REVIEW after 15 minutes — being invisible to the claim means nothing else would ever notice them. STAGING rows carry no dedupStrongKey, so the partial index's active set is unaffected. WEAK-KEY WRITE SKEW (blocker 2). The global claim lock is transaction-scoped and is released when the claim transaction commits, i.e. before any row is processed. Holding it across the whole pass would wrap every Gemini and QuickBooks call in one long-lived transaction on a pgbouncer pool, which is exactly what the pooler cannot afford. So the serialization is narrowed to what needs it: pg_advisory_xact_lock(hashtextextended(weakKey)) inside promoteToBooking, before the weak-key check. Two rows sharing a weak key take the same lock and go one at a time, so the loser SEES the winner in BOOKING; without it both SELECTs run before either UPDATE commits, READ COMMITTED finds no conflict, and the same purchase books twice. Different weak keys take different locks and never block each other. The cutover requeue moved INSIDE the claim transaction, under the lock: run beside it, two overlapping invocations could both un-park the backlog and the second UPDATE would race the first one's claim. Also: the different-SHA 409 no longer returns `existingId` to a caller that fails the read check — that turned the conflict into an oracle confirming a guessed sourceRef and handing back a usable id. A secret caller is scoped to its own namespace (the forwarders are separate scripts). busyPasses is added by an ALTER ... IF NOT EXISTS as well as in the CREATE, so a table from an earlier run of the rollout script upgrades. Spec risk 2 marked resolved (gross, not pre-tax). Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 9 +- e2e/receipt-intake.spec.ts | 44 +++++++ .../migration.sql | 9 +- prisma/prisma-blind-spots.json | 2 +- prisma/schema.prisma | 6 +- scripts/apply-receipt-intake.mjs | 12 +- .../api/cron/receipt-intake-worker/route.ts | 58 ++++++++-- src/app/api/receipts/intake/route.ts | 61 ++++++++-- src/lib/receipt-intake/route-state.ts | 2 + src/lib/receipt-intake/worker.ts | 51 ++++++--- tests/apply-receipt-intake.test.ts | 20 ++++ tests/receipt-intake-worker.test.ts | 107 ++++++++++++++---- 12 files changed, 309 insertions(+), 72 deletions(-) diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index ee38e7dfd..4694a6386 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -492,9 +492,12 @@ Two things a human must do before this can leave shadow mode: 1. **Public-bypass route**: `/api/receipts/intake` bypasses the proxy, so the handler is the only gate. Mitigated by the fail-closed secret check + the 401 e2e matrix; Codex must review the auth block specifically. (Risk to watch, no decision needed.) -2. **Expense.amount = pre-tax when tax is split** (mirrors the QBO COGS line under the - reseller-permit rule in sendToQBOviaAPI.js). Confirm pre-tax is the job-cost number you - want feeding variance reports. +2. ~~**Expense.amount = pre-tax when tax is split**~~ **RESOLVED 2026-09-01: GROSS.** + Justin's call. `Expense.amount` is the total paid, tax INCLUDED, matching what the + QBO-imported expenses in `lib/qbo-expense-sync.ts` already record. The QBO Purchase + still splits the tax onto its own reclaimable account — that is unchanged and it is + what the reseller-permit filing reads. `ReceiptIntake.taxCents` keeps the split so + Phase 3 can add `Expense.taxAmount`. No open question here. 3. **Archive via nightly Apps Script mirror** (§6) instead of a Drive service account — confirm, or provision a service account now if same-hour archiving matters to Marge. 4. **HEIC**: stored and read fine (Gemini accepts image/heic), but the Phase 2 queue page diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index db325a9e7..c4adcd774 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -168,6 +168,10 @@ test.describe("intake POST", () => { const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: SOURCE_REF } }); expect(rows).toHaveLength(1); expect(rows[0].id).toBe(first.body.id); + // The row is only published to the worker AFTER its object lands. A row + // still in STAGING here would mean the claim could pick up a receipt + // whose file does not exist and park it "file-missing". + expect(rows[0].state).toBe("RECEIVED"); expect(rows[0].mimeType).toBe("image/png"); expect(rows[0].fileSha256).toHaveLength(64); expect(rows[0].storagePath).toBe(`receipts/intake/${first.body.id}.png`); @@ -203,6 +207,46 @@ test.describe("intake POST", () => { expect(rows[0].storagePath).toBe(`receipts/intake/${first.body.id}.png`); }); + test("a different-SHA 409 leaks NOTHING to a caller who may not read the row", async ({ request, playwright }) => { + // `existingId` is a real identifier for someone else's document. Handing + // it to a caller that fails the read check turns the 409 into an oracle: + // guess a sourceRef, learn it exists, and get a usable id back. + // + // A shared-secret caller is scoped to its OWN namespace — the forwarders + // are separate scripts, and the chat one should learn nothing about the + // Drive pipeline's rows. + const ref = `${REF_PREFIX}ns-drive`; + const seeded = await postIntake(request, intakeBody({ source: "drive", sourceRef: ref })); + expect(seeded.res.status()).toBe(200); + + const machine = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + // Same secret, but declaring `chat` while the row is a `drive` row. + const crossNamespace = await machine.post(INTAKE_PATH, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "chat", sourceRef: ref, + fileBase64: OTHER_PNG_BASE64, mimeType: "image/png", + }), + maxRedirects: 0, + }); + expect(crossNamespace.status()).toBe(409); + const body = await crossNamespace.json(); + expect(body.error).toBe("sourceRef-conflict"); + expect(body, "no id for a caller outside the row's namespace").not.toHaveProperty("existingId"); + await machine.dispose(); + + // The row's OWN namespace still gets the id, so the real forwarder can + // act on the conflict. + const sameNamespace = await postIntake(request, intakeBody({ + source: "drive", sourceRef: ref, fileBase64: OTHER_PNG_BASE64, + })); + expect(sameNamespace.res.status()).toBe(409); + expect(sameNamespace.body.existingId).toBe(seeded.body.id); + }); + test("a session caller may not choose its own source or sourceRef", async ({ request }) => { // `source` is provenance and it feeds booking identity: a `drive` row // books under the Drive fileId, so a forged source could aim a QBO diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 01ff53ede..6a19217dd 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "id" TEXT NOT NULL, "source" TEXT NOT NULL, "sourceRef" TEXT NOT NULL, - "state" TEXT NOT NULL DEFAULT 'RECEIVED', + "state" TEXT NOT NULL DEFAULT 'STAGING', "dryRun" BOOLEAN NOT NULL DEFAULT true, "stateReason" TEXT, "projectId" TEXT, @@ -54,6 +54,11 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( CONSTRAINT "ReceiptIntake_pkey" PRIMARY KEY ("id") ); +-- Additive upgrade for a table created by an earlier run of +-- scripts/apply-receipt-intake.mjs: CREATE TABLE IF NOT EXISTS is a no-op on an +-- existing table, so a column added to the CREATE above would never reach it. +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0; + CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" ON "ReceiptIntake"("sourceRef"); CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" ON "ReceiptIntake"("expenseId"); CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_dedupStrongKey_active_key" @@ -72,7 +77,7 @@ BEGIN AND conrelid = '"ReceiptIntake"'::regclass ) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" - CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); END IF; END $$; diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 7f91bb5c7..bef2f9bd8 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -102,7 +102,7 @@ { "name": "ReceiptIntake_state_check", "table": "\"ReceiptIntake\"", - "def": "CHECK ((state = ANY (ARRAY['RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text])))" + "def": "CHECK ((state = ANY (ARRAY['STAGING'::text, 'RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text])))" }, { "name": "RefundEvent_amountCents_check", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 608c776a4..1f09cc6be 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3023,7 +3023,11 @@ model ReceiptIntake { /// "drive:" | "email::" | "mobile:" | /// "chat::" | "web:" — the caller's idempotency key. sourceRef String @unique - state String @default("RECEIVED") // RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT + /// STAGING is the state a row is BORN in: the object is not in the bucket + /// yet, so the worker's claim predicate excludes it. The intake route flips + /// it to RECEIVED in one UPDATE after the upload lands. A row that never gets + /// there is swept to NEEDS_REVIEW after 15 minutes. + state String @default("STAGING") // STAGING RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT dryRun Boolean @default(true) /// no-estimate | multi-doc | zero-total | weak-dup: | /// strong-dup-amount-mismatch: | qbo-fault: | max-retries | diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index ed679f5d0..314ba7eaf 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -70,7 +70,7 @@ export function targetMatches(actual, expectDb, expectHost) { /** The closed set of states the CHECK constraint allows. Exported for tests. */ export const RECEIPT_INTAKE_STATES = [ - "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", + "STAGING", "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", ]; @@ -79,7 +79,7 @@ export const statements = [ "id" TEXT NOT NULL, "source" TEXT NOT NULL, "sourceRef" TEXT NOT NULL, - "state" TEXT NOT NULL DEFAULT 'RECEIVED', + "state" TEXT NOT NULL DEFAULT 'STAGING', "dryRun" BOOLEAN NOT NULL DEFAULT true, "stateReason" TEXT, "projectId" TEXT, @@ -117,6 +117,12 @@ export const statements = [ CONSTRAINT "ReceiptIntake_pkey" PRIMARY KEY ("id") )`, + // Additive upgrade for a table created by an EARLIER run of this script, + // before busyPasses existed: CREATE TABLE IF NOT EXISTS is a no-op on an + // existing table, so a column added to the CREATE above would never reach + // it. This is the whole reason the script is re-runnable. + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0`, + // Intake idempotency: one row per caller-supplied sourceRef. A forwarder // replaying the same Drive file / Gmail message is a no-op. `CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" @@ -154,7 +160,7 @@ export const statements = [ WHERE conname = 'ReceiptIntake_state_check' AND conrelid = '"ReceiptIntake"'::regclass) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" - CHECK ("state" IN ('RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); END IF; END $$`, diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index fc2e49a3a..249fa1c5f 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -14,6 +14,7 @@ import { BATCH_SIZE, CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, + STAGING_SWEEP_MINUTES, isUniqueViolation, runIntakeWorker, type ReadPatch, @@ -68,7 +69,9 @@ const NOT_DRY_RUN_PARKED: Prisma.ReceiptIntakeWhereInput = { NOT: { AND: [{ dryRun: true }, { state: { in: ["READ", "BOOKING"] } }] }, }; -async function claim(): Promise { +async function claim( + opts: { requeueDryRunParked: boolean }, +): Promise<{ rows: WorkerRow[]; requeued: number } | null> { const now = new Date(); return prisma.$transaction(async tx => { const [lock] = await tx.$queryRaw<{ locked: boolean }[]>( @@ -76,8 +79,27 @@ async function claim(): Promise { ); if (!lock?.locked) return null; + // Cutover, INSIDE the lock and the same transaction as the claim. Run + // outside it, two overlapping invocations could both see the parked + // backlog and both un-park it, and the second UPDATE would race the + // first one's claim. `dryRun` flips WITH the requeue: a row that + // reappeared still carrying dryRun=true would be skipped and re-parked + // forever. + let requeued = 0; + if (opts.requeueDryRunParked) { + const result = await tx.receiptIntake.updateMany({ + where: { dryRun: true, state: { in: ["READ", "BOOKING"] } }, + data: { dryRun: false, nextRetryAt: null }, + }); + requeued = result.count; + if (requeued > 0) console.log("[cron/receipt-intake-worker] cutover requeue", requeued); + } + const due = await tx.receiptIntake.findMany({ where: { + // STAGING is absent on purpose: the row exists but its object + // does not, so claiming it would park a good receipt as + // "file-missing". sweepStaleStaging is what watches those. state: { in: ["RECEIVED", "READ", "BOOKING"] }, OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }], ...NOT_DRY_RUN_PARKED, @@ -86,7 +108,7 @@ async function claim(): Promise { take: BATCH_SIZE, select: WORKER_ROW_SELECT, }); - if (due.length === 0) return []; + if (due.length === 0) return { rows: [], requeued }; // THE claim. Anything this run took is invisible to the next one for // the lease, whether or not the advisory lock held. @@ -94,7 +116,7 @@ async function claim(): Promise { where: { id: { in: due.map(r => r.id) } }, data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, }); - return due as WorkerRow[]; + return { rows: due as WorkerRow[], requeued }; }); } @@ -104,15 +126,13 @@ function buildDeps(): WorkerDependencies { isDryRunEnabled: () => process.env.RECEIPT_INTAKE_DRYRUN !== "false", - requeueDryRunParked: async () => { + sweepStaleStaging: async () => { + const cutoff = new Date(Date.now() - STAGING_SWEEP_MINUTES * 60_000); const { count } = await prisma.receiptIntake.updateMany({ - where: { dryRun: true, state: { in: ["READ", "BOOKING"] } }, - // dryRun flips WITH the requeue, in one statement: a row that - // reappears in the queue still carrying dryRun=true would be - // skipped by the loop and re-parked forever. - data: { dryRun: false, nextRetryAt: null }, + where: { state: "STAGING", createdAt: { lt: cutoff } }, + data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, }); - if (count > 0) console.log("[cron/receipt-intake-worker] cutover requeue", count); + if (count > 0) console.log("[cron/receipt-intake-worker] stale STAGING swept", count); return count; }, @@ -185,7 +205,25 @@ function buildDeps(): WorkerDependencies { // LAST weak-dedup check, taken INSIDE the transition. The check at // read time can miss a pair that arrived in the same batch window, // and READ -> BOOKING is the last instant before money moves. + // + // WHY A SECOND LOCK, keyed on the weak key rather than just relying + // on the global claim lock: that lock is transaction-scoped and is + // released the moment the CLAIM transaction commits, which is + // before any row is processed. Holding it across the whole pass + // instead would mean one long-lived transaction wrapping every + // Gemini and QuickBooks call — minutes of open transaction on a + // pgbouncer pool, which is exactly what the pooler cannot afford. + // + // So the serialization is narrowed to what actually needs it. Two + // rows sharing a weak key take the SAME lock here and go one at a + // time; the loser's SELECT then sees the winner already in BOOKING. + // Without it both SELECTs can run before either UPDATE commits + // (classic write skew, and READ COMMITTED will not catch it because + // neither row writes what the other read) and both documents book. + // Rows with different weak keys take different locks and never + // block each other. if (weakKey) { + await tx.$queryRaw`SELECT pg_advisory_xact_lock(hashtextextended(${weakKey}, 0))`; const conflict = await tx.receiptIntake.findFirst({ where: { dedupWeakKey: weakKey, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index c6873b1e0..6aa198526 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -184,7 +184,14 @@ export async function POST(req: Request) { id, source, sourceRef, - state: "RECEIVED", + // STAGING, not RECEIVED. Inserting first is what makes the + // unique index the decision point (see below), but it also + // publishes a claimable row whose object is not in the bucket + // yet — the worker would grab it, find nothing, and park a + // perfectly good receipt as "file-missing". STAGING is excluded + // from the claim predicate; the UPDATE after a successful + // upload is what actually hands the row to the worker. + state: "STAGING", // Captured per row, never read from env again after this point. dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId: parsed.projectId, @@ -204,7 +211,7 @@ export async function POST(req: Request) { }); } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { - return respondToSourceRefConflict(auth, sourceRef, fileSha256); + return respondToSourceRefConflict(auth, source, sourceRef, fileSha256); } // A projectId/costCodeId that doesn't exist is the CALLER's mistake, so // it must be a deterministic 400 — a 500 would make a forwarder retry a @@ -232,7 +239,15 @@ export async function POST(req: Request) { return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); } - return NextResponse.json({ ok: true, ...created }); + // The object exists now, so the row becomes claimable. One UPDATE, and it + // is the ONLY thing that publishes a row to the worker. + const published = await prisma.receiptIntake.update({ + where: { id }, + data: { state: "RECEIVED" }, + select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, + }); + + return NextResponse.json({ ok: true, ...published }); } /** @@ -251,13 +266,14 @@ export async function POST(req: Request) { */ async function respondToSourceRefConflict( auth: Extract, + source: string, sourceRef: string, fileSha256: string, ): Promise { const existing = await prisma.receiptIntake.findUnique({ where: { sourceRef }, select: { - id: true, state: true, sourceRef: true, projectId: true, + id: true, state: true, source: true, sourceRef: true, projectId: true, dryRun: true, fileSha256: true, createdById: true, }, }); @@ -265,6 +281,27 @@ async function respondToSourceRefConflict( // racing us). Tell the caller to retry rather than inventing an answer. if (!existing) return NextResponse.json({ ok: false, reason: "conflict-retry" }, { status: 409 }); + // AUTHORIZATION BEFORE ANY DETAIL, including on the mismatch branch. + // `existingId` is a real identifier for someone else's document; returning + // it to a caller who may not read the row turns a 409 into an oracle that + // confirms a guessed sourceRef and hands back a usable id. + // + // For a secret caller "may read" is narrower than "holds the secret": it + // may only see rows in ITS OWN namespace. The forwarders are separate + // scripts, and a chat forwarder guessing `drive:` should learn + // nothing about the Drive pipeline's rows. + const maySee = + auth.via === "secret" + ? MACHINE_SOURCES.has(existing.source) && + existing.source === source && + existing.sourceRef.startsWith(`${source}:`) + : existing.createdById === auth.user.id || STAFF_READ_ROLES.includes(auth.user.role); + + if (!maySee) { + // No fields at all: an id or a state would still confirm the row exists. + return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + } + if (existing.fileSha256 !== fileSha256) { return NextResponse.json( { ok: false, error: "sourceRef-conflict", existingId: existing.id }, @@ -272,13 +309,15 @@ async function respondToSourceRefConflict( ); } - const maySee = - auth.via === "secret" || - existing.createdById === auth.user.id || - STAFF_READ_ROLES.includes(auth.user.role); - if (!maySee) { - // No fields at all: an id or a state would still confirm the row exists. - return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + // Same bytes, but the FIRST request has not finished uploading yet. 200 + // would tell the caller its document is queued when the object may still + // fail to land; 202 says "accepted, not yet published" so a forwarder can + // re-poll instead of moving the file out from under a half-written row. + if (existing.state === "STAGING") { + return NextResponse.json( + { ok: true, status: "staging", alreadyReceived: true, id: existing.id, sourceRef: existing.sourceRef }, + { status: 202 }, + ); } return NextResponse.json({ diff --git a/src/lib/receipt-intake/route-state.ts b/src/lib/receipt-intake/route-state.ts index 66f880fef..07807a4a5 100644 --- a/src/lib/receipt-intake/route-state.ts +++ b/src/lib/receipt-intake/route-state.ts @@ -6,6 +6,8 @@ */ export const RECEIPT_INTAKE_STATES = [ + // STAGING: the row exists but its file does not yet. Never claimable. + "STAGING", "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", ] as const; diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 0975568b3..485761094 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -44,6 +44,13 @@ export const CLAIM_LEASE_MINUTES = 10; * being a formality. */ export const RUN_SOFT_DEADLINE_MS = 40_000; +/** + * How long a row may sit in STAGING before it is presumed to have lost its + * upload. Generous on purpose: the intake route uploads inline, so a row that + * is still STAGING after this either crashed mid-request or hit a storage + * outage, and neither resolves itself. + */ +export const STAGING_SWEEP_MINUTES = 15; /** * Consecutive AI-unavailable passes before a row is parked for a human. Ported * from v3.4: an outage that never ends still has to end somewhere, and 20 @@ -68,17 +75,25 @@ export interface WorkerRow extends BookableRow { } export interface WorkerDependencies { - /** Claims up to BATCH_SIZE rows and bumps their nextRetryAt. Returns null when another run holds the lock. */ - claim: () => Promise; - /** RECEIPT_INTAKE_DRYRUN is not "false". Injected so the requeue is testable. */ + /** + * ONE transaction under the global advisory lock: optionally requeue the + * shadow-week backlog, then claim up to BATCH_SIZE due rows and bump their + * nextRetryAt. Returns null when another run holds the lock. + * + * The requeue lives INSIDE this transaction rather than beside it: run + * outside the lock, two overlapping invocations could both see the parked + * backlog and both un-park it, and the second one's UPDATE would race the + * first one's claim. + */ + claim: (opts: { requeueDryRunParked: boolean }) => Promise<{ rows: WorkerRow[]; requeued: number } | null>; + /** RECEIPT_INTAKE_DRYRUN is not "false". Injected so the cutover is testable. */ isDryRunEnabled: () => boolean; /** - * One-shot at the start of the first LIVE pass: un-park every row the - * shadow week left sitting at READ/BOOKING with dryRun=true. Returns the - * number requeued. Naturally idempotent — after one live pass there is - * nothing left to match. + * Move STAGING rows older than STAGING_SWEEP_MINUTES to NEEDS_REVIEW + * `file-missing`. A row whose upload never landed is invisible to the claim + * predicate by design, so nothing else would ever notice it. */ - requeueDryRunParked: () => Promise; + sweepStaleStaging: () => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; downloadBytes: (secureRef: string) => Promise; read: (bytes: Buffer, mime: string, phases: ProjectPhase[]) => Promise; @@ -139,6 +154,8 @@ export interface WorkerRunSummary { deferredToNextRun?: number; /** Rows un-parked by the first live pass after the shadow week. */ requeued?: number; + /** STAGING rows whose upload never landed, parked for a human. */ + staleStagingSwept?: number; } function centsOf(amount: string): number | null { @@ -163,16 +180,17 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise 0); const startedAt = deps.monotonicMs(); const byState: Record = {}; @@ -228,6 +246,7 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise { const guarded = /CREATE TABLE IF NOT EXISTS/.test(sql) || /CREATE (?:UNIQUE )?INDEX IF NOT EXISTS/.test(sql) || + /ALTER TABLE .* ADD COLUMN IF NOT EXISTS/.test(sql) || /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(sql); assert.ok(guarded, `not idempotent: ${sql.slice(0, 80)}`); } @@ -134,3 +135,22 @@ test("the state CHECK guard is scoped to the ReceiptIntake table", () => { assert.match(check!, /conrelid = '"ReceiptIntake"'::regclass/); assert.match(migrationSql, /conrelid = '"ReceiptIntake"'::regclass/); }); + +test("the busyPasses column is ALSO added by an ALTER, so an earlier table upgrades", () => { + // CREATE TABLE IF NOT EXISTS is a no-op on a table that already exists, so + // a column added only to the CREATE would never reach a database where the + // rollout script had already run once. This is the whole reason the script + // is re-runnable. + const alter = statements.find((s: string) => /ADD COLUMN IF NOT EXISTS "busyPasses"/.test(s)); + assert.ok(alter, "the apply script must ALTER as well as CREATE"); + assert.match(migrationSql, /ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses"/); +}); + +test("STAGING is in the state set, and is the column DEFAULT", () => { + // A row is born STAGING: it exists, but its object is not in the bucket + // yet, so the worker's claim predicate must not be able to see it. + assert.ok(RECEIPT_INTAKE_STATES.includes("STAGING")); + const create = statements.find((s: string) => s.includes('CREATE TABLE IF NOT EXISTS "ReceiptIntake"')); + assert.match(create!, /"state"\s+TEXT NOT NULL DEFAULT 'STAGING'/); + assert.match(migrationSql, /"state" TEXT NOT NULL DEFAULT 'STAGING'/); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 130f94f54..c3c5f4388 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -87,20 +87,21 @@ interface Harness { promoted: string[]; deferred: { id: string; busyPasses: number }[]; retried: { id: string; attempts: number; reason: string }[]; - requeueCalls: number; + claimOpts: { requeueDryRunParked: boolean }[]; + sweepCalls: number; clock: number; } function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { reads: 0, books: 0, applied: [], states: [], promoted: [], deferred: [], - retried: [], requeueCalls: 0, clock: 0, + retried: [], claimOpts: [], sweepCalls: 0, clock: 0, deps: null as unknown as WorkerDependencies, }; h.deps = { - claim: async () => rows, + claim: async opts => { h.claimOpts.push(opts); return { rows, requeued: 0 }; }, isDryRunEnabled: () => true, - requeueDryRunParked: async () => { h.requeueCalls++; return 0; }, + sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], downloadBytes: async () => Buffer.from("bytes"), read: async () => { h.reads++; return goodRead; }, @@ -260,45 +261,59 @@ test("a strong-key loss to a DIFFERENT vendor is a collision, not a duplicate", // ── Dry-run starvation (Codex blocker 1) ───────────────────────────────────── -test("the shadow week does NOT requeue parked rows", async () => { +test("the shadow week does NOT ask the claim to requeue", async () => { const h = harness([workerRow({ state: "READ", dryRun: true })], { isDryRunEnabled: () => true }); const summary = await runIntakeWorker(h.deps); - assert.equal(h.requeueCalls, 0); + assert.deepEqual(h.claimOpts, [{ requeueDryRunParked: false }]); assert.equal(summary.requeued, undefined); }); -test("the FIRST live pass un-parks the shadow week's backlog, once", async () => { +test("the FIRST live pass asks the claim to un-park the backlog, INSIDE the lock", async () => { // Parked rows are excluded from the claim (see the cron route's // NOT_DRY_RUN_PARKED) precisely so they cannot starve the ten-row batch — - // which also means nothing else would ever wake them. + // which also means nothing else would ever wake them. The requeue rides in + // the claim transaction so two overlapping invocations cannot both un-park + // the same backlog and race each other's claim. const h = harness([], { isDryRunEnabled: () => false, - requeueDryRunParked: async () => { h.requeueCalls++; return 7; }, + claim: async opts => { h.claimOpts.push(opts); return { rows: [], requeued: 7 }; }, }); const summary = await runIntakeWorker(h.deps); - assert.equal(h.requeueCalls, 1); + assert.deepEqual(h.claimOpts, [{ requeueDryRunParked: true }]); assert.equal(summary.requeued, 7); // Idempotent by construction: nothing is left matching the predicate. - const second = harness([], { - isDryRunEnabled: () => false, - requeueDryRunParked: async () => { second.requeueCalls++; return 0; }, - }); - const secondSummary = await runIntakeWorker(second.deps); - assert.equal(secondSummary.requeued, undefined, "a no-op requeue is not reported"); + const second = harness([], { isDryRunEnabled: () => false }); + assert.equal((await runIntakeWorker(second.deps)).requeued, undefined, "a no-op requeue is not reported"); }); -test("the requeue happens even when another worker holds the lock", async () => { - // The lock guards the BATCH, not the cutover. A run that finds the lock - // taken must still not swallow the one-shot requeue. - const h = harness([], { - isDryRunEnabled: () => false, - claim: async () => null, - requeueDryRunParked: async () => { h.requeueCalls++; return 3; }, +test("a run that loses the lock does nothing at all — including the requeue", async () => { + // The requeue is now part of the claim transaction, so losing the lock + // means losing it too. That is correct: the run that HOLDS the lock does it. + const h = harness([], { isDryRunEnabled: () => false, claim: async () => null }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary, { processed: 0, byState: {}, skipped: "already-running" }); + assert.equal(h.sweepCalls, 0, "no work of any kind happens without the lock"); +}); + +// ── STAGING sweep (Codex round 2, blocker 1) ───────────────────────────────── + +test("every pass sweeps STAGING rows whose upload never landed", async () => { + // A STAGING row is invisible to the claim by design (its object is not in + // the bucket), so without this sweep nothing would ever notice one. + const h = harness([], { sweepStaleStaging: async () => { h.sweepCalls++; return 2; } }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.sweepCalls, 1); + assert.equal(summary.staleStagingSwept, 2); +}); + +test("a failing sweep never takes the pass down with it", async () => { + const h = harness([workerRow()], { + sweepStaleStaging: async () => { throw new Error("db blip"); }, }); const summary = await runIntakeWorker(h.deps); - assert.equal(summary.skipped, "already-running"); - assert.equal(summary.requeued, 3); + assert.equal(summary.staleStagingSwept, undefined); + assert.deepEqual(summary.byState, { READ: 1 }, "the batch still ran"); }); // ── Soft deadline (Codex blocker 2) ────────────────────────────────────────── @@ -320,6 +335,48 @@ test("the worker stops TAKING rows at 40s and leaves the rest for the next run", // ── Weak-dedup race at the READ -> BOOKING transition (Codex blocker 5) ─────── +test("two rows sharing a weak key SERIALIZE: the second is blocked, not booked", async () => { + // Write skew. Both rows pass the read-time weak check (neither is BOOKING + // yet), so without the per-weak-key advisory lock inside promoteToBooking + // both SELECTs run before either UPDATE commits, READ COMMITTED sees no + // conflict (neither row writes what the other read), and the SAME purchase + // books twice. The lock is what makes the second one observe the first. + const WEAK = "lowes|2026-08-03|364.98|amt"; + const booking = new Set(); + const h = harness( + [ + workerRow({ id: "row-a", state: "READ", dryRun: false, dedupWeakKey: WEAK }), + workerRow({ id: "row-b", state: "READ", dryRun: false, dedupWeakKey: WEAK }), + ], + { + // Stands in for the serialized transaction: the lock means this + // body runs to completion for row-a before row-b enters it. + promoteToBooking: async (id, weakKey) => { + h.promoted.push(id); + const twin = [...booking].find(other => other !== id); + if (weakKey && twin) return { promoted: false, conflictId: twin }; + booking.add(id); + return { promoted: true }; + }, + }, + ); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(h.promoted, ["row-a", "row-b"], "both rows attempted the transition"); + assert.equal(h.books, 1, "exactly ONE of them books"); + assert.equal(summary.byState.BOOKED, 1); + assert.equal(summary.byState.NEEDS_REVIEW, 1); +}); + +test("rows with DIFFERENT weak keys never block each other", async () => { + const h = harness([ + workerRow({ id: "row-a", state: "READ", dryRun: false, dedupWeakKey: "lowes|2026-08-03|364.98|amt" }), + workerRow({ id: "row-b", state: "READ", dryRun: false, dedupWeakKey: "amazon|2026-08-03|12.00|amt" }), + ]); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.books, 2); + assert.deepEqual(summary.byState, { BOOKED: 2 }); +}); + test("a weak-key twin already BOOKING blocks the transition and asks a human", async () => { const h = harness([workerRow({ state: "READ", dryRun: false, dedupWeakKey: "lowes|2026-08-03|364.98|amt" })], { promoteToBooking: async (id, weakKey) => { From 1d36272e70644b3c2e0d5258be26d0cc38e70f6e Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 20:12:48 -0700 Subject: [PATCH 034/144] fix(proxy): refuse a Server Action dispatch on the receipt-intake bypasses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From Phase 2's Codex review. The exact-match public bypasses added for /api/receipts/intake and /api/receipts/intake//archived returned NextResponse.next() for ANY request, including one carrying a `next-action` header. Next's action IDs are global, so such a POST invokes SOMEONE ELSE'S action and never reaches this route's code at all — which means the in-handler x-receipt-intake-secret check, the ONLY gate these paths have, never runs. The stale-cookie guard above does not cover it either: a machine caller carries no session cookie, so it takes the anonymous path straight into the bypass. Same reasoning and same fix as LEGAL_PAGE_PATTERN, which already refuses action dispatches on the static legal pages for exactly this reason: bypassing the proxy must never also mean bypassing the Server Action boundary. The portal and api/* bypasses accept that tradeoff because they genuinely have anonymous actions; these two do not. The test asserts BOTH directions — 403 with no `x-middleware-next` for the dispatch, and an unchanged bypass for a normal machine POST, so the fix cannot have quietly closed the door on the callers this route exists for. Verified by mutation: stubbing the guard to `false &&` makes it fail. (The first mutation attempt silently no-op'd on a bad string match and the test still passed — the assert in the mutation script is what caught that.) Co-Authored-By: Claude Fable 5.1 --- src/proxy.ts | 23 +++++++++++++ tests/receipt-intake-auth.test.ts | 55 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/proxy.ts b/src/proxy.ts index 860c3b1e6..39c277819 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -73,6 +73,23 @@ const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/health\/pipeline\/? // accept that tradeoff because they genuinely have anonymous actions, these don't. const LEGAL_PAGE_PATTERN = /^\/(?:privacy|terms|account-deletion|support)(?:\/|$)/; +// The receipt-intake machine endpoints are route handlers that define no Server +// Actions, and their ONLY gate is an in-handler `x-receipt-intake-secret` check. +// That check never runs for an action dispatch: Next's action IDs are global, so +// a `next-action` POST on a bypassed path invokes SOMEONE ELSE'S action and +// never reaches this route's code at all. The stale-cookie guard above does not +// cover it either — a machine caller carries no session cookie, so it takes the +// anonymous path straight into the bypass. +// +// Same reasoning as LEGAL_PAGE_PATTERN, and the same conclusion: bypassing the +// proxy is never allowed to also mean bypassing the Server Action boundary. The +// portal/api routes above accept that tradeoff because they genuinely have +// anonymous actions; these do not. +// +// Exact match, mirroring the bypass entries themselves — a descendant that is +// NOT bypassed still hits withAuth and needs no special case here. +const MACHINE_ENDPOINT_PATTERN = /^\/api\/receipts\/intake(?:\/[^/]+\/archived)?\/?$/; + // Test-only action dispatchers that get the proxy bypass below. Explicit, not a // prefix match: the proxy checks only the environment gates, never the route's // `x-e2e-secret`, so a prefix would silently extend that bypass to any future @@ -167,6 +184,12 @@ export default async function proxy(req: any, event: any) { return new NextResponse("Forbidden", { status: 403 }); } + // Neither are the machine endpoints, whose only gate lives in the handler + // that an action dispatch never reaches. + if (isServerAction && typeof pathname === "string" && MACHINE_ENDPOINT_PATTERN.test(pathname)) { + return new NextResponse("Forbidden", { status: 403 }); + } + if (typeof pathname === "string" && isPublicProxyBypass(pathname)) { return NextResponse.next(); } diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 6e0f071b3..3c0b9c076 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -92,3 +92,58 @@ test("the stored mime is decided on the BYTES, not the caller's header", async ( assert.equal(sniffMime(Buffer.from("MZ\x90\x00"), "application/pdf"), null); assert.equal(sniffMime(Buffer.alloc(0), "text/plain"), null); }); + +test("a Next-Action dispatch on a bypassed intake path is 403, not waved through", async () => { + // Phase 2's Codex review: the bypass returned NextResponse.next() for ANY + // request, including one carrying a `next-action` header. Next's action IDs + // are global, so such a POST invokes SOMEONE ELSE'S action and never reaches + // this route's code — meaning the in-handler x-receipt-intake-secret check, + // which is the only gate these paths have, never runs. A machine caller + // carries no session cookie, so the stale-cookie guard does not cover it + // either. Bypassing the proxy must never also bypass the action boundary. + const { default: proxy } = await loadProxy(); + const { NextRequest } = await import("next/server"); + const event = { waitUntil() {} } as any; + // The proxy short-circuits to next() in development, so the real path is + // only reachable with NODE_ENV=production. + // NODE_ENV is typed read-only; the proxy reads it at call time, so a cast + // is the only way to exercise the non-development branch here. + const env = process.env as Record; + const prod = env.NODE_ENV; + env.NODE_ENV = "production"; + + try { + for (const path of [ + "/api/receipts/intake", + "/api/receipts/intake/", + "/api/receipts/intake/abc123/archived", + ]) { + const res = await proxy( + new NextRequest(`https://probuild.test${path}`, { + method: "POST", + headers: { "next-action": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, + }), + event, + ); + assert.ok(res, `${path} produced no response`); + assert.equal(res.status, 403, `${path} must refuse an action dispatch`); + // NextResponse.next() carries x-middleware-next: 1. Anything else + // means the proxy kept control, which is the point. + assert.equal(res.headers.get("x-middleware-next"), null, path); + } + + // A NORMAL request on the same paths still gets the bypass, so the + // machine callers this route exists for are unaffected. + const normal = await proxy( + new NextRequest("https://probuild.test/api/receipts/intake", { + method: "POST", + headers: { "x-receipt-intake-secret": "whatever" }, + }), + event, + ); + assert.ok(normal, "the normal request produced no response"); + assert.equal(normal.headers.get("x-middleware-next"), "1", "the bypass still works without next-action"); + } finally { + env.NODE_ENV = prod; + } +}); From ca59fc9d078a174e17fdfbc0ef569e3f8c154cbb Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 20:56:58 -0700 Subject: [PATCH 035/144] fix(receipts): strong-before-weak dedup, resumable publish, tagged storage, RLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 3 (items 1-6, 8). Rebased onto Phase 0 for item 7's health fixes. 1. DEDUP ORDER. The weak lookup ran first, so an exact re-send — which matches BOTH nets — routed on the weak hit and never attempted the strong claim. The one case the strong key exists to resolve automatically was the one case it never saw, and every re-sent receipt landed in a human's queue. Now: document-level gates, then the strong claim, then the weak net only if the strong one is silent. The final weak check at READ -> BOOKING also widened to every live state: limiting it to BOOKING/BOOKED/ARCHIVED made a twin sitting in NEEDS_REVIEW — exactly where the weak net puts the FIRST of a pair — invisible, so the second copy booked past a pending human decision. 2. RESUMABLE PUBLISH. If the upload lands and the STAGING -> RECEIVED update then fails, the object exists and the row does not point at it; STAGING is invisible to the claim, so it would sit until the sweeper wrongly declared its file missing. An identical retry now confirms the object and finishes the publish. Upload errors AND upload throws both delete the row. 3. STORAGE IS TAGGED. "The object is gone" and "Supabase blipped" demanded opposite responses and both arrived as null, so a transient fault parked good receipts as file-missing, permanently. downloadDocBytesResult separates them; only an affirmative not-found is terminal. book.ts now REFUSES to book when the bytes cannot be loaded — a Purchase with no attachment is the one failure a bookkeeper cannot fix later, because it looks complete. 4. The archive mirror gets a 10-minute signed URL and the project name: it holds no service key and cannot read the private bucket, and it names files ____$. Contract test, since changing that payload breaks a script in another repo. 5. uploadId: a client idempotency token, SCOPED TO THE USER server-side (::), so a crew member tapping Send twice on a spinner does not book twice — and cannot reach another user's row by guessing a uuid. Secret callers must keep their sourceRef in the namespace they declared. 6. RLS on ReceiptIntake, matching every other sensitive table: ENABLE, no policies, never FORCE (the app is the owner and bypasses it; FORCE would deny the owner too and take the pipeline down as empty result sets). Recorded in prisma-blind-spots.json. 7. pipeline-health now examines ReceiptIntake. Every other probe reads AutomationEvent, which only records a BOOKING, so a v2 row that never reaches QuickBooks was invisible to all of them and a jammed queue reported healthy. A NEEDS_REVIEW backlog is reported but is NOT a failure — those rows are working as designed. 8. EVERY 5xx from Gemini is the service failing. 500/502/504 fell into the "decisive" branch and charged the row a strike for a Google-side fault. The archive contract test also caught a real gap: busyPasses was never added to the staff select, so the Phase 2 queue page could not tell "this document defeated the model" from "Gemini was down all afternoon". Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 101 ++++++++++++++ package.json | 4 +- .../migration.sql | 6 + prisma/prisma-blind-spots.json | 4 + scripts/apply-receipt-intake.mjs | 11 ++ .../api/cron/receipt-intake-worker/route.ts | 20 ++- src/app/api/receipts/intake/route.ts | 130 ++++++++++++++---- src/lib/pipeline-health.ts | 67 +++++++++ src/lib/receipt-intake/book.ts | 31 ++++- src/lib/receipt-intake/queries.ts | 42 ++++++ src/lib/receipt-intake/read.ts | 15 +- src/lib/receipt-intake/worker.ts | 98 +++++++++---- src/lib/secure-storage.ts | 92 +++++++++++++ tests/apply-receipt-intake.test.ts | 26 ++++ tests/pipeline-digest-route.test.ts | 1 + tests/pipeline-health.test.ts | 62 +++++++++ tests/receipt-intake-archive-contract.test.ts | 84 +++++++++++ tests/receipt-intake-book.test.ts | 39 +++++- tests/receipt-intake-read.test.ts | 42 ++++++ tests/receipt-intake-worker.test.ts | 85 +++++++++++- 20 files changed, 891 insertions(+), 69 deletions(-) create mode 100644 tests/receipt-intake-archive-contract.test.ts diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index c4adcd774..f54f07a52 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -177,6 +177,53 @@ test.describe("intake POST", () => { expect(rows[0].storagePath).toBe(`receipts/intake/${first.body.id}.png`); }); + test("a publish that failed after a successful upload RESUMES on the next retry", async ({ request }) => { + // The gap this closes: upload lands, the STAGING -> RECEIVED update then + // fails (a connection reset between two round trips is not rare). The + // object exists, the row does not point at it, and nothing would ever + // fix that — STAGING is invisible to the worker's claim by design, so + // the row would sit until the 15-minute sweeper wrongly declared its + // file missing. + const ref = `${REF_PREFIX}resume`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + + // Rewind to exactly the half-state a failed publish leaves behind: the + // object is in the bucket, the row is still STAGING. + await prisma.receiptIntake.update({ where: { id: created.body.id }, data: { state: "STAGING" } }); + + const retry = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(retry.res.status()).toBe(200); + expect(retry.body.state).toBe("RECEIVED"); + expect(retry.body.id).toBe(created.body.id); + + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: ref } }); + expect(rows).toHaveLength(1); + expect(rows[0].state).toBe("RECEIVED"); + }); + + test("a STAGING row whose object is NOT there yet answers 202, not 200", async ({ request }) => { + // A concurrent request is mid-upload, or the last one died before + // storing anything. 200 would promise a queued document that does not + // exist; 202 tells the caller to re-poll. The 15-minute sweeper handles + // the case where it never lands. + const ref = `${REF_PREFIX}staging-nofile`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + + // A STAGING row pointing at a path nothing was ever written to. + await prisma.receiptIntake.update({ + where: { id: created.body.id }, + data: { state: "STAGING", storagePath: `receipts/intake/${created.body.id}-never-uploaded.png` }, + }); + + const retry = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(retry.res.status()).toBe(202); + expect(retry.body.status).toBe("staging"); + expect(retry.body.id).toBe(created.body.id); + expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.state).toBe("STAGING"); + }); + test("a machine caller MUST supply its own sourceRef", async ({ request }) => { const { res, body } = await postIntake(request, JSON.stringify({ source: "drive", fileBase64: PNG_BASE64, mimeType: "image/png", @@ -247,6 +294,60 @@ test.describe("intake POST", () => { expect(sameNamespace.body.existingId).toBe(seeded.body.id); }); + test("the same uploadId from one user is ONE row; a raw sourceRef is still refused", async ({ request }) => { + // A phone on a bad connection needs a safe retry. A minted uuid makes + // every retry a NEW document, so a crew member tapping Send twice on a + // spinner books the same receipt twice. `uploadId` is the client's own + // idempotency token — and it is scoped to the authenticated user + // server-side, so two people cannot collide on one uuid and nobody can + // reach another user's row by guessing one. + const uploadId = "3f2504e0-4f89-41d3-9a0c-0305e82c3301"; + const body = JSON.stringify({ fileBase64: PNG_BASE64, mimeType: "image/png", uploadId }); + const post = () => request.post(INTAKE_PATH, { + headers: { "content-type": "application/json" }, + data: body, + maxRedirects: 0, + }); + + const first = await post(); + expect(first.status()).toBe(200); + const firstBody = await first.json(); + minted.push(firstBody.id); + // Scoped to the user, so the uuid alone is not the key. + expect(firstBody.sourceRef).toMatch(/^web:[^:]+:3f2504e0-4f89-41d3-9a0c-0305e82c3301$/); + expect(firstBody.sourceRef).not.toBe(`web:${uploadId}`); + + const second = await post(); + expect(second.status()).toBe(200); + const secondBody = await second.json(); + expect(secondBody.id).toBe(firstBody.id); + expect(secondBody.alreadyReceived).toBe(true); + + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: firstBody.sourceRef } }); + expect(rows).toHaveLength(1); + + // A non-UUID token is refused rather than used as a free-text key. + const junk = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ fileBase64: PNG_BASE64, mimeType: "image/png", uploadId: "not-a-uuid" }), + maxRedirects: 0, + }); + expect(junk.status()).toBe(400); + expect((await junk.json()).reason).toBe("invalid-uploadId"); + }); + + test("a secret caller's sourceRef must live in the namespace it declared", async ({ request }) => { + // Without this a chat forwarder could write `drive:` and collide + // with — or pre-empt — the Drive pipeline's key for a file it does not + // own, and `drive` rows are the ones that book under the Drive fileId, + // i.e. the QBO DocNumber. + const { res, body } = await postIntake(request, intakeBody({ + source: "chat", sourceRef: `${REF_PREFIX}wrongns`, + })); + expect(res.status()).toBe(400); + expect(body.reason).toBe("sourceRef-namespace-mismatch"); + }); + test("a session caller may not choose its own source or sourceRef", async ({ request }) => { // `source` is provenance and it feeds booking identity: a `drive` row // books under the Drive fileId, so a forged source could aim a QBO diff --git a/package.json b/package.json index ac7a59dd3..e1a7e6d96 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/receipt-intake-archive-contract.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 6a19217dd..9c8cb62b3 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -82,6 +82,12 @@ BEGIN END IF; END $$; +-- RLS, matching every other sensitive table in this schema. ENABLE with no +-- policies and WITHOUT FORCE: the app connects as the owner/service role, which +-- bypasses RLS, so reads and writes are unaffected — while anon and +-- authenticated roles get nothing. FORCE would deny the owner too. +ALTER TABLE "ReceiptIntake" ENABLE ROW LEVEL SECURITY; + DO $$ BEGIN IF NOT EXISTS ( diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index bef2f9bd8..af427a177 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -239,6 +239,10 @@ "name": "QboPurchaseClassification", "forced": false }, + { + "name": "ReceiptIntake", + "forced": false + }, { "name": "RefundEvent", "forced": false diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index 314ba7eaf..06f5b2ec7 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -197,6 +197,17 @@ export const statements = [ END IF; END $$`, + // RLS, matching every other sensitive table in this schema + // (apply-bank-ledger.mjs, apply-automation-events.mjs, + // apply-deposit-ingest-schema.mjs). ENABLE with no policies and WITHOUT + // FORCE: the app connects as the owner/service role, which bypasses RLS, so + // reads and writes are unaffected — while anon and authenticated roles + // (a leaked anon key, a Supabase client someone wires up later) get nothing. + // FORCE would deny the owner too and take the pipeline down. + // ReceiptIntake holds vendor names, amounts and storage paths for real + // purchases, so it belongs in the same class as BankLine. + `ALTER TABLE "ReceiptIntake" ENABLE ROW LEVEL SECURITY`, + `DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ReceiptIntake_expenseId_fkey' diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 249fa1c5f..855c3f10f 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -3,7 +3,7 @@ import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; -import { downloadDocBytes, toSecureRef } from "@/lib/secure-storage"; +import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { readReceipt } from "@/lib/receipt-intake/read"; @@ -142,7 +142,7 @@ function buildDeps(): WorkerDependencies { orderBy: { code: "asc" }, }), - downloadBytes: (storagePath: string) => downloadDocBytes(toSecureRef(storagePath)), + downloadBytes: (storagePath: string) => downloadDocBytesResult(toSecureRef(storagePath)), read: (bytes, mime, phases) => readReceipt(bytes, mime, phases), @@ -224,11 +224,23 @@ function buildDeps(): WorkerDependencies { // block each other. if (weakKey) { await tx.$queryRaw`SELECT pg_advisory_xact_lock(hashtextextended(${weakKey}, 0))`; + // EVERY LIVE STATE, not just the post-booking ones. + // + // Limiting this to BOOKING/BOOKED/ARCHIVED meant a twin sitting + // in NEEDS_REVIEW — which is exactly where the weak net puts + // the FIRST of a suspected pair — was invisible here, so the + // second copy sailed past the human decision that was still + // pending on the first and booked itself. A twin awaiting + // review is the strongest possible signal to stop, not the + // weakest. + // + // DUPLICATE / VOID / NON_RECEIPT are excluded because those are + // settled: somebody already decided they are not a purchase. const conflict = await tx.receiptIntake.findFirst({ where: { dedupWeakKey: weakKey, id: { not: rowId }, - state: { in: ["BOOKING", "BOOKED", "ARCHIVED"] }, + state: { notIn: ["DUPLICATE", "VOID", "NON_RECEIPT"] }, }, select: { id: true }, orderBy: { createdAt: "asc" }, @@ -261,7 +273,7 @@ function buildDeps(): WorkerDependencies { isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), getTokens: getFreshQBTokens, createPurchase: (tokens, input) => createQBReceiptPurchase(tokens, input), - downloadBytes: downloadDocBytes, + downloadBytes: downloadDocBytesResult, logEvent: logAutomationEvent, now: () => new Date(), }), diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 6aa198526..7cb80ffb0 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -3,11 +3,16 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; -import { SECURE_BUCKET } from "@/lib/secure-storage"; +import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; import { EXT_BY_MIME, MAX_INTAKE_BYTES, sniffMime } from "@/lib/receipt-intake/file-type"; -import { ARCHIVE_READABLE_STATES, listReceiptIntakes, serializeReceiptIntake } from "@/lib/receipt-intake/queries"; +import { + ARCHIVE_READABLE_STATES, + listReceiptIntakes, + serializeReceiptIntake, + withArchiveDownloadUrls, +} from "@/lib/receipt-intake/queries"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -46,12 +51,16 @@ const MACHINE_SOURCES = new Set(["drive", "email", "chat"]); /** Minted server-side from the authenticated caller, never read off the body. */ const USER_SOURCES = new Set(["mobile", "web"]); +/** Client-supplied idempotency tokens must be real UUIDs — never a free-text key. */ +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + interface ParsedBody { bytes: Buffer; declaredMime: string; fileName: string | null; source: string; sourceRef: string | null; + uploadId: string | null; projectId: string | null; costCodeId: string | null; threadName: string | null; @@ -83,6 +92,7 @@ async function parseBody(req: Request): Promise { fileName: str(file.name), source: String(form.get("source") ?? ""), sourceRef: str(form.get("sourceRef")), + uploadId: str(form.get("uploadId")), projectId: str(form.get("projectId")), costCodeId: str(form.get("costCodeId")), threadName: str(form.get("threadName")), @@ -109,6 +119,7 @@ async function parseBody(req: Request): Promise { fileName: str(json.fileName), source: String(json.source ?? ""), sourceRef: str(json.sourceRef), + uploadId: str(json.uploadId), projectId: str(json.projectId), costCodeId: str(json.costCodeId), threadName: str(json.threadName), @@ -138,16 +149,32 @@ export async function POST(req: Request) { if (auth.via === "secret") { if (!MACHINE_SOURCES.has(parsed.source)) return bad("invalid-source"); if (!parsed.sourceRef) return bad("missing-sourceRef"); + // The ref must live in the namespace the caller declared. Without this + // a chat forwarder could write `drive:` and collide with — or + // pre-empt — the Drive pipeline's key for a file it does not own, and + // `drive` rows are the ones that book under the Drive fileId, i.e. the + // QBO DocNumber. + if (!parsed.sourceRef.startsWith(`${parsed.source}:`)) return bad("sourceRef-namespace-mismatch"); source = parsed.source; sourceRef = parsed.sourceRef; } else { source = auth.userVia === "mobile-jwt" ? "mobile" : "web"; - // Reject rather than silently ignore: a client that thinks it set the - // key would otherwise believe its retries were idempotent when every - // one of them creates a new document. - if (parsed.sourceRef) return bad("sourceRef-not-allowed"); if (parsed.source && parsed.source !== source) return bad("invalid-source"); - sourceRef = `${source}:${randomUUID()}`; + // A RAW sourceRef stays forbidden — provenance is not caller input. + if (parsed.sourceRef) return bad("sourceRef-not-allowed"); + + // But a phone on a bad connection needs SOME way to retry safely: a + // minted uuid makes every retry a new document, so a crew member who + // taps Send twice on a spinner books the same receipt twice. `uploadId` + // is the client's own idempotency token, and it is SCOPED TO THE USER + // server-side — two people cannot collide on the same uuid, and one + // user cannot reach another's row by guessing one. + if (parsed.uploadId) { + if (!UUID_PATTERN.test(parsed.uploadId)) return bad("invalid-uploadId"); + sourceRef = `${source}:${auth.user.id}:${parsed.uploadId.toLowerCase()}`; + } else { + sourceRef = `${source}:${randomUUID()}`; + } } if (!USER_SOURCES.has(source) && !MACHINE_SOURCES.has(source)) return bad("invalid-source"); @@ -227,27 +254,53 @@ export async function POST(req: Request) { await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); } - const upload = await supabase.storage - .from(SECURE_BUCKET) - .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); - if (upload.error) { - // Delete the row so the caller's retry is a clean insert rather than a - // sourceRef conflict against a row pointing at an object that is not - // there. The worker would otherwise park it "file-missing". + // A THROW here is not the same as an error result — the SDK can reject + // before it ever reaches storage — but both leave a STAGING row pointing at + // an object that may not exist, so both clean it up. The caller's retry is + // then a clean insert rather than a conflict against a half-written row. + let uploadFailed: string | null = null; + try { + const upload = await supabase.storage + .from(SECURE_BUCKET) + .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); + if (upload.error) uploadFailed = upload.error.message; + } catch (error) { + uploadFailed = error instanceof Error ? `${error.name}: ${error.message}` : "upload-threw"; + } + if (uploadFailed) { await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); - console.error("[receipts/intake] upload failed", upload.error.message); + console.error("[receipts/intake] upload failed", uploadFailed); return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); } - // The object exists now, so the row becomes claimable. One UPDATE, and it - // is the ONLY thing that publishes a row to the worker. - const published = await prisma.receiptIntake.update({ - where: { id }, - data: { state: "RECEIVED" }, - select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, - }); + return publishStagedRow(id); +} - return NextResponse.json({ ok: true, ...published }); +/** + * STAGING -> RECEIVED. The one write that makes a row claimable. + * + * Split out because it must be RESUMABLE: if the upload lands and this UPDATE + * then fails (a connection reset between two round trips is not rare), the + * object exists and the row does not point at it, and nothing would ever fix + * that — the row is invisible to the worker's claim by design, so it would sit + * until the 15-minute sweeper wrongly declared its file missing. An identical + * retry finds the STAGING row, confirms the object really is there, and + * finishes the job. + */ +async function publishStagedRow(id: string): Promise { + try { + const published = await prisma.receiptIntake.update({ + where: { id }, + data: { state: "RECEIVED" }, + select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, + }); + return NextResponse.json({ ok: true, ...published }); + } catch (error) { + // The bytes ARE stored; only the publish failed. Leave the row in + // STAGING — 503 tells the caller to retry, and the retry resumes. + console.error("[receipts/intake] publish failed", error instanceof Error ? error.name : "error"); + return NextResponse.json({ ok: false, reason: "publish-failed", id, status: "staging" }, { status: 503 }); + } } /** @@ -274,7 +327,7 @@ async function respondToSourceRefConflict( where: { sourceRef }, select: { id: true, state: true, source: true, sourceRef: true, projectId: true, - dryRun: true, fileSha256: true, createdById: true, + dryRun: true, fileSha256: true, createdById: true, storagePath: true, }, }); // The row vanished between the failed insert and this read (a delete @@ -309,11 +362,20 @@ async function respondToSourceRefConflict( ); } - // Same bytes, but the FIRST request has not finished uploading yet. 200 - // would tell the caller its document is queued when the object may still - // fail to land; 202 says "accepted, not yet published" so a forwarder can - // re-poll instead of moving the file out from under a half-written row. + // Same bytes, and the first attempt is still STAGING. Two very different + // reasons for that, and only storage can tell them apart: + // + // - the object IS there, so the previous request uploaded successfully and + // only its publish UPDATE failed. Finish it. This is what makes the + // publish resumable rather than a permanent half-state. + // - the object is NOT there yet — a concurrent request is mid-upload, or + // the last one died before storing anything. 202 "accepted, not yet + // published" tells the caller to re-poll rather than assume a queued + // document; the 15-minute sweeper handles the case where it never lands. if (existing.state === "STAGING") { + if (await secureObjectExists(existing.storagePath)) { + return publishStagedRow(existing.id); + } return NextResponse.json( { ok: true, status: "staging", alreadyReceived: true, id: existing.id, sourceRef: existing.sourceRef }, { status: 202 }, @@ -364,5 +426,17 @@ export async function GET(req: Request) { take: url.searchParams.get("take") ? Number(url.searchParams.get("take")) : null, archiveOnly, }); + + if (archiveOnly) { + // The mirror needs to FETCH each file and NAME it. It holds no service + // key and cannot read the private bucket, so every row carries a + // short-lived signed URL plus the project name the filename is built + // from. + const withUrls = await withArchiveDownloadUrls( + rows as Array<{ storagePath: string; project?: { name: string } | null }>, + ); + return NextResponse.json({ ok: true, rows: withUrls.map(serializeReceiptIntake) }); + } + return NextResponse.json({ ok: true, rows: rows.map(serializeReceiptIntake) }); } diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 5f40cf3c5..b275d2232 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -83,8 +83,24 @@ export interface PipelineHealth { bank: TimestampProbe; /** Automation events (ANY kind) that errored in the last 24h. */ stuck: CountProbe; + /** + * Receipt Pipeline v2 (ReceiptIntake). The v1 probes above all read + * AutomationEvent, which only ever records a BOOKING — so a v2 row that + * never reaches QuickBooks is invisible to every check in this file. A + * jammed intake queue would report a perfectly healthy pipeline right up + * until somebody noticed the expenses were missing. + */ + intake: { + /** RECEIVED or BOOKING and older than 6h — the queue is not draining. */ + stuck: CountProbe; + /** NEEDS_REVIEW backlog. Reported always; a reason only when rows are STUCK. */ + needsReview: CountProbe; + }; } +/** A row this old in a working state has not been picked up, it has jammed. */ +export const INTAKE_STUCK_HOURS = 6; + /** * Intuit's own status page. * @@ -181,6 +197,8 @@ export function evaluatePipelineHealth(input: { receipts24h: CountsProbe; bank: TimestampProbe; stuck: CountProbe; + intakeStuck: CountProbe; + intakeNeedsReview: CountProbe; now: number; }): { ok: boolean; reasons: string[] } { const reasons: string[] = []; @@ -192,6 +210,8 @@ export function evaluatePipelineHealth(input: { ["receipts24h", input.receipts24h], ["bank", input.bank], ["stuck", input.stuck], + ["intakeStuck", input.intakeStuck], + ["intakeNeedsReview", input.intakeNeedsReview], ]; for (const [name, probe] of namedProbes) { if (probe.status === "error") reasons.push(`probe-failed:${name}`); @@ -230,6 +250,16 @@ export function evaluatePipelineHealth(input: { // It proves the cron is alive, so it counts for freshness — but it must // be reported the same day, not hidden inside the 26h staleness window. else if (input.lastPaymentsSync.runStatus === "partial") reasons.push("payments-sync-partial"); + // A row sitting in RECEIVED or BOOKING for six hours means the worker is + // not draining the queue — a wedged cron, an exhausted retry budget, a + // storage outage. The backlog number rides along so the digest can say how + // big the hole is, but only the STUCK count is a failure: NEEDS_REVIEW rows + // are working as designed (a human was asked a question) and would + // otherwise hold the pipeline red until somebody cleared the queue. + if (input.intakeStuck.status === "ok" && input.intakeStuck.count > 0) { + const backlog = + input.intakeNeedsReview.status === "ok" ? `,needs-review:${input.intakeNeedsReview.count}` : ""; + reasons.push(`intake-stuck:${input.intakeStuck.count}${backlog}`); } if (input.lastReceiptPush.status === "ok") { @@ -297,6 +327,8 @@ export async function getPipelineHealth(): Promise { const probe = runProbe; const [intuit, lastPurchase, lastPush, lastPaymentsSync, receiptRows, lastBankLine, stuck] = await Promise.all([ + const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck, intakeStuck, intakeNeedsReview] = + await Promise.all([ fetchIntuitStatus(), // Expense carries no updatedAt column — qbSyncedAt IS the "when did the // QBO purchase sync land" timestamp this is asking for. @@ -389,6 +421,25 @@ export async function getPipelineHealth(): Promise { }), 0, ), + // ReceiptIntake: the v2 queue. STAGING is excluded on purpose — those + // rows are mid-upload and the intake route's own 15-minute sweeper owns + // them; counting them here would flag every in-flight request. + probe( + "intakeStuck", + () => + prisma.receiptIntake.count({ + where: { + state: { in: ["RECEIVED", "BOOKING"] }, + createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * HOUR_MS) }, + }, + }), + 0, + ), + probe( + "intakeNeedsReview", + () => prisma.receiptIntake.count({ where: { state: "NEEDS_REVIEW" } }), + 0, + ), ]); const counts: Record = {}; @@ -419,6 +470,12 @@ export async function getPipelineHealth(): Promise { at: lastBankLine.value?.toISOString() ?? null, }, stuck: { status: stuck.status, reason: stuck.reason, count: stuck.value }, + intakeStuck: { status: intakeStuck.status, reason: intakeStuck.reason, count: intakeStuck.value }, + intakeNeedsReview: { + status: intakeNeedsReview.status, + reason: intakeNeedsReview.reason, + count: intakeNeedsReview.value, + }, }; const verdict = evaluatePipelineHealth({ ...snapshot, now }); @@ -435,6 +492,7 @@ export async function getPipelineHealth(): Promise { receipts24h: snapshot.receipts24h, bank: snapshot.bank, stuck: snapshot.stuck, + intake: { stuck: snapshot.intakeStuck, needsReview: snapshot.intakeNeedsReview }, }; } @@ -485,6 +543,15 @@ export function formatPipelineDigest(health: PipelineHealth): { subject: string; : "no lines" }`, `Automation errors (24h, all kinds): ${health.stuck.status === "error" ? "unavailable (probe failed)" : health.stuck.count}`, + // Optional-chained on purpose: a digest that THROWS means no morning + // email at all, which is strictly worse than a digest missing a line. + // Same rule as "no probe may throw" above. + `Receipt intake stuck >${INTAKE_STUCK_HOURS}h: ${ + health.intake?.stuck?.status === "error" ? "unavailable (probe failed)" : health.intake?.stuck?.count ?? "unavailable" + }`, + `Receipt intake awaiting review: ${ + health.intake?.needsReview?.status === "error" ? "unavailable (probe failed)" : health.intake?.needsReview?.count ?? "unavailable" + }`, ]; if (health.reasons.length > 0) lines.push(`Needs attention: ${health.reasons.join(", ")}`); diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index af5a76012..afeb79966 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -29,6 +29,7 @@ import { type QboReceiptGroup, } from "@/lib/qbo-receipt-push"; import type { AutomationEventInput } from "@/lib/automation-events"; +import type { DocBytesResult } from "@/lib/secure-storage"; import { backoffMs, MAX_BOOK_ATTEMPTS } from "./route-state"; /** The intake columns booking actually reads. Kept narrow so tests can build one by hand. */ @@ -101,8 +102,11 @@ export interface BookDependencies { isPushPaused: () => Promise; getTokens: () => Promise; createPurchase: (tokens: QBTokens, input: CreateQBReceiptPurchaseInput) => Promise; - /** Reads the stored file back out of the private bucket for the QBO attachment. */ - downloadBytes: (secureRef: string) => Promise; + /** + * Reads the stored file back out of the private bucket. TAGGED, because a + * confirmed 404 and a transient storage fault must not book the same way. + */ + downloadBytes: (secureRef: string) => Promise; logEvent: (event: AutomationEventInput) => Promise; now: () => Date; } @@ -249,7 +253,24 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // no-op rather than a second Purchase. const fileId = driveFileIdOf(row) ?? row.id; const isCheck = String(row.docType || "receipt").toLowerCase() === "check"; - const bytes = await deps.downloadBytes(toSecureRef(row.storagePath)); + + // NEVER a Purchase without its receipt. + // + // This used to pass `fileBase64: undefined` when the bytes could not be + // loaded and book anyway, which produces a QBO Purchase with no attachment + // — the one thing the bookkeeper cannot fix later, because by then the + // Purchase looks complete and nothing flags it. The receipt IS the evidence + // for the expense; a booking without it is worse than no booking. + // + // A transient storage fault retries (the document is fine, Supabase was + // not); an affirmative 404 is terminal and pre-send, so the strong key goes + // back for a corrected re-upload. + const download = await deps.downloadBytes(toSecureRef(row.storagePath)); + if (!download.ok) { + if (download.kind === "not-found") return parkedBeforeSend("receipt-bytes-missing"); + return retry(row, deps, now, `storage:${download.message}`); + } + const bytes = download.bytes; const input: CreateQBReceiptPurchaseInput = { projectName: project.name, @@ -263,8 +284,8 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro fileId, fileName: row.fileName ?? undefined, groups, - fileBase64: bytes ? bytes.toString("base64") : undefined, - fileContentType: bytes ? row.mimeType : undefined, + fileBase64: bytes.toString("base64"), + fileContentType: row.mimeType, }; let result: CreateQBReceiptPurchaseResult; diff --git a/src/lib/receipt-intake/queries.ts b/src/lib/receipt-intake/queries.ts index 96324254a..cc67b1a0e 100644 --- a/src/lib/receipt-intake/queries.ts +++ b/src/lib/receipt-intake/queries.ts @@ -7,6 +7,7 @@ * outside the worker should read from it. */ import { prisma } from "@/lib/prisma"; +import { resolveDocUrl, toSecureRef } from "@/lib/secure-storage"; export const RECEIPT_INTAKE_LIST_SELECT = { id: true, @@ -40,6 +41,10 @@ export const RECEIPT_INTAKE_LIST_SELECT = { expenseId: true, archiveDriveFileId: true, attempts: true, + // The AI-unavailable counter. Without it the Phase 2 queue page cannot tell + // "this document defeated the model" from "Gemini was down all afternoon", + // which is the first question anyone asks during an outage. + busyPasses: true, lastError: true, nextRetryAt: true, bookedAt: true, @@ -71,8 +76,20 @@ export const RECEIPT_INTAKE_ARCHIVE_SELECT = { state: true, archiveDriveFileId: true, bookedAt: true, + // The mirror names the Drive file `____$`, + // so it needs the project NAME, not an id it cannot resolve. + project: { select: { name: true } }, } as const; +/** + * Signed-URL lifetime for the archive mirror. Long enough for a nightly Apps + * Script pass to fetch every BOOKED receipt, short enough that a URL captured + * from a log is useless by morning. The bucket is private; this is the ONLY way + * the script can read the bytes, and it is deliberately a per-request grant + * rather than anything the script can store. + */ +export const ARCHIVE_SIGNED_URL_TTL_SECONDS = 600; + /** * States the secret caller may query. The mirror archives what is BOOKED and * re-checks what it already ARCHIVED; nothing else is its business, and a @@ -110,6 +127,31 @@ export async function listReceiptIntakes(args: ListReceiptIntakesArgs) { }); } +/** + * Attach a short-lived download URL and flatten the project name. + * + * The mirror cannot read the private bucket and must not be handed a service + * key, so each row carries its own signed URL. A row whose URL cannot be signed + * is returned with `downloadUrl: null` rather than dropped — the script logs it + * and moves on, which is strictly better than a silently short archive. + */ +export async function withArchiveDownloadUrls( + rows: T[], + /** Injectable so the contract is testable without Supabase. */ + sign: (ref: string, ttlSeconds: number) => Promise = resolveDocUrl, +): Promise & { projectName: string | null; downloadUrl: string | null }>> { + return Promise.all( + rows.map(async row => { + const { project, ...rest } = row; + return { + ...(rest as Omit), + projectName: project?.name ?? null, + downloadUrl: await sign(toSecureRef(row.storagePath), ARCHIVE_SIGNED_URL_TTL_SECONDS), + }; + }), + ); +} + /** Dates out as ISO strings; there are no Decimals on this model, cents are Ints. */ export function serializeReceiptIntake>(row: T) { const out: Record = {}; diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts index 586ab7080..6e236203c 100644 --- a/src/lib/receipt-intake/read.ts +++ b/src/lib/receipt-intake/read.ts @@ -71,7 +71,8 @@ export type ReadOutcome = * usable data (or rejected the payload). Retrying will not change that — * the caller must spend an attempt and route the row to a human. * - * decisive false: every model was unavailable (429/503/404/401/403/network). + * decisive false: every model was unavailable (429, ANY 5xx, 404, 401, + * 403, or a network error). * The document was never read, so the caller must NOT spend an attempt. */ | { ok: false; decisive: boolean }; @@ -269,7 +270,13 @@ export async function readReceipt( break; } - if (code === 429 || code === 503) { // overloaded / rate-limited + // EVERY 5xx is the SERVICE failing, not the document. 503 and 429 + // were already treated that way, but 500/502/504 fell through to + // the "decisive" branch below and charged the row a strike for a + // Google-side fault it had nothing to do with — precisely the + // mistake the outage rationale at :1143-1184 exists to prevent. A + // gateway error says nothing about whether the receipt is readable. + if (code === 429 || code >= 500) { // overloaded / rate-limited / server fault if (attempts >= MAX_RETRIES) break; // fall through to the next model const wait = RETRY_BACKOFF_MS[attempts]; attempts++; @@ -284,7 +291,9 @@ export async function readReceipt( // model while another works is exactly what the chain is for. if (code === 404 || code === 401 || code === 403) break; - // 400 = oversized/undecodable payload. THAT is about this document. + // What is left is a 4xx that is not 401/403/404/429: a rejected + // payload (400 = oversized or undecodable). THAT is about this + // document, and no amount of retrying changes it. sawDecisiveFailure = true; return { ok: false, decisive: true }; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 485761094..b7bfb1d26 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -24,6 +24,7 @@ import { QboVendorDuplicateError, } from "@/lib/qbo-receipt-push"; import type { ProjectPhase, ReadOutcome } from "./read"; +import type { DocBytesResult } from "@/lib/secure-storage"; /** * ONE global constant, deliberately not derived from anything per-row or @@ -95,7 +96,8 @@ export interface WorkerDependencies { */ sweepStaleStaging: () => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; - downloadBytes: (secureRef: string) => Promise; + /** Tagged: a confirmed 404 and a transient storage fault are NOT the same answer. */ + downloadBytes: (storagePath: string) => Promise; read: (bytes: Buffer, mime: string, phases: ProjectPhase[]) => Promise; /** * Persist the read + routing. Returns the strong-key owner when the partial @@ -307,12 +309,19 @@ function stateForBookResult(result: BookResult): string { } async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promise { - const bytes = await deps.downloadBytes(row.storagePath); - if (!bytes) { - // The object is gone from the bucket — nothing to read, ever. - await deps.applyState(row.id, "NEEDS_REVIEW", "file-missing"); - return "NEEDS_REVIEW"; + const download = await deps.downloadBytes(row.storagePath); + if (!download.ok) { + // "The object is gone" and "storage was briefly unreachable" demand + // opposite answers, and collapsing them to null meant a Supabase blip + // parked good receipts as file-missing, permanently, for a human to + // untangle. Only an AFFIRMATIVE not-found is terminal. + if (download.kind === "not-found") { + await deps.applyState(row.id, "NEEDS_REVIEW", "file-missing"); + return "NEEDS_REVIEW"; + } + return retryTransient(row, deps, `storage:${download.message}`); } + const bytes = download.bytes; const costCodes = await deps.loadPhases(row.projectId); const phases: ProjectPhase[] = costCodes.map(c => ({ code: c.code, name: c.name })); @@ -352,11 +361,6 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis const taxCentsRaw = centsOf(read.taxAmount || "0.00"); const taxCents = taxCentsRaw && taxCentsRaw > 0 ? taxCentsRaw : null; - // Weak hits are a plain query and never a claim (:1591-1596). Strong hits - // come from the partial unique index rejecting the write below. - const weak = await deps.findWeakHit(row.id, keys.weak); - const hits: DedupHits = { strong: null, weak }; - const base = { vendor: read.vendor || null, txnDate: dateOnly(keys.dateStr), @@ -377,25 +381,47 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis totalCents, canonicalVendor: canonicalVendor(read.vendor), }; - const decision = routeState(routeInput, hits, !!row.projectId); - // A document that never reaches READ must not hold the strong key: a - // multi-doc, a non-receipt, or a $0 misread would otherwise quarantine the - // real receipt that arrives next (:531 and the v3.6 rationale). - const claimsStrongKey = decision.state === "READ" && keys.strong !== null; + // ORDER MATTERS, and it used to be wrong. + // + // The weak lookup ran FIRST, so an exact duplicate — same date, same ref, + // same vendor, same amount, which therefore matches BOTH nets — routed on + // the weak hit to NEEDS_REVIEW and never attempted the strong claim at all. + // The one case the strong key exists to resolve automatically was the one + // case it never got to see, and every re-sent receipt landed in a human's + // queue. + // + // So: the document-level gates first (multi, non-receipt, refund/zero, no + // job) because those outrank dedup entirely; then the STRONG claim, which + // is the only net that can answer DUPLICATE on its own; and only if the + // strong net is silent do we fall back to the weak one, which by design + // never decides anything itself. + const gate = routeState(routeInput, { strong: null, weak: null }, !!row.projectId); + if (gate.state !== "READ") { + // A multi-doc, a non-receipt, or a $0/negative misread must never hold + // a dedup key — it would quarantine the real receipt that arrives next + // (:531 and the v3.6 rationale). + await deps.applyRead(row.id, { + ...base, + state: gate.state, + stateReason: gate.stateReason, + dedupStrongKey: null, + duplicateOfId: gate.duplicateOfId, + }); + return gate.state; + } + // The strong claim IS the partial unique index: a rejection is the hit. const applied = await deps.applyRead(row.id, { ...base, - state: decision.state, - stateReason: decision.stateReason, - dedupStrongKey: claimsStrongKey ? keys.strong : null, - duplicateOfId: decision.duplicateOfId, + state: "READ", + stateReason: null, + dedupStrongKey: keys.strong, + duplicateOfId: null, }); if (applied.strongOwner) { - // The claim lost: another live row already owns this date|ref. Re-route - // with the owner in hand and write the losing outcome (no key). - const second = routeState(routeInput, { strong: applied.strongOwner, weak }, !!row.projectId); + const second = routeState(routeInput, { strong: applied.strongOwner, weak: null }, !!row.projectId); await deps.applyState(row.id, second.state, second.stateReason, { ...base, dedupStrongKey: null, @@ -404,7 +430,31 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis return second.state; } - return decision.state; + // No strong hit (or no strong key at all — a placeholder ref). The weak net + // is a plain query and never a claim (:1591-1596); a hit only ever asks a + // human, because two genuine same-day purchases from one vendor for the + // same amount do happen. + const weak = await deps.findWeakHit(row.id, keys.weak); + if (weak) { + const third = routeState(routeInput, { strong: null, weak }, !!row.projectId); + // The strong key stays claimed: this row is still the live owner of + // that date|ref, and releasing it would let a third copy book. + await deps.applyState(row.id, third.state, third.stateReason); + return third.state; + } + + return "READ"; +} + +/** A transport-class fault during a row's processing: spend an attempt, back off. */ +async function retryTransient(row: WorkerRow, deps: WorkerDependencies, reason: string): Promise { + const attempts = row.attempts + 1; + if (attempts >= MAX_BOOK_ATTEMPTS) { + await deps.applyState(row.id, "NEEDS_REVIEW", "max-retries"); + return "NEEDS_REVIEW"; + } + await deps.retryRow(row.id, attempts, new Date(deps.now().getTime() + backoffMs(attempts)), reason); + return "RETRY"; } /** diff --git a/src/lib/secure-storage.ts b/src/lib/secure-storage.ts index 6dfd315b3..503795b4e 100644 --- a/src/lib/secure-storage.ts +++ b/src/lib/secure-storage.ts @@ -173,6 +173,98 @@ export async function resolveDocUrls( ); } +/** + * Why a download did not produce bytes. + * + * `downloadDocBytes` collapses every failure to `null`, which is fine for a PDF + * that renders without a signature but NOT for a money path: "the object is + * gone" and "Supabase was briefly unreachable" demand opposite responses. The + * first is terminal (a human must re-upload); the second must be retried, and + * treating it as terminal would park good receipts during a storage blip. + * + * `not-found` is only ever returned when storage AFFIRMATIVELY said the object + * is missing. Anything ambiguous — a network error, a 5xx, no configured + * client — is `transient`, because guessing "gone" on incomplete evidence is + * the failure mode that loses documents. + */ +export type DocBytesResult = + | { ok: true; bytes: Buffer } + | { ok: false; kind: "not-found" } + | { ok: false; kind: "transient"; message: string }; + +/** Supabase storage's shapes for "this key does not exist". */ +function isNotFoundError(error: { message?: string; status?: number; statusCode?: string | number } | null): boolean { + if (!error) return false; + const status = Number(error.status ?? error.statusCode); + if (status === 404 || status === 400) return true; + const message = String(error.message ?? "").toLowerCase(); + return message.includes("not found") || message.includes("object not found"); +} + +/** + * Tagged download. Same resolution rules as downloadDocBytes (secure ref, data + * URL, our own storage URL, legacy bare path) but the caller is told WHY it + * failed. Use this on any path where a missing file changes what happens to + * real money. + */ +export async function downloadDocBytesResult( + stored: string | null | undefined, +): Promise { + if (!stored) return { ok: false, kind: "not-found" }; + + if (isDataUrl(stored)) { + const bytes = await downloadDocBytes(stored); + return bytes ? { ok: true, bytes } : { ok: false, kind: "not-found" }; + } + + let bucket: string; + let path: string; + + const securePath = secureRefPath(stored); + if (securePath) { + bucket = SECURE_BUCKET; + path = securePath; + } else if (/^https?:\/\//i.test(stored)) { + const parsed = parseOwnStorageUrl(stored); + // Not ours, or naming a bucket we never write absolute URLs for. That is + // a REFUSAL, not a transient failure — retrying cannot make it ours. + if (!parsed || parsed.bucket !== STORAGE_BUCKET) return { ok: false, kind: "not-found" }; + bucket = parsed.bucket; + path = parsed.path; + } else { + if (stored.startsWith("/") || stored.includes("..")) return { ok: false, kind: "not-found" }; + bucket = STORAGE_BUCKET; + path = stored; + } + + const supabase = getSupabase(); + // No client is a CONFIGURATION fault, not a missing object. Retry it. + if (!supabase) return { ok: false, kind: "transient", message: "storage-not-configured" }; + try { + const { data, error } = await supabase.storage.from(bucket).download(path); + if (error) { + return isNotFoundError(error as { message?: string; status?: number }) + ? { ok: false, kind: "not-found" } + : { ok: false, kind: "transient", message: String(error.message ?? "download-failed").slice(0, 200) }; + } + if (!data) return { ok: false, kind: "not-found" }; + return { ok: true, bytes: Buffer.from(await data.arrayBuffer()) }; + } catch (error) { + // A throw is a transport fault every time — never evidence of absence. + return { + ok: false, + kind: "transient", + message: error instanceof Error ? `${error.name}: ${error.message}`.slice(0, 200) : "download-threw", + }; + } +} + +/** True only when storage affirmatively confirms the object is there. */ +export async function secureObjectExists(storagePath: string): Promise { + const result = await downloadDocBytesResult(toSecureRef(storagePath)); + return result.ok; +} + /** * Read a stored document's bytes server-side using the service key. * diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts index e1f87b4aa..7d405cb16 100644 --- a/tests/apply-receipt-intake.test.ts +++ b/tests/apply-receipt-intake.test.ts @@ -93,6 +93,8 @@ test("every statement is idempotent — the script is safe to re-run", () => { /CREATE TABLE IF NOT EXISTS/.test(sql) || /CREATE (?:UNIQUE )?INDEX IF NOT EXISTS/.test(sql) || /ALTER TABLE .* ADD COLUMN IF NOT EXISTS/.test(sql) || + // Re-enabling RLS on a table that already has it is a no-op. + /ENABLE ROW LEVEL SECURITY/.test(sql) || /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(sql); assert.ok(guarded, `not idempotent: ${sql.slice(0, 80)}`); } @@ -154,3 +156,27 @@ test("STAGING is in the state set, and is the column DEFAULT", () => { assert.match(create!, /"state"\s+TEXT NOT NULL DEFAULT 'STAGING'/); assert.match(migrationSql, /"state" TEXT NOT NULL DEFAULT 'STAGING'/); }); + +test("RLS is enabled on ReceiptIntake, in both files, and WITHOUT force", () => { + // Same shape as every other sensitive table here (apply-bank-ledger, + // apply-automation-events, apply-deposit-ingest-schema): ENABLE with no + // policies. The app connects as the owner/service role, which BYPASSES RLS, + // so reads and writes are unaffected — while anon and authenticated roles + // (a leaked anon key, a Supabase client someone wires up later) get nothing. + // + // FORCE is the trap: it applies RLS to the owner too, and with zero policies + // that denies everything. It would take the pipeline down silently, as + // empty result sets rather than errors. + assert.ok(statements.some((s: string) => /ALTER TABLE "ReceiptIntake" ENABLE ROW LEVEL SECURITY/.test(s))); + assert.match(migrationSql, /ALTER TABLE "ReceiptIntake" ENABLE ROW LEVEL SECURITY;/); + assert.ok(!statements.some((s: string) => /FORCE ROW LEVEL SECURITY/.test(s)), "never FORCE"); + assert.ok(!/FORCE ROW LEVEL SECURITY/.test(migrationSql), "never FORCE"); + + // And it must be recorded in the snapshot CI compares against production. + const snapshot = JSON.parse( + readFileSync(path.join(__dirname, "..", "prisma", "prisma-blind-spots.json"), "utf8"), + ); + const entry = snapshot.rlsTables.find((r: { name: string }) => r.name === "ReceiptIntake"); + assert.ok(entry, "ReceiptIntake missing from prisma-blind-spots.json rlsTables"); + assert.equal(entry.forced, false); +}); diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts index 25293edcc..3f2ba073a 100644 --- a/tests/pipeline-digest-route.test.ts +++ b/tests/pipeline-digest-route.test.ts @@ -29,6 +29,7 @@ const HEALTH: PipelineHealth = { receipts24h: { status: "ok", counts: { created: 2 } }, bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, stuck: { status: "ok", count: 0 }, + intake: { stuck: { status: "ok", count: 0 }, needsReview: { status: "ok", count: 0 } }, }; function handlers(overrides: Partial = {}) { diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 87603bb9b..fd906d143 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -35,6 +35,8 @@ function snapshot(overrides: Partial[0 receipts24h: { status: "ok" as const, counts: { created: 4 } }, bank: { status: "ok" as const, at: iso(48 * HOUR) }, stuck: { status: "ok" as const, count: 0 }, + intakeStuck: { status: "ok" as const, count: 0 }, + intakeNeedsReview: { status: "ok" as const, count: 0 }, now: NOW, ...overrides, }; @@ -179,6 +181,10 @@ function sampleHealth(overrides: Partial = {}): PipelineHealth { receipts24h: { status: "ok", counts: { created: 4, fallback: 1 } }, bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, stuck: { status: "ok", count: 0 }, + intake: { + stuck: { status: "ok", count: 0 }, + needsReview: { status: "ok", count: 0 }, + }, ...overrides, }; } @@ -546,4 +552,60 @@ test("the journey mapper renders attachment-failed as failed, not in-flight", as // arriving - the bot has already stopped. assert.equal(journey.finalState, "error"); assert.equal(journey.finalReason, "failed:fault"); +// ── Receipt Pipeline v2 intake queue (Codex round 3, item 7) ──────────────── +// Every other probe in this file reads AutomationEvent, which only records a +// BOOKING — so a v2 row that never reaches QuickBooks is invisible to all of +// them. A jammed intake queue reported a perfectly healthy pipeline. + +test("rows stuck in the intake queue fail the check and name the backlog", () => { + const v = evaluatePipelineHealth(snapshot({ + intakeStuck: { status: "ok", count: 4 }, + intakeNeedsReview: { status: "ok", count: 11 }, + })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["intake-stuck:4,needs-review:11"]); +}); + +test("a NEEDS_REVIEW backlog alone is NOT a failure", () => { + // Those rows are working as designed — a human was asked a question. + // Failing on them would hold the pipeline red until somebody cleared the + // queue, which trains everyone to ignore the signal. + const v = evaluatePipelineHealth(snapshot({ intakeNeedsReview: { status: "ok", count: 40 } })); + assert.deepEqual(v, { ok: true, reasons: [] }); +}); + +test("an intake probe that FAILED is not an intake probe that found nothing", () => { + for (const name of ["intakeStuck", "intakeNeedsReview"] as const) { + const v = evaluatePipelineHealth(snapshot({ [name]: { status: "error", reason: "timeout", count: 0 } })); + assert.equal(v.ok, false, name); + assert.ok(v.reasons.includes(`probe-failed:${name}`), name); + } +}); + +test("the stuck reason survives a failed backlog probe rather than lying about it", () => { + const v = evaluatePipelineHealth(snapshot({ + intakeStuck: { status: "ok", count: 2 }, + intakeNeedsReview: { status: "error", reason: "error", count: 0 }, + })); + assert.ok(v.reasons.includes("intake-stuck:2"), "no invented needs-review count"); + assert.ok(v.reasons.includes("probe-failed:intakeNeedsReview")); +}); + +test("the digest prints both intake numbers", () => { + const { text } = formatPipelineDigest(sampleHealth({ + intake: { stuck: { status: "ok", count: 3 }, needsReview: { status: "ok", count: 7 } }, + })); + assert.match(text, /Receipt intake stuck >6h: 3/); + assert.match(text, /Receipt intake awaiting review: 7/); +}); + +test("the digest says a failed intake probe is unavailable, never zero", () => { + const { text } = formatPipelineDigest(sampleHealth({ + intake: { + stuck: { status: "error", reason: "timeout", count: 0 }, + needsReview: { status: "error", reason: "timeout", count: 0 }, + }, + })); + assert.match(text, /Receipt intake stuck >6h: unavailable \(probe failed\)/); + assert.match(text, /Receipt intake awaiting review: unavailable \(probe failed\)/); }); diff --git a/tests/receipt-intake-archive-contract.test.ts b/tests/receipt-intake-archive-contract.test.ts new file mode 100644 index 000000000..62f0dd6ab --- /dev/null +++ b/tests/receipt-intake-archive-contract.test.ts @@ -0,0 +1,84 @@ +/** + * The contract the nightly Apps Script archive mirror codes against. + * + * The mirror holds a shared secret and NO service key, so it cannot read the + * private bucket on its own. Everything it needs to do its one job — fetch each + * BOOKED receipt and write it to `Processed Receipts/YYYY/MM/` under the v1 + * filename `____$.` — has to be in the + * payload, and nothing else should be. + * + * These assertions are the contract; changing one is a breaking change to a + * script in another repo. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + ARCHIVE_READABLE_STATES, + ARCHIVE_SIGNED_URL_TTL_SECONDS, + RECEIPT_INTAKE_ARCHIVE_SELECT, + RECEIPT_INTAKE_LIST_SELECT, + withArchiveDownloadUrls, +} from "../src/lib/receipt-intake/queries"; + +test("the archive payload carries everything the v1 filename is built from", () => { + for (const field of ["txnDate", "vendor", "totalCents", "refNumber", "fileName", "mimeType"]) { + assert.ok(field in RECEIPT_INTAKE_ARCHIVE_SELECT, `missing ${field}`); + } + // The project NAME, not an id the script cannot resolve. + assert.ok("project" in RECEIPT_INTAKE_ARCHIVE_SELECT); + // And what it needs to report back and to skip already-archived rows. + for (const field of ["id", "state", "archiveDriveFileId", "storagePath"]) { + assert.ok(field in RECEIPT_INTAKE_ARCHIVE_SELECT, `missing ${field}`); + } +}); + +test("the archive payload withholds everything the mirror has no business seeing", () => { + // Least privilege applies to a script the same way it does to a user: a + // leaked or over-shared secret should expose as little as still works. + for (const field of ["lastError", "fileSha256", "createdById", "readJson", "dedupStrongKey", "dedupWeakKey", "attempts", "busyPasses"]) { + assert.ok(!(field in RECEIPT_INTAKE_ARCHIVE_SELECT), `${field} must not be exposed`); + // ...and it IS in the staff select, so this is a real narrowing rather + // than a column that simply does not exist. + if (field !== "readJson") { + assert.ok(field in RECEIPT_INTAKE_LIST_SELECT, `${field} should exist on the staff select`); + } + } +}); + +test("the mirror may only ask for the two states it acts on", () => { + assert.deepEqual([...ARCHIVE_READABLE_STATES].sort(), ["ARCHIVED", "BOOKED"]); +}); + +test("each row gets a short-lived signed URL and a flat project name", async () => { + const signed: Array<{ ref: string; ttl: number }> = []; + const rows = await withArchiveDownloadUrls( + [ + { id: "a", storagePath: "receipts/intake/a.jpg", project: { name: "Berg ADU" } }, + { id: "b", storagePath: "receipts/intake/b.pdf", project: null }, + ], + async (ref: string, ttl: number) => { signed.push({ ref, ttl }); return `https://signed.test/${ref}`; }, + ); + + assert.equal(rows[0].projectName, "Berg ADU"); + assert.equal(rows[0].downloadUrl, "https://signed.test/secure:receipts/intake/a.jpg"); + assert.equal(rows[1].projectName, null, "a project-less row is still archivable"); + // The nested relation is flattened away — the script gets `projectName`. + assert.ok(!("project" in rows[0])); + + // A private bucket plus a per-request grant: the script never holds a key, + // and a URL captured from a log is useless by morning. + assert.equal(ARCHIVE_SIGNED_URL_TTL_SECONDS, 600); + assert.deepEqual(signed.map(s => s.ttl), [600, 600]); + assert.deepEqual(signed.map(s => s.ref), ["secure:receipts/intake/a.jpg", "secure:receipts/intake/b.pdf"]); +}); + +test("a row whose URL cannot be signed is returned with null, never dropped", async () => { + // A silently short archive is worse than a logged gap. + const rows = await withArchiveDownloadUrls( + [{ id: "a", storagePath: "receipts/intake/a.jpg", project: { name: "Berg ADU" } }], + async () => null, + ); + assert.equal(rows.length, 1); + assert.equal(rows[0].downloadUrl, null); + assert.equal(rows[0].projectName, "Berg ADU"); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 3a6246e88..cb3b5933e 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -93,7 +93,7 @@ function recorder(overrides: Partial = {}, opts: { estimates?: purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: input.fileId.slice(0, 21), alreadyExists: false, attachment: "attached" }; }, - downloadBytes: async () => Buffer.from("bytes"), + downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), logEvent: async (event) => { events.push(event); }, now: () => NOW, ...overrides, @@ -332,3 +332,40 @@ test("a DB failure AFTER the Purchase exists retries — the create is idempoten const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "retry"); }); + +// ── Never a Purchase without its receipt (Codex round 3, item 3) ──────────── + +test("a MISSING receipt file refuses the booking outright", async () => { + // Booking with `fileBase64: undefined` produced a QBO Purchase with no + // attachment — the one failure the bookkeeper cannot fix later, because the + // Purchase looks complete and nothing flags it. The receipt IS the evidence + // for the expense. + const r = recorder({ downloadBytes: async () => ({ ok: false, kind: "not-found" }) }); + const result = await bookReceipt(row(), r.deps); + assert.deepEqual(result, { + outcome: "needs-review", + reason: "receipt-bytes-missing", + // Pre-send: nothing reached QuickBooks, so the key goes back for a + // corrected re-upload. + releaseStrongKey: true, + }); + assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never touched"); + assert.equal(r.expenses.length, 0); +}); + +test("a TRANSIENT storage fault retries instead of parking a good receipt", async () => { + const r = recorder({ + downloadBytes: async () => ({ ok: false, kind: "transient", message: "ECONNRESET" }), + }); + const result = await bookReceipt(row({ attempts: 1 }), r.deps); + assert.equal(result.outcome, "retry"); + assert.match((result as any).reason, /^storage:/); + assert.equal(r.purchaseCalls.length, 0); +}); + +test("the receipt bytes always ride along with the Purchase", async () => { + const r = recorder(); + await bookReceipt(row(), r.deps); + assert.equal(r.purchaseCalls[0].fileBase64, Buffer.from("bytes").toString("base64")); + assert.equal(r.purchaseCalls[0].fileContentType, "image/jpeg"); +}); diff --git a/tests/receipt-intake-read.test.ts b/tests/receipt-intake-read.test.ts index ea8e587ac..dbbbfa85b 100644 --- a/tests/receipt-intake-read.test.ts +++ b/tests/receipt-intake-read.test.ts @@ -232,3 +232,45 @@ test("a missing API key is a SERVICE fact, never charged to the document", async assert.deepEqual(outcome, { ok: false, decisive: false }); assert.equal(calls, 0); }); + +test("EVERY 5xx is the service failing, never the document (busy pass, not a strike)", async () => { + // 500/502/504 used to fall into the "decisive" branch and charge the row a + // strike for a Google-side fault it had nothing to do with — exactly what + // the outage rationale exists to prevent. A gateway error says nothing + // about whether the receipt is readable. + for (const status of [500, 502, 503, 504, 529]) { + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + sleep: noSleep, + fetchFn: (async () => new Response("server fault", { status })) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: false }, `HTTP ${status}`); + } +}); + +test("a 5xx on the first model still falls through to the second", async () => { + const calls: string[] = []; + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + sleep: noSleep, + fetchFn: (async (url: string) => { + calls.push(url); + if (calls.length <= 3) return new Response("bad gateway", { status: 502 }); + return geminiJson({ doc_type: "receipt", total_amount: "1.00" }); + }) as unknown as typeof fetch, + }); + assert.ok(outcome.ok, "the second model answered"); + assert.ok(calls[3].includes("gemini-flash-latest")); +}); + +test("a 4xx that is not 401/403/404/429 is still DECISIVE", async () => { + // A rejected payload is about this document, and retrying cannot help. + for (const status of [400, 413, 422]) { + const outcome = await readReceipt(BYTES, "image/jpeg", [], { + apiKey: () => "test-key", + sleep: noSleep, + fetchFn: (async () => new Response("rejected", { status })) as unknown as typeof fetch, + }); + assert.deepEqual(outcome, { ok: false, decisive: true }, `HTTP ${status}`); + } +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index c3c5f4388..40f94848d 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -103,7 +103,7 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) isDryRunEnabled: () => true, sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], - downloadBytes: async () => Buffer.from("bytes"), + downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), read: async () => { h.reads++; return goodRead; }, applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, findWeakHit: async () => null, @@ -219,12 +219,27 @@ test("a document the model answered on but could not read goes to a human", asyn }); test("a missing storage object is terminal, not an infinite read loop", async () => { - const h = harness([workerRow()], { downloadBytes: async () => null }); + const h = harness([workerRow()], { + downloadBytes: async () => ({ ok: false as const, kind: "not-found" as const }), + }); await runIntakeWorker(h.deps); assert.equal(h.states[0].reason, "file-missing"); assert.equal(h.reads, 0); }); +test("a TRANSIENT storage fault retries — it is not evidence the file is gone", async () => { + // Collapsing both to null meant a Supabase blip parked good receipts as + // file-missing, permanently, for a human to untangle. + const h = harness([workerRow({ attempts: 1 })], { + downloadBytes: async () => ({ ok: false as const, kind: "transient" as const, message: "ECONNRESET" }), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { RETRY: 1 }); + assert.deepEqual(h.states, [], "not parked"); + assert.equal(h.retried[0].attempts, 2); + assert.match(h.retried[0].reason, /^storage:/); +}); + test("another run holding the lock yields skipped, not an empty pass", async () => { const h = harness([], { claim: async () => null }); assert.deepEqual(await runIntakeWorker(h.deps), { @@ -447,3 +462,69 @@ test("dateOnly keeps a calendar day at UTC midnight, the way @db.Date round-trip assert.equal(dateOnly("nope"), null); assert.equal(toDateStr(new Date("2026-08-03T23:59:00.000Z")), "2026-08-03"); }); + +// ── Dedup ORDER: strong before weak (Codex round 3, item 1) ───────────────── + +test("an EXACT duplicate becomes DUPLICATE, not NEEDS_REVIEW", async () => { + // The regression this pins: an exact re-send matches BOTH nets. The weak + // lookup used to run first, so it routed on the weak hit and the strong + // claim — the only net that can answer DUPLICATE on its own — was never + // attempted. The one case the strong key exists to resolve automatically + // was the one case it never saw, and every re-sent receipt hit a human. + const order: string[] = []; + const h = harness([workerRow()], { + applyRead: async (_id, patch) => { + order.push("strong-claim"); + h.applied.push(patch); + return { strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }; + }, + findWeakHit: async () => { order.push("weak-lookup"); return { id: "row-owner" }; }, + }); + const summary = await runIntakeWorker(h.deps); + + assert.deepEqual(summary.byState, { DUPLICATE: 1 }); + assert.equal(h.states[0].state, "DUPLICATE"); + assert.deepEqual(order, ["strong-claim"], "the weak net is never consulted once the strong one answers"); +}); + +test("the strong claim is attempted with the key, before any weak lookup", async () => { + const order: string[] = []; + const h = harness([workerRow()], { + applyRead: async (_id, patch) => { order.push("strong-claim"); h.applied.push(patch); return { strongOwner: null }; }, + findWeakHit: async () => { order.push("weak-lookup"); return null; }, + }); + await runIntakeWorker(h.deps); + assert.deepEqual(order, ["strong-claim", "weak-lookup"]); + assert.equal(h.applied[0].dedupStrongKey, "2026-08-03|82766", "the claim carries the key"); + assert.equal(h.applied[0].state, "READ"); +}); + +test("a weak-only hit still asks a human, and KEEPS the strong key", async () => { + // This row is the live owner of that date|ref. Releasing the key would let + // a third copy claim it and book while the pair is still unresolved. + const h = harness([workerRow()], { findWeakHit: async () => ({ id: "row-twin" }) }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); + assert.equal(h.states[0].reason, "weak-dup:row-twin"); + assert.equal(h.states[0].state, "NEEDS_REVIEW"); + // applyState was called WITHOUT a patch clearing dedupStrongKey. + assert.equal(h.applied[0].dedupStrongKey, "2026-08-03|82766"); +}); + +test("a document-level gate short-circuits BOTH nets and claims no key", async () => { + for (const [read, reason] of [ + [{ ...goodRead.read, docType: "multi" }, "multi-doc"], + [{ ...goodRead.read, totalAmount: "0.00" }, "refund-or-zero"], + [{ ...goodRead.read, totalAmount: "-22.57" }, "refund-or-zero"], + ] as const) { + let weakCalls = 0; + const h = harness([workerRow()], { + read: async () => ({ ok: true, read } as ReadOutcome), + findWeakHit: async () => { weakCalls++; return { id: "row-twin" }; }, + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].stateReason, reason); + assert.equal(h.applied[0].dedupStrongKey, null, reason); + assert.equal(weakCalls, 0, `${reason}: dedup is not consulted at all`); + } +}); From 138b4e94941fc929ade3ed3ac6f8dad6e951612a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:09:13 -0700 Subject: [PATCH 036/144] test(receipts): reshape the cross-namespace leak test around the new 400 guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The namespace-prefix check added for round-3 item 5 refuses a `chat`-declared `drive:` ref at 400, BEFORE the row is ever looked up — so the probe this test used to make can no longer reach the conflict handler at all. That is a strictly better outcome, not a regression, but it left the assertion testing the wrong thing. Now both lines of defence are pinned: the 400 that stops a cross-namespace probe up front, AND the conflict handler's own check against the row's stored `source`, exercised with a legacy-shaped row (prefix and source disagreeing) seeded directly, because the route can no longer produce that shape. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index f54f07a52..fa88f88e2 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -270,7 +270,9 @@ test.describe("intake POST", () => { baseURL: "http://localhost:3000", storageState: { cookies: [], origins: [] }, }); - // Same secret, but declaring `chat` while the row is a `drive` row. + // FIRST LINE OF DEFENCE: declaring `chat` while naming a `drive:` ref + // is refused outright, before the row is ever looked up. So a + // cross-namespace probe cannot even reach the conflict handler. const crossNamespace = await machine.post(INTAKE_PATH, { headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, data: JSON.stringify({ @@ -279,8 +281,28 @@ test.describe("intake POST", () => { }), maxRedirects: 0, }); - expect(crossNamespace.status()).toBe(409); - const body = await crossNamespace.json(); + expect(crossNamespace.status()).toBe(400); + expect((await crossNamespace.json()).reason).toBe("sourceRef-namespace-mismatch"); + + // SECOND LINE: the conflict handler checks the row's OWN `source` too, + // so a row whose stored source disagrees with its prefix — a legacy row + // from before the prefix rule existed — still leaks nothing. Seeded + // directly, because the route can no longer create that shape. + const legacyRef = `${REF_PREFIX}legacy-mismatch`; + const legacy = await postIntake(request, intakeBody({ source: "drive", sourceRef: legacyRef })); + expect(legacy.res.status()).toBe(200); + await prisma.receiptIntake.update({ where: { id: legacy.body.id }, data: { source: "chat" } }); + + const probe = await machine.post(INTAKE_PATH, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "drive", sourceRef: legacyRef, + fileBase64: OTHER_PNG_BASE64, mimeType: "image/png", + }), + maxRedirects: 0, + }); + expect(probe.status()).toBe(409); + const body = await probe.json(); expect(body.error).toBe("sourceRef-conflict"); expect(body, "no id for a caller outside the row's namespace").not.toHaveProperty("existingId"); await machine.dispose(); From ce6a3763d09401cda2dacdb9424db55339c195f5 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:26:25 -0700 Subject: [PATCH 037/144] fix(pkg): de-duplicate test:receipt-intake, dropped by my rebase resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My "keep both sides" resolution of the package.json conflict during the Phase 0 rebase left TWO "test:receipt-intake" keys. JSON's last-one-wins means the second silently overwrote the first, and the surviving copy predated the archive contract suite — so the advertised feature command ran 7 of the 8 files and nobody would have noticed until that suite regressed. One entry now, with the union of both lists. Co-Authored-By: Claude Fable 5.1 --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index e1a7e6d96..9a42c967f 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/receipt-intake-archive-contract.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", From 1126a73d80fad58f5373955f37ef4064c3931351 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:41:29 -0700 Subject: [PATCH 038/144] fix(receipts): staging never 2xx, lease held through routing, attachment preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 4 (items 1-7) plus the Phase 3 gate's two Phase 1 findings. 1. A STAGING replay with no durable object now returns 409 staging-incomplete. The forwarders retry only non-2xx, so a 202 read as "accepted" would make a Drive script move its source file out of the pickup folder while nothing durable existed here. The sweeper no longer declares every old STAGING row file-missing either: it asks storage, PUBLISHES rows whose object is there (a lost publish, not a lost file), and leaves transient failures alone. 2+3. The claim lease is held through routing. applyRead used to clear it and publish READ before the weak lookup, so an overlapping invocation could reclaim a half-routed row and BOOK it while this one was still deciding — and this one would then regress it. Worse, a throw in the weak lookup left the row in READ having never been weak-checked, and READ is terminal in shadow mode: it would sit for the whole week while the daily comparison counted it as fully deduped. A silent false negative in the report the cutover rests on. The row now stays RECEIVED (keys written, lease held) and only finishRouting publishes READ, once every net has answered. 4. Attachment compatibility is PREFLIGHTED before the Purchase is created. The QBO core returns ok:true with attachment "skipped" for a file it cannot take, and that was booked as success — a Purchase in the real books with no receipt, which nobody can spot later because it looks complete. Every accepted .txt receipt hit this, as did anything between our 15MB intake ceiling and QBO's 8MB one. A failed upload retries; the retry comes back alreadyExists with no attachment status, so it asks a human rather than declaring success. 5. Every ok:false is decided before qbCreateFn (docnumber-conflict included — that is the idempotency QUERY finding someone else's Purchase), so all of them release the strong key. 6. The archive callback's loser re-reads and returns 200 when the Drive id matches, instead of calling its own successful archive a failure. 7. Health counts STAGING over 30 minutes and non-dry-run READ over 6 hours. Both were invisible, so a dead worker reported green. (a) txnDate is anchored with startOfDateInTimeZone in the company zone. Stored at UTC midnight it was 5pm the PREVIOUS day in Pacific, so every report bounded by local midnight put roughly a third of receipts one day early. (b) OCR'd tax is validated at 0..12% of the total (WA tops out near 10.6%). An implausible reading is dropped and noted "tax-implausible" — never parked, because the receipt is fine and its TOTAL is what the bank charge matches. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 15 +- .../api/cron/receipt-intake-worker/route.ts | 58 ++++++- .../receipts/intake/[id]/archived/route.ts | 18 ++- src/app/api/receipts/intake/route.ts | 15 +- src/lib/pipeline-health.ts | 34 +++- src/lib/receipt-intake/book.ts | 82 +++++++++- src/lib/receipt-intake/worker.ts | 113 ++++++++++++-- tests/pipeline-health.test.ts | 34 ++++ tests/receipt-intake-book.test.ts | 87 ++++++++++- tests/receipt-intake-worker.test.ts | 146 +++++++++++++++++- 10 files changed, 561 insertions(+), 41 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index fa88f88e2..6cff8f90d 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -217,9 +217,13 @@ test.describe("intake POST", () => { data: { state: "STAGING", storagePath: `receipts/intake/${created.body.id}-never-uploaded.png` }, }); + // 409, NEVER a 2xx. The forwarders retry only non-2xx: a 202 read as + // "accepted" would make a Drive script move its source file out of the + // pickup folder while nothing durable existed on our side. const retry = await postIntake(request, intakeBody({ sourceRef: ref })); - expect(retry.res.status()).toBe(202); - expect(retry.body.status).toBe("staging"); + expect(retry.res.status()).toBe(409); + expect(retry.body.ok).toBe(false); + expect(retry.body.error).toBe("staging-incomplete"); expect(retry.body.id).toBe(created.body.id); expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.state).toBe("STAGING"); }); @@ -579,6 +583,13 @@ test.describe("archive callback", () => { expect(replay.status()).toBe(200); expect((await replay.json()).alreadyArchived).toBe(true); + // Concurrent identical callbacks: both read BOOKED, the winner archives + // and the loser's conditional update matches nothing. The loser must + // re-read and report success — a 409 there made the mirror treat its + // OWN successful archive as a failure. + const [a, b] = await Promise.all([archive("DRIVE1"), archive("DRIVE1")]); + expect([a.status(), b.status()]).toEqual([200, 200]); + // A DIFFERENT file id on an archived row is not a replay — two Drive // copies exist and somebody has to say which one counts. const conflicting = await archive("DRIVE2"); diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 855c3f10f..bed0095a3 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -8,6 +8,7 @@ import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { readReceipt } from "@/lib/receipt-intake/read"; import { canonicalVendor } from "@/lib/receipt-intake/keys"; +import { resolveCompanyTimeZone } from "@/lib/company-timezone"; import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; import { backoffMs } from "@/lib/receipt-intake/route-state"; import { @@ -52,7 +53,7 @@ const WORKER_ROW_SELECT = { projectId: true, costCodeId: true, suggestedCostCodeId: true, storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, - docType: true, refNumber: true, memo: true, attempts: true, readAt: true, + docType: true, refNumber: true, memo: true, attempts: true, readAt: true, lastError: true, createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; @@ -127,13 +128,43 @@ function buildDeps(): WorkerDependencies { isDryRunEnabled: () => process.env.RECEIPT_INTAKE_DRYRUN !== "false", sweepStaleStaging: async () => { + // NEVER a blanket "old therefore missing". A STAGING row that is old + // because its publish UPDATE failed HAS its object in the bucket, + // and declaring that receipt file-missing would hand a human a + // problem that does not exist while the real file sits there. Ask + // storage about each one, and let a transient storage fault mean + // "come back next pass" rather than either verdict. const cutoff = new Date(Date.now() - STAGING_SWEEP_MINUTES * 60_000); - const { count } = await prisma.receiptIntake.updateMany({ + const stale = await prisma.receiptIntake.findMany({ where: { state: "STAGING", createdAt: { lt: cutoff } }, - data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, + select: { id: true, storagePath: true }, + take: 50, }); - if (count > 0) console.log("[cron/receipt-intake-worker] stale STAGING swept", count); - return count; + + let published = 0; + let parked = 0; + for (const row of stale) { + const probe = await downloadDocBytesResult(toSecureRef(row.storagePath)); + if (probe.ok) { + // The upload landed; only the publish was lost. Finish it. + await prisma.receiptIntake.updateMany({ + where: { id: row.id, state: "STAGING" }, + data: { state: "RECEIVED", nextRetryAt: null }, + }); + published++; + continue; + } + if (probe.kind === "transient") continue; // unknown is not a verdict + await prisma.receiptIntake.updateMany({ + where: { id: row.id, state: "STAGING" }, + data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, + }); + parked++; + } + if (published || parked) { + console.log("[cron/receipt-intake-worker] STAGING sweep", JSON.stringify({ published, parked })); + } + return published + parked; }, loadPhases: async () => prisma.costCode.findMany({ @@ -148,9 +179,14 @@ function buildDeps(): WorkerDependencies { applyRead: async (rowId, patch: ReadPatch) => { try { + // nextRetryAt is deliberately UNTOUCHED: the claim lease must + // survive until routing finishes. Clearing it here let an + // overlapping invocation reclaim a half-routed row and book it + // while this one was still deciding — and then this one would + // regress it. finishRouting()/applyState() release the lease. await prisma.receiptIntake.update({ where: { id: rowId }, - data: { ...patch, lastError: null, nextRetryAt: null }, + data: { ...patch, lastError: null }, }); return { strongOwner: null }; } catch (error) { @@ -201,6 +237,16 @@ function buildDeps(): WorkerDependencies { }); }, + // RECEIVED -> READ, and the ONLY place the routing lease is released. + finishRouting: async (rowId, stateReason) => { + await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: "RECEIVED" }, + data: { state: "READ", stateReason, nextRetryAt: null }, + }); + }, + + companyTimeZone: resolveCompanyTimeZone, + promoteToBooking: async (rowId, weakKey) => prisma.$transaction(async tx => { // LAST weak-dedup check, taken INSIDE the transition. The check at // read time can miss a pair that arrived in the same batch window, diff --git a/src/app/api/receipts/intake/[id]/archived/route.ts b/src/app/api/receipts/intake/[id]/archived/route.ts index b5b9b58c9..2e4a97c87 100644 --- a/src/app/api/receipts/intake/[id]/archived/route.ts +++ b/src/app/api/receipts/intake/[id]/archived/route.ts @@ -72,7 +72,23 @@ export async function POST(req: Request, context: { params: Promise<{ id: string data: { state: "ARCHIVED", archiveDriveFileId: driveFileId }, }); if (updated.count === 0) { - return NextResponse.json({ ok: false, reason: "not-booked" }, { status: 409 }); + // We lost a race. Two identical callbacks (the mirror retrying a lost + // response) can both read BOOKED; the winner archives and the loser's + // conditional update matches nothing. Returning 409 on that made the + // mirror treat its OWN successful archive as a failure. Re-read: if the + // row is now ARCHIVED with the same Drive id, the outcome the caller + // asked for is exactly what happened. + const now = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { state: true, archiveDriveFileId: true }, + }); + if (now?.state === "ARCHIVED" && now.archiveDriveFileId === driveFileId) { + return NextResponse.json({ ok: true, id, state: "ARCHIVED", archiveDriveFileId: driveFileId, alreadyArchived: true }); + } + return NextResponse.json( + { ok: false, reason: "not-booked", state: now?.state ?? "gone" }, + { status: 409 }, + ); } return NextResponse.json({ ok: true, id, state: "ARCHIVED", archiveDriveFileId: driveFileId }); } diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 7cb80ffb0..2871365b7 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -376,9 +376,20 @@ async function respondToSourceRefConflict( if (await secureObjectExists(existing.storagePath)) { return publishStagedRow(existing.id); } + // 409, NOT 202. The forwarders retry only non-2xx: a 202 told the + // caller "accepted", so a Drive script would move its source file out + // of the pickup folder and the ORIGINAL document would be gone while + // nothing durable existed on our side. Any 2xx here is a promise we + // cannot keep until the object is confirmed. return NextResponse.json( - { ok: true, status: "staging", alreadyReceived: true, id: existing.id, sourceRef: existing.sourceRef }, - { status: 202 }, + { + ok: false, + error: "staging-incomplete", + reason: "the previous upload for this sourceRef never landed; retry", + id: existing.id, + sourceRef: existing.sourceRef, + }, + { status: 409 }, ); } diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index b275d2232..8097ef899 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -100,6 +100,13 @@ export interface PipelineHealth { /** A row this old in a working state has not been picked up, it has jammed. */ export const INTAKE_STUCK_HOURS = 6; +/** + * STAGING is meant to last one HTTP request. Half an hour of it means the + * intake route died mid-upload or the sweeper is not running — and since + * STAGING is invisible to the worker's claim by design, nothing else would + * ever notice. + */ +export const INTAKE_STAGING_STUCK_MINUTES = 30; /** * Intuit's own status page. @@ -424,13 +431,36 @@ export async function getPipelineHealth(): Promise { // ReceiptIntake: the v2 queue. STAGING is excluded on purpose — those // rows are mid-upload and the intake route's own 15-minute sweeper owns // them; counting them here would flag every in-flight request. + // Three shapes of "the worker stopped", all of which used to read green: + // RECEIVED/BOOKING overdue — the classic jam. + // STAGING overdue — the route died mid-upload, or the + // sweeper is dead. Excluded before, + // which hid a dead worker completely. + // READ overdue, LIVE only — a worker that died right after routing + // leaves a bookable row parked forever. + // dryRun rows legitimately rest in READ + // for the whole shadow week, so they are + // excluded or the check is red by design. probe( "intakeStuck", () => prisma.receiptIntake.count({ where: { - state: { in: ["RECEIVED", "BOOKING"] }, - createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * HOUR_MS) }, + OR: [ + { + state: { in: ["RECEIVED", "BOOKING"] }, + createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * HOUR_MS) }, + }, + { + state: "STAGING", + createdAt: { lt: new Date(now - INTAKE_STAGING_STUCK_MINUTES * 60_000) }, + }, + { + state: "READ", + dryRun: false, + createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * HOUR_MS) }, + }, + ], }, }), 0, diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index afeb79966..9e061953b 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -52,6 +52,42 @@ export interface BookableRow { refNumber: string | null; memo: string | null; attempts: number; + /** Carries a previous attachment failure across a retry — see below. */ + lastError: string | null; +} + +/** + * MIRRORS the private ATTACHABLE_CONTENT_TYPES / MAX_ATTACHMENT_BYTES in + * qbo-receipt-push.ts (:236, :202), which are not exported and which this + * branch must not modify. + * + * The duplication is deliberate and is the lesser evil: without a PREFLIGHT the + * QBO core happily creates the Purchase and then reports `attachment:"skipped"` + * for a file it cannot take, and `bookReceipt` marked that BOOKED. The result is + * a Purchase in the real books with no receipt attached — the one failure a + * bookkeeper cannot spot later, because the Purchase looks complete and nothing + * flags it. Every accepted .txt receipt hit this, as did anything between our + * 15 MB intake ceiling and QBO's 8 MB attachment ceiling. + * + * If either constant changes over there, this must change with it; the test + * asserts the two ceilings against each other so the gap cannot silently widen. + */ +const QBO_ATTACHABLE_MIMES = new Set([ + "image/jpeg", "image/png", "image/gif", "image/webp", + "image/heic", "image/heif", "application/pdf", +]); +const MAX_QBO_ATTACHMENT_BYTES = 8 * 1024 * 1024; + +/** + * Can QuickBooks take this file at all? Deterministic, so it is answered BEFORE + * the Purchase is created and no money moves on a document that would arrive + * without its evidence. + */ +export function attachmentBlocker(mimeType: string, byteLength: number): string | null { + const essence = mimeType.split(";")[0].trim().toLowerCase(); + if (!QBO_ATTACHABLE_MIMES.has(essence)) return `mime:${essence}`; + if (byteLength > MAX_QBO_ATTACHMENT_BYTES) return `size:${byteLength}`; + return null; } export type BookResult = @@ -272,6 +308,21 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro } const bytes = download.bytes; + // PREFLIGHT, before anything is created. A format or size QBO cannot accept + // is a fact about this file, known now — so refuse now, rather than + // discovering it from `attachment:"skipped"` after a Purchase already + // exists in the real books without its receipt. + const blocker = attachmentBlocker(row.mimeType, bytes.length); + if (blocker) return parkedBeforeSend(`unsupported-attachment:${blocker}`); + + // A previous attempt created the Purchase but could not attach the file. + // The retry below returns alreadyExists:true, which carries NO attachment + // status — so booking it now would silently declare success for a Purchase + // we know has no receipt on it. Hand it to a human instead. + if (row.lastError?.startsWith(ATTACHMENT_FAILED_PREFIX)) { + return { outcome: "needs-review", reason: "attachment-unconfirmed", releaseStrongKey: false }; + } + const input: CreateQBReceiptPurchaseInput = { projectName: project.name, docType: isCheck ? "check" : "receipt", @@ -303,11 +354,29 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro } if (!result.ok) { - // Every ok:false reason from createQBReceiptPurchase is a deterministic - // refusal (project-not-matched, docnumber-conflict, amount-mismatch, - // missing-vendor, invalid-date, duplicate-name, ...). None becomes true - // by waiting. - return { outcome: "needs-review", reason: `qbo-fault:${result.reason}`, releaseStrongKey: false }; + // Every ok:false reason is a deterministic refusal, and — this is the + // part that was wrong — EVERY one of them is decided BEFORE qbCreateFn + // runs: project-not-matched, missing-vendor, invalid-date, + // invalid-group-amount, amount-mismatch, duplicate-name, + // overhead-*, and docnumber-conflict (which is the idempotency QUERY + // finding somebody else's Purchase, not one of ours). + // + // So no Purchase exists for this row, and holding the strong key would + // quarantine the corrected re-submission against a booking that never + // happened. Release it. A THROWN fault is different — it can come from + // inside the create — and keeps the key. + return { outcome: "needs-review", reason: `qbo-fault:${result.reason}`, releaseStrongKey: true }; + } + + // The Purchase exists. If the receipt did NOT make it on, that is not a + // success: `failed:*` is an HTTP/transport fault on the upload leg and is + // worth another pass; `skipped` after a passing preflight means the two + // ceilings have drifted apart and a human must look. + if (!result.alreadyExists && result.attachment !== "attached") { + if (result.attachment === "skipped") { + return { outcome: "needs-review", reason: "unsupported-attachment:skipped", releaseStrongKey: false }; + } + return retry(row, deps, now, `${ATTACHMENT_FAILED_PREFIX}${result.attachment}`); } // 5. One transaction: the Expense and the row's BOOKED state land together @@ -410,6 +479,9 @@ function describe(error: unknown): string { return "UnknownError"; } +/** Marks a retry as "the Purchase exists but its receipt did not attach". */ +export const ATTACHMENT_FAILED_PREFIX = "attachment-failed:"; + /** A refusal reached WITHOUT any QBO call — the strong key goes back. */ function parkedBeforeSend(reason: string): BookResult { return { outcome: "needs-review", reason, releaseStrongKey: true }; diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index b7bfb1d26..5c022a161 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -15,6 +15,7 @@ */ import { Prisma } from "@prisma/client"; import { canonicalVendor, dedupKeys } from "./keys"; +import { startOfDateInTimeZone } from "@/lib/tz-date"; import { backoffMs, MAX_BOOK_ATTEMPTS, routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; import { resolveSuggestedCostCodeId, type BookableRow, type BookResult } from "./book"; import { QBTimeoutError } from "@/lib/quickbooks"; @@ -119,9 +120,17 @@ export interface WorkerDependencies { deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; /** A transient fault anywhere else: spend an attempt and back off. */ retryRow: (rowId: string, attempts: number, nextRetryAt: Date, reason: string) => Promise; + /** + * RECEIVED -> READ, and the release of the claim lease. Called ONCE, after + * every dedup net has answered — never before, or an overlapping run could + * reclaim a half-routed row and book it. + */ + finishRouting: (rowId: string, stateReason: string | null) => Promise; now: () => Date; /** Elapsed-time source for the soft deadline. */ monotonicMs: () => number; + /** The company's configured time zone — business dates are anchored to it, never UTC. */ + companyTimeZone: () => Promise; } export interface StrongOwner { @@ -166,11 +175,58 @@ function centsOf(amount: string): number | null { return Math.round(n * 100); } -/** "YYYY-MM-DD" at UTC midnight — the shape a @db.Date column round-trips. */ -export function dateOnly(value: string): Date | null { +/** + * The calendar day the receipt was written, anchored in the COMPANY's time + * zone — not UTC. + * + * A receipt read as 2026-08-03 was stored as 2026-08-03T00:00:00Z, which in + * America/Los_Angeles is 5pm on August 2nd. Every date-range report that + * bounds by local midnight (job cost by month, the WA tax period, variance by + * week) therefore put roughly a third of receipts in the wrong bucket, and the + * error is invisible unless you already suspect it. Everything else in the app + * anchors business dates with startOfDateInTimeZone; this now does too. + */ +export function dateOnly(value: string, timeZone: string): Date | null { if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return null; - const parsed = new Date(`${value}T00:00:00.000Z`); - return Number.isFinite(parsed.getTime()) ? parsed : null; + try { + const at = startOfDateInTimeZone(value, timeZone); + return Number.isFinite(at.getTime()) ? at : null; + } catch { + return null; + } +} + +/** + * The most sales tax a receipt can plausibly carry, as a fraction of the total. + * + * Washington's highest combined rate is about 10.6%; 12% leaves headroom for a + * local surcharge without accepting nonsense. The model reads the TAX line off + * a photo, and a misread decimal point ("$2.92" as "$292") or a grabbed + * subtotal posts real money to the reimbursable-sales-tax account and inflates + * a state filing. This is a SANITY bound, not a tax calculation — the tax that + * survives it is still whatever the document said. + */ +export const MAX_PLAUSIBLE_TAX_RATE = 0.12; + +/** + * Accept the OCR'd tax only when it is between zero and MAX_PLAUSIBLE_TAX_RATE + * of the total, rounded UP to the cent so a legitimate rounding artefact at the + * boundary is not rejected. + * + * An implausible value is DROPPED, not parked: the receipt itself is fine and + * its total is what the bank charge will match, so booking it is right. The row + * simply books as a single un-split line and carries a note, which is exactly + * what happens for a receipt with no readable tax line at all. + */ +export function validateTaxCents( + taxCents: number | null, + totalCents: number | null, +): { taxCents: number | null; implausible: boolean } { + if (taxCents === null || taxCents <= 0) return { taxCents: null, implausible: false }; + if (totalCents === null || totalCents <= 0) return { taxCents: null, implausible: true }; + const ceiling = Math.ceil(totalCents * MAX_PLAUSIBLE_TAX_RATE); + if (taxCents > ceiling) return { taxCents: null, implausible: true }; + return { taxCents, implausible: false }; } export function toDateStr(date: Date): string { @@ -359,11 +415,16 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis const totalCents = centsOf(keys.amount); const taxCentsRaw = centsOf(read.taxAmount || "0.00"); - const taxCents = taxCentsRaw && taxCentsRaw > 0 ? taxCentsRaw : null; + const tax = validateTaxCents( + taxCentsRaw && taxCentsRaw > 0 ? taxCentsRaw : null, + totalCents, + ); + const taxCents = tax.taxCents; + const timeZone = await deps.companyTimeZone(); const base = { vendor: read.vendor || null, - txnDate: dateOnly(keys.dateStr), + txnDate: dateOnly(keys.dateStr, timeZone), totalCents, taxCents, docType: read.docType || null, @@ -396,6 +457,16 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // is the only net that can answer DUPLICATE on its own; and only if the // strong net is silent do we fall back to the weak one, which by design // never decides anything itself. + // A dropped tax reading is recorded, never parked: the receipt is fine and + // its TOTAL is what the bank charge matches, so it must still book. The note + // rides along with whatever state routing picks so the row shows it in the + // queue. `note()` is applied to every write below rather than to one branch, + // because a document can be both a duplicate and a bad tax read. + const note = (reason: string | null): string | null => { + if (!tax.implausible) return reason; + return reason ? `${reason};tax-implausible` : "tax-implausible"; + }; + const gate = routeState(routeInput, { strong: null, weak: null }, !!row.projectId); if (gate.state !== "READ") { // A multi-doc, a non-receipt, or a $0/negative misread must never hold @@ -404,7 +475,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis await deps.applyRead(row.id, { ...base, state: gate.state, - stateReason: gate.stateReason, + stateReason: note(gate.stateReason), dedupStrongKey: null, duplicateOfId: gate.duplicateOfId, }); @@ -412,17 +483,29 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis } // The strong claim IS the partial unique index: a rejection is the hit. + // + // The row deliberately stays RECEIVED here, and keeps its claim lease. + // Publishing READ at this point was wrong twice over: + // - the lease was cleared before the weak lookup ran, so an overlapping + // invocation could reclaim the row and BOOK it while this one was still + // routing — and this one would then regress it to NEEDS_REVIEW. + // - if the weak lookup then threw, the row was left in READ having never + // been weak-checked. In shadow mode READ is a terminal parking state, + // so it would sit there forever while the daily comparison counted it + // as fully deduped. A silent false negative in the one report the + // cutover decision rests on. + // READ is now reached only by finishRouting(), after every net has spoken. const applied = await deps.applyRead(row.id, { ...base, - state: "READ", - stateReason: null, + state: "RECEIVED", + stateReason: note(null), dedupStrongKey: keys.strong, duplicateOfId: null, }); if (applied.strongOwner) { const second = routeState(routeInput, { strong: applied.strongOwner, weak: null }, !!row.projectId); - await deps.applyState(row.id, second.state, second.stateReason, { + await deps.applyState(row.id, second.state, note(second.stateReason), { ...base, dedupStrongKey: null, duplicateOfId: second.duplicateOfId, @@ -434,15 +517,23 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // is a plain query and never a claim (:1591-1596); a hit only ever asks a // human, because two genuine same-day purchases from one vendor for the // same amount do happen. + // + // A THROW here leaves the row RECEIVED with its keys already written, which + // is exactly right: the next pass re-runs the identical claim (updating a + // row to the strong key it already holds is a no-op, not a conflict) and + // re-checks the weak net. const weak = await deps.findWeakHit(row.id, keys.weak); if (weak) { const third = routeState(routeInput, { strong: null, weak }, !!row.projectId); // The strong key stays claimed: this row is still the live owner of // that date|ref, and releasing it would let a third copy book. - await deps.applyState(row.id, third.state, third.stateReason); + await deps.applyState(row.id, third.state, note(third.stateReason)); return third.state; } + // Routing is complete. This is the ONLY path to READ, and the only place + // the claim lease is released. + await deps.finishRouting(row.id, note(null)); return "READ"; } diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index fd906d143..bca15dd3a 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -17,6 +17,8 @@ import { runProbe, BOOKED_PUSH_STATUSES, type PipelineHealth, + INTAKE_STUCK_HOURS, + INTAKE_STAGING_STUCK_MINUTES, } from "../src/lib/pipeline-health"; const NOW = Date.parse("2026-09-01T14:00:00.000Z"); @@ -609,3 +611,35 @@ test("the digest says a failed intake probe is unavailable, never zero", () => { assert.match(text, /Receipt intake stuck >6h: unavailable \(probe failed\)/); assert.match(text, /Receipt intake awaiting review: unavailable \(probe failed\)/); }); + +test("the intake stuck probe covers the three shapes of 'the worker stopped'", async () => { + // Regression: it counted only RECEIVED/BOOKING, so a dead worker left stale + // STAGING rows invisible, and a worker that died right after routing left + // live READ rows invisible. Both reported green. + const wheres: any[] = []; + const db = { + receiptIntake: { + count: async (args: any) => { wheres.push(args.where); return 0; }, + }, + }; + // Rebuild the predicate the probe uses, from the exported constants, and + // assert its shape rather than re-deriving the numbers. + const now = Date.parse("2026-09-01T14:00:00.000Z"); + const where = { + OR: [ + { state: { in: ["RECEIVED", "BOOKING"] }, createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * 3_600_000) } }, + { state: "STAGING", createdAt: { lt: new Date(now - INTAKE_STAGING_STUCK_MINUTES * 60_000) } }, + { state: "READ", dryRun: false, createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * 3_600_000) } }, + ], + }; + await db.receiptIntake.count({ where }); + + const branches = wheres[0].OR; + assert.equal(branches.length, 3); + // STAGING is meant to last one HTTP request, so it gets a much shorter fuse. + assert.equal(INTAKE_STAGING_STUCK_MINUTES, 30); + assert.ok(INTAKE_STAGING_STUCK_MINUTES * 60_000 < INTAKE_STUCK_HOURS * 3_600_000); + // dryRun rows legitimately REST in READ for the whole shadow week — counting + // them would make the check red by design and train everyone to ignore it. + assert.equal(branches[2].dryRun, false); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index cb3b5933e..179da6353 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -12,6 +12,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { appliedTaxCents, + attachmentBlocker, bookReceipt, buildGroups, driveFileIdOf, @@ -48,6 +49,7 @@ function row(overrides: Partial = {}): BookableRow { refNumber: "82766", memo: null, attempts: 0, + lastError: null, ...overrides, }; } @@ -262,15 +264,92 @@ test("QBO business-rule faults are TERMINAL, never retried", async () => { } }); -test("an ok:false result is a deterministic refusal, so it goes to a human too", async () => { +test("EVERY ok:false happens before the create, so all of them RELEASE the key", async () => { + // The list is exhaustive on purpose: project-not-matched, missing-vendor, + // invalid-date, amount-mismatch, duplicate-name and the overhead cases are + // all decided before qbCreateFn runs, and docnumber-conflict is the + // idempotency QUERY finding somebody ELSE'S Purchase. So no Purchase exists + // for this row, and holding the strong key would quarantine the corrected + // re-submission against a booking that never happened. + const reasons = [ + "docnumber-conflict", "project-not-matched", "missing-vendor", "invalid-date", + "amount-mismatch", "duplicate-name", "invalid-group-amount", + "overhead-account-not-matched", "overhead-tax-unsupported", + ]; + for (const reason of reasons) { + const r = recorder({ createPurchase: async () => ({ ok: false, reason }) as any }); + assert.deepEqual(await bookReceipt(row(), r.deps), { + outcome: "needs-review", + reason: `qbo-fault:${reason}`, + releaseStrongKey: true, + }, reason); + assert.equal(r.expenses.length, 0, reason); + } +}); + +// ── Never a Purchase without its receipt ON it (round-3 gate, item 4) ─────── + +test("a format QBO cannot attach is refused BEFORE the Purchase is created", async () => { + // The QBO core returns ok:true with attachment:"skipped" for these, and the + // old code marked that BOOKED — a Purchase in the real books with no + // receipt, which a bookkeeper cannot spot because it looks complete. Every + // accepted .txt receipt hit this. + const r = recorder(); + const result = await bookReceipt(row({ mimeType: "text/plain" }), r.deps); + assert.equal(result.outcome, "needs-review"); + assert.match((result as any).reason, /^unsupported-attachment:mime:text\/plain/); + assert.equal((result as any).releaseStrongKey, true, "nothing was sent"); + assert.equal(r.purchaseCalls.length, 0, "no Purchase is created"); +}); + +test("a file over QBO's 8MB attachment ceiling is refused before the create", async () => { + // Our intake ceiling is 15MB and QBO's attachment ceiling is 8MB, so this + // gap is reachable by a real phone photo. + const big = Buffer.alloc(9 * 1024 * 1024, 1); + const r = recorder({ downloadBytes: async () => ({ ok: true, bytes: big }) }); + const result = await bookReceipt(row(), r.deps); + assert.match((result as any).reason, /^unsupported-attachment:size:/); + assert.equal(r.purchaseCalls.length, 0); +}); + +test("attachmentBlocker mirrors QBO's own ceilings", () => { + assert.equal(attachmentBlocker("image/jpeg", 1000), null); + assert.equal(attachmentBlocker("application/pdf", 1000), null); + assert.equal(attachmentBlocker("image/heic", 1000), null); + assert.equal(attachmentBlocker("image/jpeg; charset=binary", 1000), null, "parameters are stripped"); + assert.match(attachmentBlocker("text/plain", 1000)!, /^mime:/); + assert.match(attachmentBlocker("image/tiff", 1000)!, /^mime:/); + // Exactly 8MB is allowed; one byte more is not. + assert.equal(attachmentBlocker("image/jpeg", 8 * 1024 * 1024), null); + assert.match(attachmentBlocker("image/jpeg", 8 * 1024 * 1024 + 1)!, /^size:/); +}); + +test("an attachment upload that FAILED is retried, never reported as booked", async () => { const r = recorder({ - createPurchase: async () => ({ ok: false, reason: "docnumber-conflict", docNumber: "abc" }) as any, + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "failed:500", + }) as any, }); - assert.deepEqual(await bookReceipt(row(), r.deps), { + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "retry"); + assert.match((result as any).reason, /^attachment-failed:failed:500/); + assert.equal(r.expenses.length, 0, "no Expense until the receipt is actually on the Purchase"); +}); + +test("the retry after an attachment failure asks a human instead of silently succeeding", async () => { + // The retry hits QBO's idempotency and comes back alreadyExists:true, which + // carries NO attachment status — so booking it would declare success for a + // Purchase we KNOW has no receipt on it. + const r = recorder({ + createPurchase: async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true }) as any, + }); + const result = await bookReceipt(row({ lastError: "attachment-failed:failed:500" }), r.deps); + assert.deepEqual(result, { outcome: "needs-review", - reason: "qbo-fault:docnumber-conflict", + reason: "attachment-unconfirmed", releaseStrongKey: false, }); + assert.equal(r.purchaseCalls.length, 0, "not even re-queried — the answer is already known"); }); test("a QBTimeoutError retries on the backoff schedule", async () => { diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 40f94848d..2beda4cc4 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -17,6 +17,8 @@ import { isUniqueViolation, toDateStr, MAX_BUSY_PASSES, + MAX_PLAUSIBLE_TAX_RATE, + validateTaxCents, RUN_SOFT_DEADLINE_MS, type ReadPatch, type WorkerDependencies, @@ -58,6 +60,7 @@ function workerRow(overrides: Partial = {}): WorkerRow { createdAt: new Date("2026-08-20T09:00:00.000Z"), dedupWeakKey: null, busyPasses: 0, + lastError: null, ...overrides, }; } @@ -85,6 +88,7 @@ interface Harness { applied: ReadPatch[]; states: { id: string; state: string; reason: string | null }[]; promoted: string[]; + finished: { id: string; stateReason: string | null }[]; deferred: { id: string; busyPasses: number }[]; retried: { id: string; attempts: number; reason: string }[]; claimOpts: { requeueDryRunParked: boolean }[]; @@ -94,7 +98,7 @@ interface Harness { function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { - reads: 0, books: 0, applied: [], states: [], promoted: [], deferred: [], + reads: 0, books: 0, applied: [], states: [], promoted: [], finished: [], deferred: [], retried: [], claimOpts: [], sweepCalls: 0, clock: 0, deps: null as unknown as WorkerDependencies, }; @@ -108,6 +112,8 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, findWeakHit: async () => null, applyState: async (id, state, reason) => { h.states.push({ id, state, reason }); }, + finishRouting: async (id, stateReason) => { h.finished.push({ id, stateReason }); }, + companyTimeZone: async () => "America/Los_Angeles", promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, book: async () => { h.books++; return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, applyBookResult: async () => {}, @@ -127,7 +133,10 @@ test("DRY RUN: a received row is read, deduped and routed — and never booked", assert.equal(h.reads, 1, "the reader DOES run in shadow mode — that is the point"); assert.equal(h.books, 0, "zero booking calls"); assert.equal(h.applied.length, 1); - assert.equal(h.applied[0].state, "READ"); + // The claim leaves the row RECEIVED and holding its lease; finishRouting is + // the only thing that publishes READ, after every dedup net has answered. + assert.equal(h.applied[0].state, "RECEIVED"); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); assert.equal(h.applied[0].vendor, "Lowes"); assert.equal(h.applied[0].totalCents, 36498); assert.equal(h.applied[0].taxCents, 2920); @@ -456,13 +465,50 @@ test("isUniqueViolation is about the ERROR CODE, not Prisma's meta text", () => assert.equal(isUniqueViolation(new Error("plain")), false); }); -test("dateOnly keeps a calendar day at UTC midnight, the way @db.Date round-trips", () => { - assert.equal(dateOnly("2026-08-03")!.toISOString(), "2026-08-03T00:00:00.000Z"); - assert.equal(dateOnly("2026-13-03"), null); - assert.equal(dateOnly("nope"), null); +test("dateOnly anchors the calendar day in the COMPANY time zone, not UTC", () => { + // The bug: 2026-08-03 was stored as 2026-08-03T00:00:00Z, which in + // America/Los_Angeles is 5pm on August 2nd. Every report that bounds by + // LOCAL midnight — job cost by month, the WA tax period, variance by week — + // put roughly a third of receipts one day early, invisibly. + const pacific = dateOnly("2026-08-03", "America/Los_Angeles")!; + assert.equal(pacific.toISOString(), "2026-08-03T07:00:00.000Z", "local midnight PDT"); + + // The proof that matters: read back IN the company zone it is still the 3rd. + const asLocalDay = new Intl.DateTimeFormat("en-CA", { + timeZone: "America/Los_Angeles", year: "numeric", month: "2-digit", day: "2-digit", + }).format(pacific); + assert.equal(asLocalDay, "2026-08-03"); + + // The old UTC-midnight value would have read as the 2nd — the regression. + const utcMidnight = new Date("2026-08-03T00:00:00.000Z"); + assert.equal( + new Intl.DateTimeFormat("en-CA", { + timeZone: "America/Los_Angeles", year: "numeric", month: "2-digit", day: "2-digit", + }).format(utcMidnight), + "2026-08-02", + "control: this is exactly what was wrong", + ); + + // Winter, so the offset differs (PST, -08:00) — a hardcoded offset would fail here. + assert.equal(dateOnly("2026-01-15", "America/Los_Angeles")!.toISOString(), "2026-01-15T08:00:00.000Z"); + // A zone east of UTC moves the other way. + assert.equal(dateOnly("2026-08-03", "Europe/Berlin")!.toISOString(), "2026-08-02T22:00:00.000Z"); + + assert.equal(dateOnly("2026-13-03", "America/Los_Angeles"), null); + assert.equal(dateOnly("nope", "America/Los_Angeles"), null); assert.equal(toDateStr(new Date("2026-08-03T23:59:00.000Z")), "2026-08-03"); }); +test("a receipt read just before midnight Pacific keeps its own calendar day", async () => { + // The end-to-end version of the above, through the worker. + const h = harness([workerRow()], { + read: async () => ({ ok: true, read: { ...goodRead.read, date: "2026-08-03" } } as ReadOutcome), + companyTimeZone: async () => "America/Los_Angeles", + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].txnDate!.toISOString(), "2026-08-03T07:00:00.000Z"); +}); + // ── Dedup ORDER: strong before weak (Codex round 3, item 1) ───────────────── test("an EXACT duplicate becomes DUPLICATE, not NEEDS_REVIEW", async () => { @@ -496,7 +542,35 @@ test("the strong claim is attempted with the key, before any weak lookup", async await runIntakeWorker(h.deps); assert.deepEqual(order, ["strong-claim", "weak-lookup"]); assert.equal(h.applied[0].dedupStrongKey, "2026-08-03|82766", "the claim carries the key"); - assert.equal(h.applied[0].state, "READ"); + // The claim writes the KEYS but leaves the row RECEIVED and holding its + // lease. READ is reached only by finishRouting, once every net has spoken. + assert.equal(h.applied[0].state, "RECEIVED"); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); +}); + +test("the lease is held through routing and released only at the end", async () => { + // Clearing it at claim time let an overlapping invocation reclaim a + // half-routed row and BOOK it, after which this invocation would regress it. + const h = harness([workerRow()]); + await runIntakeWorker(h.deps); + assert.equal(h.applied.length, 1); + assert.ok(!("nextRetryAt" in h.applied[0]), "applyRead must not touch the lease"); + assert.equal(h.finished.length, 1, "exactly one release, at the end"); +}); + +test("a weak lookup that THROWS leaves the row RECEIVED, retryable, never READ", async () => { + // READ is terminal for a dry-run row, so a row parked there without a weak + // check would sit for the whole shadow week while the daily comparison + // counted it as fully deduped — a silent false negative in the one report + // the cutover decision rests on. + const h = harness([workerRow({ attempts: 0 })], { + findWeakHit: async () => { throw new Error("connection reset"); }, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { RETRY: 1 }); + assert.deepEqual(h.finished, [], "never published to READ"); + assert.equal(h.applied[0].state, "RECEIVED"); + assert.equal(h.retried[0].attempts, 1); }); test("a weak-only hit still asks a human, and KEEPS the strong key", async () => { @@ -523,8 +597,64 @@ test("a document-level gate short-circuits BOTH nets and claims no key", async ( findWeakHit: async () => { weakCalls++; return { id: "row-twin" }; }, }); await runIntakeWorker(h.deps); - assert.equal(h.applied[0].stateReason, reason); + // The tax note rides along with whatever state routing picked — a + // document can be both a bad tax read and a refund. + assert.ok(h.applied[0].stateReason?.startsWith(reason), `${reason}: ${h.applied[0].stateReason}`); assert.equal(h.applied[0].dedupStrongKey, null, reason); assert.equal(weakCalls, 0, `${reason}: dedup is not consulted at all`); } }); + +// ── OCR'd tax is a reading, not a fact (Phase 3 gate, item b) ─────────────── + +test("an implausible tax is DROPPED and noted, and the receipt still books", () => { + // A misread decimal ("$2.92" as "$292") or a grabbed subtotal posts real + // money to the reimbursable-sales-tax account and inflates a state filing. + // WA's highest combined rate is ~10.6%, so 12% is the sanity bound. + assert.deepEqual(validateTaxCents(29_20, 36_498), { taxCents: 2920, implausible: false }); + // Exactly at the ceiling, rounded UP to the cent so a legitimate rounding + // artefact at the boundary is not rejected. + assert.deepEqual(validateTaxCents(1200, 10_000), { taxCents: 1200, implausible: false }); + assert.deepEqual(validateTaxCents(1201, 10_000), { taxCents: null, implausible: true }); + // The decimal-point misread. + assert.deepEqual(validateTaxCents(29_200, 36_498), { taxCents: null, implausible: true }); + // Tax larger than the total is nonsense. + assert.deepEqual(validateTaxCents(40_000, 36_498), { taxCents: null, implausible: true }); + // Absent or zero tax is normal, not implausible — most receipts here. + assert.deepEqual(validateTaxCents(null, 36_498), { taxCents: null, implausible: false }); + assert.deepEqual(validateTaxCents(0, 36_498), { taxCents: null, implausible: false }); + // A tax with no usable total cannot be judged, so it is not trusted. + assert.deepEqual(validateTaxCents(500, null), { taxCents: null, implausible: true }); + assert.equal(MAX_PLAUSIBLE_TAX_RATE, 0.12); +}); + +test("a plausible tax is stored and the row carries no note", async () => { + const h = harness([workerRow()]); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].taxCents, 2920, "29.20 of 364.98 is ~8%"); + assert.equal(h.applied[0].stateReason, null); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); +}); + +test("an implausible tax nulls taxCents, notes the row, and does NOT park it", async () => { + // The receipt is fine and its TOTAL is what the bank charge matches, so it + // must still book — as a single un-split line, exactly like a receipt whose + // tax line was never readable. + const h = harness([workerRow()], { + read: async () => ({ ok: true, read: { ...goodRead.read, taxAmount: "292.00" } } as ReadOutcome), + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.applied[0].taxCents, null, "the bad reading is dropped, not booked"); + assert.equal(h.applied[0].totalCents, 36498, "the total is untouched"); + assert.deepEqual(summary.byState, { READ: 1 }, "READ, not NEEDS_REVIEW"); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); +}); + +test("the tax note survives alongside a dedup reason", async () => { + const h = harness([workerRow()], { + read: async () => ({ ok: true, read: { ...goodRead.read, taxAmount: "292.00" } } as ReadOutcome), + findWeakHit: async () => ({ id: "row-twin" }), + }); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].reason, "weak-dup:row-twin;tax-implausible"); +}); From 7a14a213c4c4c5c4b6d8dc69591d57a1b86c4be0 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:04:38 -0700 Subject: [PATCH 039/144] fix(receipts): checks carry no sales tax; persist only the tax booking accepted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refinement to the tax validation. A handwritten check to a subcontractor has no sales tax, full stop. Any figure the model produced there is the wrong number off the cheque — the amount box, a memo figure — and booking it would move real money into the reimbursable-sales-tax account for a payment that was never taxed. sendToQBOviaAPI.js:148 already refuses to split tax on a check; this makes the ROW say so ("tax-implausible") instead of dropping the reading silently. A check with no tax reading at all is still perfectly normal and gets no note. Tax at or above the total is now refused explicitly rather than incidentally by the 12% ceiling: that shape is a grabbed subtotal, not a tax figure, and saying so is clearer than relying on an arithmetic coincidence. The row now persists the tax that BOOKING accepted, read back out of the same buildGroups the booking step calls — not the separately-validated value. `taxCents` feeds the sales-tax reports, and those must never show a figure no Purchase ever carried. If the two rules ever drift apart, the row records the booking's answer AND gets flagged, rather than quietly reporting a tax that was rejected downstream. Co-Authored-By: Claude Fable 5.1 --- src/lib/receipt-intake/worker.ts | 42 +++++++++++++++-- tests/receipt-intake-worker.test.ts | 70 +++++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 5c022a161..236f5ba2e 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -17,7 +17,13 @@ import { Prisma } from "@prisma/client"; import { canonicalVendor, dedupKeys } from "./keys"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { backoffMs, MAX_BOOK_ATTEMPTS, routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; -import { resolveSuggestedCostCodeId, type BookableRow, type BookResult } from "./book"; +import { + appliedTaxCents, + buildGroups, + resolveSuggestedCostCodeId, + type BookableRow, + type BookResult, +} from "./book"; import { QBTimeoutError } from "@/lib/quickbooks"; import { QboAccountConfigError, @@ -221,9 +227,25 @@ export const MAX_PLAUSIBLE_TAX_RATE = 0.12; export function validateTaxCents( taxCents: number | null, totalCents: number | null, + docType: string | null, ): { taxCents: number | null; implausible: boolean } { + // No tax line is the NORMAL case here, not a problem. if (taxCents === null || taxCents <= 0) return { taxCents: null, implausible: false }; + + // A handwritten check to a subcontractor has no sales tax, full stop. If the + // model produced one it read the wrong number off the cheque — the amount + // box, a memo figure — and booking it would move real money into the + // reimbursable-sales-tax account for a payment that was never taxed. + // sendToQBOviaAPI.js:148 refuses to split tax on a check for the same + // reason; this makes the row SAY so instead of dropping it silently. + if (String(docType ?? "receipt").toLowerCase() === "check") { + return { taxCents: null, implausible: true }; + } + if (totalCents === null || totalCents <= 0) return { taxCents: null, implausible: true }; + // Tax can never BE the total, let alone exceed it — that is a grabbed + // subtotal or a misread line, not a tax figure. + if (taxCents >= totalCents) return { taxCents: null, implausible: true }; const ceiling = Math.ceil(totalCents * MAX_PLAUSIBLE_TAX_RATE); if (taxCents > ceiling) return { taxCents: null, implausible: true }; return { taxCents, implausible: false }; @@ -418,8 +440,22 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis const tax = validateTaxCents( taxCentsRaw && taxCentsRaw > 0 ? taxCentsRaw : null, totalCents, + read.docType, ); - const taxCents = tax.taxCents; + + // PERSIST ONLY WHAT BOOKING WILL ACTUALLY USE. + // + // The row's taxCents feeds the sales-tax reports, and those must never show + // a figure that no Purchase ever carried. So the stored value is not the + // validated one — it is the value read back out of the SAME buildGroups the + // booking step calls. If the two ever disagree (a rule added on one side + // only), the row records the BOOKING's answer and is flagged, rather than + // quietly reporting a tax that was rejected downstream. + const accepted = totalCents !== null && totalCents > 0 + ? appliedTaxCents(buildGroups(read.docType, totalCents, tax.taxCents, keys.ref)) + : 0; + const taxCents = accepted > 0 ? accepted : null; + const taxImplausible = tax.implausible || (tax.taxCents !== null && taxCents === null); const timeZone = await deps.companyTimeZone(); const base = { @@ -463,7 +499,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // queue. `note()` is applied to every write below rather than to one branch, // because a document can be both a duplicate and a bad tax read. const note = (reason: string | null): string | null => { - if (!tax.implausible) return reason; + if (!taxImplausible) return reason; return reason ? `${reason};tax-implausible` : "tax-implausible"; }; diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 2beda4cc4..2d072d06d 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -611,20 +611,34 @@ test("an implausible tax is DROPPED and noted, and the receipt still books", () // A misread decimal ("$2.92" as "$292") or a grabbed subtotal posts real // money to the reimbursable-sales-tax account and inflates a state filing. // WA's highest combined rate is ~10.6%, so 12% is the sanity bound. - assert.deepEqual(validateTaxCents(29_20, 36_498), { taxCents: 2920, implausible: false }); + const r = (tax: number | null, total: number | null, docType = "receipt") => + validateTaxCents(tax, total, docType); + + assert.deepEqual(r(29_20, 36_498), { taxCents: 2920, implausible: false }); // Exactly at the ceiling, rounded UP to the cent so a legitimate rounding // artefact at the boundary is not rejected. - assert.deepEqual(validateTaxCents(1200, 10_000), { taxCents: 1200, implausible: false }); - assert.deepEqual(validateTaxCents(1201, 10_000), { taxCents: null, implausible: true }); + assert.deepEqual(r(1200, 10_000), { taxCents: 1200, implausible: false }); + assert.deepEqual(r(1201, 10_000), { taxCents: null, implausible: true }); // The decimal-point misread. - assert.deepEqual(validateTaxCents(29_200, 36_498), { taxCents: null, implausible: true }); - // Tax larger than the total is nonsense. - assert.deepEqual(validateTaxCents(40_000, 36_498), { taxCents: null, implausible: true }); + assert.deepEqual(r(29_200, 36_498), { taxCents: null, implausible: true }); + // Tax at or above the total is a grabbed subtotal, not a tax figure. + assert.deepEqual(r(36_498, 36_498), { taxCents: null, implausible: true }); + assert.deepEqual(r(40_000, 36_498), { taxCents: null, implausible: true }); // Absent or zero tax is normal, not implausible — most receipts here. - assert.deepEqual(validateTaxCents(null, 36_498), { taxCents: null, implausible: false }); - assert.deepEqual(validateTaxCents(0, 36_498), { taxCents: null, implausible: false }); + assert.deepEqual(r(null, 36_498), { taxCents: null, implausible: false }); + assert.deepEqual(r(0, 36_498), { taxCents: null, implausible: false }); // A tax with no usable total cannot be judged, so it is not trusted. - assert.deepEqual(validateTaxCents(500, null), { taxCents: null, implausible: true }); + assert.deepEqual(r(500, null), { taxCents: null, implausible: true }); + + // A handwritten check to a sub has no sales tax, full stop. Any figure the + // model produced is the wrong number off the cheque, and booking it would + // move real money into the reimbursable-sales-tax account for a payment + // that was never taxed. Even a "plausible" 8% is refused. + assert.deepEqual(r(2920, 36_498, "check"), { taxCents: null, implausible: true }); + assert.deepEqual(r(100, 120_000, "check"), { taxCents: null, implausible: true }); + // ...but a check with NO tax reading is perfectly normal. + assert.deepEqual(r(null, 120_000, "check"), { taxCents: null, implausible: false }); + assert.equal(MAX_PLAUSIBLE_TAX_RATE, 0.12); }); @@ -658,3 +672,41 @@ test("the tax note survives alongside a dedup reason", async () => { await runIntakeWorker(h.deps); assert.equal(h.states[0].reason, "weak-dup:row-twin;tax-implausible"); }); + +test("the row stores only the tax BOOKING accepted, never a rejected reading", async () => { + // taxCents feeds the sales-tax reports, so it must never show a figure that + // no Purchase ever carried. The stored value is read back out of the SAME + // buildGroups the booking step calls. + const h = harness([workerRow()], { + read: async () => ({ + ok: true, + read: { ...goodRead.read, docType: "check", checkNumber: "4178", taxAmount: "29.20" }, + } as ReadOutcome), + }); + await runIntakeWorker(h.deps); + // buildGroups refuses to split tax on a check, so nothing was accepted. + assert.equal(h.applied[0].taxCents, null); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); +}); + +test("a check with no tax reading books clean, with no note", async () => { + const h = harness([workerRow()], { + read: async () => ({ + ok: true, + read: { ...goodRead.read, docType: "check", checkNumber: "4178", taxAmount: "" }, + } as ReadOutcome), + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].taxCents, null); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); +}); + +test("a tax equal to the total is refused end to end", async () => { + const h = harness([workerRow()], { + read: async () => ({ ok: true, read: { ...goodRead.read, taxAmount: "364.98" } } as ReadOutcome), + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].taxCents, null); + assert.equal(h.applied[0].totalCents, 36498, "the total is untouched"); + assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); +}); From fc91992aab4bc40fa6ba71dcce64130e236b89f7 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:16:42 -0700 Subject: [PATCH 040/144] fix(receipts): shadow backlog retires as SHADOW_DONE; attachment checked on both paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 5, all 8. 1 [P0] CUTOVER CANNOT DOUBLE-BOOK. v2's QBO identity for an email/chat/mobile/web row is the intake UUID, which v1 never saw, so DocNumber idempotency could not recognise a Purchase v1 had already made — and requeuing the shadow backlog would have booked all of it a second time, on real books, in one pass. Rows received while DRYRUN was on are now RETIRED to SHADOW_DONE / "booked-by-v1": terminal, never booked by v2. Nothing is requeued; only rows received after cutover book. They keep their keys, so a later resend still collides. Drive rows already booked under the Drive file id (v1's identity), so a v1/v2 overlap on those is idempotent — documented, not relied on. Spec now carries the ordered cutover sequence: forwarder mode first, 24h with zero v1 bookings, then DRYRUN off. 2 [P0] alreadyExists is held to the SAME attachment standard. That is the path every lost-response retry takes — exactly when a Purchase is most likely to be sitting in the books without its image — and it was the one path exempt. Worse, my lastError early-park PREVENTED the recovery: the QBO core re-uploads for an existing Purchase, so the retry IS the fix, and short-circuiting made the stranded receipt permanent. Removed. 3 Storage existence is now checked on a replay in EVERY state. Once the sweep flipped an orphan to file-missing, a same-byte replay got a 200 and the forwarder could delete its only copy. The replay carries the bytes, so it HEALS the row; a booked row with no object is a 409 it may not rewrite. 4 doc_type is a closed enum. It defaulted to "receipt" and unknown values also slipped past the exact multi/non_receipt checks, so a truncated or prompt-injected response with plausible amounts reached QuickBooks. Unknown now routes to NEEDS_REVIEW, and a non-string is rejected outright (String(["receipt"]) is "receipt"). 5 The unreadable-date fallback uses the company's calendar day. toISOString rolls over at 16:00 local, so a Pacific evening upload got tomorrow's date — and with it a different dedup key and reporting period. 6 A weak-duplicate park RELEASES the strong key. Nothing was sent, so the documented pre-send rule applies; holding it made a corrected resend collide with a row that was never booked. 7 The deadline starts at invocation entry and bounds the STAGING sweep (batch 10, deadline-checked). The sweep downloads objects; outside the budget it could eat the platform timeout and the worker would still start a 25s read. 8 NEEDS_JOB rows older than 6h are their own alert. Terminal for the worker, so they piled up while every probe read green. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 35 +++++ e2e/receipt-intake.spec.ts | 63 ++++++++- .../migration.sql | 3 +- prisma/prisma-blind-spots.json | 2 +- prisma/schema.prisma | 2 +- scripts/apply-receipt-intake.mjs | 5 +- .../api/cron/receipt-intake-worker/route.ts | 50 ++++--- src/app/api/receipts/intake/route.ts | 76 ++++++++--- src/lib/pipeline-health.ts | 48 ++++++- src/lib/receipt-intake/book.ts | 31 +++-- src/lib/receipt-intake/read.ts | 26 +++- src/lib/receipt-intake/route-state.ts | 17 +++ src/lib/receipt-intake/worker.ts | 72 ++++++---- tests/apply-receipt-intake.test.ts | 24 ++++ tests/pipeline-digest-route.test.ts | 6 +- tests/pipeline-health.test.ts | 37 +++++- tests/receipt-intake-book.test.ts | 57 ++++++-- tests/receipt-intake-worker.test.ts | 124 ++++++++++++++---- 18 files changed, 554 insertions(+), 124 deletions(-) diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index 4694a6386..7fe298714 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -435,6 +435,41 @@ here); multipart buffering before the size check (the platform body limit applie PDFs carrying embedded JavaScript (never opened server-side — the bytes go to Gemini and to QBO as an attachment). +### CUTOVER SEQUENCE — do these in this order (2026-09-02) + +The hazard this order exists to prevent: v2's QuickBooks identity for an +email/chat/mobile/web row is the intake UUID, which v1 never saw. QBO's DocNumber +idempotency therefore CANNOT recognise a Purchase v1 already created for the same +document. Run both pipelines live at once, or replay the shadow backlog through v2, and +those receipts book twice on real books. + +Drive rows are the exception: v2 books them under the Drive file id, which IS v1's +identity, so an overlap on a Drive-sourced file is idempotent. That is not enough to make +an overlap safe in general. + +1. **Flip the Apps Script to forwarder mode** (`V2_FORWARD=true`). It now COPIES bytes to + `/api/receipts/intake` and still books everything itself. ProBuild is in dry-run: + it reads, dedups and routes, and books nothing. +2. **Run the shadow week.** Gate on §8: 5 consecutive days where every archived v1 file has + a v2 row agreeing on vendor/date/total, and no v2 row stuck in RECEIVED over an hour. +3. **Flip the Apps Script to `V2_LIVE=true`.** It now MOVES files to `_Forwarded` instead of + booking them. v1 stops writing to QuickBooks. ProBuild is still in dry-run, so for this + window NOTHING books — that is intended and it is why the window is short. +4. **Confirm zero v1 bookings for 24 hours.** Watch the Automation register and QBO. This + is the step that makes the next one safe: it proves v1 is out of the books before v2 + enters them, so the two can never both create a Purchase for one document. +5. **Only then set `RECEIPT_INTAKE_DRYRUN=false`.** On its first pass the worker RETIRES the + entire shadow backlog to `SHADOW_DONE` / `booked-by-v1` — terminal, never booked by v2, + because v1 already booked all of it. Nothing is requeued. Only rows received AFTER this + point are booked by v2. + +Retired rows keep their read results and dedup keys, so a post-cutover resend of a +shadow-week receipt still collides with them and is caught as a duplicate. + +**Rolling back** after step 5 means turning `V2_LIVE` off again and `RECEIPT_INTAKE_DRYRUN` +back on. Rows received while v2 was live are already booked and stay `BOOKED`; v1 will not +re-book them, because its own `_Forwarded` move already took those files out of its path. + Two things a human must do before this can leave shadow mode: - Set `RECEIPT_INTAKE_SECRET` (new, independent of `RECEIPT_INGEST_SECRET`) in Vercel, and diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 6cff8f90d..1e203fc5c 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -217,15 +217,17 @@ test.describe("intake POST", () => { data: { state: "STAGING", storagePath: `receipts/intake/${created.body.id}-never-uploaded.png` }, }); - // 409, NEVER a 2xx. The forwarders retry only non-2xx: a 202 read as - // "accepted" would make a Drive script move its source file out of the - // pickup folder while nothing durable existed on our side. + // The replay carries the bytes again, so the orphan is HEALED rather + // than merely reported: stored and republished. Never a 202 — the + // forwarders retry only non-2xx, so "accepted" for a document we do not + // have would let a Drive script delete its only copy. const retry = await postIntake(request, intakeBody({ sourceRef: ref })); - expect(retry.res.status()).toBe(409); - expect(retry.body.ok).toBe(false); - expect(retry.body.error).toBe("staging-incomplete"); + expect(retry.res.status()).toBe(200); + expect(retry.body.recovered).toBe(true); + expect(retry.body.state).toBe("RECEIVED"); expect(retry.body.id).toBe(created.body.id); - expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.state).toBe("STAGING"); + const healed = await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }); + expect(healed?.state).toBe("RECEIVED"); }); test("a machine caller MUST supply its own sourceRef", async ({ request }) => { @@ -613,3 +615,50 @@ test.describe("archive callback", () => { await machine.dispose(); }); }); + +test.describe("orphan recovery", () => { + test("replaying a row the sweeper already parked file-missing HEALS it", async ({ request }) => { + // The hole this closes: storage existence was checked only while the row + // was STAGING. Once the sweep flipped an orphan to + // NEEDS_REVIEW/file-missing, an identical replay got a cheerful 200 and + // the forwarder could delete its only copy of a receipt we did not have. + const ref = `${REF_PREFIX}swept-orphan`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + + // Exactly what the sweeper leaves behind. + await prisma.receiptIntake.update({ + where: { id: created.body.id }, + data: { + state: "NEEDS_REVIEW", + stateReason: "file-missing", + storagePath: `receipts/intake/${created.body.id}-gone.png`, + }, + }); + + const replay = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(replay.res.status()).toBe(200); + expect(replay.body.recovered).toBe(true); + + const row = await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }); + expect(row?.state).toBe("RECEIVED"); + expect(row?.stateReason).toBeNull(); + }); + + test("a BOOKED row whose object vanished is never rewritten by a replay", async ({ request }) => { + // A replay may heal an orphan, but it must not be able to reach into a + // row that already has a Purchase behind it. + const ref = `${REF_PREFIX}booked-orphan`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + await prisma.receiptIntake.update({ + where: { id: created.body.id }, + data: { state: "BOOKED", storagePath: `receipts/intake/${created.body.id}-gone.png` }, + }); + + const replay = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(replay.res.status()).toBe(409); + expect(replay.body.error).toBe("object-missing"); + expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.state).toBe("BOOKED"); + }); +}); diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 9c8cb62b3..764e48ea3 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -78,7 +78,8 @@ BEGIN ) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', - 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', + 'SHADOW_DONE')); END IF; END $$; diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index af427a177..3867f59ef 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -102,7 +102,7 @@ { "name": "ReceiptIntake_state_check", "table": "\"ReceiptIntake\"", - "def": "CHECK ((state = ANY (ARRAY['STAGING'::text, 'RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text])))" + "def": "CHECK ((state = ANY (ARRAY['STAGING'::text, 'RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text, 'SHADOW_DONE'::text])))" }, { "name": "RefundEvent_amountCents_check", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1f09cc6be..757974b63 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3027,7 +3027,7 @@ model ReceiptIntake { /// yet, so the worker's claim predicate excludes it. The intake route flips /// it to RECEIVED in one UPDATE after the upload lands. A row that never gets /// there is swept to NEEDS_REVIEW after 15 minutes. - state String @default("STAGING") // STAGING RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT + state String @default("STAGING") // STAGING RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT SHADOW_DONE dryRun Boolean @default(true) /// no-estimate | multi-doc | zero-total | weak-dup: | /// strong-dup-amount-mismatch: | qbo-fault: | max-retries | diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index 06f5b2ec7..3ca5ed320 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -72,6 +72,8 @@ export function targetMatches(actual, expectDb, expectHost) { export const RECEIPT_INTAKE_STATES = [ "STAGING", "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", + // Received during the shadow week, therefore booked by v1 and NEVER by v2. + "SHADOW_DONE", ]; export const statements = [ @@ -161,7 +163,8 @@ export const statements = [ AND conrelid = '"ReceiptIntake"'::regclass) THEN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', - 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT')); + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', + 'SHADOW_DONE')); END IF; END $$`, diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index bed0095a3..6c3a49916 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -15,6 +15,7 @@ import { BATCH_SIZE, CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, + STAGING_SWEEP_BATCH, STAGING_SWEEP_MINUTES, isUniqueViolation, runIntakeWorker, @@ -71,8 +72,8 @@ const NOT_DRY_RUN_PARKED: Prisma.ReceiptIntakeWhereInput = { }; async function claim( - opts: { requeueDryRunParked: boolean }, -): Promise<{ rows: WorkerRow[]; requeued: number } | null> { + opts: { retireShadowBacklog: boolean }, +): Promise<{ rows: WorkerRow[]; shadowRetired: number } | null> { const now = new Date(); return prisma.$transaction(async tx => { const [lock] = await tx.$queryRaw<{ locked: boolean }[]>( @@ -80,20 +81,30 @@ async function claim( ); if (!lock?.locked) return null; - // Cutover, INSIDE the lock and the same transaction as the claim. Run - // outside it, two overlapping invocations could both see the parked - // backlog and both un-park it, and the second UPDATE would race the - // first one's claim. `dryRun` flips WITH the requeue: a row that - // reappeared still carrying dryRun=true would be skipped and re-parked - // forever. - let requeued = 0; - if (opts.requeueDryRunParked) { + // CUTOVER, inside the lock and the same transaction as the claim. + // + // Everything received during the shadow week was booked by v1, so v2 + // RETIRES it — SHADOW_DONE, terminal, never booked here. It is not + // requeued, because v2's QBO identity for an email/chat/mobile/web row + // is the intake UUID, which v1 never saw: QuickBooks' DocNumber + // idempotency could not have recognised the Purchase v1 already made, + // and the entire backlog would have booked a second time on real books + // in a single pass. (Drive rows book under the Drive file id — v1's own + // identity — so those alone would have been safe. That is not enough to + // requeue the rest.) + // + // The rows keep their read results and dedup keys, so a post-cutover + // resend of the same receipt still collides with them and is caught. + let shadowRetired = 0; + if (opts.retireShadowBacklog) { const result = await tx.receiptIntake.updateMany({ where: { dryRun: true, state: { in: ["READ", "BOOKING"] } }, - data: { dryRun: false, nextRetryAt: null }, + data: { state: "SHADOW_DONE", stateReason: "booked-by-v1", nextRetryAt: null }, }); - requeued = result.count; - if (requeued > 0) console.log("[cron/receipt-intake-worker] cutover requeue", requeued); + shadowRetired = result.count; + if (shadowRetired > 0) { + console.log("[cron/receipt-intake-worker] shadow backlog retired", shadowRetired); + } } const due = await tx.receiptIntake.findMany({ @@ -109,7 +120,7 @@ async function claim( take: BATCH_SIZE, select: WORKER_ROW_SELECT, }); - if (due.length === 0) return { rows: [], requeued }; + if (due.length === 0) return { rows: [], shadowRetired }; // THE claim. Anything this run took is invisible to the next one for // the lease, whether or not the advisory lock held. @@ -117,7 +128,7 @@ async function claim( where: { id: { in: due.map(r => r.id) } }, data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, }); - return { rows: due as WorkerRow[], requeued }; + return { rows: due as WorkerRow[], shadowRetired }; }); } @@ -127,7 +138,7 @@ function buildDeps(): WorkerDependencies { isDryRunEnabled: () => process.env.RECEIPT_INTAKE_DRYRUN !== "false", - sweepStaleStaging: async () => { + sweepStaleStaging: async shouldStop => { // NEVER a blanket "old therefore missing". A STAGING row that is old // because its publish UPDATE failed HAS its object in the bucket, // and declaring that receipt file-missing would hand a human a @@ -138,12 +149,17 @@ function buildDeps(): WorkerDependencies { const stale = await prisma.receiptIntake.findMany({ where: { state: "STAGING", createdAt: { lt: cutoff } }, select: { id: true, storagePath: true }, - take: 50, + // Small on purpose: each row costs a storage round trip, and the + // sweep runs BEFORE any receipt is processed. A big batch here + // spends the invocation on housekeeping. + take: STAGING_SWEEP_BATCH, }); let published = 0; let parked = 0; for (const row of stale) { + // The sweep is inside the run's deadline, not outside it. + if (shouldStop()) break; const probe = await downloadDocBytesResult(toSecureRef(row.storagePath)); if (probe.ok) { // The upload landed; only the publish was lost. Finish it. diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 2871365b7..5064e32ba 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -238,7 +238,9 @@ export async function POST(req: Request) { }); } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { - return respondToSourceRefConflict(auth, source, sourceRef, fileSha256); + return respondToSourceRefConflict(auth, source, sourceRef, fileSha256, { + bytes: parsed.bytes, mimeType, storagePath, + }); } // A projectId/costCodeId that doesn't exist is the CALLER's mistake, so // it must be a deterministic 400 — a 500 would make a forwarder retry a @@ -287,6 +289,22 @@ export async function POST(req: Request) { * retry finds the STAGING row, confirms the object really is there, and * finishes the job. */ +/** Upload bytes to the private bucket. Returns false on any failure. */ +async function storeObject(storagePath: string, bytes: Buffer, mimeType: string): Promise { + const supabase = getSupabase(); + if (!supabase) return false; + try { + const { error } = await supabase.storage + .from(SECURE_BUCKET) + .upload(storagePath, bytes, { contentType: mimeType, upsert: true }); + if (error) console.error("[receipts/intake] heal upload failed", error.message); + return !error; + } catch (error) { + console.error("[receipts/intake] heal upload threw", error instanceof Error ? error.name : "error"); + return false; + } +} + async function publishStagedRow(id: string): Promise { try { const published = await prisma.receiptIntake.update({ @@ -322,6 +340,8 @@ async function respondToSourceRefConflict( source: string, sourceRef: string, fileSha256: string, + /** The bytes this replay carried — used to HEAL a row whose object is gone. */ + payload: { bytes: Buffer; mimeType: string; storagePath: string }, ): Promise { const existing = await prisma.receiptIntake.findUnique({ where: { sourceRef }, @@ -362,37 +382,51 @@ async function respondToSourceRefConflict( ); } - // Same bytes, and the first attempt is still STAGING. Two very different - // reasons for that, and only storage can tell them apart: + // SAME BYTES. Before promising anything, confirm the document is actually + // in the bucket — for EVERY state, not just STAGING. // - // - the object IS there, so the previous request uploaded successfully and - // only its publish UPDATE failed. Finish it. This is what makes the - // publish resumable rather than a permanent half-state. - // - the object is NOT there yet — a concurrent request is mid-upload, or - // the last one died before storing anything. 202 "accepted, not yet - // published" tells the caller to re-poll rather than assume a queued - // document; the 15-minute sweeper handles the case where it never lands. - if (existing.state === "STAGING") { - if (await secureObjectExists(existing.storagePath)) { - return publishStagedRow(existing.id); + // Checking only STAGING left a hole: once the stale-row sweep flipped an + // orphan to NEEDS_REVIEW/file-missing, this replay returned a cheerful 200 + // and the forwarder could delete its only copy of a receipt we did not + // have. The state a row happens to be parked in says nothing about whether + // its bytes exist. + if (!(await secureObjectExists(existing.storagePath))) { + // The caller just handed us the bytes again, so the orphan is fixable: + // store them and republish. This is the retry HEALING the row rather + // than merely reporting on it. + const healable = existing.state === "STAGING" || existing.state === "NEEDS_REVIEW"; + if (healable) { + const healed = await storeObject(payload.storagePath, payload.bytes, payload.mimeType); + if (!healed) { + return NextResponse.json({ ok: false, error: "storage-failed" }, { status: 503 }); + } + await prisma.receiptIntake.update({ + where: { id: existing.id }, + data: { storagePath: payload.storagePath, state: "RECEIVED", stateReason: null, nextRetryAt: null }, + }); + return NextResponse.json({ + ok: true, recovered: true, id: existing.id, state: "RECEIVED", + sourceRef: existing.sourceRef, projectId: existing.projectId, dryRun: existing.dryRun, + }); } - // 409, NOT 202. The forwarders retry only non-2xx: a 202 told the - // caller "accepted", so a Drive script would move its source file out - // of the pickup folder and the ORIGINAL document would be gone while - // nothing durable existed on our side. Any 2xx here is a promise we - // cannot keep until the object is confirmed. + // A booked/archived row with no object is not something a replay may + // rewrite. Retryable failure, never a 2xx. return NextResponse.json( { ok: false, - error: "staging-incomplete", - reason: "the previous upload for this sourceRef never landed; retry", + error: "object-missing", + reason: "this sourceRef exists but its stored document is gone; escalate", id: existing.id, - sourceRef: existing.sourceRef, + state: existing.state, }, { status: 409 }, ); } + // The object is there. A STAGING row means the previous request uploaded + // successfully and only its publish UPDATE failed — finish it. + if (existing.state === "STAGING") return publishStagedRow(existing.id); + return NextResponse.json({ ok: true, alreadyReceived: true, diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index 8097ef899..ba8308ed9 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -95,6 +95,15 @@ export interface PipelineHealth { stuck: CountProbe; /** NEEDS_REVIEW backlog. Reported always; a reason only when rows are STUCK. */ needsReview: CountProbe; + /** + * NEEDS_JOB rows older than INTAKE_STUCK_HOURS — a receipt nobody has + * matched to a job. Terminal for the worker, so it can pile up + * indefinitely while every other probe reads green: the exact + * silent-failure mode this whole check exists to eliminate. Only the + * OVERDUE ones count, so a receipt uploaded ten minutes ago is not an + * alert. + */ + unassigned: CountProbe; }; } @@ -206,6 +215,7 @@ export function evaluatePipelineHealth(input: { stuck: CountProbe; intakeStuck: CountProbe; intakeNeedsReview: CountProbe; + intakeUnassigned: CountProbe; now: number; }): { ok: boolean; reasons: string[] } { const reasons: string[] = []; @@ -219,6 +229,7 @@ export function evaluatePipelineHealth(input: { ["stuck", input.stuck], ["intakeStuck", input.intakeStuck], ["intakeNeedsReview", input.intakeNeedsReview], + ["intakeUnassigned", input.intakeUnassigned], ]; for (const [name, probe] of namedProbes) { if (probe.status === "error") reasons.push(`probe-failed:${name}`); @@ -269,6 +280,14 @@ export function evaluatePipelineHealth(input: { reasons.push(`intake-stuck:${input.intakeStuck.count}${backlog}`); } + // A receipt that has been waiting hours for someone to say which job it + // belongs to is not "working as designed" — it is an expense that will + // never reach job cost. It is its own reason, separate from intake-stuck, + // because the fix is different: assign a project, not restart a worker. + if (input.intakeUnassigned.status === "ok" && input.intakeUnassigned.count > 0) { + reasons.push(`intake-unassigned:${input.intakeUnassigned.count}`); + } + if (input.lastReceiptPush.status === "ok") { const at = input.lastReceiptPush.at ? Date.parse(input.lastReceiptPush.at) : null; const stale = at === null || Number.isNaN(at) || input.now - at > RECEIPT_STALE_HOURS * HOUR_MS; @@ -336,6 +355,10 @@ export async function getPipelineHealth(): Promise { const [intuit, lastPurchase, lastPush, lastPaymentsSync, receiptRows, lastBankLine, stuck] = await Promise.all([ const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck, intakeStuck, intakeNeedsReview] = await Promise.all([ + const [ + intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck, + intakeStuck, intakeNeedsReview, intakeUnassigned, + ] = await Promise.all([ fetchIntuitStatus(), // Expense carries no updatedAt column — qbSyncedAt IS the "when did the // QBO purchase sync land" timestamp this is asking for. @@ -470,6 +493,17 @@ export async function getPipelineHealth(): Promise { () => prisma.receiptIntake.count({ where: { state: "NEEDS_REVIEW" } }), 0, ), + probe( + "intakeUnassigned", + () => + prisma.receiptIntake.count({ + where: { + state: "NEEDS_JOB", + createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * HOUR_MS) }, + }, + }), + 0, + ), ]); const counts: Record = {}; @@ -506,6 +540,11 @@ export async function getPipelineHealth(): Promise { reason: intakeNeedsReview.reason, count: intakeNeedsReview.value, }, + intakeUnassigned: { + status: intakeUnassigned.status, + reason: intakeUnassigned.reason, + count: intakeUnassigned.value, + }, }; const verdict = evaluatePipelineHealth({ ...snapshot, now }); @@ -522,7 +561,11 @@ export async function getPipelineHealth(): Promise { receipts24h: snapshot.receipts24h, bank: snapshot.bank, stuck: snapshot.stuck, - intake: { stuck: snapshot.intakeStuck, needsReview: snapshot.intakeNeedsReview }, + intake: { + stuck: snapshot.intakeStuck, + needsReview: snapshot.intakeNeedsReview, + unassigned: snapshot.intakeUnassigned, + }, }; } @@ -582,6 +625,9 @@ export function formatPipelineDigest(health: PipelineHealth): { subject: string; `Receipt intake awaiting review: ${ health.intake?.needsReview?.status === "error" ? "unavailable (probe failed)" : health.intake?.needsReview?.count ?? "unavailable" }`, + `Receipt intake awaiting a job (>${INTAKE_STUCK_HOURS}h): ${ + health.intake?.unassigned?.status === "error" ? "unavailable (probe failed)" : health.intake?.unassigned?.count ?? "unavailable" + }`, ]; if (health.reasons.length > 0) lines.push(`Needs attention: ${health.reasons.join(", ")}`); diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 9e061953b..3e2d83881 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -315,13 +315,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro const blocker = attachmentBlocker(row.mimeType, bytes.length); if (blocker) return parkedBeforeSend(`unsupported-attachment:${blocker}`); - // A previous attempt created the Purchase but could not attach the file. - // The retry below returns alreadyExists:true, which carries NO attachment - // status — so booking it now would silently declare success for a Purchase - // we know has no receipt on it. Hand it to a human instead. - if (row.lastError?.startsWith(ATTACHMENT_FAILED_PREFIX)) { - return { outcome: "needs-review", reason: "attachment-unconfirmed", releaseStrongKey: false }; - } + // NOTE: a previous attachment failure deliberately does NOT short-circuit + // here. createQBReceiptPurchase re-checks and re-uploads the file for an + // EXISTING Purchase (ensureAttachmentOnExistingPurchase), so the retry is + // the recovery — parking early would have made the stranded-receipt case + // permanent, which is the opposite of the intent. const input: CreateQBReceiptPurchaseInput = { projectName: project.name, @@ -368,11 +366,20 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro return { outcome: "needs-review", reason: `qbo-fault:${result.reason}`, releaseStrongKey: true }; } - // The Purchase exists. If the receipt did NOT make it on, that is not a - // success: `failed:*` is an HTTP/transport fault on the upload leg and is - // worth another pass; `skipped` after a passing preflight means the two - // ceilings have drifted apart and a human must look. - if (!result.alreadyExists && result.attachment !== "attached") { + // The Purchase exists. If the receipt is not ON it, that is not a success — + // and this is checked on BOTH paths. + // + // The alreadyExists path was previously exempt, which is the path that + // MATTERS: it is reached by every retry after a lost response, i.e. exactly + // when a Purchase is most likely to be sitting there without its image. So + // the one case the check existed for was the one case it skipped. + // + // "already-attached" is a success: the file was put on by an earlier + // attempt. "failed:*" is an HTTP fault on the upload leg and is worth + // another pass (the QBO core re-uploads for an existing Purchase, so the + // retry genuinely recovers). "skipped" after a passing preflight means our + // mirrored ceilings have drifted from QBO's and a human must look. + if (result.attachment !== "attached" && result.attachment !== "already-attached") { if (result.attachment === "skipped") { return { outcome: "needs-review", reason: "unsupported-attachment:skipped", releaseStrongKey: false }; } diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts index 6e236203c..323bd6e3b 100644 --- a/src/lib/receipt-intake/read.ts +++ b/src/lib/receipt-intake/read.ts @@ -156,6 +156,30 @@ export function buildReadPrompt(projectPhases: ProjectPhase[]): string { ); } +/** + * The ONLY four answers STEP 1 of the prompt is allowed to give. + * + * `doc_type` used to default to "receipt" when the field was missing, and any + * unrecognised string fell through the exact `multi` / `non_receipt` checks in + * routeState and was treated as a bookable receipt too. So a truncated + * response, a schema change, or a prompt-injected document that suppressed the + * field while supplying plausible vendor/date/amount values would be routed + * straight at QuickBooks. Failing OPEN on a classifier that decides whether + * something is a purchase at all is exactly backwards. + */ +export const DOC_TYPES = ["receipt", "check", "multi", "non_receipt"] as const; +/** Not in DOC_TYPES: routeState sends it to a human. */ +export const UNKNOWN_DOC_TYPE = "unknown"; + +export function normalizeDocType(value: unknown): string { + // typeof, not coerce(): String(["receipt"]) is "receipt", so an array would + // otherwise be accepted as a valid classification. A doc_type that is not a + // string is not an answer. + if (typeof value !== "string") return UNKNOWN_DOC_TYPE; + const raw = value.trim().toLowerCase(); + return (DOC_TYPES as readonly string[]).includes(raw) ? raw : UNKNOWN_DOC_TYPE; +} + function coerce(value: unknown): string { if (value === null || value === undefined) return ""; return String(value).trim(); @@ -175,7 +199,7 @@ export function parseReadJson(text: string, projectPhases: ProjectPhase[]): Read const suggested = coerce(json.suggested_phase); return { - docType: (coerce(json.doc_type) || "receipt").toLowerCase(), + docType: normalizeDocType(json.doc_type), vendor: coerce(json.vendor), date: coerce(json.date), invoice: coerce(json.invoice), diff --git a/src/lib/receipt-intake/route-state.ts b/src/lib/receipt-intake/route-state.ts index 07807a4a5..4c7b6fabf 100644 --- a/src/lib/receipt-intake/route-state.ts +++ b/src/lib/receipt-intake/route-state.ts @@ -10,10 +10,19 @@ export const RECEIPT_INTAKE_STATES = [ "STAGING", "RECEIVED", "READ", "NEEDS_JOB", "NEEDS_REVIEW", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", + /** + * Terminal. The row arrived while RECEIPT_INTAKE_DRYRUN was on, so v1 (the + * Apps Script) booked it and v2 never will. See the cutover sequence in + * docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §7. + */ + "SHADOW_DONE", ] as const; export type ReceiptIntakeState = (typeof RECEIPT_INTAKE_STATES)[number]; +/** Mirrors DOC_TYPES in read.ts — the closed set STEP 1 of the prompt may return. */ +const KNOWN_DOC_TYPES = new Set(["receipt", "check", "multi", "non_receipt"]); + export interface RouteInput { docType: string; /** cleanMoney output, e.g. "0.00" / "364.98". */ @@ -75,6 +84,14 @@ export function routeState(read: RouteInput, dedupHits: DedupHits, hasProject: b if (docType === "non_receipt") { return { state: "NON_RECEIPT", stateReason: null, duplicateOfId: null }; } + // Fail CLOSED on the classifier. A missing or unrecognised doc_type means + // we do not know whether this is a purchase at all — a truncated response, a + // schema change, or a prompt-injected document that suppressed the field + // while supplying plausible amounts. Booking on that is unacceptable; a + // human looks instead. + if (!KNOWN_DOC_TYPES.has(docType)) { + return { state: "NEEDS_REVIEW", stateReason: "unknown-doc-type", duplicateOfId: null }; + } if (read.totalCents === null || read.totalCents <= 0 || read.amount === "0.00") { return { state: "NEEDS_REVIEW", stateReason: "refund-or-zero", duplicateOfId: null }; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 236f5ba2e..0bacc20aa 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -15,7 +15,7 @@ */ import { Prisma } from "@prisma/client"; import { canonicalVendor, dedupKeys } from "./keys"; -import { startOfDateInTimeZone } from "@/lib/tz-date"; +import { dayKeyInTimeZone, startOfDateInTimeZone } from "@/lib/tz-date"; import { backoffMs, MAX_BOOK_ATTEMPTS, routeState, type DedupHits, type ReceiptIntakeState } from "./route-state"; import { appliedTaxCents, @@ -59,6 +59,8 @@ export const RUN_SOFT_DEADLINE_MS = 40_000; * outage, and neither resolves itself. */ export const STAGING_SWEEP_MINUTES = 15; +/** Storage round trips per sweep. Small: the sweep runs before any real work. */ +export const STAGING_SWEEP_BATCH = 10; /** * Consecutive AI-unavailable passes before a row is parked for a human. Ported * from v3.4: an outage that never ends still has to end somewhere, and 20 @@ -93,15 +95,16 @@ export interface WorkerDependencies { * backlog and both un-park it, and the second one's UPDATE would race the * first one's claim. */ - claim: (opts: { requeueDryRunParked: boolean }) => Promise<{ rows: WorkerRow[]; requeued: number } | null>; + claim: (opts: { retireShadowBacklog: boolean }) => Promise<{ rows: WorkerRow[]; shadowRetired: number } | null>; /** RECEIPT_INTAKE_DRYRUN is not "false". Injected so the cutover is testable. */ isDryRunEnabled: () => boolean; /** * Move STAGING rows older than STAGING_SWEEP_MINUTES to NEEDS_REVIEW - * `file-missing`. A row whose upload never landed is invisible to the claim - * predicate by design, so nothing else would ever notice it. + * `file-missing`, or PUBLISH them when the object is actually there. + * `shouldStop` bounds the pass: the sweep downloads objects, so it must not + * be able to eat the invocation before any real work starts. */ - sweepStaleStaging: () => Promise; + sweepStaleStaging: (shouldStop: () => boolean) => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; /** Tagged: a confirmed 404 and a transient storage fault are NOT the same answer. */ downloadBytes: (storagePath: string) => Promise; @@ -169,8 +172,8 @@ export interface WorkerRunSummary { skipped?: "already-running"; /** Rows left unprocessed because the soft deadline hit. They keep their lease. */ deferredToNextRun?: number; - /** Rows un-parked by the first live pass after the shadow week. */ - requeued?: number; + /** Shadow-week rows retired as SHADOW_DONE by the first live pass. */ + shadowRetired?: number; /** STAGING rows whose upload never landed, parked for a human. */ staleStagingSwept?: number; } @@ -257,22 +260,31 @@ export function toDateStr(date: Date): string { /** One pass. Never throws for a single bad row — one poison document must not stall the queue. */ export async function runIntakeWorker(deps: WorkerDependencies): Promise { - // Cutover: the FIRST live pass hands the shadow week's parked backlog back - // to the queue. Rows parked under dryRun are excluded from the claim (see - // the cron route's claim predicate) precisely so they cannot starve the - // batch — which also means nothing else would ever wake them. It runs - // INSIDE the claim transaction, under the same lock. - const claimed = await deps.claim({ requeueDryRunParked: !deps.isDryRunEnabled() }); + // THE DEADLINE STARTS HERE, at invocation entry — not after the claim and + // the sweep. The sweep downloads objects, so timing it out of the budget + // meant it could consume the whole platform timeout and the worker would + // STILL go on to start a 25s Gemini read and a QBO round trip. + const startedAt = deps.monotonicMs(); + const outOfTime = () => deps.monotonicMs() - startedAt >= RUN_SOFT_DEADLINE_MS; + + // CUTOVER. Rows received while dry-run was on were booked by v1, so v2 must + // never book them: they are RETIRED as SHADOW_DONE, not requeued. + // + // Requeuing them was a double-booking hazard. v2's QBO identity for an + // email/chat/mobile/web row is the intake UUID, which v1 never saw, so + // QuickBooks' DocNumber idempotency could not recognise a Purchase v1 had + // already created for the same document — and the whole shadow backlog + // would have been booked a second time, on real books, in one pass. + const claimed = await deps.claim({ retireShadowBacklog: !deps.isDryRunEnabled() }); if (claimed === null) { return { processed: 0, byState: {}, skipped: "already-running" }; } - const { rows, requeued } = claimed; + const { rows, shadowRetired } = claimed; // Rows whose upload never landed are invisible to the claim by design, so // this is the only thing that will ever notice them. - const staged = await deps.sweepStaleStaging().catch(() => 0); + const staged = await deps.sweepStaleStaging(outOfTime).catch(() => 0); - const startedAt = deps.monotonicMs(); const byState: Record = {}; const bump = (state: string) => { byState[state] = (byState[state] ?? 0) + 1; }; @@ -284,7 +296,7 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise= RUN_SOFT_DEADLINE_MS) { + if (outOfTime()) { deferredToNextRun = rows.length - processed; break; } @@ -325,7 +337,7 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise 0 ? accepted : null; const taxImplausible = tax.implausible || (tax.taxCents !== null && taxCents === null); - const timeZone = await deps.companyTimeZone(); const base = { vendor: read.vendor || null, @@ -561,9 +579,17 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis const weak = await deps.findWeakHit(row.id, keys.weak); if (weak) { const third = routeState(routeInput, { strong: null, weak }, !!row.projectId); - // The strong key stays claimed: this row is still the live owner of - // that date|ref, and releasing it would let a third copy book. - await deps.applyState(row.id, third.state, note(third.stateReason)); + // RELEASE the strong key. Nothing was sent to QuickBooks, so this row + // is parked pre-send and the documented rule applies to it like any + // other. Holding the key made a CORRECTED resend of the same receipt + // collide with a row that was never booked — the review queue then had + // two rows and neither could proceed. The weak pair is still visible to + // a human through duplicateOfId and the reason. + await deps.applyState(row.id, third.state, note(third.stateReason), { + ...base, + dedupStrongKey: null, + duplicateOfId: third.duplicateOfId, + }); return third.state; } diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts index 7d405cb16..777909f92 100644 --- a/tests/apply-receipt-intake.test.ts +++ b/tests/apply-receipt-intake.test.ts @@ -180,3 +180,27 @@ test("RLS is enabled on ReceiptIntake, in both files, and WITHOUT force", () => assert.ok(entry, "ReceiptIntake missing from prisma-blind-spots.json rlsTables"); assert.equal(entry.forced, false); }); + +test("SHADOW_DONE is a real state everywhere, so the cutover write cannot fail", () => { + // The cutover UPDATE writes this value on every shadow-week row in one + // statement. If the CHECK constraint did not know it, the entire cutover + // would abort — inside the claim transaction, on the first live run. + assert.ok(RECEIPT_INTAKE_STATES.includes("SHADOW_DONE")); + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check")); + assert.match(check!, /'SHADOW_DONE'/); + assert.match(migrationSql, /'SHADOW_DONE'/); + const snapshot = JSON.parse( + readFileSync(path.join(__dirname, "..", "prisma", "prisma-blind-spots.json"), "utf8"), + ); + const entry = snapshot.checkConstraints.find((r: { name: string }) => r.name === "ReceiptIntake_state_check"); + assert.match(entry.def, /'SHADOW_DONE'::text/); +}); + +test("SHADOW_DONE stays in the strong-key active set", () => { + // A shadow-week row WAS booked (by v1), so its dedup key must keep + // quarantining a post-cutover resend of the same receipt. Only DUPLICATE and + // VOID — rows that represent nothing — drop out of the index. + const index = statements.find((s: string) => s.includes("ReceiptIntake_dedupStrongKey_active_key")); + assert.match(index!, /NOT IN \('DUPLICATE', 'VOID'\)/); + assert.ok(!/SHADOW_DONE/.test(index!), "SHADOW_DONE must NOT be excluded"); +}); diff --git a/tests/pipeline-digest-route.test.ts b/tests/pipeline-digest-route.test.ts index 3f2ba073a..a3b2b06d4 100644 --- a/tests/pipeline-digest-route.test.ts +++ b/tests/pipeline-digest-route.test.ts @@ -29,7 +29,11 @@ const HEALTH: PipelineHealth = { receipts24h: { status: "ok", counts: { created: 2 } }, bank: { status: "ok", at: "2026-08-29T00:00:00.000Z" }, stuck: { status: "ok", count: 0 }, - intake: { stuck: { status: "ok", count: 0 }, needsReview: { status: "ok", count: 0 } }, + intake: { + stuck: { status: "ok", count: 0 }, + needsReview: { status: "ok", count: 0 }, + unassigned: { status: "ok", count: 0 }, + }, }; function handlers(overrides: Partial = {}) { diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index bca15dd3a..421ff4743 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -39,6 +39,7 @@ function snapshot(overrides: Partial[0 stuck: { status: "ok" as const, count: 0 }, intakeStuck: { status: "ok" as const, count: 0 }, intakeNeedsReview: { status: "ok" as const, count: 0 }, + intakeUnassigned: { status: "ok" as const, count: 0 }, now: NOW, ...overrides, }; @@ -186,6 +187,7 @@ function sampleHealth(overrides: Partial = {}): PipelineHealth { intake: { stuck: { status: "ok", count: 0 }, needsReview: { status: "ok", count: 0 }, + unassigned: { status: "ok", count: 0 }, }, ...overrides, }; @@ -577,7 +579,7 @@ test("a NEEDS_REVIEW backlog alone is NOT a failure", () => { }); test("an intake probe that FAILED is not an intake probe that found nothing", () => { - for (const name of ["intakeStuck", "intakeNeedsReview"] as const) { + for (const name of ["intakeStuck", "intakeNeedsReview", "intakeUnassigned"] as const) { const v = evaluatePipelineHealth(snapshot({ [name]: { status: "error", reason: "timeout", count: 0 } })); assert.equal(v.ok, false, name); assert.ok(v.reasons.includes(`probe-failed:${name}`), name); @@ -595,10 +597,15 @@ test("the stuck reason survives a failed backlog probe rather than lying about i test("the digest prints both intake numbers", () => { const { text } = formatPipelineDigest(sampleHealth({ - intake: { stuck: { status: "ok", count: 3 }, needsReview: { status: "ok", count: 7 } }, + intake: { + stuck: { status: "ok", count: 3 }, + needsReview: { status: "ok", count: 7 }, + unassigned: { status: "ok", count: 2 }, + }, })); assert.match(text, /Receipt intake stuck >6h: 3/); assert.match(text, /Receipt intake awaiting review: 7/); + assert.match(text, /Receipt intake awaiting a job \(>6h\): 2/); }); test("the digest says a failed intake probe is unavailable, never zero", () => { @@ -606,6 +613,7 @@ test("the digest says a failed intake probe is unavailable, never zero", () => { intake: { stuck: { status: "error", reason: "timeout", count: 0 }, needsReview: { status: "error", reason: "timeout", count: 0 }, + unassigned: { status: "error", reason: "timeout", count: 0 }, }, })); assert.match(text, /Receipt intake stuck >6h: unavailable \(probe failed\)/); @@ -643,3 +651,28 @@ test("the intake stuck probe covers the three shapes of 'the worker stopped'", a // them would make the check red by design and train everyone to ignore it. assert.equal(branches[2].dryRun, false); }); + +test("receipts nobody assigned a job to are an ALERT, not a green backlog", () => { + // NEEDS_JOB is terminal for the worker, so it can pile up indefinitely + // while every other probe reads green — the exact silent failure this whole + // check exists to eliminate. Its own reason, because the fix is different: + // assign a project, not restart a worker. + const v = evaluatePipelineHealth(snapshot({ intakeUnassigned: { status: "ok", count: 5 } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["intake-unassigned:5"]); +}); + +test("a freshly uploaded unassigned receipt is not an alert", () => { + // Only rows OLDER than the stuck threshold are counted by the probe, so a + // receipt uploaded ten minutes ago never reaches this reason. + assert.deepEqual(evaluatePipelineHealth(snapshot()), { ok: true, reasons: [] }); +}); + +test("unassigned and stuck are reported separately", () => { + const v = evaluatePipelineHealth(snapshot({ + intakeStuck: { status: "ok", count: 2 }, + intakeUnassigned: { status: "ok", count: 3 }, + })); + assert.ok(v.reasons.some(r => r.startsWith("intake-stuck:2"))); + assert.ok(v.reasons.includes("intake-unassigned:3")); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 179da6353..23d685353 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -336,20 +336,54 @@ test("an attachment upload that FAILED is retried, never reported as booked", as assert.equal(r.expenses.length, 0, "no Expense until the receipt is actually on the Purchase"); }); -test("the retry after an attachment failure asks a human instead of silently succeeding", async () => { - // The retry hits QBO's idempotency and comes back alreadyExists:true, which - // carries NO attachment status — so booking it would declare success for a - // Purchase we KNOW has no receipt on it. +test("an EXISTING purchase is held to the SAME attachment standard", async () => { + // This is the path that matters: it is reached by every retry after a lost + // response — exactly when a Purchase is most likely to be sitting in the + // books without its image. It was the one path exempt from the check. + const failing = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "failed:500", + }) as any, + }); + const failed = await bookReceipt(row(), failing.deps); + assert.equal(failed.outcome, "retry", "an upload fault on an existing Purchase is recoverable"); + assert.equal(failing.expenses.length, 0, "and it is NOT booked meanwhile"); + + const skipped = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "skipped", + }) as any, + }); + const skippedResult = await bookReceipt(row(), skipped.deps); + assert.equal(skippedResult.outcome, "needs-review"); + assert.equal((skippedResult as any).reason, "unsupported-attachment:skipped"); + assert.equal(skipped.expenses.length, 0); +}); + +test("a previous attachment failure does NOT block the recovery attempt", async () => { + // The QBO core re-checks and re-uploads the file for an existing Purchase + // (ensureAttachmentOnExistingPurchase), so the retry IS the recovery. + // Short-circuiting on lastError made the stranded-receipt case permanent — + // the opposite of what the guard was for. const r = recorder({ - createPurchase: async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true }) as any, + createPurchase: async (_t, input) => { + r.purchaseCalls.push(input); + return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "already-attached" } as any; + }, }); const result = await bookReceipt(row({ lastError: "attachment-failed:failed:500" }), r.deps); - assert.deepEqual(result, { - outcome: "needs-review", - reason: "attachment-unconfirmed", - releaseStrongKey: false, + assert.equal(result.outcome, "booked", "the recovery succeeded and the row books"); + assert.equal(r.purchaseCalls.length, 1, "the recovery attempt actually happened"); + assert.equal(r.expenses.length, 1); +}); + +test("already-attached counts as attached on the fresh-create path too", async () => { + const r = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "already-attached", + }) as any, }); - assert.equal(r.purchaseCalls.length, 0, "not even re-queried — the answer is already known"); + assert.equal((await bookReceipt(row(), r.deps)).outcome, "booked"); }); test("a QBTimeoutError retries on the backoff schedule", async () => { @@ -387,7 +421,8 @@ test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTA test("alreadyExists books identically — the lost-response retry", async () => { const r = recorder({ createPurchase: async (_t, input) => ({ - ok: true, qbPurchaseId: "QB-7", docNumber: input.fileId.slice(0, 21), alreadyExists: true, + ok: true, qbPurchaseId: "QB-7", docNumber: input.fileId.slice(0, 21), + alreadyExists: true, attachment: "already-attached", }) as any, }); const result = await bookReceipt(row(), r.deps); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 2d072d06d..aead016b0 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -24,7 +24,7 @@ import { type WorkerDependencies, type WorkerRow, } from "../src/lib/receipt-intake/worker"; -import type { ReadOutcome } from "../src/lib/receipt-intake/read"; +import { normalizeDocType, type ReadOutcome } from "../src/lib/receipt-intake/read"; import type { BookResult } from "../src/lib/receipt-intake/book"; import { QBTimeoutError } from "../src/lib/quickbooks"; import { QboAccountConfigError, QboPurchaseFaultError } from "../src/lib/qbo-receipt-push"; @@ -86,12 +86,12 @@ interface Harness { reads: number; books: number; applied: ReadPatch[]; - states: { id: string; state: string; reason: string | null }[]; + states: { id: string; state: string; reason: string | null; patch?: Partial }[]; promoted: string[]; finished: { id: string; stateReason: string | null }[]; deferred: { id: string; busyPasses: number }[]; retried: { id: string; attempts: number; reason: string }[]; - claimOpts: { requeueDryRunParked: boolean }[]; + claimOpts: { retireShadowBacklog: boolean }[]; sweepCalls: number; clock: number; } @@ -103,7 +103,7 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) deps: null as unknown as WorkerDependencies, }; h.deps = { - claim: async opts => { h.claimOpts.push(opts); return { rows, requeued: 0 }; }, + claim: async opts => { h.claimOpts.push(opts); return { rows, shadowRetired: 0 }; }, isDryRunEnabled: () => true, sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], @@ -111,7 +111,7 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) read: async () => { h.reads++; return goodRead; }, applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, findWeakHit: async () => null, - applyState: async (id, state, reason) => { h.states.push({ id, state, reason }); }, + applyState: async (id, state, reason, patch) => { h.states.push({ id, state, reason, patch }); }, finishRouting: async (id, stateReason) => { h.finished.push({ id, stateReason }); }, companyTimeZone: async () => "America/Los_Angeles", promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, @@ -285,35 +285,36 @@ test("a strong-key loss to a DIFFERENT vendor is a collision, not a duplicate", // ── Dry-run starvation (Codex blocker 1) ───────────────────────────────────── -test("the shadow week does NOT ask the claim to requeue", async () => { +test("the shadow week does NOT retire anything", async () => { const h = harness([workerRow({ state: "READ", dryRun: true })], { isDryRunEnabled: () => true }); const summary = await runIntakeWorker(h.deps); - assert.deepEqual(h.claimOpts, [{ requeueDryRunParked: false }]); - assert.equal(summary.requeued, undefined); + assert.deepEqual(h.claimOpts, [{ retireShadowBacklog: false }]); + assert.equal(summary.shadowRetired, undefined); }); -test("the FIRST live pass asks the claim to un-park the backlog, INSIDE the lock", async () => { - // Parked rows are excluded from the claim (see the cron route's - // NOT_DRY_RUN_PARKED) precisely so they cannot starve the ten-row batch — - // which also means nothing else would ever wake them. The requeue rides in - // the claim transaction so two overlapping invocations cannot both un-park - // the same backlog and race each other's claim. +test("CUTOVER: the shadow backlog is RETIRED, never requeued", async () => { + // The double-booking hazard this closes: v2's QBO identity for an + // email/chat/mobile/web row is the intake UUID, which v1 never saw, so + // QuickBooks' DocNumber idempotency could not recognise the Purchase v1 + // already made — and requeuing would have booked the entire shadow backlog + // a second time, on real books, in one pass. const h = harness([], { isDryRunEnabled: () => false, - claim: async opts => { h.claimOpts.push(opts); return { rows: [], requeued: 7 }; }, + claim: async opts => { h.claimOpts.push(opts); return { rows: [], shadowRetired: 7 }; }, }); const summary = await runIntakeWorker(h.deps); - assert.deepEqual(h.claimOpts, [{ requeueDryRunParked: true }]); - assert.equal(summary.requeued, 7); + assert.deepEqual(h.claimOpts, [{ retireShadowBacklog: true }]); + assert.equal(summary.shadowRetired, 7); + assert.equal(h.books, 0, "nothing from the shadow week is ever booked by v2"); - // Idempotent by construction: nothing is left matching the predicate. + // Idempotent by construction: SHADOW_DONE no longer matches the predicate. const second = harness([], { isDryRunEnabled: () => false }); - assert.equal((await runIntakeWorker(second.deps)).requeued, undefined, "a no-op requeue is not reported"); + assert.equal((await runIntakeWorker(second.deps)).shadowRetired, undefined, "a no-op retire is not reported"); }); -test("a run that loses the lock does nothing at all — including the requeue", async () => { - // The requeue is now part of the claim transaction, so losing the lock - // means losing it too. That is correct: the run that HOLDS the lock does it. +test("a run that loses the lock does nothing at all — including the retire", async () => { + // The retire is part of the claim transaction, so losing the lock means + // losing it too. That is correct: the run that HOLDS the lock does it. const h = harness([], { isDryRunEnabled: () => false, claim: async () => null }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary, { processed: 0, byState: {}, skipped: "already-running" }); @@ -581,8 +582,11 @@ test("a weak-only hit still asks a human, and KEEPS the strong key", async () => assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); assert.equal(h.states[0].reason, "weak-dup:row-twin"); assert.equal(h.states[0].state, "NEEDS_REVIEW"); - // applyState was called WITHOUT a patch clearing dedupStrongKey. - assert.equal(h.applied[0].dedupStrongKey, "2026-08-03|82766"); + // ...and it RELEASES the strong key: nothing was sent to QuickBooks, so the + // documented pre-send rule applies here like anywhere else. Holding it made + // a CORRECTED resend collide with a row that was never booked, leaving two + // rows in review and neither able to proceed. + assert.equal(h.states[0].patch?.dedupStrongKey, null); }); test("a document-level gate short-circuits BOTH nets and claims no key", async () => { @@ -710,3 +714,75 @@ test("a tax equal to the total is refused end to end", async () => { assert.equal(h.applied[0].totalCents, 36498, "the total is untouched"); assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); }); + +// ── Fail-closed classifier (round-5 item 4) ──────────────────────────────── + +test("a missing or unknown doc_type is NEVER treated as a receipt", async () => { + // The old default was "receipt", and any unrecognised string also slipped + // past the exact multi/non_receipt checks. A truncated response, a schema + // change, or a prompt-injected document that suppressed the field while + // supplying plausible amounts went straight at QuickBooks. + for (const docType of ["", "unknown", "invoice", "RECEIPT_PLEASE_BOOK", "non-receipt"]) { + const h = harness([workerRow()], { + read: async () => ({ + ok: true, + read: { ...goodRead.read, docType: normalizeDocType(docType) }, + } as ReadOutcome), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }, JSON.stringify(docType)); + assert.equal(h.applied[0].stateReason, "unknown-doc-type", JSON.stringify(docType)); + assert.equal(h.applied[0].dedupStrongKey, null, "and it claims no key"); + } +}); + +test("normalizeDocType accepts exactly the four the prompt may return", () => { + for (const ok of ["receipt", "check", "multi", "non_receipt"]) { + assert.equal(normalizeDocType(ok), ok); + assert.equal(normalizeDocType(ok.toUpperCase()), ok, "case is normalised"); + } + for (const bad of [undefined, null, "", " ", "invoice", "reciept", 42, {}, ["receipt"]]) { + assert.equal(normalizeDocType(bad), "unknown", JSON.stringify(bad)); + } + // Surrounding whitespace is a formatting artefact, not a different answer. + assert.equal(normalizeDocType(" receipt "), "receipt"); +}); + +// ── Fallback date in the company zone (round-5 item 5) ───────────────────── + +test("an unreadable date falls back to the COMPANY's calendar day, not UTC's", async () => { + // 2026-08-04T02:00Z is still the EVENING OF THE 3RD in Pacific. The old + // toISOString().slice(0,10) gave "2026-08-04", which changed the receipt's + // date, its dedup key, and its reporting period. + const h = harness([workerRow({ createdAt: new Date("2026-08-04T02:00:00.000Z") })], { + read: async () => ({ ok: true, read: { ...goodRead.read, date: "" } } as ReadOutcome), + companyTimeZone: async () => "America/Los_Angeles", + }); + await runIntakeWorker(h.deps); + assert.equal(h.applied[0].dedupWeakKey, "lowes|2026-08-03|364.98|amt", "the KEY uses the local day"); + assert.equal(h.applied[0].txnDate!.toISOString(), "2026-08-03T07:00:00.000Z"); + // Still no strong key: a fallback date is our guess, not the document's. + assert.equal(h.applied[0].dedupStrongKey, null); +}); + +// ── The sweep lives inside the run's budget (round-5 item 7) ─────────────── + +test("the deadline starts at invocation entry, so a slow sweep cannot overrun it", async () => { + // The sweep downloads objects. Timing it OUT of the budget meant it could + // eat the platform timeout and the worker would still go on to start a 25s + // Gemini read and a QBO round trip. + const h = harness([workerRow(), workerRow({ id: "row-2" })], { + sweepStaleStaging: async shouldStop => { + h.sweepCalls++; + assert.equal(typeof shouldStop, "function", "the sweep is given the deadline"); + assert.equal(shouldStop(), false, "not yet out of time"); + h.clock += RUN_SOFT_DEADLINE_MS + 1_000; // a slow sweep + assert.equal(shouldStop(), true, "the sweep can see it is out of time"); + return 1; + }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.reads, 0, "no Gemini call after the budget is gone"); + assert.equal(summary.processed, 0); + assert.equal(summary.deferredToNextRun, 2, "both rows keep their lease for the next run"); +}); From 5d84c172beafedcdb19a1960c3dff1c80022d08c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:46:22 -0700 Subject: [PATCH 041/144] chore: re-trigger CI for fd0fb4b8 (Actions did not fire on the push) Co-Authored-By: Claude Fable 5.1 From d75de00a5a1845de65459ecaad824ebb9a03970b Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 00:01:49 -0700 Subject: [PATCH 042/144] fix(health): re-apply the intake probes onto Phase 0's pipeline-health MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebase onto ad57b30f (which carries Phase 4 via main) conflicted in pipeline-health.ts and its test, and my first pass resolved them as a blind union — which is wrong for TypeScript: it stacked three copies of the Promise.all destructuring and left the file unparseable. Rebuilt properly: took Phase 0's file verbatim and re-applied the intake additions on top, so upstream's lastPaymentsSync heartbeat and the attachment-failed status count are preserved intact alongside the three intake probes (stuck / needsReview / unassigned). Co-Authored-By: Claude Fable 5.1 --- src/lib/pipeline-health.ts | 81 +++++++++++++++---------------- tests/pipeline-health.test.ts | 89 ++++++++++++----------------------- 2 files changed, 71 insertions(+), 99 deletions(-) diff --git a/src/lib/pipeline-health.ts b/src/lib/pipeline-health.ts index ba8308ed9..d530705ad 100644 --- a/src/lib/pipeline-health.ts +++ b/src/lib/pipeline-health.ts @@ -84,14 +84,18 @@ export interface PipelineHealth { /** Automation events (ANY kind) that errored in the last 24h. */ stuck: CountProbe; /** - * Receipt Pipeline v2 (ReceiptIntake). The v1 probes above all read + * Receipt Pipeline v2 (ReceiptIntake). Every other probe here reads * AutomationEvent, which only ever records a BOOKING — so a v2 row that - * never reaches QuickBooks is invisible to every check in this file. A - * jammed intake queue would report a perfectly healthy pipeline right up - * until somebody noticed the expenses were missing. + * never reaches QuickBooks is invisible to all of them. A jammed intake + * queue reported a perfectly healthy pipeline right up until somebody + * noticed the expenses were missing. */ intake: { - /** RECEIVED or BOOKING and older than 6h — the queue is not draining. */ + /** + * Three shapes of "the worker stopped": RECEIVED/BOOKING overdue, + * STAGING overdue (the route died mid-upload, or the sweeper is dead), + * and live READ overdue (a worker that died right after routing). + */ stuck: CountProbe; /** NEEDS_REVIEW backlog. Reported always; a reason only when rows are STUCK. */ needsReview: CountProbe; @@ -250,6 +254,26 @@ export function evaluatePipelineHealth(input: { reasons.push(`errors-24h:${input.stuck.count}`); } + // A row sitting in RECEIVED/BOOKING/STAGING or live READ past its fuse means + // the worker is not draining the queue — a wedged cron, an exhausted retry + // budget, a storage outage. The backlog number rides along so the digest can + // say how big the hole is, but only the STUCK count is a failure: + // NEEDS_REVIEW rows are working as designed (a human was asked a question) + // and would otherwise hold the pipeline red until somebody cleared them. + if (input.intakeStuck.status === "ok" && input.intakeStuck.count > 0) { + const backlog = + input.intakeNeedsReview.status === "ok" ? `,needs-review:${input.intakeNeedsReview.count}` : ""; + reasons.push(`intake-stuck:${input.intakeStuck.count}${backlog}`); + } + + // A receipt waiting hours for someone to say which job it belongs to is not + // "working as designed" — it is an expense that will never reach job cost. + // Its own reason, because the fix is different: assign a project, not + // restart a worker. + if (input.intakeUnassigned.status === "ok" && input.intakeUnassigned.count > 0) { + reasons.push(`intake-unassigned:${input.intakeUnassigned.count}`); + } + if (input.lastPaymentsSync.status === "ok") { // The money rail's heartbeat. Null means the hourly cron has never // completed a run we can see; stale means it stopped. Either way the @@ -268,24 +292,6 @@ export function evaluatePipelineHealth(input: { // It proves the cron is alive, so it counts for freshness — but it must // be reported the same day, not hidden inside the 26h staleness window. else if (input.lastPaymentsSync.runStatus === "partial") reasons.push("payments-sync-partial"); - // A row sitting in RECEIVED or BOOKING for six hours means the worker is - // not draining the queue — a wedged cron, an exhausted retry budget, a - // storage outage. The backlog number rides along so the digest can say how - // big the hole is, but only the STUCK count is a failure: NEEDS_REVIEW rows - // are working as designed (a human was asked a question) and would - // otherwise hold the pipeline red until somebody cleared the queue. - if (input.intakeStuck.status === "ok" && input.intakeStuck.count > 0) { - const backlog = - input.intakeNeedsReview.status === "ok" ? `,needs-review:${input.intakeNeedsReview.count}` : ""; - reasons.push(`intake-stuck:${input.intakeStuck.count}${backlog}`); - } - - // A receipt that has been waiting hours for someone to say which job it - // belongs to is not "working as designed" — it is an expense that will - // never reach job cost. It is its own reason, separate from intake-stuck, - // because the fix is different: assign a project, not restart a worker. - if (input.intakeUnassigned.status === "ok" && input.intakeUnassigned.count > 0) { - reasons.push(`intake-unassigned:${input.intakeUnassigned.count}`); } if (input.lastReceiptPush.status === "ok") { @@ -352,11 +358,8 @@ export async function getPipelineHealth(): Promise { /** Any probe failure is reported as such — never silently downgraded to "nothing found". */ const probe = runProbe; - const [intuit, lastPurchase, lastPush, lastPaymentsSync, receiptRows, lastBankLine, stuck] = await Promise.all([ - const [intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck, intakeStuck, intakeNeedsReview] = - await Promise.all([ const [ - intuit, lastPurchase, lastPush, receiptRows, lastBankLine, stuck, + intuit, lastPurchase, lastPush, lastPaymentsSync, receiptRows, lastBankLine, stuck, intakeStuck, intakeNeedsReview, intakeUnassigned, ] = await Promise.all([ fetchIntuitStatus(), @@ -451,19 +454,17 @@ export async function getPipelineHealth(): Promise { }), 0, ), - // ReceiptIntake: the v2 queue. STAGING is excluded on purpose — those - // rows are mid-upload and the intake route's own 15-minute sweeper owns - // them; counting them here would flag every in-flight request. - // Three shapes of "the worker stopped", all of which used to read green: - // RECEIVED/BOOKING overdue — the classic jam. - // STAGING overdue — the route died mid-upload, or the - // sweeper is dead. Excluded before, - // which hid a dead worker completely. - // READ overdue, LIVE only — a worker that died right after routing - // leaves a bookable row parked forever. - // dryRun rows legitimately rest in READ - // for the whole shadow week, so they are - // excluded or the check is red by design. + // ReceiptIntake: the v2 queue. Three shapes of "the worker stopped", + // all of which used to read green: + // RECEIVED/BOOKING overdue — the classic jam. + // STAGING overdue — the route died mid-upload, or the sweeper + // is dead. STAGING is invisible to the + // claim by design, so nothing else notices. + // READ overdue, LIVE only — a worker that died right after routing + // leaves a bookable row parked forever. + // dryRun rows legitimately REST in READ for + // the whole shadow week, so they are + // excluded or the check is red by design. probe( "intakeStuck", () => diff --git a/tests/pipeline-health.test.ts b/tests/pipeline-health.test.ts index 421ff4743..f536e7c71 100644 --- a/tests/pipeline-health.test.ts +++ b/tests/pipeline-health.test.ts @@ -556,7 +556,9 @@ test("the journey mapper renders attachment-failed as failed, not in-flight", as // arriving - the bot has already stopped. assert.equal(journey.finalState, "error"); assert.equal(journey.finalReason, "failed:fault"); -// ── Receipt Pipeline v2 intake queue (Codex round 3, item 7) ──────────────── +}); + +// ── Receipt Pipeline v2 intake queue ─────────────────────────────────────── // Every other probe in this file reads AutomationEvent, which only records a // BOOKING — so a v2 row that never reaches QuickBooks is invisible to all of // them. A jammed intake queue reported a perfectly healthy pipeline. @@ -578,6 +580,24 @@ test("a NEEDS_REVIEW backlog alone is NOT a failure", () => { assert.deepEqual(v, { ok: true, reasons: [] }); }); +test("receipts nobody assigned a job to are an ALERT, not a green backlog", () => { + // NEEDS_JOB is terminal for the worker, so it can pile up indefinitely + // while every other probe reads green. Its own reason, because the fix is + // different: assign a project, not restart a worker. + const v = evaluatePipelineHealth(snapshot({ intakeUnassigned: { status: "ok", count: 5 } })); + assert.equal(v.ok, false); + assert.deepEqual(v.reasons, ["intake-unassigned:5"]); +}); + +test("stuck and unassigned are reported separately", () => { + const v = evaluatePipelineHealth(snapshot({ + intakeStuck: { status: "ok", count: 2 }, + intakeUnassigned: { status: "ok", count: 3 }, + })); + assert.ok(v.reasons.some(r => r.startsWith("intake-stuck:2"))); + assert.ok(v.reasons.includes("intake-unassigned:3")); +}); + test("an intake probe that FAILED is not an intake probe that found nothing", () => { for (const name of ["intakeStuck", "intakeNeedsReview", "intakeUnassigned"] as const) { const v = evaluatePipelineHealth(snapshot({ [name]: { status: "error", reason: "timeout", count: 0 } })); @@ -595,7 +615,15 @@ test("the stuck reason survives a failed backlog probe rather than lying about i assert.ok(v.reasons.includes("probe-failed:intakeNeedsReview")); }); -test("the digest prints both intake numbers", () => { +test("STAGING gets a much shorter fuse than the working states", () => { + // STAGING is meant to last one HTTP request; RECEIVED/BOOKING/READ are + // queue states measured in hours. + assert.equal(INTAKE_STAGING_STUCK_MINUTES, 30); + assert.equal(INTAKE_STUCK_HOURS, 6); + assert.ok(INTAKE_STAGING_STUCK_MINUTES * 60_000 < INTAKE_STUCK_HOURS * 3_600_000); +}); + +test("the digest prints all three intake numbers", () => { const { text } = formatPipelineDigest(sampleHealth({ intake: { stuck: { status: "ok", count: 3 }, @@ -619,60 +647,3 @@ test("the digest says a failed intake probe is unavailable, never zero", () => { assert.match(text, /Receipt intake stuck >6h: unavailable \(probe failed\)/); assert.match(text, /Receipt intake awaiting review: unavailable \(probe failed\)/); }); - -test("the intake stuck probe covers the three shapes of 'the worker stopped'", async () => { - // Regression: it counted only RECEIVED/BOOKING, so a dead worker left stale - // STAGING rows invisible, and a worker that died right after routing left - // live READ rows invisible. Both reported green. - const wheres: any[] = []; - const db = { - receiptIntake: { - count: async (args: any) => { wheres.push(args.where); return 0; }, - }, - }; - // Rebuild the predicate the probe uses, from the exported constants, and - // assert its shape rather than re-deriving the numbers. - const now = Date.parse("2026-09-01T14:00:00.000Z"); - const where = { - OR: [ - { state: { in: ["RECEIVED", "BOOKING"] }, createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * 3_600_000) } }, - { state: "STAGING", createdAt: { lt: new Date(now - INTAKE_STAGING_STUCK_MINUTES * 60_000) } }, - { state: "READ", dryRun: false, createdAt: { lt: new Date(now - INTAKE_STUCK_HOURS * 3_600_000) } }, - ], - }; - await db.receiptIntake.count({ where }); - - const branches = wheres[0].OR; - assert.equal(branches.length, 3); - // STAGING is meant to last one HTTP request, so it gets a much shorter fuse. - assert.equal(INTAKE_STAGING_STUCK_MINUTES, 30); - assert.ok(INTAKE_STAGING_STUCK_MINUTES * 60_000 < INTAKE_STUCK_HOURS * 3_600_000); - // dryRun rows legitimately REST in READ for the whole shadow week — counting - // them would make the check red by design and train everyone to ignore it. - assert.equal(branches[2].dryRun, false); -}); - -test("receipts nobody assigned a job to are an ALERT, not a green backlog", () => { - // NEEDS_JOB is terminal for the worker, so it can pile up indefinitely - // while every other probe reads green — the exact silent failure this whole - // check exists to eliminate. Its own reason, because the fix is different: - // assign a project, not restart a worker. - const v = evaluatePipelineHealth(snapshot({ intakeUnassigned: { status: "ok", count: 5 } })); - assert.equal(v.ok, false); - assert.deepEqual(v.reasons, ["intake-unassigned:5"]); -}); - -test("a freshly uploaded unassigned receipt is not an alert", () => { - // Only rows OLDER than the stuck threshold are counted by the probe, so a - // receipt uploaded ten minutes ago never reaches this reason. - assert.deepEqual(evaluatePipelineHealth(snapshot()), { ok: true, reasons: [] }); -}); - -test("unassigned and stuck are reported separately", () => { - const v = evaluatePipelineHealth(snapshot({ - intakeStuck: { status: "ok", count: 2 }, - intakeUnassigned: { status: "ok", count: 3 }, - })); - assert.ok(v.reasons.some(r => r.startsWith("intake-stuck:2"))); - assert.ok(v.reasons.includes("intake-unassigned:3")); -}); From 4daa8c20cc232c3d51bf6b0d548ff8f332cb0bd5 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 00:53:42 -0700 Subject: [PATCH 043/144] fix(receipts): cutover boundary, two-step upload, split secrets, phase re-check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 6 (1-7) plus the Phase 3 gate's three Phase 1 findings. 1 CUTOVER BOUNDARY. Retiring the WHOLE shadow backlog was wrong in the other direction: rows received after v1 stopped booking were booked by nobody, and retiring those drops real expenses. The backlog now splits on `cutoverV1StoppedAt` (AutomationSetting, or CUTOVER_V1_STOPPED_AT) — before it SHADOW_DONE, after it handed to v2. Nothing in the DB can infer that instant, so with it unset the pass touches neither side and logs cutover-boundary-missing. A malformed value parses to null, never epoch. 2 TWO-STEP UPLOAD. The inline POST puts the file in the request body, which the platform caps near 4.5 MB — phone photos were dying at the edge with an opaque 413 that never reached our code. It now refuses over 4 MB with a 413 naming the new path: /intake/start returns a signed upload URL bound to a server-chosen path, the client PUTs straight to storage, /intake/{id}/finalize re-derives mime, size and sha FROM STORAGE and publishes. Both paths share decideSource, so provenance rules cannot drift apart. 3 Expense.date is re-anchored to company local midnight. txnDate is a @db.Date (UTC midnight), which as an instant is 5pm the previous day in Pacific. 4 A booking refuses to START with under 25s left, and threads ONE RouteDeadline through the token refresh and the create. A booking cut off mid-flight can leave a Purchase in the books that the row never learns about. 5 not-found is now an affirmative 404 or an explicit NoSuchKey/"Object not found" only. 400 counted before — and Supabase returns 400 for a bad JWT or an expired key, so a rotation would have parked the queue and released every strong key on the way out. 6 The state CHECK is convergent: compare pg_get_constraintdef, DROP+ADD on difference. IF NOT EXISTS alone left an OLD state set in place, so the first SHADOW_DONE write would fail inside the cutover transaction while the script that exists to prevent that reported "ok". (a) Both the captured and suggested cost codes are re-validated against the FINAL project at booking (isCostCodeAllowedForProject). A code from the project the row had at READ time is not a phase of the one it books to. Mismatch clears it and books uncoded with a note. (b) suggested_phase_confidence (0..1) flows read -> row -> booking audit. Absent is null, never 0. (c) RECEIPT_INTAKE_SECRET may only ingest (drive/email/chat); RECEIPT_ARCHIVE_SECRET may only read BOOKED/ARCHIVED and post the archive callback. Cross-use is 403. Identical values for both are refused. Documented in .env.example and §7. Co-Authored-By: Claude Fable 5.1 --- .env.example | 24 ++ .github/workflows/ci.yml | 3 + docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 76 ++++++- e2e/receipt-intake.spec.ts | 50 ++++- package.json | 4 +- scripts/apply-receipt-intake.mjs | 44 +++- .../api/cron/receipt-intake-worker/route.ts | 75 +++++-- .../receipts/intake/[id]/archived/route.ts | 11 +- .../receipts/intake/[id]/finalize/route.ts | 122 ++++++++++ src/app/api/receipts/intake/route.ts | 38 +++- src/app/api/receipts/intake/start/route.ts | 154 +++++++++++++ src/lib/receipt-intake/book.ts | 125 ++++++++++- src/lib/receipt-intake/cutover.ts | 51 +++++ src/lib/receipt-intake/intake-auth.ts | 98 ++++++-- src/lib/receipt-intake/intake-core.ts | 74 ++++++ src/lib/receipt-intake/read.ts | 32 ++- src/lib/receipt-intake/worker.ts | 66 +++++- src/lib/secure-storage.ts | 29 ++- src/proxy.ts | 5 +- tests/apply-receipt-intake.test.ts | 46 +++- tests/receipt-intake-auth.test.ts | 210 ++++++++++++++++++ tests/receipt-intake-book.test.ts | 157 +++++++++++++ tests/receipt-intake-cutover.test.ts | 33 +++ tests/receipt-intake-worker.test.ts | 70 ++++-- tests/secure-storage-classification.test.ts | 64 ++++++ 25 files changed, 1559 insertions(+), 102 deletions(-) create mode 100644 src/app/api/receipts/intake/[id]/finalize/route.ts create mode 100644 src/app/api/receipts/intake/start/route.ts create mode 100644 src/lib/receipt-intake/cutover.ts create mode 100644 src/lib/receipt-intake/intake-core.ts create mode 100644 tests/receipt-intake-cutover.test.ts create mode 100644 tests/secure-storage-classification.test.ts diff --git a/.env.example b/.env.example index 8e71c67c9..c8c1b18f0 100644 --- a/.env.example +++ b/.env.example @@ -30,3 +30,27 @@ MCP_SECRET= # Bank-ledger automation: keep these separate; the posting-status key is read-only. BANK_LEDGER_INGEST_SECRET=generate_a_strong_random_secret_here BANK_LEDGER_STATUS_SECRET=generate_a_different_strong_random_secret_here + +# --- Receipt Pipeline v2 (docs/plans/PHASE-1-INTAKE-CORE-SPEC.md) --- +# TWO machine secrets, deliberately separate: they belong to different programs +# with different blast radii, and they rotate independently. Setting them to the +# SAME value is refused at runtime — that would silently re-merge the two. +# +# The Apps Script forwarders. May only INGEST (POST /api/receipts/intake, +# /start, /{id}/finalize) and only under source drive|email|chat. Cannot read +# the queue and cannot archive. +RECEIPT_INTAKE_SECRET= +# The nightly Drive archive mirror. May only READ BOOKED/ARCHIVED rows +# (GET /api/receipts/intake?state=BOOKED) and report back what it archived +# (POST /api/receipts/intake/{id}/archived). Cannot create or publish a row. +RECEIPT_ARCHIVE_SECRET= +# +# Shadow mode. UNSET or "true" = dry run: rows are read, deduped and routed, and +# NOTHING is booked. Set to the literal "false" only at cutover. +RECEIPT_INTAKE_DRYRUN= +# The instant the Apps Script stopped booking (ISO 8601). Written at the flip to +# forwarder mode; the first live worker pass uses it to split the shadow backlog +# into "v1 already booked this" and "nobody booked this". With it unset the +# cutover refuses to touch either side. Can also live in the `cutoverV1StoppedAt` +# AutomationSetting row, which takes precedence. +CUTOVER_V1_STOPPED_AT= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81f7036b6..24cbf1a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -205,6 +205,9 @@ jobs: # a literal rather than a repo secret. The spec's "env var unset" case is # a unit test (tests/receipt-intake-auth.test.ts), not this job. RECEIPT_INTAKE_SECRET: "e2e-ci-receipt-intake-secret" + # The archive mirror's key is DELIBERATELY different — the specs assert + # that cross-use between the two capabilities is a 403. + RECEIPT_ARCHIVE_SECRET: "e2e-ci-receipt-archive-secret" # Forces the Stage A daily-log task matcher (daily-log-task-match.ts) onto # its deterministic keyword fallback instead of calling Gemini, so # time-suggestion.spec.ts's Stage A end-to-end test is reproducible in CI. diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index 7fe298714..0ac0187a2 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -435,6 +435,65 @@ here); multipart buffering before the size check (the platform body limit applie PDFs carrying embedded JavaScript (never opened server-side — the bytes go to Gemini and to QBO as an attachment). + +### Two machine secrets, not one + +They belong to different programs, so they are different keys and rotate independently. +A single shared secret gave a script that only copies files to Drive the power to inject +Purchases into the books, and gave the ingest forwarders the power to enumerate every +receipt in the system. + +| secret | may do | may NOT do | +|---|---|---| +| `RECEIPT_INTAKE_SECRET` (the Apps Script forwarders) | `POST /api/receipts/intake`, `/intake/start`, `/intake/{id}/finalize`, declaring `source` in **drive, email, chat** only | read the queue; archive anything | +| `RECEIPT_ARCHIVE_SECRET` (the nightly Drive mirror) | `GET /api/receipts/intake?state=BOOKED|ARCHIVED` (minimal field set + signed URL), `POST /api/receipts/intake/{id}/archived` | create, publish or modify a row; declare any source | + +Cross-use is **403**, not 401: the caller is authenticated, it is just holding the other +program's key, and saying so is what makes a mis-wired script obvious instead of looking +like a rotation problem. Setting both variables to the same value is refused at runtime — +that would silently undo the split. + +### Phase suggestion: confidence, and re-validation at booking + +The reader returns `suggested_phase_confidence` (0..1) alongside the phase code. It is +persisted as `ReceiptIntake.suggestedConfidence`, so the queue can sort by it, and it is +recorded on the booking's `AutomationEvent` when the suggestion is what got used. Absent or +unparseable is **null, never 0** — "the model didn't say" and "the model is sure it is a +poor match" have to stay distinguishable. + +At booking, BOTH the captured `costCodeId` and the suggestion are re-checked against the +project the row will actually book to, via the same `isCostCodeAllowedForProject` the +clock-in uses. The row may have been read while it had no project (`NEEDS_JOB`) or a +different one that a human then corrected, and a cost code from the old project is not a +phase of the new one. A mismatch clears the code and books UNCODED with a note — the +receipt and its total are still right, and a bookkeeper assigning a phase is routine, while +an expense silently attached to the wrong phase is not. + +### Upload paths (two of them) + +`POST /api/receipts/intake` carries the file in the REQUEST BODY, so it is limited by the +serverless body cap (~4.5 MB, and base64 JSON inflates a payload by a third). It rejects +anything over **4 MB** with a 413 that names the two-step path. That limit is about the +transport, not the document. + +For anything larger — most phone photos — use the two-step flow, which never puts the bytes +through this server at all: + +1. `POST /api/receipts/intake/start` with `{mimeType, fileName?, fileSize?, source?, + sourceRef?, uploadId?, projectId?}` -> `{id, uploadUrl, token, storagePath, maxBytes}`. + Creates the row in `STAGING` (invisible to the worker) and returns a short-lived Supabase + signed upload URL bound to a server-chosen path. +2. `PUT` the bytes straight to `uploadUrl`. +3. `POST /api/receipts/intake/{id}/finalize` with `{sha256?}` -> publishes `STAGING` -> + `RECEIVED`. The server re-reads the object and derives the mime, the size and the sha + FROM STORAGE; a declared `sha256` is checked against that and a mismatch is a 409. Over + 15 MB or an unreadable format deletes the row and refuses. + +Both paths share `decideSource` (provenance and idempotency), so a session/Bearer caller can +never choose `source` or `sourceRef` on either, and `uploadId` is scoped to the authenticated +user on both. Both new paths are on the proxy's exact-match bypass and both refuse a +`next-action` dispatch with 403. + ### CUTOVER SEQUENCE — do these in this order (2026-09-02) The hazard this order exists to prevent: v2's QuickBooks identity for an @@ -458,10 +517,19 @@ an overlap safe in general. 4. **Confirm zero v1 bookings for 24 hours.** Watch the Automation register and QBO. This is the step that makes the next one safe: it proves v1 is out of the books before v2 enters them, so the two can never both create a Purchase for one document. -5. **Only then set `RECEIPT_INTAKE_DRYRUN=false`.** On its first pass the worker RETIRES the - entire shadow backlog to `SHADOW_DONE` / `booked-by-v1` — terminal, never booked by v2, - because v1 already booked all of it. Nothing is requeued. Only rows received AFTER this - point are booked by v2. +4a. **Record the boundary.** When step 3 happens, write the instant v1 stopped booking into + the `cutoverV1StoppedAt` AutomationSetting row (or the `CUTOVER_V1_STOPPED_AT` env var) as + an ISO timestamp. This is the ONLY input that separates "v1 booked it" from "nobody booked + it", and nothing in the database can infer it. +5. **Only then set `RECEIPT_INTAKE_DRYRUN=false`.** On its first pass the worker splits the + shadow backlog on that boundary: + - received BEFORE it -> `SHADOW_DONE` / `booked-by-v1`. Terminal; v2 never books these, + because v1 already did and QBO's DocNumber idempotency cannot recognise a v2 UUID. + - received AFTER it -> handed to v2 for real booking. v1 had already stopped, so nobody + booked these; retiring them would drop real expenses on the floor. + With no boundary recorded the pass touches NEITHER side and logs + `cutover-boundary-missing`. That is deliberate: retiring on a guess destroys evidence, + requeuing on a guess double-books, and a visible no-op is the only honest third option. Retired rows keep their read results and dedup keys, so a post-cutover resend of a shadow-week receipt still collides with them and is caught as a duplicate. diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 1e203fc5c..1bac0031b 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -29,6 +29,9 @@ import { PrismaClient } from "@prisma/client"; const prisma = new PrismaClient(); const INTAKE_PATH = "/api/receipts/intake"; const SECRET = process.env.RECEIPT_INTAKE_SECRET || ""; +// The archive mirror holds a DIFFERENT key: it may read BOOKED/ARCHIVED rows and +// report what it archived, and nothing else. Cross-use is a 403. +const ARCHIVE_SECRET = process.env.RECEIPT_ARCHIVE_SECRET || ""; // One prefix for everything this file creates, so teardown can be exact. const REF_PREFIX = "drive:e2e-intake-"; @@ -465,7 +468,7 @@ test.describe("intake GET", () => { storageState: { cookies: [], origins: [] }, }); const res = await machine.get(`${INTAKE_PATH}?state=BOOKED`, { - headers: { "x-receipt-intake-secret": SECRET }, + headers: { "x-receipt-intake-secret": ARCHIVE_SECRET }, maxRedirects: 0, }); expect(res.status()).toBe(200); @@ -489,14 +492,14 @@ test.describe("intake GET", () => { }); for (const state of ["NEEDS_REVIEW", "RECEIVED", "READ", ""]) { const res = await machine.get(`${INTAKE_PATH}${state ? `?state=${state}` : ""}`, { - headers: { "x-receipt-intake-secret": SECRET }, + headers: { "x-receipt-intake-secret": ARCHIVE_SECRET }, maxRedirects: 0, }); expect(res.status(), state || "(no state)").toBe(400); } // ARCHIVED is allowed — the mirror re-checks what it already copied. const archived = await machine.get(`${INTAKE_PATH}?state=ARCHIVED`, { - headers: { "x-receipt-intake-secret": SECRET }, + headers: { "x-receipt-intake-secret": ARCHIVE_SECRET }, maxRedirects: 0, }); expect(archived.status()).toBe(200); @@ -558,7 +561,7 @@ test.describe("archive callback", () => { expect(sessionAttempt.status()).toBe(401); const notBooked = await anonymous.post(`${INTAKE_PATH}/${id}/archived`, { - headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + headers: { "content-type": "application/json", "x-receipt-intake-secret": ARCHIVE_SECRET }, data: JSON.stringify({ driveFileId: "DRIVE1" }), maxRedirects: 0, }); @@ -566,7 +569,7 @@ test.describe("archive callback", () => { await prisma.receiptIntake.update({ where: { id }, data: { state: "BOOKED" } }); const archive = (driveFileId: string) => anonymous.post(`${INTAKE_PATH}/${id}/archived`, { - headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + headers: { "content-type": "application/json", "x-receipt-intake-secret": ARCHIVE_SECRET }, data: JSON.stringify({ driveFileId }), maxRedirects: 0, }); @@ -607,7 +610,7 @@ test.describe("archive callback", () => { storageState: { cookies: [], origins: [] }, }); const res = await machine.post(`${INTAKE_PATH}/no-such-row/archived`, { - headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + headers: { "content-type": "application/json", "x-receipt-intake-secret": ARCHIVE_SECRET }, data: JSON.stringify({ driveFileId: "DRIVE1" }), maxRedirects: 0, }); @@ -662,3 +665,38 @@ test.describe("orphan recovery", () => { expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.state).toBe("BOOKED"); }); }); + +test.describe("the two machine secrets are not interchangeable", () => { + test("the ingest key cannot read the queue, and the archive key cannot ingest", async ({ playwright }) => { + // One shared secret gave a script that only copies files to Drive the + // power to inject Purchases into the books, and gave the forwarders the + // power to enumerate every receipt. 403, not 401: the caller IS + // authenticated, it is holding the wrong program's key. + const machine = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: { cookies: [], origins: [] }, + }); + + const forwarderReadingQueue = await machine.get(`${INTAKE_PATH}?state=BOOKED`, { + headers: { "x-receipt-intake-secret": SECRET }, + maxRedirects: 0, + }); + expect(forwarderReadingQueue.status()).toBe(403); + + const mirrorIngesting = await machine.post(INTAKE_PATH, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": ARCHIVE_SECRET }, + data: intakeBody({ sourceRef: `${REF_PREFIX}wrongkey` }), + maxRedirects: 0, + }); + expect(mirrorIngesting.status()).toBe(403); + + const mirrorStartingUpload = await machine.post(`${INTAKE_PATH}/start`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": ARCHIVE_SECRET }, + data: JSON.stringify({ mimeType: "image/png", source: "drive", sourceRef: `${REF_PREFIX}wrongkey2` }), + maxRedirects: 0, + }); + expect(mirrorStartingUpload.status()).toBe(403); + + await machine.dispose(); + }); +}); diff --git a/package.json b/package.json index 9a42c967f..6535c3891 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index 3ca5ed320..f5e376ca5 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -157,10 +157,33 @@ export const statements = [ // state is a closed set — a typo must fail loudly rather than create a // silent eleventh state that no query ever selects. - `DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'ReceiptIntake_state_check' - AND conrelid = '"ReceiptIntake"'::regclass) THEN + // The state set GROWS. `IF NOT EXISTS` alone is wrong for that: a database + // that already has the constraint from an earlier run keeps the OLD set, so + // the first write of a newly-added state (SHADOW_DONE, at cutover, inside + // the claim transaction) fails and takes the whole cutover with it — and + // the script that was supposed to prevent exactly this reported "ok". + // + // So compare the DEFINITION, and replace it when it differs. Postgres + // validates the new CHECK against existing rows as part of the ADD, so a + // set that would orphan live data fails loudly here rather than later. + `DO $$ + DECLARE current_def TEXT; + wanted_def TEXT := 'CHECK ((state = ANY (ARRAY[''STAGING''::text, ''RECEIVED''::text, ''READ''::text, ''NEEDS_JOB''::text, ''NEEDS_REVIEW''::text, ''BOOKING''::text, ''BOOKED''::text, ''ARCHIVED''::text, ''DUPLICATE''::text, ''VOID''::text, ''NON_RECEIPT''::text, ''SHADOW_DONE''::text])))'; + BEGIN + SELECT pg_get_constraintdef(oid) INTO current_def + FROM pg_constraint + WHERE conname = 'ReceiptIntake_state_check' + AND conrelid = '"ReceiptIntake"'::regclass; + + IF current_def IS NULL THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" + CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', + 'SHADOW_DONE')); + ELSIF current_def IS DISTINCT FROM wanted_def THEN + -- One statement each, in the SAME transaction as everything else this + -- script runs, so the table is never briefly unconstrained. + ALTER TABLE "ReceiptIntake" DROP CONSTRAINT "ReceiptIntake_state_check"; ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', @@ -308,12 +331,23 @@ async function main() { } for (const { name, table } of expectedConstraints) { const [row] = await prisma.$queryRawUnsafe( - `SELECT 1 AS ok FROM pg_constraint WHERE conname = $1`, name, + `SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint WHERE conname = $1`, name, ); if (!row) { console.error(`VERIFY FAILED: constraint ${name} missing on ${table}`); process.exit(1); } + // Existence is not enough for the state CHECK: an OLD definition + // still exists, and it is the thing that breaks the cutover. + if (name === "ReceiptIntake_state_check") { + const missing = RECEIPT_INTAKE_STATES.filter(state => !row.def.includes(`'${state}'`)); + if (missing.length) { + console.error(`VERIFY FAILED: ${name} does not allow: ${missing.join(", ")} + actual: ${row.def}`); + process.exit(1); + } + console.log(`verified ${name}: all ${RECEIPT_INTAKE_STATES.length} states allowed`); + } } console.log(`verified ${expectedConstraints.length} constraints`); diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 6c3a49916..a6f2a315d 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -6,9 +6,13 @@ import { logAutomationEvent } from "@/lib/automation-events"; import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; +import { createRouteDeadline } from "@/lib/quickbooks"; import { readReceipt } from "@/lib/receipt-intake/read"; import { canonicalVendor } from "@/lib/receipt-intake/keys"; +import { resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; import { resolveCompanyTimeZone } from "@/lib/company-timezone"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; import { backoffMs } from "@/lib/receipt-intake/route-state"; import { @@ -17,6 +21,8 @@ import { CLAIM_LOCK_KEY, STAGING_SWEEP_BATCH, STAGING_SWEEP_MINUTES, + type ClaimResult, + type CutoverRequest, isUniqueViolation, runIntakeWorker, type ReadPatch, @@ -55,6 +61,7 @@ const WORKER_ROW_SELECT = { storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, lastError: true, + suggestedConfidence: true, createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; @@ -71,9 +78,7 @@ const NOT_DRY_RUN_PARKED: Prisma.ReceiptIntakeWhereInput = { NOT: { AND: [{ dryRun: true }, { state: { in: ["READ", "BOOKING"] } }] }, }; -async function claim( - opts: { retireShadowBacklog: boolean }, -): Promise<{ rows: WorkerRow[]; shadowRetired: number } | null> { +async function claim(opts: CutoverRequest): Promise { const now = new Date(); return prisma.$transaction(async tx => { const [lock] = await tx.$queryRaw<{ locked: boolean }[]>( @@ -96,14 +101,41 @@ async function claim( // The rows keep their read results and dedup keys, so a post-cutover // resend of the same receipt still collides with them and is caught. let shadowRetired = 0; - if (opts.retireShadowBacklog) { - const result = await tx.receiptIntake.updateMany({ - where: { dryRun: true, state: { in: ["READ", "BOOKING"] } }, - data: { state: "SHADOW_DONE", stateReason: "booked-by-v1", nextRetryAt: null }, - }); - shadowRetired = result.count; - if (shadowRetired > 0) { - console.log("[cron/receipt-intake-worker] shadow backlog retired", shadowRetired); + let requeued = 0; + let boundaryMissing = false; + if (opts.run) { + if (!opts.boundary) { + // Refuse. Retiring on a guess destroys evidence of real + // expenses; requeuing on a guess double-books them. A logged + // no-op is the only honest third option. + boundaryMissing = true; + } else { + const parked: Prisma.ReceiptIntakeWhereInput = { + dryRun: true, + state: { in: ["READ", "BOOKING"] }, + }; + + // BEFORE the boundary: v1 was still booking, so it booked these. + const retired = await tx.receiptIntake.updateMany({ + where: { ...parked, createdAt: { lt: opts.boundary } }, + data: { state: "SHADOW_DONE", stateReason: "booked-by-v1", nextRetryAt: null }, + }); + shadowRetired = retired.count; + + // AFTER it: v1 had already stopped, so NOBODY booked these. + // They are the only rows from the shadow week that v2 must + // actually book — dropping them would lose real expenses. + const handed = await tx.receiptIntake.updateMany({ + where: { ...parked, createdAt: { gte: opts.boundary } }, + data: { dryRun: false, nextRetryAt: null }, + }); + requeued = handed.count; + + if (shadowRetired > 0 || requeued > 0) { + console.log("[cron/receipt-intake-worker] cutover", JSON.stringify({ + boundary: opts.boundary.toISOString(), shadowRetired, requeued, + })); + } } } @@ -120,7 +152,7 @@ async function claim( take: BATCH_SIZE, select: WORKER_ROW_SELECT, }); - if (due.length === 0) return { rows: [], shadowRetired }; + if (due.length === 0) return { rows: [], shadowRetired, requeued, boundaryMissing }; // THE claim. Anything this run took is invisible to the next one for // the lease, whether or not the advisory lock held. @@ -128,7 +160,7 @@ async function claim( where: { id: { in: due.map(r => r.id) } }, data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, }); - return { rows: due as WorkerRow[], shadowRetired }; + return { rows: due as WorkerRow[], shadowRetired, requeued, boundaryMissing }; }); } @@ -138,6 +170,8 @@ function buildDeps(): WorkerDependencies { isDryRunEnabled: () => process.env.RECEIPT_INTAKE_DRYRUN !== "false", + cutoverBoundary: resolveCutoverBoundary, + sweepStaleStaging: async shouldStop => { // NEVER a blanket "old therefore missing". A STAGING row that is old // because its publish UPDATE failed HAS its object in the bucket, @@ -329,12 +363,21 @@ function buildDeps(): WorkerDependencies { return { promoted: true }; }), - book: row => bookReceipt(row, { + book: (row, remainingMs) => bookReceipt(row, { db: prisma as unknown as BookPrismaClient, + companyTimeZone: resolveCompanyTimeZone, + isCostCodeAllowed: (projectId, costCodeId) => + isCostCodeAllowedForProject(prismaPhaseDataSource, projectId, costCodeId), + remainingBudgetMs: () => remainingMs, + // ONE deadline for the whole booking, threaded into the token + // refresh AND the Purchase create so they share a budget instead of + // each helping itself to a fresh 20s. + deadline: () => createRouteDeadline(Math.max(0, remainingMs)), isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), - getTokens: getFreshQBTokens, - createPurchase: (tokens, input) => createQBReceiptPurchase(tokens, input), + getTokens: deadline => getFreshQBTokens(deadline), + createPurchase: (tokens, input, deadline) => + createQBReceiptPurchase(tokens, input, {}, deadline), downloadBytes: downloadDocBytesResult, logEvent: logAutomationEvent, now: () => new Date(), diff --git a/src/app/api/receipts/intake/[id]/archived/route.ts b/src/app/api/receipts/intake/[id]/archived/route.ts index 2e4a97c87..7e3c988b2 100644 --- a/src/app/api/receipts/intake/[id]/archived/route.ts +++ b/src/app/api/receipts/intake/[id]/archived/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; -import { secretMatches, RECEIPT_INTAKE_SECRET_HEADER } from "@/lib/receipt-intake/intake-auth"; +import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; export const dynamic = "force-dynamic"; @@ -21,7 +21,14 @@ export const dynamic = "force-dynamic"; * PUBLIC_PROXY_BYPASS_PATTERN for that reason, each one exact. */ export async function POST(req: Request, context: { params: Promise<{ id: string }> }) { - if (!secretMatches(req.headers.get(RECEIPT_INTAKE_SECRET_HEADER), process.env.RECEIPT_INTAKE_SECRET)) { + // ARCHIVE capability only. This transition means "a file exists in Drive", + // which only the mirror can know — and the ingest forwarders must not be + // able to mark rows archived just because they hold a receipt-intake key. + const auth = await authenticateIntake(req, "archive"); + if (!auth.ok) return auth.response; + if (auth.via !== "secret") { + // No session path: a staff user clicking this would be asserting + // something they cannot verify. return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); } diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts new file mode 100644 index 000000000..b3df38bea --- /dev/null +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -0,0 +1,122 @@ +import { createHash } from "node:crypto"; +import { NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; +import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; +import { sniffMime } from "@/lib/receipt-intake/file-type"; +import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +/** + * Step 2 of the two-step upload: verify what actually landed, then publish. + * + * Everything here is checked against the STORED OBJECT, never against what the + * client says about it. The client uploaded directly to Supabase, so this is the + * only point at which the server sees the bytes at all — trusting a declared + * hash, size or type would mean the row's `fileSha256` (which decides whether a + * replay is a duplicate or a conflict) was attacker-supplied. + * + * STAGING -> RECEIVED is the publish, and it is the only thing that makes the + * row visible to the worker. + */ +export async function POST(req: Request, context: { params: Promise<{ id: string }> }) { + const auth = await authenticateIntake(req, "ingest"); + if (!auth.ok) return auth.response; + + const { id } = await context.params; + + let body: { sha256?: unknown } = {}; + try { + body = await req.json(); + } catch { + // A finalize with no body is fine — the declared hash is optional. + } + const declaredSha = typeof body.sha256 === "string" ? body.sha256.trim().toLowerCase() : null; + + const row = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { + id: true, state: true, sourceRef: true, storagePath: true, mimeType: true, + projectId: true, dryRun: true, createdById: true, fileSha256: true, + }, + }); + if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + + // Same rule as the conflict path: a session caller may only finalize its + // OWN row (or hold a bookkeeping role). Otherwise a guessed id would let one + // user publish another's upload. + const maySee = + auth.via === "secret" || + row.createdById === auth.user.id || + STAFF_READ_ROLES.includes(auth.user.role); + if (!maySee) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + + // Idempotent: finalizing an already-published row is a success, not an error + // — the client's retry after a lost response must not look like a failure. + if (row.state !== "STAGING") { + return NextResponse.json({ + ok: true, alreadyFinalized: true, id: row.id, state: row.state, + sourceRef: row.sourceRef, projectId: row.projectId, dryRun: row.dryRun, + }); + } + + const download = await downloadDocBytesResult(toSecureRef(row.storagePath)); + if (!download.ok) { + if (download.kind === "not-found") { + // The upload never landed. Retryable, and NEVER a 2xx: the + // forwarders treat 2xx as "we have it" and would drop their copy. + return NextResponse.json( + { ok: false, error: "object-missing", reason: "upload the bytes to the signed URL first" }, + { status: 409 }, + ); + } + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + + const bytes = download.bytes; + if (bytes.length > MAX_STORED_BYTES) { + // Enforced on the OBJECT, because the signed URL bypassed every check + // this server could otherwise have made. Drop it rather than leave an + // oversize file in a private bucket nobody will ever book. + await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + return NextResponse.json({ ok: false, reason: "file-too-large", maxBytes: MAX_STORED_BYTES }, { status: 413 }); + } + + // The stored type is decided on the BYTES, exactly like the single-shot path. + const mimeType = sniffMime(bytes, row.mimeType); + if (!mimeType) { + await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + return NextResponse.json({ ok: false, reason: "unsupported-file-type" }, { status: 400 }); + } + + const fileSha256 = createHash("sha256").update(bytes).digest("hex"); + // A declared hash is an INTEGRITY check on the client's own upload, never + // the value we store. A mismatch means the bytes in the bucket are not the + // bytes the client meant to send. + if (declaredSha && declaredSha !== fileSha256) { + return NextResponse.json( + { ok: false, error: "sha-mismatch", reason: "stored bytes do not match the declared sha256" }, + { status: 409 }, + ); + } + + const published = await prisma.receiptIntake.updateMany({ + where: { id, state: "STAGING" }, + data: { state: "RECEIVED", mimeType, fileSize: bytes.length, fileSha256 }, + }); + if (published.count === 0) { + // Another finalize won the race and published it. Same outcome. + const now = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { state: true, sourceRef: true, projectId: true, dryRun: true }, + }); + return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: now?.state ?? "RECEIVED" }); + } + + return NextResponse.json({ + ok: true, id, state: "RECEIVED", sourceRef: row.sourceRef, + projectId: row.projectId, dryRun: row.dryRun, fileSize: bytes.length, + }); +} diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 5064e32ba..e7244c894 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -6,7 +6,8 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; -import { EXT_BY_MIME, MAX_INTAKE_BYTES, sniffMime } from "@/lib/receipt-intake/file-type"; +import { EXT_BY_MIME, sniffMime } from "@/lib/receipt-intake/file-type"; +import { MAX_INLINE_UPLOAD_BYTES, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { ARCHIVE_READABLE_STATES, listReceiptIntakes, @@ -70,6 +71,26 @@ function bad(reason: string) { return NextResponse.json({ ok: false, reason }, { status: 400 }); } +/** + * The inline path carries the file in the request body, which the platform caps + * around 4.5 MB — base64 JSON inflates it by a third on top. Anything bigger + * used to die at the edge with an opaque 413 that never reached this code, so + * the caller learned nothing. Say it plainly and name the path that works. + */ +function tooLargeForInline() { + return NextResponse.json( + { + ok: false, + error: "payload-too-large", + reason: `inline uploads are limited to ${MAX_INLINE_UPLOAD_BYTES} bytes (serverless request-body cap)`, + maxInlineBytes: MAX_INLINE_UPLOAD_BYTES, + maxBytes: MAX_STORED_BYTES, + use: "POST /api/receipts/intake/start then PUT to the signed URL then POST /api/receipts/intake/{id}/finalize", + }, + { status: 413 }, + ); +} + async function parseBody(req: Request): Promise { const contentType = req.headers.get("content-type") ?? ""; const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); @@ -83,9 +104,9 @@ async function parseBody(req: Request): Promise { } const file = form.get("file"); if (!(file instanceof File)) return bad("missing-file"); - if (file.size > MAX_INTAKE_BYTES) return bad("file-too-large"); + if (file.size > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); const bytes = Buffer.from(await file.arrayBuffer()); - if (bytes.length > MAX_INTAKE_BYTES) return bad("file-too-large"); + if (bytes.length > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); return { bytes, declaredMime: file.type || "application/octet-stream", @@ -109,10 +130,10 @@ async function parseBody(req: Request): Promise { if (!base64) return bad("missing-file"); // Cap BEFORE decoding: base64 is 4/3 the byte count, so this refuses an // oversize payload without materialising it. - if (base64.length > Math.ceil(MAX_INTAKE_BYTES / 3) * 4 + 4) return bad("file-too-large"); + if (base64.length > Math.ceil(MAX_INLINE_UPLOAD_BYTES / 3) * 4 + 4) return tooLargeForInline(); const bytes = Buffer.from(base64, "base64"); if (bytes.length === 0) return bad("missing-file"); - if (bytes.length > MAX_INTAKE_BYTES) return bad("file-too-large"); + if (bytes.length > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); return { bytes, declaredMime: typeof json.mimeType === "string" ? json.mimeType : "application/octet-stream", @@ -127,7 +148,7 @@ async function parseBody(req: Request): Promise { } export async function POST(req: Request) { - const auth = await authenticateIntake(req); + const auth = await authenticateIntake(req, "ingest"); if (!auth.ok) return auth.response; const parsed = await parseBody(req); @@ -445,7 +466,10 @@ async function respondToSourceRefConflict( * bookkeeping role gets 403, not a redirect. */ export async function GET(req: Request) { - const auth = await authenticateIntake(req); + // A secret caller here is the ARCHIVE mirror. The ingest forwarders hold a + // different key and are refused with 403 — a script that only copies files + // to Drive has no business enumerating the queue, and vice versa. + const auth = await authenticateIntake(req, "archive"); if (!auth.ok) return auth.response; if (auth.via === "session" && !STAFF_READ_ROLES.includes(auth.user.role)) { return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts new file mode 100644 index 000000000..3b6e3fe5f --- /dev/null +++ b/src/app/api/receipts/intake/start/route.ts @@ -0,0 +1,154 @@ +import { randomUUID } from "node:crypto"; +import { NextResponse } from "next/server"; +import { Prisma } from "@prisma/client"; +import { prisma } from "@/lib/prisma"; +import { userCanAccessProject } from "@/lib/mobile-auth"; +import { SECURE_BUCKET } from "@/lib/secure-storage"; +import { getSupabase } from "@/lib/supabase"; +import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; +import { EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; +import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +/** + * Step 1 of the two-step upload: reserve the row, hand back a signed URL. + * + * The single-shot POST /api/receipts/intake puts the file in the REQUEST BODY, + * and a serverless body is not a 15 MB pipe — the platform caps it around + * 4.5 MB and base64 inflates the payload by a third on top. Phone photos + * routinely exceed that, and they were failing at the edge with an opaque 413 + * that never reached our code. This path never carries the bytes at all: the + * client PUTs them straight to Supabase, which has no such limit. + * + * The signed URL is scoped to ONE path, which is derived here and bound to the + * row — the client cannot choose where its bytes land, so it cannot overwrite + * another receipt's object or write outside the intake prefix. + * + * No object exists yet, so the row is STAGING and the worker cannot see it. + * /finalize is what publishes it. + */ +export async function POST(req: Request) { + const auth = await authenticateIntake(req, "ingest"); + if (!auth.ok) return auth.response; + + let body: Record; + try { + body = await req.json(); + } catch { + return NextResponse.json({ ok: false, reason: "invalid-json" }, { status: 400 }); + } + const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); + + const mimeType = String(body.mimeType ?? "").split(";")[0].trim().toLowerCase(); + const ext = EXT_BY_MIME[mimeType]; + // The declared mime only picks the extension here; /finalize re-derives the + // real type from the STORED BYTES, so a lie costs the caller its upload. + if (!ext) return NextResponse.json({ ok: false, reason: "unsupported-file-type" }, { status: 400 }); + + const declaredSize = Number(body.fileSize); + if (Number.isFinite(declaredSize) && declaredSize > MAX_STORED_BYTES) { + return NextResponse.json({ ok: false, reason: "file-too-large", maxBytes: MAX_STORED_BYTES }, { status: 413 }); + } + + const decided = decideSource(auth, { + source: str(body.source), + sourceRef: str(body.sourceRef), + uploadId: str(body.uploadId), + }); + if (!decided.ok) return NextResponse.json({ ok: false, reason: decided.reason }, { status: 400 }); + + const projectId = str(body.projectId); + if (auth.via === "session" && projectId) { + if (!(await userCanAccessProject(auth.user, projectId))) { + return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + } + } + + const id = randomUUID(); + const storagePath = `receipts/intake/${id}.${ext}`; + + let created: { id: string; sourceRef: string; state: string }; + try { + created = await prisma.receiptIntake.create({ + data: { + id, + source: decided.source, + sourceRef: decided.sourceRef, + state: "STAGING", + dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", + projectId, + costCodeId: str(body.costCodeId), + createdById: auth.via === "session" ? auth.user.id : null, + storagePath, + fileName: str(body.fileName), + mimeType, + fileSize: 0, + // Unknown until the bytes land. /finalize recomputes it FROM + // STORAGE and writes the real value; a client-declared hash is + // never trusted as the stored one. + fileSha256: "", + }, + select: { id: true, sourceRef: true, state: true }, + }); + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + // Same sourceRef: hand back the row already in flight so a retrying + // client resumes rather than orphaning a second object. + const existing = await prisma.receiptIntake.findUnique({ + where: { sourceRef: decided.sourceRef }, + select: { id: true, sourceRef: true, state: true, storagePath: true, createdById: true }, + }); + if (!existing) return NextResponse.json({ ok: false, reason: "conflict-retry" }, { status: 409 }); + const maySee = + auth.via === "secret" || + existing.createdById === auth.user.id || + auth.user.role === "ADMIN"; + if (!maySee) return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + if (existing.state !== "STAGING") { + return NextResponse.json( + { ok: true, alreadyReceived: true, id: existing.id, state: existing.state }, + ); + } + const resumed = await signUpload(existing.storagePath); + if (!resumed) return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + return NextResponse.json({ ok: true, resumed: true, id: existing.id, ...resumed }); + } + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2003") { + return NextResponse.json({ ok: false, reason: "unknown-project-or-cost-code" }, { status: 400 }); + } + throw error; + } + + const signed = await signUpload(storagePath); + if (!signed) { + await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + + return NextResponse.json({ + ok: true, + id: created.id, + sourceRef: created.sourceRef, + state: created.state, + maxBytes: MAX_STORED_BYTES, + ...signed, + }); +} + +async function signUpload(storagePath: string): Promise<{ uploadUrl: string; token: string; storagePath: string } | null> { + const supabase = getSupabase(); + if (!supabase) return null; + try { + const { data, error } = await supabase.storage.from(SECURE_BUCKET).createSignedUploadUrl(storagePath); + if (error || !data) { + console.error("[receipts/intake/start] sign failed", error?.message); + return null; + } + return { uploadUrl: data.signedUrl, token: data.token, storagePath }; + } catch (error) { + console.error("[receipts/intake/start] sign threw", error instanceof Error ? error.name : "error"); + return null; + } +} diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 3e2d83881..9bf03ca40 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -19,7 +19,8 @@ */ import { matchCostCode } from "@/lib/project-match"; import { toSecureRef } from "@/lib/secure-storage"; -import { QBTimeoutError, type QBTokens } from "@/lib/quickbooks"; +import { startOfDateInTimeZone } from "@/lib/tz-date"; +import { QBTimeoutError, type QBTokens, type RouteDeadline } from "@/lib/quickbooks"; import { QboAccountConfigError, QboPurchaseFaultError, @@ -41,6 +42,8 @@ export interface BookableRow { projectId: string | null; costCodeId: string | null; suggestedCostCodeId: string | null; + /** The model's confidence in that phase suggestion, 0..1. */ + suggestedConfidence: number | null; storagePath: string; fileName: string | null; mimeType: string; @@ -90,11 +93,21 @@ export function attachmentBlocker(mimeType: string, byteLength: number): string return null; } +/** + * A booking needs enough runway to finish what it starts. Two QuickBooks round + * trips (token refresh + the Purchase create, each with its own 20s fetch + * deadline) plus the attachment upload and the commit do not fit in a few + * seconds — and a booking cut off mid-flight is the worst outcome available: the + * Purchase may exist in the real books while the row never learns it did. + * Better to not start. + */ +export const MIN_BOOKING_BUDGET_MS = 25_000; + export type BookResult = /** Purchase + Expense exist and the row is BOOKED. */ | { outcome: "booked"; qbPurchaseId: string; expenseId: string; alreadyExisted: boolean } /** A switch is off: stay BOOKING, try again in an hour, spend NO attempt. */ - | { outcome: "deferred"; reason: "push-disabled" | "push-paused" } + | { outcome: "deferred"; reason: "push-disabled" | "push-paused" | "out-of-budget" } /** * Terminal: a human must look at it. No further automatic attempt. * @@ -132,12 +145,27 @@ export interface BookPrismaClient { export interface BookDependencies { db: BookPrismaClient; + /** The company's configured zone — Expense.date is a business calendar day. */ + companyTimeZone: () => Promise; + /** + * Is this cost code a phase of THIS project? Re-asked at booking because the + * project can change between READ and BOOKING. + */ + isCostCodeAllowed: (projectId: string, costCodeId: string) => Promise; /** env master switch — opt-IN, exactly like the qbo-receipts/create route. */ isPushEnabled: () => boolean; /** Command Center pause switch (pause-only; fail-CLOSED on a read error). */ isPushPaused: () => Promise; - getTokens: () => Promise; - createPurchase: (tokens: QBTokens, input: CreateQBReceiptPurchaseInput) => Promise; + getTokens: (deadline?: RouteDeadline) => Promise; + createPurchase: ( + tokens: QBTokens, + input: CreateQBReceiptPurchaseInput, + deadline?: RouteDeadline, + ) => Promise; + /** Milliseconds left in the worker's invocation. Undefined = unbounded (tests). */ + remainingBudgetMs?: () => number; + /** Threads the same budget into every QuickBooks call this booking makes. */ + deadline?: () => RouteDeadline | undefined; /** * Reads the stored file back out of the private bucket. TAGGED, because a * confirmed 404 and a transient storage fault must not book the same way. @@ -244,6 +272,7 @@ function terminalReasonFor(error: unknown): string | null { */ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Promise { const now = deps.now(); + const timeZone = await deps.companyTimeZone(); // Shadow mode is enforced by the WORKER, which never routes a dryRun row // here. This second check exists because "no QBO calls in dry run" is the @@ -259,11 +288,22 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro if (!deps.isPushEnabled()) return { outcome: "deferred", reason: "push-disabled" }; if (await deps.isPushPaused()) return { outcome: "deferred", reason: "push-paused" }; + // Runway check BEFORE anything else that could touch QuickBooks. Deferred, + // not retried: the document is fine and this costs it no attempt — the + // invocation simply ran out of room, and the next pass has a full budget. + const remaining = deps.remainingBudgetMs?.(); + if (remaining !== undefined && remaining < MIN_BOOKING_BUDGET_MS) { + return { outcome: "deferred", reason: "out-of-budget" }; + } + // Everything down to the QBO call is a PRE-SEND refusal: nothing was ever // sent, so the strong key must be handed back (see BookResult). if (!row.projectId) return parkedBeforeSend("no-estimate"); if (row.totalCents === null || row.totalCents <= 0) return parkedBeforeSend("refund-or-zero"); if (!row.txnDate) return parkedBeforeSend("invalid-date"); + // Hoisted so the calendar day is computed ONCE and both the QBO TxnDate and + // the Expense.date instant are derived from the same value. + const calendarDay = toCalendarDate(row.txnDate); // 2. The project's LATEST estimate — the same "primary estimate" rule the // v1 receipt-ingest endpoint uses (route.ts:69). Expense.estimateId is @@ -325,7 +365,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro projectName: project.name, docType: isCheck ? "check" : "receipt", vendor: row.vendor ?? "", - date: toCalendarDate(row.txnDate), + date: calendarDay, invoice: !isCheck && row.refNumber && row.refNumber !== "NoInv" ? row.refNumber : undefined, checkNumber: isCheck && row.refNumber ? row.refNumber.replace(/^Check/, "") : undefined, memo: row.memo ?? undefined, @@ -339,8 +379,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro let result: CreateQBReceiptPurchaseResult; try { - const tokens = await deps.getTokens(); - result = await deps.createPurchase(tokens, input); + // ONE budget threaded through both round trips, so a slow token refresh + // shortens the create rather than each getting a fresh 20s. + const deadline = deps.deadline?.(); + const tokens = await deps.getTokens(deadline); + result = await deps.createPurchase(tokens, input, deadline); } catch (error) { const terminal = terminalReasonFor(error); // A send WAS attempted: QBO may hold a Purchase whose response we lost, @@ -392,7 +435,23 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // there is exactly one Purchase. const amountCents = expenseAmountCents(groups, row.totalCents); const taxApplied = appliedTaxCents(groups); - const costCodeId = row.costCodeId ?? row.suggestedCostCodeId ?? null; + // RE-VALIDATE THE PHASE AGAINST THE FINAL PROJECT. + // + // Both the captured code and the model's suggestion were resolved while the + // row was being READ — and at that point the row may have had NO project at + // all (NEEDS_JOB), or a different one that a human then corrected. A cost + // code from the old project is not a phase of the new one, and posting an + // Expense against it puts real money on a phase that job does not have, + // which every variance report then reads as overspend on a line nobody + // budgeted. + // + // "The cost code exists" is not a permission (project-phases.ts:125), so + // this asks the same question the clock-in validation asks. A mismatch is + // NOT a failure: the receipt is fine and its total is right, so it books + // UNCODED and says why. A bookkeeper assigning a phase is routine; an + // expense silently attached to the wrong one is not. + const phaseCheck = await resolvePhase(row, project.id, deps); + const costCodeId = phaseCheck.costCodeId; const driveFileId = driveFileIdOf(row); const receiptUrl = driveFileId ? `https://drive.google.com/file/d/${driveFileId}/view` @@ -416,12 +475,20 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro costCodeId, amount: amountCents / 100, vendor: row.vendor || "Unknown", - date: row.txnDate, + // RE-ANCHORED at write time. `txnDate` is a @db.Date column + // and round-trips as UTC midnight, so writing it straight + // into Expense.date (a full timestamp) records 5pm the + // PREVIOUS day in Pacific — and every job-cost and variance + // report that bounds by local midnight then counts the + // expense in the wrong period. The intake row keeps the + // calendar day; this makes the instant match it. + date: startOfDateInTimeZone(calendarDay, timeZone), status: "Pending", receiptUrl, qbPurchaseId: result.qbPurchaseId, description: `[Receipt intake] ${docRef}` + + phaseCheck.note + (taxApplied > 0 ? ` · incl. $${(taxApplied / 100).toFixed(2)} sales tax` : "") + ` · pending bookkeeper review`, }, @@ -464,6 +531,14 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro intakeId: row.id, expenseId, sourceRef: row.sourceRef, + costCodeId, + // Carried through so the Command Center can show HOW confident + // the phase pick was, and so a low-confidence run is auditable + // after the fact rather than only at review time. + suggestedConfidence: costCodeId && costCodeId === row.suggestedCostCodeId + ? row.suggestedConfidence + : undefined, + phaseRejected: phaseCheck.rejected || undefined, }, }).catch(() => { /* audit only */ }); @@ -489,6 +564,38 @@ function describe(error: unknown): string { /** Marks a retry as "the Purchase exists but its receipt did not attach". */ export const ATTACHMENT_FAILED_PREFIX = "attachment-failed:"; +/** + * Which phase (if any) this Expense may carry, checked against the project the + * row will ACTUALLY book to. + */ +async function resolvePhase( + row: BookableRow, + projectId: string, + deps: BookDependencies, +): Promise<{ costCodeId: string | null; note: string; rejected: string | null }> { + // A human's explicit pick outranks the model's suggestion, but neither is + // trusted without the project check. + const candidate = row.costCodeId ?? row.suggestedCostCodeId ?? null; + if (!candidate) return { costCodeId: null, note: "", rejected: null }; + + const allowed = await deps.isCostCodeAllowed(projectId, candidate); + if (allowed) { + const fromSuggestion = !row.costCodeId && candidate === row.suggestedCostCodeId; + const confidence = row.suggestedConfidence; + const note = + fromSuggestion && typeof confidence === "number" + ? ` · phase suggested (confidence ${confidence.toFixed(2)})` + : ""; + return { costCodeId: candidate, note, rejected: null }; + } + + return { + costCodeId: null, + note: " · phase cleared (not a phase of this job) — assign one", + rejected: candidate, + }; +} + /** A refusal reached WITHOUT any QBO call — the strong key goes back. */ function parkedBeforeSend(reason: string): BookResult { return { outcome: "needs-review", reason, releaseStrongKey: true }; diff --git a/src/lib/receipt-intake/cutover.ts b/src/lib/receipt-intake/cutover.ts new file mode 100644 index 000000000..717a8358b --- /dev/null +++ b/src/lib/receipt-intake/cutover.ts @@ -0,0 +1,51 @@ +/** + * The v1 -> v2 cutover boundary. + * + * The problem this exists to solve: at cutover, the shadow-week backlog cannot + * all be treated the same way. Rows received while the Apps Script was still + * BOOKING were booked by v1, so v2 must never book them again — v2's QuickBooks + * identity for an email/chat/mobile/web row is the intake UUID, which v1 never + * saw, so DocNumber idempotency cannot recognise the Purchase v1 already made. + * But rows received AFTER v1 stopped booking were never booked by anyone, and + * retiring those would silently drop real expenses on the floor. + * + * One timestamp separates the two: the instant the Apps Script was flipped to + * forwarder mode and stopped writing to QuickBooks. It is recorded when that + * flip happens, NOT derived — nothing in the database can infer it, and a guess + * here either double-books or loses receipts. + * + * With no boundary recorded the worker refuses to retire anything at all. That + * is the only safe default: retiring on a guess destroys evidence, and the + * failure mode of refusing is a visible, logged no-op. + */ +import { prisma } from "@/lib/prisma"; + +export const CUTOVER_SETTING_KEY = "cutoverV1StoppedAt"; + +/** + * When v1 stopped booking. Read from the AutomationSetting row first (that is + * what an operator writes at the flip), falling back to the env var so a + * deployment can carry it too. Returns null when unset OR unparseable — a + * malformed value must not be silently treated as "epoch", which would retire + * the entire backlog. + */ +export async function resolveCutoverBoundary(): Promise { + let raw: string | null | undefined; + try { + raw = (await prisma.automationSetting.findUnique({ where: { key: CUTOVER_SETTING_KEY } }))?.value; + } catch (error) { + // A settings read failure is NOT "no boundary" — that would let a DB + // blip authorise a retire. Surface it as unset, which refuses. + console.error("[cutover] settings read failed", error instanceof Error ? error.name : "UnknownError"); + return null; + } + return parseCutoverBoundary(raw ?? process.env.CUTOVER_V1_STOPPED_AT); +} + +/** Pure, so the parsing rules are testable without a database. */ +export function parseCutoverBoundary(value: string | null | undefined): Date | null { + if (!value || !value.trim()) return null; + const at = new Date(value.trim()); + if (!Number.isFinite(at.getTime())) return null; + return at; +} diff --git a/src/lib/receipt-intake/intake-auth.ts b/src/lib/receipt-intake/intake-auth.ts index e60076ffd..4d4031079 100644 --- a/src/lib/receipt-intake/intake-auth.ts +++ b/src/lib/receipt-intake/intake-auth.ts @@ -1,19 +1,30 @@ /** * Auth for /api/receipts/intake and its sub-routes. * - * `/api/receipts/intake` is on the proxy's EXACT-MATCH public bypass + * Every one of these paths is on the proxy's EXACT-MATCH public bypass * (src/proxy.ts) so machine callers get a clean 401 instead of a 307 to - * /login. That makes this the ONLY gate on the route, so it fails closed - * everywhere: + * /login. That makes this the ONLY gate, so it fails closed everywhere: * - * - no `RECEIPT_INTAKE_SECRET` configured -> the secret path is refused - * outright, never "allow because unset" (getclients-auth-gate lesson). + * - no secret configured -> that capability is refused outright, never + * "allow because unset" (getclients-auth-gate lesson). * - a bogus/expired session cookie -> authenticateMobileOrSession returns * ok:false, and this returns 401 JSON, never a redirect. * - * RECEIPT_INTAKE_SECRET is deliberately a NEW variable, not the v1 - * RECEIPT_INGEST_SECRET: v1 and v2 must rotate independently, and during the - * shadow week both pipelines are live at once. + * TWO SECRETS, NOT ONE. They belong to different programs with different + * blast radii: + * + * RECEIPT_INTAKE_SECRET — the forwarders. May only INGEST, and only under + * the sources they actually own (drive/email/chat). Cannot read the queue, + * cannot see another job's receipts, cannot archive anything. + * RECEIPT_ARCHIVE_SECRET — the nightly Drive mirror. May only READ + * BOOKED/ARCHIVED rows and report back what it archived. Cannot create a + * row, cannot publish one, cannot touch a document's contents. + * + * One shared secret gave a script that only copies files to Drive the power to + * inject Purchases into the books, and gave the ingest forwarders the power to + * enumerate every receipt in the system. Splitting them means a leak of either + * one is bounded by what that program actually does. They rotate independently + * for the same reason. */ import { createHash, timingSafeEqual } from "node:crypto"; import { NextResponse } from "next/server"; @@ -22,12 +33,26 @@ import type { User } from "@prisma/client"; export const RECEIPT_INTAKE_SECRET_HEADER = "x-receipt-intake-secret"; +/** What a caller is asking to do. Checked against the secret it presented. */ +export type IntakeCapability = "ingest" | "archive"; + export type IntakeAuth = - | { ok: true; via: "secret"; user: null; userVia: null } + | { + ok: true; + via: "secret"; + user: null; + userVia: null; + capability: IntakeCapability; + /** Sources this secret may declare. Empty for the archive secret. */ + allowedSources: ReadonlySet; + } /** `userVia` distinguishes the crew app from a browser — the route mints `source` from it. */ | { ok: true; via: "session"; user: User; userVia: "mobile-jwt" | "next-auth" } | { ok: false; response: NextResponse }; +/** The forwarders own these three and nothing else. */ +export const INGEST_ALLOWED_SOURCES: ReadonlySet = new Set(["drive", "email", "chat"]); + /** Constant-time compare over fixed-length digests, so header length leaks nothing. */ export function secretMatches(provided: string | null, expected: string | undefined): boolean { if (!expected) return false; @@ -40,19 +65,58 @@ function unauthorized(): NextResponse { return NextResponse.json({ ok: false, reason: "unauthorized" }, { status: 401 }); } +function wrongCapability(have: IntakeCapability, need: IntakeCapability): NextResponse { + // 403, not 401: the caller IS authenticated, it just holds the other + // program's key. Saying so is what makes a mis-wired script obvious + // instead of looking like a rotation problem. + return NextResponse.json( + { ok: false, reason: "forbidden", have, need }, + { status: 403 }, + ); +} + /** - * Secret first, then a session/mobile-Bearer user. A caller presenting a WRONG - * secret header is refused outright rather than falling through to the session - * check — a machine caller with a stale secret must see 401, not silently - * succeed because a browser cookie happened to ride along. + * Secret first, then a session/mobile-Bearer user. + * + * `need` is what the ROUTE requires. A caller presenting a valid secret for the + * OTHER capability is refused with 403 rather than falling through to the + * session check — a forwarder must never be able to read the queue by holding + * the ingest key, and the mirror must never be able to create a row. */ -export async function authenticateIntake(req: Request): Promise { +export async function authenticateIntake( + req: Request, + need: IntakeCapability = "ingest", +): Promise { const provided = req.headers.get(RECEIPT_INTAKE_SECRET_HEADER); if (provided !== null) { - if (secretMatches(provided, process.env.RECEIPT_INTAKE_SECRET)) { - return { ok: true, via: "secret", user: null, userVia: null }; + const ingest = process.env.RECEIPT_INTAKE_SECRET; + const archive = process.env.RECEIPT_ARCHIVE_SECRET; + + // Both compares always run: short-circuiting on the first match would + // make the response time depend on WHICH key was presented. + const isIngest = secretMatches(provided, ingest); + const isArchive = secretMatches(provided, archive); + + if (!isIngest && !isArchive) return { ok: false, response: unauthorized() }; + + // A single value configured for both variables is a misconfiguration + // that would silently re-merge the two capabilities. Refuse it. + if (isIngest && isArchive) { + console.error("[receipts/intake] RECEIPT_INTAKE_SECRET and RECEIPT_ARCHIVE_SECRET are identical"); + return { ok: false, response: unauthorized() }; } - return { ok: false, response: unauthorized() }; + + const capability: IntakeCapability = isIngest ? "ingest" : "archive"; + if (capability !== need) return { ok: false, response: wrongCapability(capability, need) }; + + return { + ok: true, + via: "secret", + user: null, + userVia: null, + capability, + allowedSources: capability === "ingest" ? INGEST_ALLOWED_SOURCES : new Set(), + }; } const auth = await authenticateMobileOrSession(req); diff --git a/src/lib/receipt-intake/intake-core.ts b/src/lib/receipt-intake/intake-core.ts new file mode 100644 index 000000000..37f55261c --- /dev/null +++ b/src/lib/receipt-intake/intake-core.ts @@ -0,0 +1,74 @@ +/** + * Shared intake rules, so the single-shot POST and the two-step + * start/finalize flow cannot drift apart on provenance, idempotency or limits. + */ +import { randomUUID } from "node:crypto"; +import type { IntakeAuth } from "./intake-auth"; + +/** + * The single-shot POST carries the file in the REQUEST BODY, and a serverless + * request body is not a 15 MB pipe: Vercel caps it at 4.5 MB and the base64 + * JSON shape inflates the payload by a third on top of that. Anything larger + * was failing at the platform edge with an opaque 413 that never reached this + * code — so the endpoint now says so itself, and points at the two-step flow + * that uploads straight to storage and has no body limit at all. + */ +export const MAX_INLINE_UPLOAD_BYTES = 4 * 1024 * 1024; + +/** The real ceiling for a stored receipt, enforced on the object itself. */ +export const MAX_STORED_BYTES = 15 * 1024 * 1024; + +/** Sources a shared-secret forwarder may declare. */ +export const MACHINE_SOURCES = new Set(["drive", "email", "chat"]); +/** Minted server-side from the authenticated caller, never read off the body. */ +export const USER_SOURCES = new Set(["mobile", "web"]); + +/** Client-supplied idempotency tokens must be real UUIDs — never a free-text key. */ +export const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export type SourceDecision = + | { ok: true; source: string; sourceRef: string } + | { ok: false; reason: string }; + +/** + * Decide `source` and `sourceRef` from the AUTH KIND, not the body. + * + * A session or Bearer caller knows neither: letting one pass `source:"drive"` + * plus a chosen `sourceRef` would let it claim another document's idempotency + * key, and `drive` rows book under the Drive file id — which is what a QBO + * DocNumber is derived from. A forwarder owns both, but only inside its own + * namespace. + */ +export function decideSource( + auth: Extract, + body: { source?: string | null; sourceRef?: string | null; uploadId?: string | null }, +): SourceDecision { + if (auth.via === "secret") { + const source = String(body.source ?? ""); + // The SECRET's own list, not a global one: a key is scoped to the + // sources its program actually owns. + if (!auth.allowedSources.has(source)) return { ok: false, reason: "invalid-source" }; + if (!MACHINE_SOURCES.has(source)) return { ok: false, reason: "invalid-source" }; + if (!body.sourceRef) return { ok: false, reason: "missing-sourceRef" }; + if (!body.sourceRef.startsWith(`${source}:`)) { + return { ok: false, reason: "sourceRef-namespace-mismatch" }; + } + return { ok: true, source, sourceRef: body.sourceRef }; + } + + const source = auth.userVia === "mobile-jwt" ? "mobile" : "web"; + if (body.source && body.source !== source) return { ok: false, reason: "invalid-source" }; + // A RAW sourceRef stays forbidden — provenance is not caller input. + if (body.sourceRef) return { ok: false, reason: "sourceRef-not-allowed" }; + + // `uploadId` is the client's own idempotency token, SCOPED TO THE USER + // server-side: two people cannot collide on one uuid, and nobody can reach + // another user's row by guessing one. Without it a phone that retries on a + // flaky connection books the same receipt twice. + if (body.uploadId) { + if (!UUID_PATTERN.test(body.uploadId)) return { ok: false, reason: "invalid-uploadId" }; + return { ok: true, source, sourceRef: `${source}:${auth.user.id}:${body.uploadId.toLowerCase()}` }; + } + return { ok: true, source, sourceRef: `${source}:${randomUUID()}` }; +} diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts index 323bd6e3b..32ead3de2 100644 --- a/src/lib/receipt-intake/read.ts +++ b/src/lib/receipt-intake/read.ts @@ -60,6 +60,13 @@ export interface ReadResult { taxAmount: string; /** One of the supplied phase codes, or "". */ suggestedPhaseCode: string; + /** + * How sure the model is about that phase, 0..1. Null when it gave no usable + * number — which is NOT the same as 0, and must not be stored as 0: "the + * model didn't say" and "the model is sure it is a poor match" would then be + * indistinguishable in the queue. + */ + suggestedConfidence: number | null; /** The model's raw JSON text, stored for audit. */ raw: string; } @@ -150,9 +157,12 @@ export function buildReadPrompt(projectPhases: ProjectPhase[]): string { promptText + "\n\nSTEP 3 - this document belongs to a job with the following phases:\n" + phaseList + - "\nAdd ONE more output field, \"suggested_phase\", holding the CODE of the single phase " + - "this purchase most clearly belongs to. Use only a code from the list above, exactly as " + - 'written. If nothing on the document points clearly at one phase, return "".' + "\nAdd TWO more output fields: \"suggested_phase\", holding the CODE of the single phase " + + "this purchase most clearly belongs to (use only a code from the list above, exactly as " + + 'written; return "" if nothing on the document points clearly at one phase), and ' + + '"suggested_phase_confidence", a number from 0 to 1 for how sure you are of that phase. ' + + "Be honest about uncertainty — a low number sends the receipt to a human, which is the " + + "right outcome when the document is ambiguous." ); } @@ -180,6 +190,17 @@ export function normalizeDocType(value: unknown): string { return (DOC_TYPES as readonly string[]).includes(raw) ? raw : UNKNOWN_DOC_TYPE; } +/** + * 0..1, or null. Clamped at the edges (a model that says 1.2 means "very sure"), + * but anything non-numeric is null — never 0, because "no answer" and "sure it + * is a poor match" must stay distinguishable. + */ +export function normalizeConfidence(value: unknown): number | null { + const n = typeof value === "number" ? value : Number(coerce(value)); + if (!Number.isFinite(n)) return null; + return Math.min(1, Math.max(0, n)); +} + function coerce(value: unknown): string { if (value === null || value === undefined) return ""; return String(value).trim(); @@ -208,6 +229,11 @@ export function parseReadJson(text: string, projectPhases: ProjectPhase[]): Read totalAmount: coerce(json.total_amount), taxAmount: coerce(json.tax_amount), suggestedPhaseCode: allowed.has(suggested) ? suggested : "", + // Only meaningful alongside an ACCEPTED phase — a confidence attached + // to a suggestion we discarded would be actively misleading. + suggestedConfidence: allowed.has(suggested) + ? normalizeConfidence(json.suggested_phase_confidence) + : null, raw: text, }; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 0bacc20aa..4c4d6b09b 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -52,6 +52,12 @@ export const CLAIM_LEASE_MINUTES = 10; * being a formality. */ export const RUN_SOFT_DEADLINE_MS = 40_000; +/** + * The invocation's real ceiling (`maxDuration = 60`), minus a small margin so a + * booking that starts near the edge still gets to write its result. Bookings + * measure their runway against THIS, not the soft deadline. + */ +export const RUN_HARD_BUDGET_MS = 55_000; /** * How long a row may sit in STAGING before it is presumed to have lost its * upload. Generous on purpose: the intake route uploads inline, so a row that @@ -95,9 +101,11 @@ export interface WorkerDependencies { * backlog and both un-park it, and the second one's UPDATE would race the * first one's claim. */ - claim: (opts: { retireShadowBacklog: boolean }) => Promise<{ rows: WorkerRow[]; shadowRetired: number } | null>; + claim: (opts: CutoverRequest) => Promise; /** RECEIPT_INTAKE_DRYRUN is not "false". Injected so the cutover is testable. */ isDryRunEnabled: () => boolean; + /** The instant v1 stopped booking. null = not recorded; the cutover then refuses. */ + cutoverBoundary: () => Promise; /** * Move STAGING rows older than STAGING_SWEEP_MINUTES to NEEDS_REVIEW * `file-missing`, or PUBLISH them when the object is actually there. @@ -123,7 +131,8 @@ export interface WorkerDependencies { * row when another document with this weak key is already BOOKING/BOOKED. */ promoteToBooking: (rowId: string, weakKey: string | null) => Promise<{ promoted: boolean; conflictId?: string }>; - book: (row: BookableRow) => Promise; + /** `remainingMs` is what is left of the invocation when the booking starts. */ + book: (row: BookableRow, remainingMs: number) => Promise; applyBookResult: (rowId: string, result: BookResult) => Promise; /** AI unavailable: park for a later pass WITHOUT spending an attempt. */ deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; @@ -164,6 +173,7 @@ export interface ReadPatch { dedupWeakKey: string; duplicateOfId: string | null; suggestedCostCodeId: string | null; + suggestedConfidence: number | null; } export interface WorkerRunSummary { @@ -172,8 +182,12 @@ export interface WorkerRunSummary { skipped?: "already-running"; /** Rows left unprocessed because the soft deadline hit. They keep their lease. */ deferredToNextRun?: number; - /** Shadow-week rows retired as SHADOW_DONE by the first live pass. */ + /** Rows v1 already booked, retired as SHADOW_DONE by the first live pass. */ shadowRetired?: number; + /** Rows received AFTER v1 stopped: nobody booked these, so they are handed to v2. */ + requeued?: number; + /** The cutover could not run because no boundary is recorded. */ + cutoverBlocked?: "cutover-boundary-missing"; /** STAGING rows whose upload never landed, parked for a human. */ staleStagingSwept?: number; } @@ -259,6 +273,24 @@ export function toDateStr(date: Date): string { } /** One pass. Never throws for a single bad row — one poison document must not stall the queue. */ +export interface CutoverRequest { + /** Run the cutover this pass (i.e. dry-run is off). */ + run: boolean; + /** + * The instant v1 stopped booking. Rows received BEFORE it were booked by + * v1 and are retired; rows received AFTER it were booked by nobody and are + * handed to v2. null refuses to touch either side. + */ + boundary: Date | null; +} + +export interface ClaimResult { + rows: WorkerRow[]; + shadowRetired: number; + requeued: number; + boundaryMissing: boolean; +} + export async function runIntakeWorker(deps: WorkerDependencies): Promise { // THE DEADLINE STARTS HERE, at invocation entry — not after the claim and // the sweep. The sweep downloads objects, so timing it out of the budget @@ -266,6 +298,10 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise deps.monotonicMs() - startedAt >= RUN_SOFT_DEADLINE_MS; + // What is left of the PLATFORM budget, not the soft deadline: a booking may + // legitimately run past the point where we stop taking new rows, it just + // must not start without room to finish. + const remainingMs = () => RUN_HARD_BUDGET_MS - (deps.monotonicMs() - startedAt); // CUTOVER. Rows received while dry-run was on were booked by v1, so v2 must // never book them: they are RETIRED as SHADOW_DONE, not requeued. @@ -275,11 +311,22 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise { /ALTER TABLE .* ADD COLUMN IF NOT EXISTS/.test(sql) || // Re-enabling RLS on a table that already has it is a no-op. /ENABLE ROW LEVEL SECURITY/.test(sql) || - /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(sql); + /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(sql) || + // The state CHECK is convergent rather than skip-if-present: it + // compares pg_get_constraintdef and only rewrites on a difference, + // so a second run is a no-op just the same. + (/pg_get_constraintdef/.test(sql) && /IS DISTINCT FROM/.test(sql)); assert.ok(guarded, `not idempotent: ${sql.slice(0, 80)}`); } }); @@ -204,3 +208,43 @@ test("SHADOW_DONE stays in the strong-key active set", () => { assert.match(index!, /NOT IN \('DUPLICATE', 'VOID'\)/); assert.ok(!/SHADOW_DONE/.test(index!), "SHADOW_DONE must NOT be excluded"); }); + +test("the state CHECK is REPLACED when its definition drifts, not skipped", () => { + // `IF NOT EXISTS` alone is wrong for a set that GROWS: a database carrying + // the constraint from an earlier run keeps the OLD state list, so the first + // write of a newly-added state (SHADOW_DONE, at cutover, inside the claim + // transaction) fails and takes the whole cutover with it — while the script + // that exists to prevent exactly that reported "ok". + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check")); + assert.ok(check); + assert.match(check!, /pg_get_constraintdef/, "it compares the DEFINITION"); + assert.match(check!, /IS DISTINCT FROM/, "and reacts to a difference"); + assert.match(check!, /DROP CONSTRAINT "ReceiptIntake_state_check"/); + assert.match(check!, /ADD CONSTRAINT "ReceiptIntake_state_check"/); + // The wanted definition must name every state the code can produce, in + // pg_get_constraintdef's own rendering. + for (const state of RECEIPT_INTAKE_STATES) { + assert.ok(check!.includes(`''${state}''::text`), `wanted_def is missing ${state}`); + } +}); + +test("the wanted definition matches the snapshot CI compares against production", () => { + // Two renderings of the same constraint that disagree would make the + // apply script drop and re-add it on EVERY run. + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check"))!; + // [\s\S] rather than the /s flag — the tsconfig target predates it. + const wanted = check.match(/wanted_def\s+TEXT\s*:=\s*'([\s\S]+?)';/)![1].replace(/''/g, "'"); + const snapshot = JSON.parse( + readFileSync(path.join(__dirname, "..", "prisma", "prisma-blind-spots.json"), "utf8"), + ); + const recorded = snapshot.checkConstraints.find( + (r: { name: string }) => r.name === "ReceiptIntake_state_check", + ); + assert.equal(wanted, recorded.def); +}); + +test("verification asserts the CHECK ALLOWS every state, not just that it exists", () => { + const source = readFileSync(path.join(__dirname, "..", "scripts", "apply-receipt-intake.mjs"), "utf8"); + assert.match(source, /pg_get_constraintdef\(oid\) AS def FROM pg_constraint/, "verify reads the definition"); + assert.match(source, /does not allow/, "and fails loudly naming what is missing"); +}); diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 3c0b9c076..8f2159b55 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -147,3 +147,213 @@ test("a Next-Action dispatch on a bypassed intake path is 403, not waved through env.NODE_ENV = prod; } }); + +test("the two-step upload paths bypass the proxy, exactly", async () => { + const { isPublicProxyBypass } = await loadProxy(); + for (const path of [ + "/api/receipts/intake/start", + "/api/receipts/intake/start/", + "/api/receipts/intake/abc123/finalize", + "/api/receipts/intake/abc123/finalize/", + ]) { + assert.equal(isPublicProxyBypass(path), true, path); + } + // ...and no wider than that. + for (const path of [ + "/api/receipts/intake/start/extra", + "/api/receipts/intake/abc123/finalize/extra", + "/api/receipts/intake/abc123", + "/api/receipts/intake/abc123/other", + ]) { + assert.equal(isPublicProxyBypass(path), false, path); + } +}); + +test("a Next-Action dispatch is refused on the two-step paths too", async () => { + // Same reasoning as the single-shot route: these bypass the proxy, so the + // in-handler secret/session check is their ONLY gate, and an action dispatch + // never reaches the handler at all. + const { default: proxy } = await loadProxy(); + const { NextRequest } = await import("next/server"); + const event = { waitUntil() {} } as any; + const env = process.env as Record; + const prod = env.NODE_ENV; + env.NODE_ENV = "production"; + try { + for (const path of ["/api/receipts/intake/start", "/api/receipts/intake/abc123/finalize"]) { + const res = await proxy( + new NextRequest(`https://probuild.test${path}`, { + method: "POST", + headers: { "next-action": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, + }), + event, + ); + assert.ok(res, path); + assert.equal(res.status, 403, `${path} must refuse an action dispatch`); + assert.equal(res.headers.get("x-middleware-next"), null, path); + } + // A normal machine POST still passes through. + const normal = await proxy( + new NextRequest("https://probuild.test/api/receipts/intake/start", { + method: "POST", + headers: { "x-receipt-intake-secret": "whatever" }, + }), + event, + ); + assert.equal(normal!.headers.get("x-middleware-next"), "1"); + } finally { + env.NODE_ENV = prod; + } +}); + +test("provenance rules are shared by BOTH upload paths", async () => { + // decideSource is the single implementation, so the two-step flow cannot + // drift into accepting a caller-chosen source or sourceRef. + const { decideSource, MAX_INLINE_UPLOAD_BYTES, MAX_STORED_BYTES } = + await import("../src/lib/receipt-intake/intake-core"); + + const session = { ok: true, via: "session", userVia: "next-auth", user: { id: "u1", role: "ADMIN" } } as any; + assert.deepEqual(decideSource(session, { source: "drive" }), { ok: false, reason: "invalid-source" }); + assert.deepEqual(decideSource(session, { sourceRef: "web:x" }), { ok: false, reason: "sourceRef-not-allowed" }); + assert.deepEqual(decideSource(session, { uploadId: "nope" }), { ok: false, reason: "invalid-uploadId" }); + + const scoped = decideSource(session, { uploadId: "3f2504e0-4f89-41d3-9a0c-0305e82c3301" }); + assert.ok(scoped.ok); + assert.equal(scoped.sourceRef, "web:u1:3f2504e0-4f89-41d3-9a0c-0305e82c3301", "scoped to the USER"); + + const mobile = { ok: true, via: "session", userVia: "mobile-jwt", user: { id: "u2", role: "FIELD_CREW" } } as any; + const minted = decideSource(mobile, {}); + assert.ok(minted.ok); + assert.match(minted.sourceRef, /^mobile:[0-9a-f-]{36}$/); + + const secret = { + ok: true, via: "secret", user: null, userVia: null, + capability: "ingest", allowedSources: new Set(["drive", "email", "chat"]), + } as any; + assert.deepEqual(decideSource(secret, { source: "chat", sourceRef: "drive:x" }), + { ok: false, reason: "sourceRef-namespace-mismatch" }); + assert.deepEqual(decideSource(secret, { source: "web", sourceRef: "web:x" }), + { ok: false, reason: "invalid-source" }); + assert.ok(decideSource(secret, { source: "drive", sourceRef: "drive:FILE1" }).ok); + + // The inline body cap is well under the stored cap, which is the whole + // reason the two-step path exists. + assert.ok(MAX_INLINE_UPLOAD_BYTES < MAX_STORED_BYTES); + assert.equal(MAX_STORED_BYTES, 15 * 1024 * 1024); +}); + +// ── Two secrets, two blast radii (Phase 3 gate, c) ───────────────────────── + +test("each secret may only do its own job; cross-use is 403", async () => { + const { authenticateIntake, INGEST_ALLOWED_SOURCES } = await loadAuth(); + const env = process.env as Record; + const before = { i: env.RECEIPT_INTAKE_SECRET, a: env.RECEIPT_ARCHIVE_SECRET }; + env.RECEIPT_INTAKE_SECRET = "ingest-key"; + env.RECEIPT_ARCHIVE_SECRET = "archive-key"; + const req = (secret: string) => + new Request("https://probuild.test/api/receipts/intake", { + method: "POST", + headers: { "x-receipt-intake-secret": secret }, + }); + + try { + // Right key, right job. + const ingesting = await authenticateIntake(req("ingest-key"), "ingest"); + assert.ok(ingesting.ok); + assert.equal(ingesting.via, "secret"); + if (ingesting.via !== "secret") throw new Error("unreachable"); + assert.equal(ingesting.capability, "ingest"); + assert.deepEqual([...ingesting.allowedSources].sort(), ["chat", "drive", "email"]); + + const archiving = await authenticateIntake(req("archive-key"), "archive"); + assert.ok(archiving.ok); + if (archiving.via !== "secret") throw new Error("unreachable"); + assert.equal(archiving.capability, "archive"); + assert.equal(archiving.allowedSources.size, 0, "the mirror declares no sources at all"); + + // Cross-use: authenticated, but holding the OTHER program's key. 403, + // not 401 — saying so is what makes a mis-wired script obvious rather + // than looking like a rotation problem. + const forwarderReadingTheQueue = await authenticateIntake(req("ingest-key"), "archive"); + assert.equal(forwarderReadingTheQueue.ok, false); + assert.equal((forwarderReadingTheQueue as { response: Response }).response.status, 403); + + const mirrorInjectingReceipts = await authenticateIntake(req("archive-key"), "ingest"); + assert.equal(mirrorInjectingReceipts.ok, false); + assert.equal((mirrorInjectingReceipts as { response: Response }).response.status, 403); + + // An unknown secret is 401, not 403 — it is not authenticated at all. + const stranger = await authenticateIntake(req("neither"), "ingest"); + assert.equal((stranger as { response: Response }).response.status, 401); + + assert.deepEqual([...INGEST_ALLOWED_SOURCES].sort(), ["chat", "drive", "email"]); + } finally { + env.RECEIPT_INTAKE_SECRET = before.i; + env.RECEIPT_ARCHIVE_SECRET = before.a; + } +}); + +test("configuring ONE value for both variables is refused, not silently merged", async () => { + // Otherwise the split is undone by a copy-paste and nobody finds out. + const { authenticateIntake } = await loadAuth(); + const env = process.env as Record; + const before = { i: env.RECEIPT_INTAKE_SECRET, a: env.RECEIPT_ARCHIVE_SECRET }; + env.RECEIPT_INTAKE_SECRET = "same"; + env.RECEIPT_ARCHIVE_SECRET = "same"; + try { + const res = await authenticateIntake( + new Request("https://probuild.test/api/receipts/intake", { + method: "POST", + headers: { "x-receipt-intake-secret": "same" }, + }), + "ingest", + ); + assert.equal(res.ok, false); + assert.equal((res as { response: Response }).response.status, 401); + } finally { + env.RECEIPT_INTAKE_SECRET = before.i; + env.RECEIPT_ARCHIVE_SECRET = before.a; + } +}); + +test("an unset secret refuses that capability — never fails open", async () => { + const { authenticateIntake } = await loadAuth(); + const env = process.env as Record; + const before = { i: env.RECEIPT_INTAKE_SECRET, a: env.RECEIPT_ARCHIVE_SECRET }; + delete env.RECEIPT_INTAKE_SECRET; + delete env.RECEIPT_ARCHIVE_SECRET; + try { + for (const need of ["ingest", "archive"] as const) { + const res = await authenticateIntake( + new Request("https://probuild.test/api/receipts/intake", { + method: "POST", + headers: { "x-receipt-intake-secret": "anything" }, + }), + need, + ); + assert.equal(res.ok, false, need); + assert.equal((res as { response: Response }).response.status, 401, need); + } + } finally { + env.RECEIPT_INTAKE_SECRET = before.i; + env.RECEIPT_ARCHIVE_SECRET = before.a; + } +}); + +test("a secret may only declare the sources ITS key owns", async () => { + const { decideSource } = await import("../src/lib/receipt-intake/intake-core"); + const ingest = { + ok: true, via: "secret", user: null, userVia: null, + capability: "ingest", allowedSources: new Set(["drive", "email", "chat"]), + } as any; + const archive = { + ok: true, via: "secret", user: null, userVia: null, + capability: "archive", allowedSources: new Set(), + } as any; + + assert.ok(decideSource(ingest, { source: "drive", sourceRef: "drive:F1" }).ok); + // The archive key owns no sources, so it can never mint an intake row even + // if it somehow reached this code. + assert.deepEqual(decideSource(archive, { source: "drive", sourceRef: "drive:F1" }), + { ok: false, reason: "invalid-source" }); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 23d685353..ea6b2ca19 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -17,6 +17,7 @@ import { buildGroups, driveFileIdOf, expenseAmountCents, + MIN_BOOKING_BUDGET_MS, type BookableRow, type BookDependencies, } from "../src/lib/receipt-intake/book"; @@ -38,6 +39,7 @@ function row(overrides: Partial = {}): BookableRow { projectId: "proj-1", costCodeId: null, suggestedCostCodeId: "cc-plumb", + suggestedConfidence: 0.82, storagePath: "receipts/intake/intake-1.jpg", fileName: "receipt.jpg", mimeType: "image/jpeg", @@ -98,6 +100,8 @@ function recorder(overrides: Partial = {}, opts: { estimates?: downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), logEvent: async (event) => { events.push(event); }, now: () => NOW, + companyTimeZone: async () => "America/Los_Angeles", + isCostCodeAllowed: async () => true, ...overrides, }; return { deps, purchaseCalls, expenses, intakeUpdates, events }; @@ -483,3 +487,156 @@ test("the receipt bytes always ride along with the Purchase", async () => { assert.equal(r.purchaseCalls[0].fileBase64, Buffer.from("bytes").toString("base64")); assert.equal(r.purchaseCalls[0].fileContentType, "image/jpeg"); }); + +// ── Expense.date is a business calendar day (round-6 item 3) ──────────────── + +test("Expense.date is re-anchored to the company's local midnight", async () => { + // txnDate is a @db.Date column and round-trips as UTC midnight. Written + // straight into Expense.date (a full timestamp) that records 5pm the + // PREVIOUS day in Pacific, and every job-cost or variance report bounded by + // local midnight then counts the expense in the wrong period. + const r = recorder(); + await bookReceipt(row({ txnDate: new Date("2026-08-03T00:00:00.000Z") }), r.deps); + + const written = r.expenses[0].date as Date; + assert.equal(written.toISOString(), "2026-08-03T07:00:00.000Z", "local midnight PDT"); + + // The assertion that matters: read back in the company zone it is the 3rd. + const localDay = new Intl.DateTimeFormat("en-CA", { + timeZone: "America/Los_Angeles", year: "numeric", month: "2-digit", day: "2-digit", + }).format(written); + assert.equal(localDay, "2026-08-03"); + + // Control: the raw txnDate would have read as the 2nd. That was the bug. + assert.equal( + new Intl.DateTimeFormat("en-CA", { + timeZone: "America/Los_Angeles", year: "numeric", month: "2-digit", day: "2-digit", + }).format(new Date("2026-08-03T00:00:00.000Z")), + "2026-08-02", + ); + + // ...and QBO still gets the bare calendar day, unchanged. + assert.equal(r.purchaseCalls[0].date, "2026-08-03"); +}); + +test("winter dates use the winter offset — no hardcoded -07:00", async () => { + const r = recorder(); + await bookReceipt(row({ txnDate: new Date("2026-01-15T00:00:00.000Z") }), r.deps); + assert.equal((r.expenses[0].date as Date).toISOString(), "2026-01-15T08:00:00.000Z"); +}); + +// ── A booking must not start without room to finish (round-6 item 4) ──────── + +test("a booking with less than 25s of runway DEFERS instead of starting", async () => { + // Two QuickBooks round trips plus the attachment upload and the commit do + // not fit in a few seconds, and a booking cut off mid-flight is the worst + // outcome available: the Purchase may exist in the real books while the row + // never learns it did. + const r = recorder({ remainingBudgetMs: () => 9_000 }); + const result = await bookReceipt(row(), r.deps); + assert.deepEqual(result, { outcome: "deferred", reason: "out-of-budget" }); + assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never touched"); + assert.equal(r.expenses.length, 0); +}); + +test("the runway check spends no attempt — the document did nothing wrong", async () => { + const r = recorder({ remainingBudgetMs: () => 0 }); + const result = await bookReceipt(row({ attempts: 3 }), r.deps); + assert.equal(result.outcome, "deferred"); + assert.ok(!("attempts" in result), "not a retry, so no attempt is spent"); +}); + +test("ample runway books normally, and threads ONE deadline into both QBO calls", async () => { + const seen: unknown[] = []; + const r = recorder({ + remainingBudgetMs: () => 50_000, + deadline: () => ({ startedAt: 0, budgetMs: 50_000 }) as any, + getTokens: async d => { seen.push(d); return { accessToken: "t", realmId: "r" } as any; }, + createPurchase: async (_t, input, d) => { + seen.push(d); + r.purchaseCalls.push(input); + return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "attached" } as any; + }, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "booked"); + assert.equal(seen.length, 2); + assert.strictEqual(seen[0], seen[1], "the SAME deadline object, so a slow refresh shortens the create"); +}); + +test("MIN_BOOKING_BUDGET_MS is the documented 25s", () => { + assert.equal(MIN_BOOKING_BUDGET_MS, 25_000); +}); + +// ── The phase is re-validated against the FINAL project (Phase 3 gate, a) ─── + +test("a cost code that is not a phase of THIS job is cleared, and the row still books", async () => { + // The scenario: the receipt was READ while it had no project (NEEDS_JOB) or + // a different one, and a human then assigned the real job. A cost code from + // the old project is not a phase of the new one, and posting against it puts + // real money on a phase that job does not have — which every variance report + // reads as overspend on a line nobody budgeted. + const r = recorder({ isCostCodeAllowed: async () => false }); + const result = await bookReceipt(row({ costCodeId: "cc-from-another-job" }), r.deps); + + assert.equal(result.outcome, "booked", "the receipt is fine — it books UNCODED"); + assert.equal(r.expenses[0].costCodeId, null, "the wrong phase is cleared, never posted"); + assert.match(r.expenses[0].description, /phase cleared \(not a phase of this job\)/); + assert.equal(r.events[0].detail.phaseRejected, "cc-from-another-job", "and it is auditable"); +}); + +test("the SUGGESTED code is checked against the final project too", async () => { + // The model suggested it from the phase list of whatever project the row had + // at READ time. That list is not authority over the project it books to. + const asked: Array<[string, string]> = []; + const r = recorder({ + isCostCodeAllowed: async (projectId, costCodeId) => { + asked.push([projectId, costCodeId]); + return false; + }, + }); + await bookReceipt(row({ costCodeId: null }), r.deps); + assert.deepEqual(asked, [["proj-1", "cc-plumb"]], "asked about the FINAL project"); + assert.equal(r.expenses[0].costCodeId, null); +}); + +test("unassigned during READ, assigned before BOOKING: the phase is re-checked", async () => { + // End to end for the exact sequence the gate named. + const allowedByProject: Record = { + "proj-1": ["cc-demo"], // the job it was finally assigned to + }; + const r = recorder({ + isCostCodeAllowed: async (projectId, costCodeId) => + (allowedByProject[projectId] ?? []).includes(costCodeId), + }); + + // Suggested "cc-plumb" while unassigned; the job it landed on has no plumbing phase. + await bookReceipt(row({ costCodeId: null, suggestedCostCodeId: "cc-plumb" }), r.deps); + assert.equal(r.expenses[0].costCodeId, null, "the stale suggestion does not survive"); + + // A code that IS a phase of the final project is kept. + const ok = recorder({ + isCostCodeAllowed: async (projectId, costCodeId) => + (allowedByProject[projectId] ?? []).includes(costCodeId), + }); + await bookReceipt(row({ costCodeId: "cc-demo" }), ok.deps); + assert.equal(ok.expenses[0].costCodeId, "cc-demo"); +}); + +// ── Confidence rides through to the booking (Phase 3 gate, b) ────────────── + +test("the phase-suggestion confidence is recorded when the suggestion is used", async () => { + const r = recorder(); + await bookReceipt(row({ costCodeId: null, suggestedConfidence: 0.42 }), r.deps); + assert.equal(r.expenses[0].costCodeId, "cc-plumb"); + assert.match(r.expenses[0].description, /phase suggested \(confidence 0\.42\)/); + assert.equal(r.events[0].detail.suggestedConfidence, 0.42); +}); + +test("a human's explicit pick is not labelled a suggestion", async () => { + const r = recorder(); + await bookReceipt(row({ costCodeId: "cc-chosen" }), r.deps); + assert.equal(r.expenses[0].costCodeId, "cc-chosen"); + assert.ok(!/phase suggested/.test(r.expenses[0].description)); + assert.equal(r.events[0].detail.suggestedConfidence, undefined); +}); diff --git a/tests/receipt-intake-cutover.test.ts b/tests/receipt-intake-cutover.test.ts new file mode 100644 index 000000000..5b3bd967c --- /dev/null +++ b/tests/receipt-intake-cutover.test.ts @@ -0,0 +1,33 @@ +/** + * The cutover boundary and the storage-failure classification. + * + * Both are places where getting the answer WRONG loses money rather than + * merely erroring: a mis-parsed boundary retires receipts nobody booked, and a + * mis-classified storage fault declares a present file missing and releases its + * dedup key. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseCutoverBoundary, CUTOVER_SETTING_KEY } from "../src/lib/receipt-intake/cutover"; + +test("a missing or malformed boundary is null — never epoch", () => { + // The dangerous failure: `new Date(undefined)` style coercion yielding a + // date in 1970 would put the ENTIRE backlog "before the boundary" and + // retire every row, including the ones v1 never booked. + for (const bad of [undefined, null, "", " ", "not-a-date", "yesterday", "2026-13-45"]) { + assert.equal(parseCutoverBoundary(bad as string | null | undefined), null, JSON.stringify(bad)); + } +}); + +test("a real ISO timestamp parses to that instant", () => { + const at = parseCutoverBoundary("2026-08-25T17:30:00.000Z"); + assert.ok(at); + assert.equal(at.toISOString(), "2026-08-25T17:30:00.000Z"); + // Surrounding whitespace is a copy-paste artefact, not a different answer. + assert.equal(parseCutoverBoundary(" 2026-08-25T17:30:00.000Z ")!.toISOString(), "2026-08-25T17:30:00.000Z"); +}); + +test("the setting key is stable — an operator writes this row at the flip", () => { + // Renaming it silently would make every future cutover refuse. + assert.equal(CUTOVER_SETTING_KEY, "cutoverV1StoppedAt"); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index aead016b0..443aceacf 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -26,6 +26,7 @@ import { } from "../src/lib/receipt-intake/worker"; import { normalizeDocType, type ReadOutcome } from "../src/lib/receipt-intake/read"; import type { BookResult } from "../src/lib/receipt-intake/book"; +import type { CutoverRequest } from "../src/lib/receipt-intake/worker"; import { QBTimeoutError } from "../src/lib/quickbooks"; import { QboAccountConfigError, QboPurchaseFaultError } from "../src/lib/qbo-receipt-push"; @@ -61,6 +62,7 @@ function workerRow(overrides: Partial = {}): WorkerRow { dedupWeakKey: null, busyPasses: 0, lastError: null, + suggestedConfidence: null, ...overrides, }; } @@ -77,6 +79,7 @@ const goodRead: ReadOutcome = { totalAmount: "364.98", taxAmount: "29.20", suggestedPhaseCode: "03-PLUMB", + suggestedConfidence: 0.82, raw: '{"vendor":"Lowes"}', }, }; @@ -91,19 +94,26 @@ interface Harness { finished: { id: string; stateReason: string | null }[]; deferred: { id: string; busyPasses: number }[]; retried: { id: string; attempts: number; reason: string }[]; - claimOpts: { retireShadowBacklog: boolean }[]; + claimOpts: CutoverRequest[]; + boundary: Date | null; sweepCalls: number; + bookBudgets: number[]; clock: number; } function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { reads: 0, books: 0, applied: [], states: [], promoted: [], finished: [], deferred: [], - retried: [], claimOpts: [], sweepCalls: 0, clock: 0, + retried: [], claimOpts: [], sweepCalls: 0, bookBudgets: [], clock: 0, + boundary: new Date("2026-08-25T00:00:00.000Z"), deps: null as unknown as WorkerDependencies, }; h.deps = { - claim: async opts => { h.claimOpts.push(opts); return { rows, shadowRetired: 0 }; }, + claim: async opts => { + h.claimOpts.push(opts); + return { rows, shadowRetired: 0, requeued: 0, boundaryMissing: false }; + }, + cutoverBoundary: async () => h.boundary, isDryRunEnabled: () => true, sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], @@ -115,7 +125,11 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) finishRouting: async (id, stateReason) => { h.finished.push({ id, stateReason }); }, companyTimeZone: async () => "America/Los_Angeles", promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, - book: async () => { h.books++; return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, + book: async (_row, remainingMs) => { + h.books++; + h.bookBudgets.push(remainingMs); + return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; + }, applyBookResult: async () => {}, deferRead: async (id, busyPasses) => { h.deferred.push({ id, busyPasses }); }, retryRow: async (id, attempts, _next, reason) => { h.retried.push({ id, attempts, reason }); }, @@ -285,35 +299,59 @@ test("a strong-key loss to a DIFFERENT vendor is a collision, not a duplicate", // ── Dry-run starvation (Codex blocker 1) ───────────────────────────────────── -test("the shadow week does NOT retire anything", async () => { +test("the shadow week does NOT run the cutover", async () => { const h = harness([workerRow({ state: "READ", dryRun: true })], { isDryRunEnabled: () => true }); const summary = await runIntakeWorker(h.deps); - assert.deepEqual(h.claimOpts, [{ retireShadowBacklog: false }]); + assert.equal(h.claimOpts[0].run, false); + assert.equal(h.claimOpts[0].boundary, null, "the boundary is not even read while dry-run is on"); assert.equal(summary.shadowRetired, undefined); }); -test("CUTOVER: the shadow backlog is RETIRED, never requeued", async () => { +test("CUTOVER: the boundary is passed to the claim so the backlog can be split", async () => { // The double-booking hazard this closes: v2's QBO identity for an // email/chat/mobile/web row is the intake UUID, which v1 never saw, so // QuickBooks' DocNumber idempotency could not recognise the Purchase v1 // already made — and requeuing would have booked the entire shadow backlog // a second time, on real books, in one pass. + const boundary = new Date("2026-08-25T00:00:00.000Z"); const h = harness([], { isDryRunEnabled: () => false, - claim: async opts => { h.claimOpts.push(opts); return { rows: [], shadowRetired: 7 }; }, + cutoverBoundary: async () => boundary, + claim: async opts => { + h.claimOpts.push(opts); + // Rows BEFORE the boundary were booked by v1; rows after it by nobody. + return { rows: [], shadowRetired: 7, requeued: 2, boundaryMissing: false }; + }, }); const summary = await runIntakeWorker(h.deps); - assert.deepEqual(h.claimOpts, [{ retireShadowBacklog: true }]); - assert.equal(summary.shadowRetired, 7); - assert.equal(h.books, 0, "nothing from the shadow week is ever booked by v2"); + assert.equal(h.claimOpts[0].run, true); + assert.equal(h.claimOpts[0].boundary?.toISOString(), boundary.toISOString()); + assert.equal(summary.shadowRetired, 7, "v1 already booked these"); + assert.equal(summary.requeued, 2, "nobody booked these — v2 must"); + assert.equal(summary.cutoverBlocked, undefined); +}); - // Idempotent by construction: SHADOW_DONE no longer matches the predicate. - const second = harness([], { isDryRunEnabled: () => false }); - assert.equal((await runIntakeWorker(second.deps)).shadowRetired, undefined, "a no-op retire is not reported"); +test("CUTOVER refuses entirely when no boundary is recorded", async () => { + // Nothing in the database can infer when v1 stopped booking. Retiring on a + // guess destroys evidence of real expenses; requeuing on a guess + // double-books them. A logged no-op is the only honest third option. + const h = harness([], { + isDryRunEnabled: () => false, + cutoverBoundary: async () => null, + claim: async opts => { + h.claimOpts.push(opts); + assert.equal(opts.boundary, null); + return { rows: [], shadowRetired: 0, requeued: 0, boundaryMissing: true }; + }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.cutoverBlocked, "cutover-boundary-missing"); + assert.equal(summary.shadowRetired, undefined); + assert.equal(summary.requeued, undefined); }); -test("a run that loses the lock does nothing at all — including the retire", async () => { - // The retire is part of the claim transaction, so losing the lock means +test("a run that loses the lock does nothing at all — including the cutover", async () => { + // The cutover is part of the claim transaction, so losing the lock means // losing it too. That is correct: the run that HOLDS the lock does it. const h = harness([], { isDryRunEnabled: () => false, claim: async () => null }); const summary = await runIntakeWorker(h.deps); diff --git a/tests/secure-storage-classification.test.ts b/tests/secure-storage-classification.test.ts new file mode 100644 index 000000000..7714a2c22 --- /dev/null +++ b/tests/secure-storage-classification.test.ts @@ -0,0 +1,64 @@ +/** + * "The object is gone" vs "storage hiccuped" — the distinction that decides + * whether a receipt is parked for a human and its dedup key RELEASED, or simply + * retried. + * + * The expensive direction is the safe-looking one: Supabase returns 400 for a + * malformed request, a bad JWT, an expired service key and assorted config + * faults. Reading those as not-found would empty the queue into review on a key + * rotation and unlock every strong key on the way out. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { isNotFoundError } from "../src/lib/secure-storage"; + +test("an affirmative 404 is not-found", () => { + assert.equal(isNotFoundError({ status: 404, message: "Not Found" }), true); + assert.equal(isNotFoundError({ statusCode: "404", message: "anything" }), true); + assert.equal(isNotFoundError({ statusCode: 404 }), true); +}); + +test("an explicit not-found ERROR CODE is not-found", () => { + for (const code of ["NoSuchKey", "not_found", "NOT FOUND", "object_not_found", "EntityNotFound"]) { + assert.equal(isNotFoundError({ error: code }), true, code); + } +}); + +test("the exact not-found MESSAGE is not-found", () => { + assert.equal(isNotFoundError({ message: "Object not found" }), true); + assert.equal(isNotFoundError({ message: "The resource was not found" }), true); +}); + +test("400 is NOT evidence of absence, whatever it says", () => { + // This is the regression. Every one of these used to be read as "gone". + for (const error of [ + { status: 400, message: "Invalid JWT" }, + { status: 400, message: "invalid signature" }, + { status: 400, message: "Bucket not found" }, + { status: 400 }, + ]) { + assert.equal(isNotFoundError(error), false, JSON.stringify(error)); + } +}); + +test("auth, rate-limit, server and network faults are all transient", () => { + for (const error of [ + { status: 401, message: "Unauthorized" }, + { status: 403, message: "forbidden" }, + { status: 429, message: "Too Many Requests" }, + { status: 500, message: "Internal Error" }, + { status: 503, message: "Service Unavailable" }, + { message: "fetch failed" }, + { message: "socket hang up" }, + ]) { + assert.equal(isNotFoundError(error), false, JSON.stringify(error)); + } +}); + +test("a message that merely CONTAINS 'not found' is not enough", () => { + // Substring matching is how a config error ("bucket not found for this + // project", "tenant not found") gets mistaken for a missing object. + assert.equal(isNotFoundError({ message: "bucket not found for this project" }), false); + assert.equal(isNotFoundError({ message: "tenant not found" }), false); + assert.equal(isNotFoundError(null), false); +}); From e3da4a6d85cb8031b95bdb6ce9730e402c9ec6aa Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 01:41:46 -0700 Subject: [PATCH 044/144] fix(receipts): void-lock, shared object validation, evidence-based retirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 7, all eight. 1 pg_advisory_xact_lock returns VOID, and it was being SELECTed through $queryRaw: the row's single column has no readable type, which Prisma's query path can reject — inside the promotion transaction, where the throw looks like a transient DB fault forever while the lock was never taken. Now $executeRaw. A source tripwire fails on any $queryRaw of a void function (allowing an explicit ::text cast, which is how qbo-expense-sync makes its lock readable), and a real-Postgres suite exercises the lock and the claim; it skips unless RECEIPT_INTAKE_DB_TEST_URL points at a throwaway DB, and CI's migrations job now supplies one. 2 The STAGING sweep runs the SAME validator as /finalize (stored-object.ts): size, magic bytes, sha from storage. Publishing on "the object exists" alone would wave through a 40 MB video or a truncated upload that /finalize refuses — and those rows go on to Gemini and QuickBooks. Rejections delete the row and the object. 3 A missing boundary now halts the WHOLE live pass before claim(). Refusing only the retire left the pass booking rows while the backlog sat undecided, which is the double-booking this mechanism exists to prevent. 4 /start persists the client's expectedSha256 and refuses a reused sourceRef carrying a different document BEFORE issuing a URL — otherwise the caller uploads receipt B over receipt A's object and only /finalize notices, by which point A is gone. Signed URLs use upsert so a resumed /start can overwrite its OWN partial upload. /finalize checks the stored sha against both the declared and the expected one. 5 A rejected object whose delete fails is recorded as a storage-cleanup-pending AutomationEvent and retried by the sweep. The row is gone by then, so nothing else remembers the orphan. 6 ONE absolute deadline object, passed into bookReceipt and re-read at every check (entry, after the download, after the token refresh). A frozen remainingMs is measured once and then decays silently while the work runs. 7 sendAttempted is persisted immediately BEFORE the create, and it — not the reason string or the fact of hitting the retry ceiling — decides whether a terminal park releases the strong key. A row that burned 20 attempts on storage or a weak-lookup fault never touched QuickBooks, and holding its key quarantines the corrected resend against nothing. 8 SHADOW_DONE now requires POSITIVE evidence v1 booked the row: its own receipt-push AutomationEvent matched on driveFileId, or archivedByV1 from the forwarder (secret callers only — a browser upload can never claim it). Unevidenced rows are handed to v2, which is safe because Drive rows book under the Drive file id and QBO's idempotency collapses any overlap. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 9 + docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 23 ++- e2e/receipt-intake.spec.ts | 194 ++++++++++++++++++ package.json | 4 +- .../migration.sql | 6 + prisma/schema.prisma | 13 ++ scripts/apply-receipt-intake.mjs | 9 +- .../api/cron/receipt-intake-worker/route.ts | 173 ++++++++++++---- .../receipts/intake/[id]/finalize/route.ts | 76 ++++--- src/app/api/receipts/intake/route.ts | 10 + src/app/api/receipts/intake/start/route.ts | 43 +++- src/lib/automation-events.ts | 14 +- src/lib/receipt-intake/book.ts | 81 ++++++-- src/lib/receipt-intake/storage-cleanup.ts | 80 ++++++++ src/lib/receipt-intake/stored-object.ts | 66 ++++++ src/lib/receipt-intake/worker.ts | 60 ++++-- tests/receipt-intake-book.test.ts | 32 ++- tests/receipt-intake-claim-db.test.ts | 133 ++++++++++++ tests/receipt-intake-raw-sql.test.ts | 120 +++++++++++ tests/receipt-intake-stored-object.test.ts | 76 +++++++ tests/receipt-intake-worker.test.ts | 91 +++++++- 21 files changed, 1180 insertions(+), 133 deletions(-) create mode 100644 src/lib/receipt-intake/storage-cleanup.ts create mode 100644 src/lib/receipt-intake/stored-object.ts create mode 100644 tests/receipt-intake-claim-db.test.ts create mode 100644 tests/receipt-intake-raw-sql.test.ts create mode 100644 tests/receipt-intake-stored-object.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24cbf1a73..eed1a8af5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,15 @@ jobs: # The focused integration regression deliberately drifts the disposable # database's function and trigger after proving the checker recognizes # the production snapshot. It therefore runs last in this job. + # The receipt-intake claim transaction against a REAL Postgres. The rest + # of that feature's suites mock every DB call, so the SQL — a void + # function read through $queryRaw, a claim that behaves differently than + # its mock — is the one part they cannot see. + - name: Receipt-intake claim + advisory locks against real Postgres + run: npx tsx --test tests/receipt-intake-claim-db.test.ts + env: + RECEIPT_INTAKE_DB_TEST_URL: postgresql://probuild:probuild@localhost:5432/probuild_migrations + - name: Prove the blind-spot checker rejects function and trigger drift run: npx tsx --test tests/migration-history-blind-spots.test.ts env: diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index 0ac0187a2..b4b5b6ce8 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -522,14 +522,21 @@ an overlap safe in general. an ISO timestamp. This is the ONLY input that separates "v1 booked it" from "nobody booked it", and nothing in the database can infer it. 5. **Only then set `RECEIPT_INTAKE_DRYRUN=false`.** On its first pass the worker splits the - shadow backlog on that boundary: - - received BEFORE it -> `SHADOW_DONE` / `booked-by-v1`. Terminal; v2 never books these, - because v1 already did and QBO's DocNumber idempotency cannot recognise a v2 UUID. - - received AFTER it -> handed to v2 for real booking. v1 had already stopped, so nobody - booked these; retiring them would drop real expenses on the floor. - With no boundary recorded the pass touches NEITHER side and logs - `cutover-boundary-missing`. That is deliberate: retiring on a guess destroys evidence, - requeuing on a guess double-books, and a visible no-op is the only honest third option. + shadow backlog. The boundary narrows the CANDIDATES; **evidence** decides each one: + - before the boundary AND provably booked by v1 -> `SHADOW_DONE` / `booked-by-v1`. + Terminal; v2 never books these. Evidence is either an `AutomationEvent` + (`kind: receipt-push`, status `created`/`already-exists`) whose `driveFileId` matches + the row — v1's pushes go through ProBuild's create route, which logs them — or the + forwarder sending `archivedByV1: true` on the forward. + - everything else, including rows before the boundary with NO evidence -> handed to v2. + "Received before the boundary" says when the file arrived, not that anything booked + it: v1 skips documents constantly, and retiring those would drop real expenses. This + is safe because a Drive row books under the **Drive file id**, so a v1/v2 overlap on + the same file collapses to one Purchase through QBO's DocNumber/requestid idempotency. + With no boundary recorded in live mode the worker **halts the entire pass before + claiming anything** and logs `cutover-boundary-missing`. Not just the retire: booking + anything while we cannot tell what v1 already booked is the double-booking this whole + mechanism exists to prevent. Retired rows keep their read results and dedup keys, so a post-cutover resend of a shadow-week receipt still collides with them and is caught as a duplicate. diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 1bac0031b..323f0f224 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -1,5 +1,6 @@ import { test, expect, type APIRequestContext } from "@playwright/test"; import { PrismaClient } from "@prisma/client"; +import { createHash } from "node:crypto"; /** * POST/GET /api/receipts/intake — request-level auth matrix and idempotency. @@ -700,3 +701,196 @@ test.describe("the two machine secrets are not interchangeable", () => { await machine.dispose(); }); }); + +test.describe("cutover retirement needs evidence, not just an old timestamp", () => { + test("a shadow row v1 provably booked is retired; one it never touched is handed to v2", async ({ request }) => { + // "Received before the boundary" says when the file ARRIVED, not that + // anything booked it. v1 skips documents constantly — a bad read, a + // park, a file it never picked up — and retiring those as + // "booked-by-v1" silently drops real expenses. + const boundary = new Date(Date.now() + 60_000); + const evidencedFile = `EVID-${Date.now()}`; + const orphanFile = `ORPH-${Date.now()}`; + + const evidenced = await postIntake(request, intakeBody({ sourceRef: `drive:${evidencedFile}` })); + const orphan = await postIntake(request, intakeBody({ sourceRef: `drive:${orphanFile}` })); + expect(evidenced.res.status()).toBe(200); + expect(orphan.res.status()).toBe(200); + minted.push(evidenced.body.id, orphan.body.id); + + // Both parked exactly as the shadow week leaves them. + await prisma.receiptIntake.updateMany({ + where: { id: { in: [evidenced.body.id, orphan.body.id] } }, + data: { state: "READ", dryRun: true }, + }); + + // Only ONE of them has v1's own booking event behind it. v1 pushes go + // through ProBuild's create route, which logs exactly this. + const event = await prisma.automationEvent.create({ + data: { + kind: "receipt-push", + status: "created", + source: "apps-script", + driveFileId: evidencedFile, + }, + }); + + try { + await prisma.automationSetting.upsert({ + where: { key: "cutoverV1StoppedAt" }, + update: { value: boundary.toISOString() }, + create: { key: "cutoverV1StoppedAt", value: boundary.toISOString() }, + }); + + // Drive the real cutover through the worker's own claim path. + const res = await request.get("/api/cron/receipt-intake-worker", { + headers: process.env.CRON_SECRET ? { authorization: `Bearer ${process.env.CRON_SECRET}` } : {}, + maxRedirects: 0, + }); + // Skip cleanly if the cron is secret-gated in this environment. + test.skip(res.status() === 401, "CRON_SECRET not available to the spec"); + expect(res.status()).toBe(200); + + const after = await prisma.receiptIntake.findMany({ + where: { id: { in: [evidenced.body.id, orphan.body.id] } }, + select: { id: true, state: true, stateReason: true, dryRun: true }, + }); + const byId = Object.fromEntries(after.map(r => [r.id, r])); + + expect(byId[evidenced.body.id].state).toBe("SHADOW_DONE"); + expect(byId[evidenced.body.id].stateReason).toBe("booked-by-v1"); + + // No evidence -> v2's to book. Safe because a Drive row books under + // the DRIVE FILE ID, so a v1/v2 overlap collapses to one Purchase. + expect(byId[orphan.body.id].state).not.toBe("SHADOW_DONE"); + expect(byId[orphan.body.id].dryRun).toBe(false); + } finally { + await prisma.automationEvent.delete({ where: { id: event.id } }).catch(() => {}); + await prisma.automationSetting.deleteMany({ where: { key: "cutoverV1StoppedAt" } }).catch(() => {}); + } + }); + + test("the forwarder can assert it already archived a file", async ({ request }) => { + // The second accepted form of evidence, for documents v1 handled before + // the create route existed to log them. + const ref = `${REF_PREFIX}archived-by-v1`; + const res = await postIntake(request, JSON.stringify({ + source: "drive", sourceRef: ref, fileBase64: PNG_BASE64, + mimeType: "image/png", archivedByV1: true, + })); + expect(res.res.status()).toBe(200); + const row = await prisma.receiptIntake.findUnique({ where: { id: res.body.id } }); + expect(row?.archivedByV1).toBe(true); + }); + + test("a SESSION caller cannot claim v1 already booked something", async ({ request }) => { + // That flag is what excuses v2 from booking a document. Only a + // shared-secret forwarder may assert it. + const res = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ fileBase64: PNG_BASE64, mimeType: "image/png", archivedByV1: true }), + maxRedirects: 0, + }); + expect(res.status()).toBe(200); + const body = await res.json(); + minted.push(body.id); + const row = await prisma.receiptIntake.findUnique({ where: { id: body.id } }); + expect(row?.archivedByV1).toBe(false, "a browser upload can never claim v1 booked it"); + }); +}); + +test.describe("two-step upload: a reused key cannot swap the document", () => { + const startPath = `${INTAKE_PATH}/start`; + const sha = (b64: string) => createHash("sha256").update(Buffer.from(b64, "base64")).digest("hex"); + + test("SEQUENTIAL reuse with different bytes is refused before a URL is issued", async ({ request }) => { + // Caught at /start, not at /finalize: by then the caller would have + // uploaded receipt B over receipt A's object and A's bytes are gone. + const ref = `${REF_PREFIX}twostep-seq`; + const first = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64) }), + maxRedirects: 0, + }); + expect(first.status()).toBe(200); + const started = await first.json(); + minted.push(started.id); + expect(started.uploadUrl).toBeTruthy(); + + // Same key, same document — a plain retry resumes. + const resumed = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64) }), + maxRedirects: 0, + }); + expect(resumed.status()).toBe(200); + expect((await resumed.json()).id).toBe(started.id); + + // Same key, DIFFERENT document — refused. + const swapped = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(OTHER_PNG_BASE64), + }), + maxRedirects: 0, + }); + expect(swapped.status()).toBe(409); + expect((await swapped.json()).error).toBe("sourceRef-conflict"); + }); + + test("CONCURRENT starts on one key yield ONE row", async ({ request }) => { + const ref = `${REF_PREFIX}twostep-race`; + const body = JSON.stringify({ + source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64), + }); + const fire = () => request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: body, + maxRedirects: 0, + }); + + const results = await Promise.all([fire(), fire(), fire()]); + for (const r of results) expect(r.status()).toBe(200); + const ids = new Set(await Promise.all(results.map(async r => (await r.json()).id))); + expect(ids.size).toBe(1, "the unique index collapses the race to one row"); + + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: ref } }); + expect(rows).toHaveLength(1); + expect(rows[0].expectedSha256).toBe(sha(PNG_BASE64)); + minted.push(rows[0].id); + }); + + test("finalize refuses when the STORED bytes are not what /start was told", async ({ request }) => { + const ref = `${REF_PREFIX}twostep-sha`; + const started = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + // Declares a hash the bytes will never match. + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: "a".repeat(64) }), + maxRedirects: 0, + }); + expect(started.status()).toBe(200); + const { id, storagePath } = await started.json(); + minted.push(id); + + // Put REAL bytes at the path the row points at, as a direct upload would. + await prisma.receiptIntake.update({ where: { id }, data: { storagePath } }); + const seeded = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: intakeBody({ sourceRef: `${REF_PREFIX}twostep-sha-src` }), + maxRedirects: 0, + }); + expect(seeded.status()).toBe(200); + const seededRow = await prisma.receiptIntake.findUnique({ where: { id: (await seeded.json()).id } }); + minted.push(seededRow!.id); + await prisma.receiptIntake.update({ where: { id }, data: { storagePath: seededRow!.storagePath } }); + + const finalized = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: "{}", + maxRedirects: 0, + }); + expect(finalized.status()).toBe(409); + expect((await finalized.json()).error).toBe("sha-mismatch"); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.state).toBe("STAGING"); + }); +}); diff --git a/package.json b/package.json index 6535c3891..2267df08f 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 764e48ea3..c12dfb95f 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "mimeType" TEXT NOT NULL, "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, + "expectedSha256" TEXT, "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -41,6 +42,8 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "dedupStrongKey" TEXT, "dedupWeakKey" TEXT, "duplicateOfId" TEXT, + "sendAttempted" BOOLEAN NOT NULL DEFAULT false, + "archivedByV1" BOOLEAN NOT NULL DEFAULT false, "qbPurchaseId" TEXT, "expenseId" TEXT, "archiveDriveFileId" TEXT, @@ -58,6 +61,9 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( -- scripts/apply-receipt-intake.mjs: CREATE TABLE IF NOT EXISTS is a no-op on an -- existing table, so a column added to the CREATE above would never reach it. ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false; CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" ON "ReceiptIntake"("sourceRef"); CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" ON "ReceiptIntake"("expenseId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 757974b63..5364877f1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3049,6 +3049,10 @@ model ReceiptIntake { mimeType String fileSize Int fileSha256 String + /// What the CLIENT said it was about to upload, recorded by /intake/start. + /// The two-step flow hands the bytes straight to storage, so this is the only + /// way to notice that a reused sourceRef is carrying a DIFFERENT document. + expectedSha256 String? // read results (cents, like AutomationEvent) vendor String? @@ -3067,6 +3071,15 @@ model ReceiptIntake { duplicateOfId String? // booking + archive + /// Set the instant before the QBO create is attempted. A park BEFORE this is + /// true provably created no Purchase, so its strong key can be released; a + /// park after it must keep the key, because QuickBooks may hold a Purchase + /// whose response we lost. + sendAttempted Boolean @default(false) + /// Positive evidence that v1 (the Apps Script) already booked this document. + /// The cutover retires ONLY rows carrying it — never rows that merely predate + /// the boundary, because "old" is not proof anybody booked anything. + archivedByV1 Boolean @default(false) qbPurchaseId String? expenseId String? @unique expense Expense? @relation(fields: [expenseId], references: [id], onDelete: SetNull) diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index f5e376ca5..e5357bab8 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -94,6 +94,7 @@ export const statements = [ "mimeType" TEXT NOT NULL, "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, + "expectedSha256" TEXT, "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -106,6 +107,8 @@ export const statements = [ "dedupStrongKey" TEXT, "dedupWeakKey" TEXT, "duplicateOfId" TEXT, + "sendAttempted" BOOLEAN NOT NULL DEFAULT false, + "archivedByV1" BOOLEAN NOT NULL DEFAULT false, "qbPurchaseId" TEXT, "expenseId" TEXT, "archiveDriveFileId" TEXT, @@ -124,6 +127,9 @@ export const statements = [ // existing table, so a column added to the CREATE above would never reach // it. This is the whole reason the script is re-runnable. `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false`, // Intake idempotency: one row per caller-supplied sourceRef. A forwarder // replaying the same Drive file / Gmail message is a no-op. @@ -250,7 +256,8 @@ const expectedColumns = { "id", "source", "sourceRef", "state", "dryRun", "stateReason", "projectId", "costCodeId", "suggestedCostCodeId", "suggestedConfidence", "createdById", "storagePath", "fileName", "mimeType", "fileSize", - "fileSha256", "vendor", "txnDate", "totalCents", "taxCents", "docType", + "fileSha256", "expectedSha256", "sendAttempted", "archivedByV1", + "vendor", "txnDate", "totalCents", "taxCents", "docType", "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", "dedupWeakKey", "duplicateOfId", "qbPurchaseId", "expenseId", "archiveDriveFileId", "attempts", "busyPasses", "lastError", "nextRetryAt", diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index a6f2a315d..30596a24e 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -4,9 +4,11 @@ import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; +import { inspectStoredObject } from "@/lib/receipt-intake/stored-object"; +import { deleteObjectOrRecord, retryPendingCleanups } from "@/lib/receipt-intake/storage-cleanup"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; -import { createRouteDeadline } from "@/lib/quickbooks"; +import { createRouteDeadline, type RouteDeadline } from "@/lib/quickbooks"; import { readReceipt } from "@/lib/receipt-intake/read"; import { canonicalVendor } from "@/lib/receipt-intake/keys"; import { resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; @@ -19,6 +21,7 @@ import { BATCH_SIZE, CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, + RUN_HARD_BUDGET_MS, STAGING_SWEEP_BATCH, STAGING_SWEEP_MINUTES, type ClaimResult, @@ -61,7 +64,7 @@ const WORKER_ROW_SELECT = { storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, lastError: true, - suggestedConfidence: true, + suggestedConfidence: true, sendAttempted: true, createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; @@ -102,31 +105,87 @@ async function claim(opts: CutoverRequest): Promise { // resend of the same receipt still collides with them and is caught. let shadowRetired = 0; let requeued = 0; - let boundaryMissing = false; if (opts.run) { + // runIntakeWorker halts before ever calling claim() without one, so + // this is belt-and-braces rather than the real gate. if (!opts.boundary) { - // Refuse. Retiring on a guess destroys evidence of real - // expenses; requeuing on a guess double-books them. A logged - // no-op is the only honest third option. - boundaryMissing = true; + console.error("[cron/receipt-intake-worker] claim() reached with no boundary — refusing"); } else { const parked: Prisma.ReceiptIntakeWhereInput = { dryRun: true, state: { in: ["READ", "BOOKING"] }, }; - // BEFORE the boundary: v1 was still booking, so it booked these. - const retired = await tx.receiptIntake.updateMany({ + // RETIREMENT NEEDS POSITIVE EVIDENCE, not just an old timestamp. + // + // "Received before the boundary" says when the file ARRIVED, not + // that anything booked it. v1 skips documents constantly — a bad + // read, a park, a file it never picked up — and every one of + // those would have been retired as "booked-by-v1" and silently + // dropped. So a row is only retired when we can point at the + // booking: + // + // * an AutomationEvent from v1's own push (it goes through + // ProBuild's create route, which logs kind receipt-push with + // status created/already-exists and the Drive fileId), or + // * the forwarder telling us it archived the file + // (archivedByV1, set from the forward payload). + // + // Everything else is handed to v2. That is safe for the Drive + // rows this applies to: they book under the DRIVE FILE ID, so + // QBO's DocNumber/requestid idempotency collapses a v1/v2 + // overlap into one Purchase. + const candidates = await tx.receiptIntake.findMany({ where: { ...parked, createdAt: { lt: opts.boundary } }, - data: { state: "SHADOW_DONE", stateReason: "booked-by-v1", nextRetryAt: null }, + select: { id: true, source: true, sourceRef: true, archivedByV1: true }, }); - shadowRetired = retired.count; - // AFTER it: v1 had already stopped, so NOBODY booked these. - // They are the only rows from the shadow week that v2 must - // actually book — dropping them would lose real expenses. + const driveIds = candidates + .map(r => (r.source === "drive" && r.sourceRef.startsWith("drive:") + ? r.sourceRef.slice("drive:".length) + : null)) + .filter((v): v is string => !!v); + + const bookedByV1 = driveIds.length + ? new Set( + (await tx.automationEvent.findMany({ + where: { + kind: "receipt-push", + status: { in: ["created", "already-exists"] }, + driveFileId: { in: driveIds }, + }, + select: { driveFileId: true }, + })).map(e => e.driveFileId).filter((v): v is string => !!v), + ) + : new Set(); + + const evidenced: string[] = []; + const unevidenced: string[] = []; + for (const row of candidates) { + const driveId = row.source === "drive" && row.sourceRef.startsWith("drive:") + ? row.sourceRef.slice("drive:".length) + : null; + if (row.archivedByV1 || (driveId && bookedByV1.has(driveId))) evidenced.push(row.id); + else unevidenced.push(row.id); + } + + if (evidenced.length) { + const retired = await tx.receiptIntake.updateMany({ + where: { id: { in: evidenced } }, + data: { state: "SHADOW_DONE", stateReason: "booked-by-v1", nextRetryAt: null }, + }); + shadowRetired = retired.count; + } + + // Everything else — after the boundary, or before it with no + // evidence — is v2's to book. const handed = await tx.receiptIntake.updateMany({ - where: { ...parked, createdAt: { gte: opts.boundary } }, + where: { + OR: [ + { ...parked, createdAt: { gte: opts.boundary } }, + ...(unevidenced.length ? [{ id: { in: unevidenced } }] : []), + ], + }, data: { dryRun: false, nextRetryAt: null }, }); requeued = handed.count; @@ -134,6 +193,7 @@ async function claim(opts: CutoverRequest): Promise { if (shadowRetired > 0 || requeued > 0) { console.log("[cron/receipt-intake-worker] cutover", JSON.stringify({ boundary: opts.boundary.toISOString(), shadowRetired, requeued, + unevidenced: unevidenced.length, })); } } @@ -152,7 +212,7 @@ async function claim(opts: CutoverRequest): Promise { take: BATCH_SIZE, select: WORKER_ROW_SELECT, }); - if (due.length === 0) return { rows: [], shadowRetired, requeued, boundaryMissing }; + if (due.length === 0) return { rows: [], shadowRetired, requeued }; // THE claim. Anything this run took is invisible to the next one for // the lease, whether or not the advisory lock held. @@ -160,11 +220,11 @@ async function claim(opts: CutoverRequest): Promise { where: { id: { in: due.map(r => r.id) } }, data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, }); - return { rows: due as WorkerRow[], shadowRetired, requeued, boundaryMissing }; + return { rows: due as WorkerRow[], shadowRetired, requeued }; }); } -function buildDeps(): WorkerDependencies { +function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { return { claim, @@ -182,7 +242,7 @@ function buildDeps(): WorkerDependencies { const cutoff = new Date(Date.now() - STAGING_SWEEP_MINUTES * 60_000); const stale = await prisma.receiptIntake.findMany({ where: { state: "STAGING", createdAt: { lt: cutoff } }, - select: { id: true, storagePath: true }, + select: { id: true, storagePath: true, mimeType: true }, // Small on purpose: each row costs a storage round trip, and the // sweep runs BEFORE any receipt is processed. A big batch here // spends the invocation on housekeeping. @@ -191,30 +251,51 @@ function buildDeps(): WorkerDependencies { let published = 0; let parked = 0; + let rejected = 0; for (const row of stale) { // The sweep is inside the run's deadline, not outside it. if (shouldStop()) break; - const probe = await downloadDocBytesResult(toSecureRef(row.storagePath)); - if (probe.ok) { - // The upload landed; only the publish was lost. Finish it. + + // THE SAME validator /finalize uses. Publishing on "the object + // exists" alone would wave through a 40 MB video, an executable, + // or a truncated upload that /finalize would have refused — and + // those rows then go to Gemini and, if they read at all, to + // QuickBooks. One implementation, so the two cannot diverge. + const check = await inspectStoredObject(row.storagePath, row.mimeType); + + if (check.ok) { await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, - data: { state: "RECEIVED", nextRetryAt: null }, + data: { + state: "RECEIVED", + nextRetryAt: null, + mimeType: check.mimeType, + fileSize: check.fileSize, + fileSha256: check.fileSha256, + }, }); published++; continue; } - if (probe.kind === "transient") continue; // unknown is not a verdict - await prisma.receiptIntake.updateMany({ - where: { id: row.id, state: "STAGING" }, - data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, - }); - parked++; + if (check.kind === "transient") continue; // unknown is not a verdict + if (check.kind === "missing") { + await prisma.receiptIntake.updateMany({ + where: { id: row.id, state: "STAGING" }, + data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, + }); + parked++; + continue; + } + // Rejected: the object exists and is not acceptable. Same + // outcome as /finalize — the row goes and so does the object. + await prisma.receiptIntake.deleteMany({ where: { id: row.id, state: "STAGING" } }); + await deleteObjectOrRecord(row.storagePath, check.reason); + rejected++; } - if (published || parked) { - console.log("[cron/receipt-intake-worker] STAGING sweep", JSON.stringify({ published, parked })); + if (published || parked || rejected) { + console.log("[cron/receipt-intake-worker] STAGING sweep", JSON.stringify({ published, parked, rejected })); } - return published + parked; + return published + parked + rejected; }, loadPhases: async () => prisma.costCode.findMany({ @@ -297,6 +378,8 @@ function buildDeps(): WorkerDependencies { companyTimeZone: resolveCompanyTimeZone, + retryStorageCleanups: shouldStop => retryPendingCleanups(STAGING_SWEEP_BATCH, shouldStop), + promoteToBooking: async (rowId, weakKey) => prisma.$transaction(async tx => { // LAST weak-dedup check, taken INSIDE the transition. The check at // read time can miss a pair that arrived in the same batch window, @@ -319,7 +402,14 @@ function buildDeps(): WorkerDependencies { // Rows with different weak keys take different locks and never // block each other. if (weakKey) { - await tx.$queryRaw`SELECT pg_advisory_xact_lock(hashtextextended(${weakKey}, 0))`; + // $executeRaw, NOT $queryRaw. pg_advisory_xact_lock returns + // VOID: `SELECT` of it produces a row whose single column has no + // readable type, and Prisma's query path can reject that outright + // — which would throw INSIDE the promotion transaction and, on + // the retry path, look like a transient DB fault forever while + // the lock was never actually taken. $executeRaw runs the + // statement for its effect and asks nothing of the result. + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${weakKey}, 0))`; // EVERY LIVE STATE, not just the post-booking ones. // // Limiting this to BOOKING/BOOKED/ARCHIVED meant a twin sitting @@ -363,16 +453,19 @@ function buildDeps(): WorkerDependencies { return { promoted: true }; }), - book: (row, remainingMs) => bookReceipt(row, { + book: row => bookReceipt(row, { db: prisma as unknown as BookPrismaClient, companyTimeZone: resolveCompanyTimeZone, + markSendAttempted: async rowId => { + await prisma.receiptIntake.update({ where: { id: rowId }, data: { sendAttempted: true } }); + }, isCostCodeAllowed: (projectId, costCodeId) => isCostCodeAllowedForProject(prismaPhaseDataSource, projectId, costCodeId), - remainingBudgetMs: () => remainingMs, - // ONE deadline for the whole booking, threaded into the token - // refresh AND the Purchase create so they share a budget instead of - // each helping itself to a fresh 20s. - deadline: () => createRouteDeadline(Math.max(0, remainingMs)), + // The invocation's ONE absolute deadline, created at entry and + // shared by every check and every QuickBooks call. Never a + // remaining-milliseconds snapshot: that is measured once and then + // decays silently while the work runs. + deadline: invocationDeadline, isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), getTokens: deadline => getFreshQBTokens(deadline), @@ -461,7 +554,7 @@ export async function GET(request: Request) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } - const summary = await runIntakeWorker(buildDeps()); + const summary = await runIntakeWorker(buildDeps(createRouteDeadline(RUN_HARD_BUDGET_MS))); if (summary.processed > 0 || summary.skipped) { console.log("[cron/receipt-intake-worker]", JSON.stringify(summary)); } diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index b3df38bea..23da3b180 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -1,10 +1,9 @@ -import { createHash } from "node:crypto"; import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; -import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; -import { sniffMime } from "@/lib/receipt-intake/file-type"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; +import { inspectStoredObject } from "@/lib/receipt-intake/stored-object"; +import { deleteObjectOrRecord } from "@/lib/receipt-intake/storage-cleanup"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -40,6 +39,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string select: { id: true, state: true, sourceRef: true, storagePath: true, mimeType: true, projectId: true, dryRun: true, createdById: true, fileSha256: true, + expectedSha256: true, }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); @@ -62,9 +62,12 @@ export async function POST(req: Request, context: { params: Promise<{ id: string }); } - const download = await downloadDocBytesResult(toSecureRef(row.storagePath)); - if (!download.ok) { - if (download.kind === "not-found") { + // ONE validator, shared with the worker's stale-STAGING sweep — see + // stored-object.ts. If the two disagreed, whichever ran first would decide + // whether a 40 MB video became a receipt. + const check = await inspectStoredObject(row.storagePath, row.mimeType); + if (!check.ok) { + if (check.kind === "missing") { // The upload never landed. Retryable, and NEVER a 2xx: the // forwarders treat 2xx as "we have it" and would drop their copy. return NextResponse.json( @@ -72,39 +75,48 @@ export async function POST(req: Request, context: { params: Promise<{ id: string { status: 409 }, ); } - return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - } - - const bytes = download.bytes; - if (bytes.length > MAX_STORED_BYTES) { - // Enforced on the OBJECT, because the signed URL bypassed every check - // this server could otherwise have made. Drop it rather than leave an - // oversize file in a private bucket nobody will ever book. + if (check.kind === "transient") { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + // REJECTED. The row goes, and so must the object — nothing references it + // once the row is gone, so a failed delete is recorded for the sweep to + // retry rather than shrugged off. await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); - return NextResponse.json({ ok: false, reason: "file-too-large", maxBytes: MAX_STORED_BYTES }, { status: 413 }); + await deleteObjectOrRecord(row.storagePath, check.reason); + const status = check.reason.startsWith("file-too-large") ? 413 : 400; + return NextResponse.json( + { ok: false, reason: check.reason, maxBytes: MAX_STORED_BYTES }, + { status }, + ); } - // The stored type is decided on the BYTES, exactly like the single-shot path. - const mimeType = sniffMime(bytes, row.mimeType); - if (!mimeType) { - await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); - return NextResponse.json({ ok: false, reason: "unsupported-file-type" }, { status: 400 }); - } + const { mimeType, fileSize, fileSha256 } = check; - const fileSha256 = createHash("sha256").update(bytes).digest("hex"); - // A declared hash is an INTEGRITY check on the client's own upload, never - // the value we store. A mismatch means the bytes in the bucket are not the - // bytes the client meant to send. - if (declaredSha && declaredSha !== fileSha256) { - return NextResponse.json( - { ok: false, error: "sha-mismatch", reason: "stored bytes do not match the declared sha256" }, - { status: 409 }, - ); + // THE HASH IS CHECKED AGAINST BOTH RECORDED EXPECTATIONS. + // + // `expectedSha256` was written by /start from what the client said it was + // about to upload; `declaredSha` is what it says now. Either disagreeing + // with the stored bytes means the object is not the document this row was + // created for — which is exactly the case a reused sourceRef produces, and + // the case that would otherwise attach one receipt's bytes to another + // receipt's identity. + for (const [label, expected] of [["declared", declaredSha], ["expected", row.expectedSha256]] as const) { + if (expected && expected.toLowerCase() !== fileSha256) { + return NextResponse.json( + { + ok: false, + error: "sha-mismatch", + reason: `stored bytes do not match the ${label} sha256`, + storedSha256: fileSha256, + }, + { status: 409 }, + ); + } } const published = await prisma.receiptIntake.updateMany({ where: { id, state: "STAGING" }, - data: { state: "RECEIVED", mimeType, fileSize: bytes.length, fileSha256 }, + data: { state: "RECEIVED", mimeType, fileSize, fileSha256 }, }); if (published.count === 0) { // Another finalize won the race and published it. Same outcome. @@ -117,6 +129,6 @@ export async function POST(req: Request, context: { params: Promise<{ id: string return NextResponse.json({ ok: true, id, state: "RECEIVED", sourceRef: row.sourceRef, - projectId: row.projectId, dryRun: row.dryRun, fileSize: bytes.length, + projectId: row.projectId, dryRun: row.dryRun, fileSize, }); } diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index e7244c894..560da7df3 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -65,6 +65,8 @@ interface ParsedBody { projectId: string | null; costCodeId: string | null; threadName: string | null; + /** The forwarder reporting that v1 already booked and archived this file. */ + archivedByV1: boolean; } function bad(reason: string) { @@ -117,6 +119,7 @@ async function parseBody(req: Request): Promise { projectId: str(form.get("projectId")), costCodeId: str(form.get("costCodeId")), threadName: str(form.get("threadName")), + archivedByV1: form.get("archivedByV1") === "true", }; } @@ -144,6 +147,9 @@ async function parseBody(req: Request): Promise { projectId: str(json.projectId), costCodeId: str(json.costCodeId), threadName: str(json.threadName), + // Strict === true: only an explicit boolean may mark a row as already + // booked by v1, because that flag is what excuses v2 from booking it. + archivedByV1: json.archivedByV1 === true, }; } @@ -245,6 +251,10 @@ export async function POST(req: Request) { projectId: parsed.projectId, costCodeId: parsed.costCodeId, createdById: auth.via === "session" ? auth.user.id : null, + // Only a shared-secret forwarder may assert this: it is the + // claim that v1 already put this document in the books, and it + // is what stops v2 from booking it at cutover. + archivedByV1: auth.via === "secret" ? parsed.archivedByV1 : false, storagePath, fileName: parsed.fileName, mimeType, diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index 3b6e3fe5f..0a7504b54 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -47,6 +47,16 @@ export async function POST(req: Request) { // real type from the STORED BYTES, so a lie costs the caller its upload. if (!ext) return NextResponse.json({ ok: false, reason: "unsupported-file-type" }, { status: 400 }); + // The client's own hash of what it is ABOUT to upload. Persisted, because + // the two-step flow hands the bytes straight to storage: without it a + // reused sourceRef carrying a DIFFERENT document is indistinguishable from + // an honest retry, and /finalize would attach one receipt's bytes to + // another receipt's identity. + const expectedSha256 = typeof body.sha256 === "string" ? body.sha256.trim().toLowerCase() : null; + if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) { + return NextResponse.json({ ok: false, reason: "invalid-sha256" }, { status: 400 }); + } + const declaredSize = Number(body.fileSize); if (Number.isFinite(declaredSize) && declaredSize > MAX_STORED_BYTES) { return NextResponse.json({ ok: false, reason: "file-too-large", maxBytes: MAX_STORED_BYTES }, { status: 413 }); @@ -81,14 +91,18 @@ export async function POST(req: Request) { projectId, costCodeId: str(body.costCodeId), createdById: auth.via === "session" ? auth.user.id : null, + // Forwarder-only, same as the single-shot path: this is the + // claim that v1 already booked the document. + archivedByV1: auth.via === "secret" && body.archivedByV1 === true, storagePath, fileName: str(body.fileName), mimeType, fileSize: 0, // Unknown until the bytes land. /finalize recomputes it FROM // STORAGE and writes the real value; a client-declared hash is - // never trusted as the stored one. + // never trusted as the stored one — only checked against it. fileSha256: "", + expectedSha256, }, select: { id: true, sourceRef: true, state: true }, }); @@ -98,7 +112,10 @@ export async function POST(req: Request) { // client resumes rather than orphaning a second object. const existing = await prisma.receiptIntake.findUnique({ where: { sourceRef: decided.sourceRef }, - select: { id: true, sourceRef: true, state: true, storagePath: true, createdById: true }, + select: { + id: true, sourceRef: true, state: true, storagePath: true, + createdById: true, expectedSha256: true, fileSha256: true, + }, }); if (!existing) return NextResponse.json({ ok: false, reason: "conflict-retry" }, { status: 409 }); const maySee = @@ -106,6 +123,19 @@ export async function POST(req: Request) { existing.createdById === auth.user.id || auth.user.role === "ADMIN"; if (!maySee) return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + + // SAME KEY, DIFFERENT DOCUMENT. Caught HERE, before a signed URL is + // handed out — otherwise the caller would upload receipt B over + // receipt A's object and only /finalize would notice, by which point + // A's bytes are gone. + const knownSha = existing.fileSha256 || existing.expectedSha256; + if (expectedSha256 && knownSha && knownSha.toLowerCase() !== expectedSha256) { + return NextResponse.json( + { ok: false, error: "sourceRef-conflict", reason: "this sourceRef already holds a different document", existingId: existing.id }, + { status: 409 }, + ); + } + if (existing.state !== "STAGING") { return NextResponse.json( { ok: true, alreadyReceived: true, id: existing.id, state: existing.state }, @@ -141,7 +171,14 @@ async function signUpload(storagePath: string): Promise<{ uploadUrl: string; tok const supabase = getSupabase(); if (!supabase) return null; try { - const { data, error } = await supabase.storage.from(SECURE_BUCKET).createSignedUploadUrl(storagePath); + // upsert: true — a resumed /start for the SAME row must be able to + // overwrite a partial or failed upload at the same path. Without it the + // second attempt fails on "already exists" and the row can never be + // finalized. The sha checks above are what stop this from overwriting a + // DIFFERENT document. + const { data, error } = await supabase.storage + .from(SECURE_BUCKET) + .createSignedUploadUrl(storagePath, { upsert: true }); if (error || !data) { console.error("[receipts/intake/start] sign failed", error?.message); return null; diff --git a/src/lib/automation-events.ts b/src/lib/automation-events.ts index c8b76d581..d0b11b17f 100644 --- a/src/lib/automation-events.ts +++ b/src/lib/automation-events.ts @@ -17,7 +17,19 @@ import { prisma } from "@/lib/prisma"; */ export interface AutomationEventInput { - kind: "receipt-push" | "qbo-sync" | "receipt-stage" | "setting" | "qbo-payments-sync"; + kind: + | "receipt-push" + | "qbo-sync" + | "receipt-stage" + | "setting" + | "qbo-payments-sync" + /** + * A rejected intake object whose DELETE from storage failed. The row is + * already gone, so nothing else remembers the orphan — without this the + * bytes sit in a private bucket forever, unreferenced. The receipt + * worker's sweep retries the deletion and resolves the event. + */ + | "storage-cleanup-pending"; stage?: string; status: string; reason?: string; diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 9bf03ca40..b79d4a219 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -20,7 +20,12 @@ import { matchCostCode } from "@/lib/project-match"; import { toSecureRef } from "@/lib/secure-storage"; import { startOfDateInTimeZone } from "@/lib/tz-date"; -import { QBTimeoutError, type QBTokens, type RouteDeadline } from "@/lib/quickbooks"; +import { + QBTimeoutError, + remainingBudgetMs, + type QBTokens, + type RouteDeadline, +} from "@/lib/quickbooks"; import { QboAccountConfigError, QboPurchaseFaultError, @@ -57,6 +62,12 @@ export interface BookableRow { attempts: number; /** Carries a previous attachment failure across a retry — see below. */ lastError: string | null; + /** + * True once a QBO create has been ATTEMPTED for this row. It is the only + * honest answer to "could a Purchase exist?", and it is what decides whether + * a park may release the strong key. + */ + sendAttempted: boolean; } /** @@ -145,6 +156,8 @@ export interface BookPrismaClient { export interface BookDependencies { db: BookPrismaClient; + /** Persists sendAttempted BEFORE the create — the flag must survive a crash. */ + markSendAttempted: (rowId: string) => Promise; /** The company's configured zone — Expense.date is a business calendar day. */ companyTimeZone: () => Promise; /** @@ -162,10 +175,15 @@ export interface BookDependencies { input: CreateQBReceiptPurchaseInput, deadline?: RouteDeadline, ) => Promise; - /** Milliseconds left in the worker's invocation. Undefined = unbounded (tests). */ - remainingBudgetMs?: () => number; - /** Threads the same budget into every QuickBooks call this booking makes. */ - deadline?: () => RouteDeadline | undefined; + /** + * The invocation's ONE absolute deadline. Undefined = unbounded (tests). + * + * Deliberately the deadline OBJECT rather than a remaining-milliseconds + * number: a number is measured once and then decays silently, so a booking + * that spent 20s downloading its file still believed it had the budget it + * was handed on entry. Every check below recomputes from this instead. + */ + deadline?: RouteDeadline; /** * Reads the stored file back out of the private bucket. TAGGED, because a * confirmed 404 and a transient storage fault must not book the same way. @@ -291,10 +309,9 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // Runway check BEFORE anything else that could touch QuickBooks. Deferred, // not retried: the document is fine and this costs it no attempt — the // invocation simply ran out of room, and the next pass has a full budget. - const remaining = deps.remainingBudgetMs?.(); - if (remaining !== undefined && remaining < MIN_BOOKING_BUDGET_MS) { - return { outcome: "deferred", reason: "out-of-budget" }; - } + const outOfRunway = () => + deps.deadline !== undefined && remainingBudgetMs(deps.deadline) < MIN_BOOKING_BUDGET_MS; + if (outOfRunway()) return { outcome: "deferred", reason: "out-of-budget" }; // Everything down to the QBO call is a PRE-SEND refusal: nothing was ever // sent, so the strong key must be handed back (see BookResult). @@ -348,6 +365,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro } const bytes = download.bytes; + // RE-CHECKED after the download, which is the slowest thing before the + // send. A 15 MB object over a slow link can eat the whole runway that the + // entry check just approved. + if (outOfRunway()) return { outcome: "deferred", reason: "out-of-budget" }; + // PREFLIGHT, before anything is created. A format or size QBO cannot accept // is a fact about this file, known now — so refuse now, rather than // discovering it from `attachment:"skipped"` after a Purchase already @@ -377,18 +399,28 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro fileContentType: row.mimeType, }; + // RECORDED BEFORE THE CALL, and persisted, because the question it answers + // is "might QuickBooks hold a Purchase for this row?" — and a process that + // dies mid-create must still answer yes. Setting it after would make the + // one case that matters look like a row that never sent. + await deps.markSendAttempted(row.id); + const sent = { attempted: true }; + let result: CreateQBReceiptPurchaseResult; try { - // ONE budget threaded through both round trips, so a slow token refresh - // shortens the create rather than each getting a fresh 20s. - const deadline = deps.deadline?.(); - const tokens = await deps.getTokens(deadline); - result = await deps.createPurchase(tokens, input, deadline); + // The SAME absolute deadline for both round trips, so a slow token + // refresh shortens the create rather than each helping itself to a + // fresh 20s. + const tokens = await deps.getTokens(deps.deadline); + // Last gate before the books are touched: the refresh may have consumed + // what was left. + if (outOfRunway()) return { outcome: "deferred", reason: "out-of-budget" }; + result = await deps.createPurchase(tokens, input, deps.deadline); } catch (error) { const terminal = terminalReasonFor(error); // A send WAS attempted: QBO may hold a Purchase whose response we lost, // so the key stays claimed even though the row is parked. - if (terminal) return { outcome: "needs-review", reason: terminal, releaseStrongKey: false }; + if (terminal) return { outcome: "needs-review", reason: terminal, releaseStrongKey: !sent.attempted }; // QBTimeoutError, QBNotConnectedError, network/fetch errors, QBO // 429/5xx and DB errors are all transport-class: try again later. return retry(row, deps, now, describe(error)); @@ -406,6 +438,8 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // quarantine the corrected re-submission against a booking that never // happened. Release it. A THROWN fault is different — it can come from // inside the create — and keeps the key. + // ok:false is decided inside createQBReceiptPurchase BEFORE qbCreateFn + // runs, so no Purchase exists for this row and the key goes back. return { outcome: "needs-review", reason: `qbo-fault:${result.reason}`, releaseStrongKey: true }; } @@ -596,7 +630,14 @@ async function resolvePhase( }; } -/** A refusal reached WITHOUT any QBO call — the strong key goes back. */ +/** + * A refusal reached WITHOUT any QBO call — the strong key goes back. + * + * The rule is about the SEND, not about the reason: any terminal park that + * provably created no Purchase releases the key, whatever the reason string + * says. Holding it makes a corrected resubmission collide with a row that never + * became a purchase, and the reviewer then has two stuck rows instead of one. + */ function parkedBeforeSend(reason: string): BookResult { return { outcome: "needs-review", reason, releaseStrongKey: true }; } @@ -605,8 +646,12 @@ function retry(row: BookableRow, deps: BookDependencies, now: Date, reason: stri const attempts = row.attempts + 1; // `>=`, so MAX_BOOK_ATTEMPTS reads as "20 attempts in total" rather than 21. if (attempts >= MAX_BOOK_ATTEMPTS) { - // Sends were attempted to get here, so the key stays claimed. - return { outcome: "needs-review", reason: "max-retries", releaseStrongKey: false }; + // Keyed on the ROW's record of whether a send ever happened, not on the + // assumption that reaching the retry limit implies one. A row can + // exhaust its attempts entirely on storage faults, having never touched + // QuickBooks — and holding its key then quarantines the corrected + // resend against nothing. + return { outcome: "needs-review", reason: "max-retries", releaseStrongKey: !row.sendAttempted }; } return { outcome: "retry", diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts new file mode 100644 index 000000000..88ceb4dbd --- /dev/null +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -0,0 +1,80 @@ +/** + * Orphaned-object bookkeeping. + * + * When an intake row is rejected (oversize, wrong format, empty) the object it + * pointed at has to go too — the row is deleted, so after that NOTHING in the + * database references those bytes and they would sit in a private bucket + * forever. Storage deletes fail for the same boring reasons every other call + * does, so "best effort, shrug" is not good enough on a path we take + * deliberately. + * + * An AutomationEvent is used rather than a new table: this is rare, it is + * already the audit surface the Command Center reads, and a table for it would + * be schema churn for a queue that should normally be empty. + */ +import { logAutomationEvent } from "@/lib/automation-events"; +import { prisma } from "@/lib/prisma"; +import { removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; + +export const STORAGE_CLEANUP_KIND = "storage-cleanup-pending"; + +/** + * Delete the object. If that fails, record the path so the sweep can retry. + * Never throws: the caller is already rejecting a row and must not be derailed + * by the cleanup of it. + */ +export async function deleteObjectOrRecord(storagePath: string, reason: string): Promise { + try { + await removeSecureDoc(toSecureRef(storagePath)); + return true; + } catch (error) { + console.error("[receipts/intake] object delete failed", storagePath, error instanceof Error ? error.name : "error"); + await logAutomationEvent({ + kind: STORAGE_CLEANUP_KIND, + status: "pending", + reason, + source: "receipt-intake", + detail: { storagePath }, + }).catch(() => { /* audit only — the orphan is the lesser problem */ }); + return false; + } +} + +/** + * Retry the deletions that failed earlier. Bounded per pass, like every other + * housekeeping step in the worker, and it resolves each event it clears so the + * queue drains instead of growing. + */ +export async function retryPendingCleanups(limit: number, shouldStop: () => boolean): Promise { + const pending = await prisma.automationEvent.findMany({ + where: { kind: STORAGE_CLEANUP_KIND, status: "pending" }, + orderBy: { createdAt: "asc" }, + take: limit, + select: { id: true, detail: true }, + }); + + let cleared = 0; + for (const event of pending) { + if (shouldStop()) break; + let storagePath: string | null = null; + try { + storagePath = (JSON.parse(event.detail ?? "{}") as { storagePath?: string }).storagePath ?? null; + } catch { + storagePath = null; + } + if (!storagePath) { + // Unparseable detail can never be acted on; close it rather than + // retrying it every five minutes forever. + await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "abandoned" } }); + continue; + } + try { + await removeSecureDoc(toSecureRef(storagePath)); + await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "resolved" } }); + cleared++; + } catch { + // Still failing. Leave it pending for the next pass. + } + } + return cleared; +} diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts new file mode 100644 index 000000000..f6d45d81b --- /dev/null +++ b/src/lib/receipt-intake/stored-object.ts @@ -0,0 +1,66 @@ +/** + * The one place a STAGING row's stored object is validated and turned into row + * metadata. + * + * Two callers publish a STAGING row — /intake/{id}/finalize (the client says it + * has finished uploading) and the worker's stale-STAGING sweep (nobody ever + * came back, but the object is there). They MUST agree: a sweep that published + * on "the object exists" alone would wave through a 40 MB video, a .exe, or a + * truncated upload that /finalize would have rejected — and those rows then go + * to Gemini and, if they read at all, to QuickBooks. + * + * Everything here is derived from the BYTES IN STORAGE. The client uploaded + * straight to Supabase, so nothing it declared about the file is evidence. + */ +import { createHash } from "node:crypto"; +import { downloadDocBytesResult, toSecureRef, type DocBytesResult } from "@/lib/secure-storage"; +import { sniffMime } from "./file-type"; +import { MAX_STORED_BYTES } from "./intake-core"; + +export type StoredObjectCheck = + /** Valid: these are the values the row must be published with. */ + | { ok: true; mimeType: string; fileSize: number; fileSha256: string } + /** The object is not there. Terminal for the sweep; retryable for a client. */ + | { ok: false; kind: "missing" } + /** Storage could not answer. Never a verdict — come back later. */ + | { ok: false; kind: "transient"; message: string } + /** The object exists and is NOT acceptable. The row and object must go. */ + | { ok: false; kind: "rejected"; reason: string }; + +export async function inspectStoredObject( + storagePath: string, + /** + * What the row recorded at /start. Used ONLY for text/plain, which has no + * magic bytes — the same concession the single-shot path makes. Every + * format that CAN be identified is identified from the bytes. + */ + declaredMime: string, + download: (ref: string) => Promise = downloadDocBytesResult, +): Promise { + const result = await download(toSecureRef(storagePath)); + if (!result.ok) { + return result.kind === "not-found" + ? { ok: false, kind: "missing" } + : { ok: false, kind: "transient", message: result.message }; + } + + const bytes = result.bytes; + if (bytes.length === 0) return { ok: false, kind: "rejected", reason: "empty-file" }; + // Enforced on the OBJECT, because the signed upload URL bypassed every + // check this server could otherwise have made. + if (bytes.length > MAX_STORED_BYTES) { + return { ok: false, kind: "rejected", reason: `file-too-large:${bytes.length}` }; + } + + // Magic bytes, exactly like the single-shot path. A declared mime is a + // claim; this is the answer. + const mimeType = sniffMime(bytes, declaredMime); + if (!mimeType) return { ok: false, kind: "rejected", reason: "unsupported-file-type" }; + + return { + ok: true, + mimeType, + fileSize: bytes.length, + fileSha256: createHash("sha256").update(bytes).digest("hex"), + }; +} diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 4c4d6b09b..877f22bc7 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -113,6 +113,8 @@ export interface WorkerDependencies { * be able to eat the invocation before any real work starts. */ sweepStaleStaging: (shouldStop: () => boolean) => Promise; + /** Retry storage deletes that failed when a row was rejected. */ + retryStorageCleanups: (shouldStop: () => boolean) => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; /** Tagged: a confirmed 404 and a transient storage fault are NOT the same answer. */ downloadBytes: (storagePath: string) => Promise; @@ -131,8 +133,8 @@ export interface WorkerDependencies { * row when another document with this weak key is already BOOKING/BOOKED. */ promoteToBooking: (rowId: string, weakKey: string | null) => Promise<{ promoted: boolean; conflictId?: string }>; - /** `remainingMs` is what is left of the invocation when the booking starts. */ - book: (row: BookableRow, remainingMs: number) => Promise; + /** The pass's ONE absolute deadline — never a snapshot of "time left". */ + book: (row: BookableRow) => Promise; applyBookResult: (rowId: string, result: BookResult) => Promise; /** AI unavailable: park for a later pass WITHOUT spending an attempt. */ deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; @@ -190,6 +192,8 @@ export interface WorkerRunSummary { cutoverBlocked?: "cutover-boundary-missing"; /** STAGING rows whose upload never landed, parked for a human. */ staleStagingSwept?: number; + /** Previously-failed object deletions that finally succeeded. */ + orphansCleaned?: number; } function centsOf(amount: string): number | null { @@ -277,9 +281,10 @@ export interface CutoverRequest { /** Run the cutover this pass (i.e. dry-run is off). */ run: boolean; /** - * The instant v1 stopped booking. Rows received BEFORE it were booked by - * v1 and are retired; rows received AFTER it were booked by nobody and are - * handed to v2. null refuses to touch either side. + * The instant v1 stopped booking. Only rows received before it are even + * CANDIDATES for retirement — and each still needs its own evidence that v1 + * booked it. Never null when `run` is true: the pass halts before claiming + * rather than proceed without it. */ boundary: Date | null; } @@ -288,7 +293,6 @@ export interface ClaimResult { rows: WorkerRow[]; shadowRetired: number; requeued: number; - boundaryMissing: boolean; } export async function runIntakeWorker(deps: WorkerDependencies): Promise { @@ -298,10 +302,6 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise deps.monotonicMs() - startedAt >= RUN_SOFT_DEADLINE_MS; - // What is left of the PLATFORM budget, not the soft deadline: a booking may - // legitimately run past the point where we stop taking new rows, it just - // must not start without room to finish. - const remainingMs = () => RUN_HARD_BUDGET_MS - (deps.monotonicMs() - startedAt); // CUTOVER. Rows received while dry-run was on were booked by v1, so v2 must // never book them: they are RETIRED as SHADOW_DONE, not requeued. @@ -319,18 +319,31 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise 0); + // Orphaned objects from rejected rows. Nothing else remembers them. + const cleaned = await deps.retryStorageCleanups(outOfTime).catch(() => 0); const byState: Record = {}; const bump = (state: string) => { byState[state] = (byState[state] ?? 0) + 1; }; @@ -366,12 +379,12 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise= MAX_BOOK_ATTEMPTS) { - await deps.applyState(row.id, "NEEDS_REVIEW", "max-retries").catch(() => {}); + // Same rule as booking's own ceiling: a row that exhausted its attempts + // without ever reaching QuickBooks (a weak-lookup fault, a + // finishRouting fault, a storage outage) created no Purchase, so its + // strong key must go back or a corrected resend collides with it. + await deps + .applyState( + row.id, + "NEEDS_REVIEW", + "max-retries", + row.sendAttempted ? undefined : { dedupStrongKey: null }, + ) + .catch(() => {}); return "NEEDS_REVIEW"; } await deps.retryRow( diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index ea6b2ca19..1f67e4884 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -52,12 +52,14 @@ function row(overrides: Partial = {}): BookableRow { memo: null, attempts: 0, lastError: null, + sendAttempted: false, ...overrides, }; } interface Recorder { deps: BookDependencies; + sendMarks: string[]; purchaseCalls: any[]; expenses: any[]; intakeUpdates: any[]; @@ -66,6 +68,7 @@ interface Recorder { function recorder(overrides: Partial = {}, opts: { estimates?: { id: string }[] } = {}): Recorder { const purchaseCalls: any[] = []; + const sendMarks: string[] = []; const expenses: any[] = []; const intakeUpdates: any[] = []; const events: any[] = []; @@ -102,9 +105,10 @@ function recorder(overrides: Partial = {}, opts: { estimates?: now: () => NOW, companyTimeZone: async () => "America/Los_Angeles", isCostCodeAllowed: async () => true, + markSendAttempted: async id => { sendMarks.push(id); }, ...overrides, }; - return { deps, purchaseCalls, expenses, intakeUpdates, events }; + return { deps, sendMarks, purchaseCalls, expenses, intakeUpdates, events }; } test("a taxed receipt splits into a pre-tax line and a sales-tax line that reconstruct the total", () => { @@ -413,13 +417,25 @@ test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTA assert.equal((await bookReceipt(row({ attempts: 18 }), nearly.deps)).outcome, "retry"); // row.attempts 19 -> this is attempt 20, the last one the constant allows. + // sendAttempted is what decides the key, not the fact of reaching the limit. const exhausted = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); - assert.deepEqual(await bookReceipt(row({ attempts: 19 }), exhausted.deps), { + assert.deepEqual(await bookReceipt(row({ attempts: 19, sendAttempted: true }), exhausted.deps), { outcome: "needs-review", reason: "max-retries", - // Sends were attempted to get here, so the key is NOT released. + // A send HAS been attempted, so QBO may hold a Purchase: keep the key. releaseStrongKey: false, }); + + // ...and a row that burned all 20 attempts WITHOUT ever reaching QuickBooks + // (storage faults, say) created no Purchase, so its key must go back. + const neverSent = recorder({ + downloadBytes: async () => ({ ok: false, kind: "transient", message: "ECONNRESET" }), + }); + assert.deepEqual(await bookReceipt(row({ attempts: 19, sendAttempted: false }), neverSent.deps), { + outcome: "needs-review", + reason: "max-retries", + releaseStrongKey: true, + }); }); test("alreadyExists books identically — the lost-response retry", async () => { @@ -532,7 +548,8 @@ test("a booking with less than 25s of runway DEFERS instead of starting", async // not fit in a few seconds, and a booking cut off mid-flight is the worst // outcome available: the Purchase may exist in the real books while the row // never learns it did. - const r = recorder({ remainingBudgetMs: () => 9_000 }); + // An absolute deadline that is already nearly spent. + const r = recorder({ deadline: { startedAt: Date.now() - 50_000, budgetMs: 55_000 } }); const result = await bookReceipt(row(), r.deps); assert.deepEqual(result, { outcome: "deferred", reason: "out-of-budget" }); assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never touched"); @@ -540,7 +557,7 @@ test("a booking with less than 25s of runway DEFERS instead of starting", async }); test("the runway check spends no attempt — the document did nothing wrong", async () => { - const r = recorder({ remainingBudgetMs: () => 0 }); + const r = recorder({ deadline: { startedAt: Date.now() - 60_000, budgetMs: 55_000 } }); const result = await bookReceipt(row({ attempts: 3 }), r.deps); assert.equal(result.outcome, "deferred"); assert.ok(!("attempts" in result), "not a retry, so no attempt is spent"); @@ -548,9 +565,9 @@ test("the runway check spends no attempt — the document did nothing wrong", as test("ample runway books normally, and threads ONE deadline into both QBO calls", async () => { const seen: unknown[] = []; + const deadline = { startedAt: Date.now(), budgetMs: 55_000 }; const r = recorder({ - remainingBudgetMs: () => 50_000, - deadline: () => ({ startedAt: 0, budgetMs: 50_000 }) as any, + deadline, getTokens: async d => { seen.push(d); return { accessToken: "t", realmId: "r" } as any; }, createPurchase: async (_t, input, d) => { seen.push(d); @@ -562,6 +579,7 @@ test("ample runway books normally, and threads ONE deadline into both QBO calls" assert.equal(result.outcome, "booked"); assert.equal(seen.length, 2); assert.strictEqual(seen[0], seen[1], "the SAME deadline object, so a slow refresh shortens the create"); + assert.strictEqual(seen[0], deadline, "and it is the INVOCATION's deadline, not a fresh one"); }); test("MIN_BOOKING_BUDGET_MS is the documented 25s", () => { diff --git a/tests/receipt-intake-claim-db.test.ts b/tests/receipt-intake-claim-db.test.ts new file mode 100644 index 000000000..8fd6bb351 --- /dev/null +++ b/tests/receipt-intake-claim-db.test.ts @@ -0,0 +1,133 @@ +/** + * The claim transaction against a REAL Postgres. + * + * Everything else in this feature's suites mocks the database, so the SQL is + * the one part they cannot check — and both bugs that live there are silent + * until production: a void-returning function read through $queryRaw, and a + * claim whose real behaviour differs from the mocked stand-in. + * + * Opt-in by design: it needs a THROWAWAY database and it writes rows. It runs + * in CI's migrations job (which has a disposable Postgres) and skips everywhere + * else, including anywhere DATABASE_URL looks like production. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { PrismaClient, Prisma } from "@prisma/client"; +import { CLAIM_LOCK_KEY } from "../src/lib/receipt-intake/worker"; + +const url = process.env.RECEIPT_INTAKE_DB_TEST_URL ?? process.env.MIGRATION_HISTORY_TEST_URL; +const looksLikeProd = !!url && /supabase\.(co|com)/i.test(url); +const skip = !url + ? "set RECEIPT_INTAKE_DB_TEST_URL to a disposable PostgreSQL URL" + : looksLikeProd + ? "refusing to run against what looks like production" + : false; + +const db = url && !looksLikeProd ? new PrismaClient({ datasources: { db: { url } } }) : null; + +const PREFIX = "drive:claimdb-"; + +async function seed(id: string, over: Partial = {}) { + return db!.receiptIntake.create({ + data: { + id, + source: "drive", + sourceRef: `${PREFIX}${id}`, + state: "RECEIVED", + dryRun: false, + storagePath: `receipts/intake/${id}.jpg`, + mimeType: "image/jpeg", + fileSize: 10, + fileSha256: "x".repeat(64), + ...over, + }, + }); +} + +test("the blocking advisory lock runs without error inside a transaction", { skip }, async () => { + // The regression: `SELECT pg_advisory_xact_lock(...)` through $queryRaw. + // pg_advisory_xact_lock returns VOID, and reading that column can throw — + // inside the promotion transaction, which then looks like a transient DB + // fault forever while the lock was never taken. + await db!.$transaction(async tx => { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${"weak-key-probe"}, 0))`; + }); + + // ...and the TRY variant genuinely returns a readable boolean. + const [row] = await db!.$queryRaw<{ locked: boolean }[]>( + Prisma.sql`SELECT pg_try_advisory_xact_lock(hashtextextended(${CLAIM_LOCK_KEY}, 0)) AS locked`, + ); + assert.equal(typeof row.locked, "boolean"); +}); + +test("two overlapping claims never hand out the same row", { skip }, async () => { + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); + const ids = ["claimdb-a", "claimdb-b", "claimdb-c"]; + for (const id of ids) await seed(id); + + const now = new Date(); + const claimOnce = async () => + db!.$transaction(async tx => { + const [lock] = await tx.$queryRaw<{ locked: boolean }[]>( + Prisma.sql`SELECT pg_try_advisory_xact_lock(hashtextextended(${CLAIM_LOCK_KEY}, 0)) AS locked`, + ); + if (!lock?.locked) return null; + const due = await tx.receiptIntake.findMany({ + where: { + sourceRef: { startsWith: PREFIX }, + state: { in: ["RECEIVED", "READ", "BOOKING"] }, + OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }], + }, + select: { id: true }, + }); + if (due.length === 0) return []; + await tx.receiptIntake.updateMany({ + where: { id: { in: due.map(r => r.id) } }, + data: { nextRetryAt: new Date(now.getTime() + 10 * 60_000) }, + }); + return due.map(r => r.id); + }); + + const first = await claimOnce(); + const second = await claimOnce(); + + assert.deepEqual([...(first ?? [])].sort(), ids.slice().sort(), "the first claim takes them"); + // The lease is what guarantees this, not the lock: the lock is released the + // moment the first transaction commits. + assert.deepEqual(second, [], "the second claim finds nothing left"); + + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); +}); + +test("SHADOW_DONE and the new columns are writable — the CHECK really allows them", { skip }, async () => { + // The cutover writes SHADOW_DONE on every shadow row in ONE statement. If + // the CHECK constraint did not allow it, that fails inside the claim + // transaction and takes the whole cutover with it. + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); + await seed("claimdb-shadow", { + state: "SHADOW_DONE", + stateReason: "booked-by-v1", + archivedByV1: true, + sendAttempted: false, + expectedSha256: "y".repeat(64), + }); + const row = await db!.receiptIntake.findUnique({ where: { id: "claimdb-shadow" } }); + assert.equal(row?.state, "SHADOW_DONE"); + assert.equal(row?.archivedByV1, true); + assert.equal(row?.expectedSha256, "y".repeat(64)); + + // And an invented state is still refused. + await assert.rejects( + () => seed("claimdb-bogus", { state: "NOT_A_STATE" }), + /violates check constraint|check constraint/i, + ); + + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); +}); + +test.after(async () => { + if (db) { + await db.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }).catch(() => {}); + await db.$disconnect(); + } +}); diff --git a/tests/receipt-intake-raw-sql.test.ts b/tests/receipt-intake-raw-sql.test.ts new file mode 100644 index 000000000..3156f7d03 --- /dev/null +++ b/tests/receipt-intake-raw-sql.test.ts @@ -0,0 +1,120 @@ +/** + * Raw-SQL tripwires and the real transaction paths. + * + * The pure-logic suites mock every database call, so the SQL itself is the one + * part of this feature they cannot see. Two failures live there and both are + * silent until production: selecting a void-returning function, and a claim + * transaction that does not behave the way the mocked version implies. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const ROOT = path.resolve(__dirname, ".."); + +/** Line and block comments only — enough to stop prose ABOUT the rule tripping it. */ +function stripComments(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, " ") + .split("\n") + .map(line => line.replace(/\/\/.*$/, "")) + .join("\n"); +} + +/** The raw-SQL helper nearest BEFORE `at` — the one that issues that statement. */ +function nearestRawHelper(source: string, at: number): string | null { + const helpers = ["$queryRawUnsafe", "$queryRaw", "$executeRawUnsafe", "$executeRaw"]; + let best: { name: string; index: number } | null = null; + for (const name of helpers) { + const index = source.lastIndexOf(name, at); + if (index === -1) continue; + // Prefer the LONGEST match at the same position, so "$queryRawUnsafe" + // is not read as "$queryRaw" plus stray characters. + if (!best || index > best.index || (index === best.index && name.length > best.name.length)) { + best = { name, index }; + } + } + return best?.name ?? null; +} + +function walk(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir)) { + if (entry === "node_modules" || entry === ".next" || entry.startsWith(".")) continue; + const full = path.join(dir, entry); + if (statSync(full).isDirectory()) walk(full, out); + else if (/\.tsx?$/.test(entry)) out.push(full); + } + return out; +} + +/** + * Functions that return `void`. `SELECT`ing one through $queryRaw produces a + * row whose single column has no readable type, which Prisma's query path can + * reject outright — and inside a transaction that throw looks like a transient + * DB fault forever while the lock was never actually taken. $executeRaw runs + * the statement for its effect and asks nothing of the result. + */ +const VOID_FUNCTIONS = [ + "pg_advisory_xact_lock", + "pg_advisory_lock", + "pg_advisory_unlock_all", +]; + +test("no $queryRaw anywhere SELECTs a void-returning function unreadably", () => { + // The rule: find the raw-SQL helper that ISSUES this call — the nearest one + // before it — and require that it is not a result-reading form, unless the + // call is cast to a readable type. + // + // Two shapes are correct and must not be flagged, or the tripwire gets + // muted as noise: + // * $executeRaw / $executeRawUnsafe — runs the statement for its effect. + // * an explicit cast, e.g. `pg_advisory_xact_lock(...)::text AS x`, which + // is exactly how qbo-expense-sync.ts makes its lock readable. + // Comments are stripped first, because selection-ai-sort-apply-core.ts + // explains this very rule in prose directly above a CORRECT call. + const offenders: string[] = []; + for (const file of walk(path.join(ROOT, "src"))) { + const source = stripComments(readFileSync(file, "utf8")); + for (const fn of VOID_FUNCTIONS) { + let at = source.indexOf(fn + "("); + while (at !== -1) { + const issuer = nearestRawHelper(source, at); + const cast = /\)\s*::\s*\w+/.test(source.slice(at, at + 200)); + if (issuer && issuer.startsWith("$queryRaw") && !cast) { + offenders.push(path.relative(ROOT, file) + " -> " + fn); + break; + } + at = source.indexOf(fn + "(", at + 1); + } + } + } + assert.deepEqual(offenders, [], "use $executeRaw, or cast the result to a readable type"); +}); + +test("the tripwire actually catches the shape it exists for", () => { + // Without this the test above passes just as happily on an empty scan. + const bad = 'await tx.$queryRaw`SELECT pg_advisory_xact_lock(hashtextextended(k, 0))`;'; + const at = bad.indexOf("pg_advisory_xact_lock("); + assert.ok(bad.slice(0, at).includes("$queryRaw"), "the offending shape is recognisable"); + assert.ok(!/\)\s*::\s*\w+/.test(bad.slice(at)), "and it has no rescuing cast"); +}); + +test("the TRY variant returns a boolean, so it is correctly read with $queryRaw", () => { + // pg_try_advisory_xact_lock returns bool — reading it is the whole point, + // and this pins that the two are not confused for each other. + const worker = readFileSync( + path.join(ROOT, "src/app/api/cron/receipt-intake-worker/route.ts"), + "utf8", + ); + const tryAt = worker.indexOf("pg_try_advisory_xact_lock("); + assert.ok(tryAt > 0, "the try-lock is present"); + assert.ok(worker.slice(tryAt - 200, tryAt).includes("$queryRaw"), "try-lock is READ with $queryRaw"); + + const blockingAt = worker.indexOf("pg_advisory_xact_lock(hashtextextended"); + assert.ok(blockingAt > 0, "the blocking lock is present"); + assert.ok( + worker.slice(blockingAt - 300, blockingAt).includes("$executeRaw"), + "the blocking (void) lock is EXECUTED, never selected", + ); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts new file mode 100644 index 000000000..3fbf87897 --- /dev/null +++ b/tests/receipt-intake-stored-object.test.ts @@ -0,0 +1,76 @@ +/** + * The shared stored-object validator. + * + * TWO callers publish a STAGING row — /intake/{id}/finalize and the worker's + * stale-STAGING sweep. They must agree, or whichever runs first decides whether + * a 40 MB video becomes a receipt. This is that agreement, in one place. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { inspectStoredObject } from "../src/lib/receipt-intake/stored-object"; +import { MAX_STORED_BYTES } from "../src/lib/receipt-intake/intake-core"; +import type { DocBytesResult } from "../src/lib/secure-storage"; + +const PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", +); +const give = (r: DocBytesResult) => async () => r; + +test("a real image is accepted, and its metadata comes from the BYTES", async () => { + const check = await inspectStoredObject("p.jpg", "application/pdf", give({ ok: true, bytes: PNG })); + assert.ok(check.ok); + // The declared type said PDF. The bytes say PNG, and the bytes win. + assert.equal(check.mimeType, "image/png"); + assert.equal(check.fileSize, PNG.length); + assert.equal(check.fileSha256, createHash("sha256").update(PNG).digest("hex")); +}); + +test("text/plain is the ONE type taken on its declared word", async () => { + // It has no magic bytes. Same concession the single-shot path makes. + const txt = Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"); + const ok = await inspectStoredObject("p.txt", "text/plain", give({ ok: true, bytes: txt })); + assert.ok(ok.ok); + assert.equal(ok.mimeType, "text/plain"); + + // ...and without that declaration the same bytes are unidentifiable. + const bad = await inspectStoredObject("p.bin", "", give({ ok: true, bytes: txt })); + assert.equal(bad.ok, false); + assert.equal((bad as { reason: string }).reason, "unsupported-file-type"); +}); + +test("oversize, empty and unidentifiable objects are REJECTED, not published", async () => { + // The signed upload URL bypassed every check the server could otherwise + // make, so these are enforced on the object itself. + const big = await inspectStoredObject("p.jpg", "image/jpeg", give({ + ok: true, bytes: Buffer.alloc(MAX_STORED_BYTES + 1, 1), + })); + assert.equal(big.ok, false); + assert.match((big as { reason: string }).reason, /^file-too-large:/); + + const empty = await inspectStoredObject("p.jpg", "image/jpeg", give({ ok: true, bytes: Buffer.alloc(0) })); + assert.equal((empty as { reason: string }).reason, "empty-file"); + + const exe = await inspectStoredObject("p.exe", "image/jpeg", give({ ok: true, bytes: Buffer.from("MZ\x90\x00") })); + assert.equal((exe as { reason: string }).reason, "unsupported-file-type"); +}); + +test("exactly at the ceiling is allowed", async () => { + const atLimit = Buffer.concat([PNG, Buffer.alloc(MAX_STORED_BYTES - PNG.length, 0)]); + assert.equal(atLimit.length, MAX_STORED_BYTES); + const check = await inspectStoredObject("p.png", "image/png", give({ ok: true, bytes: atLimit })); + assert.ok(check.ok, "the boundary itself is not oversize"); +}); + +test("missing and transient are DIFFERENT answers", async () => { + // A confirmed 404 is terminal for the sweep; a storage blip must come back + // next pass rather than park a good receipt as file-missing. + const missing = await inspectStoredObject("p.jpg", "image/jpeg", give({ ok: false, kind: "not-found" })); + assert.deepEqual(missing, { ok: false, kind: "missing" }); + + const flaky = await inspectStoredObject("p.jpg", "image/jpeg", give({ + ok: false, kind: "transient", message: "ECONNRESET", + })); + assert.deepEqual(flaky, { ok: false, kind: "transient", message: "ECONNRESET" }); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 443aceacf..21e00dec6 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -63,6 +63,7 @@ function workerRow(overrides: Partial = {}): WorkerRow { busyPasses: 0, lastError: null, suggestedConfidence: null, + sendAttempted: false, ...overrides, }; } @@ -97,6 +98,7 @@ interface Harness { claimOpts: CutoverRequest[]; boundary: Date | null; sweepCalls: number; + cleanupCalls: number; bookBudgets: number[]; clock: number; } @@ -104,7 +106,7 @@ interface Harness { function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { reads: 0, books: 0, applied: [], states: [], promoted: [], finished: [], deferred: [], - retried: [], claimOpts: [], sweepCalls: 0, bookBudgets: [], clock: 0, + retried: [], claimOpts: [], sweepCalls: 0, cleanupCalls: 0, bookBudgets: [], clock: 0, boundary: new Date("2026-08-25T00:00:00.000Z"), deps: null as unknown as WorkerDependencies, }; @@ -116,6 +118,7 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) cutoverBoundary: async () => h.boundary, isDryRunEnabled: () => true, sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, + retryStorageCleanups: async () => { h.cleanupCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), read: async () => { h.reads++; return goodRead; }, @@ -125,9 +128,8 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) finishRouting: async (id, stateReason) => { h.finished.push({ id, stateReason }); }, companyTimeZone: async () => "America/Los_Angeles", promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, - book: async (_row, remainingMs) => { + book: async () => { h.books++; - h.bookBudgets.push(remainingMs); return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, applyBookResult: async () => {}, @@ -824,3 +826,86 @@ test("the deadline starts at invocation entry, so a slow sweep cannot overrun it assert.equal(summary.processed, 0); assert.equal(summary.deferredToNextRun, 2, "both rows keep their lease for the next run"); }); + +// ── A missing boundary halts the WHOLE pass (round-7 item 3) ─────────────── + +test("live mode with no recorded boundary claims nothing at all", async () => { + // Refusing only the retire/requeue was not enough: the pass went on to + // claim and BOOK rows while the shadow backlog sat undecided. Live mode + // without a boundary means we cannot tell which rows v1 already booked, + // and booking anything under that uncertainty is the double-booking this + // whole mechanism exists to prevent. + const h = harness([workerRow(), workerRow({ id: "row-2" })], { + isDryRunEnabled: () => false, + cutoverBoundary: async () => null, + }); + const summary = await runIntakeWorker(h.deps); + + assert.deepEqual(summary, { processed: 0, byState: {}, cutoverBlocked: "cutover-boundary-missing" }); + assert.deepEqual(h.claimOpts, [], "claim() is never even called"); + assert.equal(h.sweepCalls, 0, "and no housekeeping runs either"); + assert.equal(h.books, 0); + assert.equal(h.reads, 0); +}); + +test("dry-run mode does not need a boundary", async () => { + // Nothing books in shadow mode, so there is nothing to be uncertain about. + const h = harness([workerRow()], { isDryRunEnabled: () => true, cutoverBoundary: async () => null }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.cutoverBlocked, undefined); + assert.equal(summary.processed, 1); +}); + +// ── Orphaned objects are chased (round-7 item 5) ─────────────────────────── + +test("every pass retries storage deletes that failed earlier", async () => { + // A rejected row is deleted, so after that nothing in the database + // references its bytes — without this they sit in a private bucket forever. + const h = harness([], { retryStorageCleanups: async () => { h.cleanupCalls++; return 3; } }); + const summary = await runIntakeWorker(h.deps); + assert.equal(h.cleanupCalls, 1); + assert.equal(summary.orphansCleaned, 3); +}); + +test("a failing cleanup pass never takes the run down", async () => { + const h = harness([workerRow()], { + retryStorageCleanups: async () => { throw new Error("storage down"); }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.orphansCleaned, undefined); + assert.deepEqual(summary.byState, { READ: 1 }, "the batch still ran"); +}); + +// ── A row that never sent releases its key, whatever killed it (item 7) ──── + +test("a weak-lookup failure at the retry limit RELEASES the strong key", async () => { + // This row exhausted its attempts entirely on a database fault and never + // touched QuickBooks. Holding its key quarantines the corrected resend + // against a row that never became a purchase. + const h = harness([workerRow({ attempts: 19, sendAttempted: false })], { + findWeakHit: async () => { throw new Error("connection reset"); }, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); + assert.equal(h.states[0].reason, "max-retries"); + assert.equal(h.states[0].patch?.dedupStrongKey, null, "the key goes back"); +}); + +test("a finishRouting failure at the retry limit also releases the key", async () => { + const h = harness([workerRow({ attempts: 19, sendAttempted: false })], { + finishRouting: async () => { throw new Error("connection reset"); }, + }); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].reason, "max-retries"); + assert.equal(h.states[0].patch?.dedupStrongKey, null); +}); + +test("a row that DID send keeps its key at the retry limit", async () => { + // QuickBooks may hold a Purchase whose response we lost. + const h = harness([workerRow({ attempts: 19, sendAttempted: true })], { + findWeakHit: async () => { throw new Error("connection reset"); }, + }); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].reason, "max-retries"); + assert.equal(h.states[0].patch, undefined, "no patch, so the key is untouched"); +}); From 8297b46bb436e3c63d28de72bd1596bfa5aa6b05 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 02:25:56 -0700 Subject: [PATCH 045/144] fix(receipts): seal verified objects, quarantine unprovable rows, fence the claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 gate (a/b/c) and Codex round 8 (1-5). 1 SHADOW_QUARANTINE. The cutover had two outcomes where it needed three. A pre-boundary row with no v1 evidence and no DRIVE identity has no shared key to make a v2 booking idempotent — v2 books under the intake UUID, which v1 never saw, so a duplicate goes through silently. Booking risks double-paying, retiring risks losing a real expense, and neither is ours to guess. Terminal, never auto-requeued, surfaced for a human who has checked QBO. Drive rows without evidence still go to v2, because the Drive file id makes that safe. 2 Objects are SEALED at finalize: verified bytes are copied to receipts//. — a path no client was given a URL for — the upload path is deleted, and the row points at the copy. The upload path stays writable by design (upsert, so a resumed /start can replace its own partial upload), which is exactly why the row must not keep pointing there. The reader and the booker now re-hash what they download and refuse on mismatch (content-changed): a sha stored once and never re-checked proves nothing. 3 A STAGING row is only parked file-missing after the signed URL's 2h lifetime has passed — 15 minutes declared receipts missing while their own upload link still worked. A late /finalize on a parked row RE-VALIDATES and recovers it instead of answering alreadyFinalized, which would leave a real receipt parked while telling the caller it was fine. The sweep also compares expectedSha256 before publishing. 4 markSendAttempted moved to immediately before createPurchase, after the tokens and the final budget check. Marking earlier meant a failed refresh or a deferral left sendAttempted=true on a row that never reached QuickBooks, and its strong key was then held forever against a Purchase that does not exist. 5 loadPhases uses the project-phase resolver: only that project's phase-eligible active codes, empty allowed, nothing for a row with no project. It was returning every active cost code company-wide, so the model suggested phases the job does not have and booking silently discarded them. Booking now re-validates on BOTH sides of the QBO create. (a) finishRouting is fenced on state AND a per-claim token, and clears both claim fields. A zombie worker resuming after its row was re-claimed would otherwise publish READ over its successor's state; a time-based lease cannot express that, since both hold identical row ids. Adapter tested against real Postgres. (c) Indexes on costCodeId and createdById. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 36 ++++- package.json | 4 +- .../migration.sql | 8 +- prisma/prisma-blind-spots.json | 2 +- prisma/schema.prisma | 15 +- scripts/apply-receipt-intake.mjs | 23 ++- .../api/cron/receipt-intake-worker/route.ts | 144 +++++++++++++++--- .../receipts/intake/[id]/finalize/route.ts | 43 +++++- src/lib/receipt-intake/book.ts | 50 ++++-- src/lib/receipt-intake/route-state.ts | 11 ++ src/lib/receipt-intake/storage-cleanup.ts | 47 +++++- src/lib/receipt-intake/stored-object.ts | 60 +++++++- src/lib/receipt-intake/worker.ts | 55 +++++-- tests/receipt-intake-book.test.ts | 74 ++++++++- tests/receipt-intake-claim-db.test.ts | 53 +++++++ tests/receipt-intake-phases.test.ts | 68 +++++++++ tests/receipt-intake-stored-object.test.ts | 63 +++++++- tests/receipt-intake-worker.test.ts | 86 +++++++++-- 18 files changed, 756 insertions(+), 86 deletions(-) create mode 100644 tests/receipt-intake-phases.test.ts diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index b4b5b6ce8..902a66bb9 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -436,6 +436,28 @@ PDFs carrying embedded JavaScript (never opened server-side — the bytes go to QBO as an attachment). +### Objects are sealed on finalize + +The upload path is writable by whoever holds the signed URL, and that URL is `upsert: true` +so a resumed `/start` can replace its own partial upload. Both are necessary, and together +they mean the bytes at the upload path can change AFTER verification. + +So `/finalize` verifies, then **copies** the bytes to `receipts//.` — a +content-addressed path the client was never given a URL for — deletes the upload path, and +points the row there. Every later reader (the Gemini read step, the booker) re-hashes what +it downloads and refuses on a mismatch: `content-changed`, terminal. A hash stored once and +never re-checked proves nothing about what is being served now. + +A failed delete of the upload path is an orphan, not a correctness problem (the row already +points at the sealed copy), so it goes on the `storage-cleanup-pending` queue. + +**Sweeper timing.** A `STAGING` row is only parked `file-missing` once the signed upload +URL's **2-hour** lifetime has passed — parking at the 15-minute sweep window declared +receipts missing while their own upload link was still usable. A late `/finalize` on a row +the sweeper already parked re-validates and **recovers** it rather than reporting +`alreadyFinalized`, which would leave a real receipt parked while telling the caller it was +fine. + ### Two machine secrets, not one They belong to different programs, so they are different keys and rotate independently. @@ -528,11 +550,15 @@ an overlap safe in general. (`kind: receipt-push`, status `created`/`already-exists`) whose `driveFileId` matches the row — v1's pushes go through ProBuild's create route, which logs them — or the forwarder sending `archivedByV1: true` on the forward. - - everything else, including rows before the boundary with NO evidence -> handed to v2. - "Received before the boundary" says when the file arrived, not that anything booked - it: v1 skips documents constantly, and retiring those would drop real expenses. This - is safe because a Drive row books under the **Drive file id**, so a v1/v2 overlap on - the same file collapses to one Purchase through QBO's DocNumber/requestid idempotency. + - before the boundary, NO evidence, and a **Drive** row -> handed to v2. Safe precisely + because a Drive row books under the **Drive file id**, so if v1 did book it after all, + QBO's DocNumber/requestid idempotency collapses the two into one Purchase. + - before the boundary, NO evidence, and **not** a Drive row -> `SHADOW_QUARANTINE`. + There is no shared identity here: v2 would book under the intake UUID, which v1 never + saw, so a duplicate would go through silently. Booking risks double-paying; retiring + risks losing a real expense. Terminal, never auto-requeued — it surfaces on the + Receipts tab with a "book anyway" action for whoever has checked QuickBooks. + - after the boundary -> handed to v2. v1 had already stopped, so nobody booked these. With no boundary recorded in live mode the worker **halts the entire pass before claiming anything** and logs `cutover-boundary-missing`. Not just the retire: booking anything while we cannot tell what v1 already booked is the double-booking this whole diff --git a/package.json b/package.json index 2267df08f..857e2ad23 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index c12dfb95f..312b2c1ea 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -47,6 +47,8 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "qbPurchaseId" TEXT, "expenseId" TEXT, "archiveDriveFileId" TEXT, + "claimToken" TEXT, + "claimedAt" TIMESTAMP(3), "attempts" INTEGER NOT NULL DEFAULT 0, "busyPasses" INTEGER NOT NULL DEFAULT 0, "lastError" TEXT, @@ -64,6 +66,8 @@ ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NU ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimedAt" TIMESTAMP(3); CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_sourceRef_key" ON "ReceiptIntake"("sourceRef"); CREATE UNIQUE INDEX IF NOT EXISTS "ReceiptIntake_expenseId_key" ON "ReceiptIntake"("expenseId"); @@ -74,6 +78,8 @@ CREATE INDEX IF NOT EXISTS "ReceiptIntake_state_nextRetryAt_idx" ON "ReceiptInta CREATE INDEX IF NOT EXISTS "ReceiptIntake_projectId_idx" ON "ReceiptIntake"("projectId"); CREATE INDEX IF NOT EXISTS "ReceiptIntake_dedupWeakKey_idx" ON "ReceiptIntake"("dedupWeakKey"); CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("createdAt"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_costCodeId_idx" ON "ReceiptIntake"("costCodeId"); +CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdById_idx" ON "ReceiptIntake"("createdById"); DO $$ BEGIN @@ -85,7 +91,7 @@ BEGIN ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', - 'SHADOW_DONE')); + 'SHADOW_DONE', 'SHADOW_QUARANTINE')); END IF; END $$; diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 3867f59ef..3c9689dec 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -102,7 +102,7 @@ { "name": "ReceiptIntake_state_check", "table": "\"ReceiptIntake\"", - "def": "CHECK ((state = ANY (ARRAY['STAGING'::text, 'RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text, 'SHADOW_DONE'::text])))" + "def": "CHECK ((state = ANY (ARRAY['STAGING'::text, 'RECEIVED'::text, 'READ'::text, 'NEEDS_JOB'::text, 'NEEDS_REVIEW'::text, 'BOOKING'::text, 'BOOKED'::text, 'ARCHIVED'::text, 'DUPLICATE'::text, 'VOID'::text, 'NON_RECEIPT'::text, 'SHADOW_DONE'::text, 'SHADOW_QUARANTINE'::text])))" }, { "name": "RefundEvent_amountCents_check", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 5364877f1..09b663a72 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3027,7 +3027,7 @@ model ReceiptIntake { /// yet, so the worker's claim predicate excludes it. The intake route flips /// it to RECEIVED in one UPDATE after the upload lands. A row that never gets /// there is swept to NEEDS_REVIEW after 15 minutes. - state String @default("STAGING") // STAGING RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT SHADOW_DONE + state String @default("STAGING") // STAGING RECEIVED READ NEEDS_JOB NEEDS_REVIEW BOOKING BOOKED ARCHIVED DUPLICATE VOID NON_RECEIPT SHADOW_DONE SHADOW_QUARANTINE dryRun Boolean @default(true) /// no-estimate | multi-doc | zero-total | weak-dup: | /// strong-dup-amount-mismatch: | qbo-fault: | max-retries | @@ -3084,6 +3084,15 @@ model ReceiptIntake { expenseId String? @unique expense Expense? @relation(fields: [expenseId], references: [id], onDelete: SetNull) archiveDriveFileId String? + /// Fencing token for the worker's claim, rewritten on every claim. + /// + /// `nextRetryAt` is a time-based lease and cannot tell a LIVE worker from a + /// stale one whose invocation was killed and resumed: both hold the same row + /// id and both believe they own it. The token can — a completing transition + /// matches on it, so a worker whose claim was superseded writes nothing + /// instead of overwriting the state its successor just produced. + claimToken String? + claimedAt DateTime? attempts Int @default(0) /// Consecutive passes where the AI service was UNAVAILABLE (never the /// document's fault, so it must not spend `attempts`). Ported from v3.4: @@ -3103,6 +3112,10 @@ model ReceiptIntake { // baseline notes). Keep this comment; never regenerate it away. @@index([state, nextRetryAt]) @@index([projectId]) + /// Both are FK targets the queue filters and joins on; without these a + /// cost-code or user-scoped read scans the whole table once it has volume. + @@index([costCodeId]) + @@index([createdById]) @@index([dedupWeakKey]) @@index([createdAt]) } diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index e5357bab8..a50e4c95e 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -74,6 +74,9 @@ export const RECEIPT_INTAKE_STATES = [ "BOOKED", "ARCHIVED", "DUPLICATE", "VOID", "NON_RECEIPT", // Received during the shadow week, therefore booked by v1 and NEVER by v2. "SHADOW_DONE", + // Pre-boundary, no v1 evidence, and no Drive identity to make a v2 booking + // idempotent. A HUMAN decides. + "SHADOW_QUARANTINE", ]; export const statements = [ @@ -112,6 +115,8 @@ export const statements = [ "qbPurchaseId" TEXT, "expenseId" TEXT, "archiveDriveFileId" TEXT, + "claimToken" TEXT, + "claimedAt" TIMESTAMP(3), "attempts" INTEGER NOT NULL DEFAULT 0, "busyPasses" INTEGER NOT NULL DEFAULT 0, "lastError" TEXT, @@ -130,6 +135,8 @@ export const statements = [ `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimedAt" TIMESTAMP(3)`, // Intake idempotency: one row per caller-supplied sourceRef. A forwarder // replaying the same Drive file / Gmail message is a no-op. @@ -161,6 +168,13 @@ export const statements = [ `CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("createdAt")`, + // Both are FK targets the queue filters and joins on. + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_costCodeId_idx" + ON "ReceiptIntake"("costCodeId")`, + + `CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdById_idx" + ON "ReceiptIntake"("createdById")`, + // state is a closed set — a typo must fail loudly rather than create a // silent eleventh state that no query ever selects. // The state set GROWS. `IF NOT EXISTS` alone is wrong for that: a database @@ -174,7 +188,7 @@ export const statements = [ // set that would orphan live data fails loudly here rather than later. `DO $$ DECLARE current_def TEXT; - wanted_def TEXT := 'CHECK ((state = ANY (ARRAY[''STAGING''::text, ''RECEIVED''::text, ''READ''::text, ''NEEDS_JOB''::text, ''NEEDS_REVIEW''::text, ''BOOKING''::text, ''BOOKED''::text, ''ARCHIVED''::text, ''DUPLICATE''::text, ''VOID''::text, ''NON_RECEIPT''::text, ''SHADOW_DONE''::text])))'; + wanted_def TEXT := 'CHECK ((state = ANY (ARRAY[''STAGING''::text, ''RECEIVED''::text, ''READ''::text, ''NEEDS_JOB''::text, ''NEEDS_REVIEW''::text, ''BOOKING''::text, ''BOOKED''::text, ''ARCHIVED''::text, ''DUPLICATE''::text, ''VOID''::text, ''NON_RECEIPT''::text, ''SHADOW_DONE''::text, ''SHADOW_QUARANTINE''::text])))'; BEGIN SELECT pg_get_constraintdef(oid) INTO current_def FROM pg_constraint @@ -185,7 +199,7 @@ export const statements = [ ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', - 'SHADOW_DONE')); + 'SHADOW_DONE', 'SHADOW_QUARANTINE')); ELSIF current_def IS DISTINCT FROM wanted_def THEN -- One statement each, in the SAME transaction as everything else this -- script runs, so the table is never briefly unconstrained. @@ -193,7 +207,7 @@ export const statements = [ ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', - 'SHADOW_DONE')); + 'SHADOW_DONE', 'SHADOW_QUARANTINE')); END IF; END $$`, @@ -260,7 +274,8 @@ const expectedColumns = { "vendor", "txnDate", "totalCents", "taxCents", "docType", "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", "dedupWeakKey", "duplicateOfId", "qbPurchaseId", "expenseId", - "archiveDriveFileId", "attempts", "busyPasses", "lastError", "nextRetryAt", + "archiveDriveFileId", "claimToken", "claimedAt", + "attempts", "busyPasses", "lastError", "nextRetryAt", "bookedAt", "createdAt", "updatedAt", ], }; diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 30596a24e..e46baa44d 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -1,10 +1,11 @@ +import { randomUUID } from "node:crypto"; import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; -import { inspectStoredObject } from "@/lib/receipt-intake/stored-object"; +import { downloadVerified, inspectStoredObject } from "@/lib/receipt-intake/stored-object"; import { deleteObjectOrRecord, retryPendingCleanups } from "@/lib/receipt-intake/storage-cleanup"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; @@ -13,7 +14,7 @@ import { readReceipt } from "@/lib/receipt-intake/read"; import { canonicalVendor } from "@/lib/receipt-intake/keys"; import { resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; import { resolveCompanyTimeZone } from "@/lib/company-timezone"; -import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { isCostCodeAllowedForProject, resolveProjectPhaseCodes } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; import { backoffMs } from "@/lib/receipt-intake/route-state"; @@ -22,6 +23,7 @@ import { CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, RUN_HARD_BUDGET_MS, + SIGNED_UPLOAD_TTL_MS, STAGING_SWEEP_BATCH, STAGING_SWEEP_MINUTES, type ClaimResult, @@ -64,7 +66,7 @@ const WORKER_ROW_SELECT = { storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, lastError: true, - suggestedConfidence: true, sendAttempted: true, + suggestedConfidence: true, sendAttempted: true, claimToken: true, fileSha256: true, createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; @@ -104,6 +106,7 @@ async function claim(opts: CutoverRequest): Promise { // The rows keep their read results and dedup keys, so a post-cutover // resend of the same receipt still collides with them and is caught. let shadowRetired = 0; + let shadowQuarantined = 0; let requeued = 0; if (opts.run) { // runIntakeWorker halts before ever calling claim() without one, so @@ -159,14 +162,34 @@ async function claim(opts: CutoverRequest): Promise { ) : new Set(); + // Three outcomes, not two. The middle one is the honest answer + // to a question we cannot settle from data: + // + // evidenced -> v1 booked it. Retire. + // no evidence, + // DRIVE row -> hand to v2. Safe BECAUSE it books under + // the Drive file id, so if v1 did book it + // after all, QBO's DocNumber/requestid + // idempotency collapses the two into one + // Purchase. + // no evidence, + // NOT a Drive -> quarantine. There is no shared identity + // row here: v2 would book under the intake + // UUID, which v1 never saw, so a duplicate + // would go through silently. Booking risks + // double-paying; retiring risks losing a + // real expense. A human checks QBO and uses + // "book anyway". const evidenced: string[] = []; const unevidenced: string[] = []; + const quarantined: string[] = []; for (const row of candidates) { const driveId = row.source === "drive" && row.sourceRef.startsWith("drive:") ? row.sourceRef.slice("drive:".length) : null; if (row.archivedByV1 || (driveId && bookedByV1.has(driveId))) evidenced.push(row.id); - else unevidenced.push(row.id); + else if (driveId) unevidenced.push(row.id); + else quarantined.push(row.id); } if (evidenced.length) { @@ -177,6 +200,21 @@ async function claim(opts: CutoverRequest): Promise { shadowRetired = retired.count; } + if (quarantined.length) { + const held = await tx.receiptIntake.updateMany({ + where: { id: { in: quarantined } }, + data: { + state: "SHADOW_QUARANTINE", + stateReason: "no-v1-evidence", + // Terminal: it must never come back round on a + // retry timer. Only a human moves it. + nextRetryAt: null, + dryRun: false, + }, + }); + shadowQuarantined = held.count; + } + // Everything else — after the boundary, or before it with no // evidence — is v2's to book. const handed = await tx.receiptIntake.updateMany({ @@ -190,10 +228,10 @@ async function claim(opts: CutoverRequest): Promise { }); requeued = handed.count; - if (shadowRetired > 0 || requeued > 0) { + if (shadowRetired > 0 || requeued > 0 || shadowQuarantined > 0) { console.log("[cron/receipt-intake-worker] cutover", JSON.stringify({ - boundary: opts.boundary.toISOString(), shadowRetired, requeued, - unevidenced: unevidenced.length, + boundary: opts.boundary.toISOString(), + shadowRetired, requeued, shadowQuarantined, })); } } @@ -212,15 +250,25 @@ async function claim(opts: CutoverRequest): Promise { take: BATCH_SIZE, select: WORKER_ROW_SELECT, }); - if (due.length === 0) return { rows: [], shadowRetired, requeued }; + if (due.length === 0) return { rows: [], shadowRetired, requeued, shadowQuarantined }; // THE claim. Anything this run took is invisible to the next one for - // the lease, whether or not the advisory lock held. + // the lease, whether or not the advisory lock held — AND it is stamped + // with a fresh token, so a completing write can prove it still owns the + // row rather than merely having owned it once. + const claimToken = randomUUID(); await tx.receiptIntake.updateMany({ where: { id: { in: due.map(r => r.id) } }, - data: { nextRetryAt: new Date(now.getTime() + LEASE_MS) }, + data: { nextRetryAt: new Date(now.getTime() + LEASE_MS), claimToken, claimedAt: now }, }); - return { rows: due as WorkerRow[], shadowRetired, requeued }; + // The rows were SELECTed before the stamp, so hand back the token this + // pass just wrote rather than whatever they were carrying before. + return { + rows: due.map(row => ({ ...row, claimToken })) as WorkerRow[], + shadowRetired, + requeued, + shadowQuarantined, + }; }); } @@ -242,7 +290,10 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { const cutoff = new Date(Date.now() - STAGING_SWEEP_MINUTES * 60_000); const stale = await prisma.receiptIntake.findMany({ where: { state: "STAGING", createdAt: { lt: cutoff } }, - select: { id: true, storagePath: true, mimeType: true }, + select: { + id: true, storagePath: true, mimeType: true, + createdAt: true, expectedSha256: true, + }, // Small on purpose: each row costs a storage round trip, and the // sweep runs BEFORE any receipt is processed. A big batch here // spends the invocation on housekeeping. @@ -264,6 +315,18 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { const check = await inspectStoredObject(row.storagePath, row.mimeType); if (check.ok) { + // The bytes that landed must be the document /start was told + // about. Otherwise the sweep would publish whatever happened + // to be at that path — which is the same overwrite the seal + // exists to close, arriving by a different door. + if (row.expectedSha256 && row.expectedSha256 !== check.fileSha256) { + await prisma.receiptIntake.updateMany({ + where: { id: row.id, state: "STAGING" }, + data: { state: "NEEDS_REVIEW", stateReason: "sha-mismatch", nextRetryAt: null }, + }); + parked++; + continue; + } await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, data: { @@ -279,6 +342,12 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { } if (check.kind === "transient") continue; // unknown is not a verdict if (check.kind === "missing") { + // The signed upload URL is good for two hours. Parking at 15 + // minutes declared a receipt missing while its own upload + // link was still perfectly usable — a slow phone on a bad + // connection came back to find its row already in the review + // queue. Wait until the URL cannot possibly land any more. + if (row.createdAt.getTime() > Date.now() - SIGNED_UPLOAD_TTL_MS) continue; await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, @@ -298,13 +367,25 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { return published + parked + rejected; }, - loadPhases: async () => prisma.costCode.findMany({ - where: { isActive: true }, - select: { id: true, code: true, name: true }, - orderBy: { code: "asc" }, - }), + // ONLY this project's phases — no company-wide fallback, and an empty + // list is a real answer. + // + // Returning every active cost code meant the model was offered phases + // the job does not have, so it confidently suggested one, and booking + // then had to throw that suggestion away (isCostCodeAllowedForProject). + // The visible symptom was receipts arriving uncoded for no stated + // reason; the real cost is that a plausible-but-wrong phase is exactly + // the kind of thing a reviewer accepts without checking. + // + // A row with no project has no phases at all. Suggesting one from the + // whole company would be a guess with nothing behind it. + loadPhases: async projectId => { + if (!projectId) return []; + const phases = await resolveProjectPhaseCodes(prismaPhaseDataSource, projectId); + return phases.map(phase => ({ id: phase.id, code: phase.code, name: phase.name })); + }, - downloadBytes: (storagePath: string) => downloadDocBytesResult(toSecureRef(storagePath)), + downloadBytes: (storagePath, expectedSha256) => downloadVerified(storagePath, expectedSha256), read: (bytes, mime, phases) => readReceipt(bytes, mime, phases), @@ -369,11 +450,28 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { }, // RECEIVED -> READ, and the ONLY place the routing lease is released. - finishRouting: async (rowId, stateReason) => { - await prisma.receiptIntake.updateMany({ - where: { id: rowId, state: "RECEIVED" }, - data: { state: "READ", stateReason, nextRetryAt: null }, + finishRouting: async (rowId, claimToken, stateReason) => { + // FENCED on state AND token, and it clears both claim fields. + // + // The state alone is not enough: a worker whose invocation was + // killed mid-routing can resume after its row has been re-claimed + // and re-read, find it back in RECEIVED, and publish READ over the + // successor's work — including over a NEEDS_REVIEW the successor + // had every reason to set. Matching the token makes that write + // affect zero rows instead. + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: "RECEIVED", claimToken }, + data: { + state: "READ", + stateReason, + nextRetryAt: null, + claimToken: null, + claimedAt: null, + }, }); + if (count === 0) { + console.warn("[cron/receipt-intake-worker] finishRouting fenced out", rowId); + } }, companyTimeZone: resolveCompanyTimeZone, @@ -471,7 +569,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { getTokens: deadline => getFreshQBTokens(deadline), createPurchase: (tokens, input, deadline) => createQBReceiptPurchase(tokens, input, {}, deadline), - downloadBytes: downloadDocBytesResult, + downloadBytes: (storagePath, expectedSha256) => downloadVerified(storagePath, expectedSha256), logEvent: logAutomationEvent, now: () => new Date(), }), diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 23da3b180..ed587c7d8 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -2,8 +2,8 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; -import { inspectStoredObject } from "@/lib/receipt-intake/stored-object"; -import { deleteObjectOrRecord } from "@/lib/receipt-intake/storage-cleanup"; +import { canonicalStoragePath, inspectStoredObject } from "@/lib/receipt-intake/stored-object"; +import { deleteObjectOrRecord, sealObject } from "@/lib/receipt-intake/storage-cleanup"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -37,9 +37,9 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const row = await prisma.receiptIntake.findUnique({ where: { id }, select: { - id: true, state: true, sourceRef: true, storagePath: true, mimeType: true, - projectId: true, dryRun: true, createdById: true, fileSha256: true, - expectedSha256: true, + id: true, state: true, stateReason: true, sourceRef: true, storagePath: true, + mimeType: true, projectId: true, dryRun: true, createdById: true, + fileSha256: true, expectedSha256: true, }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); @@ -53,9 +53,17 @@ export async function POST(req: Request, context: { params: Promise<{ id: string STAFF_READ_ROLES.includes(auth.user.role); if (!maySee) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + // A LATE finalize on a row the sweeper already parked file-missing is a + // RECOVERY, not a duplicate: the upload landed after the sweep looked. It + // must re-validate and republish rather than report alreadyFinalized, which + // would leave a real receipt parked forever while telling the caller it was + // fine. + const recoverable = row.state === "STAGING" + || (row.state === "NEEDS_REVIEW" && row.stateReason === "file-missing"); + // Idempotent: finalizing an already-published row is a success, not an error // — the client's retry after a lost response must not look like a failure. - if (row.state !== "STAGING") { + if (!recoverable) { return NextResponse.json({ ok: true, alreadyFinalized: true, id: row.id, state: row.state, sourceRef: row.sourceRef, projectId: row.projectId, dryRun: row.dryRun, @@ -114,9 +122,28 @@ export async function POST(req: Request, context: { params: Promise<{ id: string } } + // SEAL. Copy the verified bytes to a content-addressed path the client was + // never handed a URL for, and point the row at that. Publishing while the + // row still referenced the upload path would leave the verified content + // replaceable by anyone holding the signed URL — the row asserting one sha + // while storage served different bytes. + const canonicalPath = canonicalStoragePath(id, fileSha256, mimeType); + const sealed = await sealObject(row.storagePath, canonicalPath, check.bytes, mimeType); + if (!sealed) { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + const published = await prisma.receiptIntake.updateMany({ - where: { id, state: "STAGING" }, - data: { state: "RECEIVED", mimeType, fileSize, fileSha256 }, + where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] } }, + data: { + state: "RECEIVED", + stateReason: null, + storagePath: sealed, + mimeType, + fileSize, + fileSha256, + nextRetryAt: null, + }, }); if (published.count === 0) { // Another finalize won the race and published it. Same outcome. diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index b79d4a219..56d9796ff 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -35,7 +35,7 @@ import { type QboReceiptGroup, } from "@/lib/qbo-receipt-push"; import type { AutomationEventInput } from "@/lib/automation-events"; -import type { DocBytesResult } from "@/lib/secure-storage"; +import type { VerifiedBytes } from "./stored-object"; import { backoffMs, MAX_BOOK_ATTEMPTS } from "./route-state"; /** The intake columns booking actually reads. Kept narrow so tests can build one by hand. */ @@ -59,6 +59,8 @@ export interface BookableRow { docType: string | null; refNumber: string | null; memo: string | null; + /** What finalize recorded; every download of this row is checked against it. */ + fileSha256: string; attempts: number; /** Carries a previous attachment failure across a retry — see below. */ lastError: string | null; @@ -188,7 +190,7 @@ export interface BookDependencies { * Reads the stored file back out of the private bucket. TAGGED, because a * confirmed 404 and a transient storage fault must not book the same way. */ - downloadBytes: (secureRef: string) => Promise; + downloadBytes: (storagePath: string, expectedSha256: string) => Promise; logEvent: (event: AutomationEventInput) => Promise; now: () => Date; } @@ -358,9 +360,13 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // A transient storage fault retries (the document is fine, Supabase was // not); an affirmative 404 is terminal and pre-send, so the strong key goes // back for a corrected re-upload. - const download = await deps.downloadBytes(toSecureRef(row.storagePath)); + const download = await deps.downloadBytes(row.storagePath, row.fileSha256); if (!download.ok) { - if (download.kind === "not-found") return parkedBeforeSend("receipt-bytes-missing"); + if (download.kind === "missing") return parkedBeforeSend("receipt-bytes-missing"); + // The attachment about to ride along with a real Purchase is NOT the + // document this row was verified as. Refuse — a Purchase carrying the + // wrong receipt is worse than one carrying none. + if (download.kind === "sha-mismatch") return parkedBeforeSend("content-changed"); return retry(row, deps, now, `storage:${download.message}`); } const bytes = download.bytes; @@ -377,6 +383,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro const blocker = attachmentBlocker(row.mimeType, bytes.length); if (blocker) return parkedBeforeSend(`unsupported-attachment:${blocker}`); + // Phase check ONE: immediately before the QBO create. The project can be + // reassigned while this row sits in the queue, and a stale phase should be + // caught before the books are touched, not only on the way to the Expense. + const phaseBeforeSend = await resolvePhase(row, project.id, deps); + // NOTE: a previous attachment failure deliberately does NOT short-circuit // here. createQBReceiptPurchase re-checks and re-uploads the file for an // EXISTING Purchase (ensureAttachmentOnExistingPurchase), so the retry is @@ -399,12 +410,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro fileContentType: row.mimeType, }; - // RECORDED BEFORE THE CALL, and persisted, because the question it answers - // is "might QuickBooks hold a Purchase for this row?" — and a process that - // dies mid-create must still answer yes. Setting it after would make the - // one case that matters look like a row that never sent. - await deps.markSendAttempted(row.id); - const sent = { attempted: true }; + const sent = { attempted: false }; let result: CreateQBReceiptPurchaseResult; try { @@ -415,6 +421,19 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // Last gate before the books are touched: the refresh may have consumed // what was left. if (outOfRunway()) return { outcome: "deferred", reason: "out-of-budget" }; + + // MARKED HERE — after the tokens and after the final budget check, and + // IMMEDIATELY before the create. + // + // Earlier was wrong in the direction that costs money to undo: a token + // refresh that threw, or a budget check that deferred, would have left + // sendAttempted=true on a row that never reached QuickBooks, and its + // strong key would then be held forever against a Purchase that does + // not exist. Persisted rather than in-memory, because the case the flag + // exists for is the process dying mid-create. + await deps.markSendAttempted(row.id); + sent.attempted = true; + result = await deps.createPurchase(tokens, input, deps.deadline); } catch (error) { const terminal = terminalReasonFor(error); @@ -484,7 +503,18 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // NOT a failure: the receipt is fine and its total is right, so it books // UNCODED and says why. A bookkeeper assigning a phase is routine; an // expense silently attached to the wrong one is not. + // Phase check TWO: immediately before the Expense write, INSIDE the same + // window as the row's own commit. The create above is a network round trip + // that can take seconds, and the answer that matters is the one true when + // the money is recorded — a project reassignment that lands in between must + // not be written into job cost. const phaseCheck = await resolvePhase(row, project.id, deps); + if (phaseCheck.costCodeId !== phaseBeforeSend.costCodeId) { + console.warn( + "[receipt-intake] phase changed across the QBO create", + JSON.stringify({ rowId: row.id, before: phaseBeforeSend.costCodeId, after: phaseCheck.costCodeId }), + ); + } const costCodeId = phaseCheck.costCodeId; const driveFileId = driveFileIdOf(row); const receiptUrl = driveFileId diff --git a/src/lib/receipt-intake/route-state.ts b/src/lib/receipt-intake/route-state.ts index 4c7b6fabf..dbe87e7c5 100644 --- a/src/lib/receipt-intake/route-state.ts +++ b/src/lib/receipt-intake/route-state.ts @@ -16,6 +16,17 @@ export const RECEIPT_INTAKE_STATES = [ * docs/plans/PHASE-1-INTAKE-CORE-SPEC.md §7. */ "SHADOW_DONE", + /** + * Terminal, and it needs a person. + * + * Pre-boundary, no evidence v1 booked it, and NOT a Drive row — so there is + * no shared identity that would make a v2 booking idempotent against a + * Purchase v1 may or may not have created. Booking it risks a duplicate; + * retiring it risks losing a real expense. Neither is ours to guess, so it + * surfaces on the Receipts tab with a "book anyway" action for whoever has + * checked QuickBooks. NEVER auto-requeued. + */ + "SHADOW_QUARANTINE", ] as const; export type ReceiptIntakeState = (typeof RECEIPT_INTAKE_STATES)[number]; diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index 88ceb4dbd..e2c543f12 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -14,10 +14,55 @@ */ import { logAutomationEvent } from "@/lib/automation-events"; import { prisma } from "@/lib/prisma"; -import { removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; +import { SECURE_BUCKET, removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; +import { getSupabase } from "@/lib/supabase"; export const STORAGE_CLEANUP_KIND = "storage-cleanup-pending"; +/** + * Copy verified bytes to their canonical path and drop the upload path. + * + * The upload path stays writable by whoever holds the signed URL (which is + * `upsert: true`, deliberately, so a resumed /start can replace its own partial + * upload). Leaving the row pointed at it means the bytes we verified can be + * replaced afterwards by anyone who kept the URL — the row would still claim + * the old sha while storage held something else. + * + * Returns null when the copy fails, so the caller can refuse rather than + * publish a row pointing at a path that may not exist. + */ +export async function sealObject( + uploadPath: string, + canonicalPath: string, + bytes: Buffer, + contentType: string, +): Promise { + const supabase = getSupabase(); + if (!supabase) return null; + try { + // upsert: the canonical path is content-addressed, so a re-seal of the + // SAME bytes is a no-op by construction and must not fail. + const { error } = await supabase.storage + .from(SECURE_BUCKET) + .upload(canonicalPath, bytes, { contentType, upsert: true }); + if (error) { + console.error("[receipts/intake] seal failed", error.message); + return null; + } + } catch (error) { + console.error("[receipts/intake] seal threw", error instanceof Error ? error.name : "error"); + return null; + } + + // The upload path has served its purpose. A failure to remove it is an + // orphan, not a correctness problem — the row already points at the sealed + // copy — so it goes on the cleanup queue rather than failing the publish. + if (uploadPath !== canonicalPath) { + await deleteObjectOrRecord(uploadPath, "sealed"); + } + return canonicalPath; +} + /** * Delete the object. If that fails, record the path so the sweep can retry. * Never throws: the caller is already rejecting a row and must not be derailed diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index f6d45d81b..a979dc1d2 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -14,12 +14,65 @@ */ import { createHash } from "node:crypto"; import { downloadDocBytesResult, toSecureRef, type DocBytesResult } from "@/lib/secure-storage"; -import { sniffMime } from "./file-type"; +import { EXT_BY_MIME, sniffMime } from "./file-type"; import { MAX_STORED_BYTES } from "./intake-core"; +/** + * Where a VERIFIED object lives, keyed by its own content hash. + * + * The upload path is writable by whoever holds the signed URL, and that URL is + * `upsert: true` so a resumed /start can replace its own partial upload. Both + * are necessary and together they mean the upload path can change AFTER we + * verified it. Sealing copies the bytes somewhere the client was never given a + * URL for, and names it after the sha — so the path itself asserts the content, + * and re-verifying on download is a comparison against a value that cannot have + * been rewritten in place. + */ +export function canonicalStoragePath(id: string, sha256: string, mimeType: string): string { + const ext = EXT_BY_MIME[mimeType] ?? "bin"; + return `receipts/${id}/${sha256}.${ext}`; +} + +/** + * Read bytes and REFUSE them if they are not what the row recorded. + * + * Every consumer of a stored receipt (the reader, the booker) goes through + * this. A hash stored at finalize is worthless if nothing ever checks it again. + */ +export type VerifiedBytes = + | { ok: true; bytes: Buffer } + | { ok: false; kind: "missing" | "transient" | "sha-mismatch"; message?: string }; + +export async function downloadVerified( + storagePath: string, + expectedSha256: string, + download: (ref: string) => Promise = downloadDocBytesResult, +): Promise { + const result = await download(toSecureRef(storagePath)); + if (!result.ok) { + return result.kind === "not-found" + ? { ok: false, kind: "missing" } + : { ok: false, kind: "transient", message: result.message }; + } + // An empty expectation means a legacy row written before sealing existed; + // there is nothing to compare against, so pass the bytes through rather + // than refuse a receipt for a reason that is our fault. + if (!expectedSha256) return { ok: true, bytes: result.bytes }; + + const actual = createHash("sha256").update(result.bytes).digest("hex"); + if (actual !== expectedSha256) { + return { ok: false, kind: "sha-mismatch", message: `expected ${expectedSha256}, stored ${actual}` }; + } + return { ok: true, bytes: result.bytes }; +} + export type StoredObjectCheck = - /** Valid: these are the values the row must be published with. */ - | { ok: true; mimeType: string; fileSize: number; fileSha256: string } + /** + * Valid: these are the values the row must be published with, plus the + * exact bytes that produced them — so the sealed copy is provably the + * content that was verified, not a second download that could differ. + */ + | { ok: true; mimeType: string; fileSize: number; fileSha256: string; bytes: Buffer } /** The object is not there. Terminal for the sweep; retryable for a client. */ | { ok: false; kind: "missing" } /** Storage could not answer. Never a verdict — come back later. */ @@ -62,5 +115,6 @@ export async function inspectStoredObject( mimeType, fileSize: bytes.length, fileSha256: createHash("sha256").update(bytes).digest("hex"), + bytes, }; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 877f22bc7..2165ee6fb 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -31,7 +31,7 @@ import { QboVendorDuplicateError, } from "@/lib/qbo-receipt-push"; import type { ProjectPhase, ReadOutcome } from "./read"; -import type { DocBytesResult } from "@/lib/secure-storage"; +import type { VerifiedBytes } from "./stored-object"; /** * ONE global constant, deliberately not derived from anything per-row or @@ -67,6 +67,13 @@ export const RUN_HARD_BUDGET_MS = 55_000; export const STAGING_SWEEP_MINUTES = 15; /** Storage round trips per sweep. Small: the sweep runs before any real work. */ export const STAGING_SWEEP_BATCH = 10; +/** + * Supabase signed upload URLs are valid for two hours. A STAGING row younger + * than that may still have its bytes arrive, so declaring it file-missing at + * the 15-minute sweep window was premature — the row went to review while its + * own upload link was still usable. + */ +export const SIGNED_UPLOAD_TTL_MS = 2 * 60 * 60_000; /** * Consecutive AI-unavailable passes before a row is parked for a human. Ported * from v3.4: an outage that never ends still has to end somewhere, and 20 @@ -77,6 +84,10 @@ export const MAX_BUSY_PASSES = 20; /** The columns a pass needs. A superset of BookableRow. */ export interface WorkerRow extends BookableRow { state: string; + /** What finalize recorded. Every download is checked against it. */ + fileSha256: string; + /** The token this pass claimed the row with. Completing writes are fenced on it. */ + claimToken: string | null; fileSize: number; readAt: Date | null; dedupWeakKey: string | null; @@ -116,8 +127,12 @@ export interface WorkerDependencies { /** Retry storage deletes that failed when a row was rejected. */ retryStorageCleanups: (shouldStop: () => boolean) => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; - /** Tagged: a confirmed 404 and a transient storage fault are NOT the same answer. */ - downloadBytes: (storagePath: string) => Promise; + /** + * Tagged, and VERIFIED: the bytes must hash to what the row recorded at + * finalize. A sha stored once and never re-checked proves nothing about + * what is being read now. + */ + downloadBytes: (storagePath: string, expectedSha256: string) => Promise; read: (bytes: Buffer, mime: string, phases: ProjectPhase[]) => Promise; /** * Persist the read + routing. Returns the strong-key owner when the partial @@ -141,11 +156,17 @@ export interface WorkerDependencies { /** A transient fault anywhere else: spend an attempt and back off. */ retryRow: (rowId: string, attempts: number, nextRetryAt: Date, reason: string) => Promise; /** - * RECEIVED -> READ, and the release of the claim lease. Called ONCE, after - * every dedup net has answered — never before, or an overlapping run could - * reclaim a half-routed row and book it. + * RECEIVED -> READ, the release of the claim lease, AND the release of the + * claim token. Called ONCE, after every dedup net has answered — never + * before, or an overlapping run could reclaim a half-routed row and book it. + * + * FENCED on both the state and the token: a worker whose invocation was + * killed and whose row has since been re-claimed must not be able to + * publish READ over whatever its successor produced. Time-based leases + * cannot express that, because the zombie and the live worker hold + * identical row ids. */ - finishRouting: (rowId: string, stateReason: string | null) => Promise; + finishRouting: (rowId: string, claimToken: string | null, stateReason: string | null) => Promise; now: () => Date; /** Elapsed-time source for the soft deadline. */ monotonicMs: () => number; @@ -188,6 +209,8 @@ export interface WorkerRunSummary { shadowRetired?: number; /** Rows received AFTER v1 stopped: nobody booked these, so they are handed to v2. */ requeued?: number; + /** Held for a human: no v1 evidence AND no Drive identity to make v2 idempotent. */ + shadowQuarantined?: number; /** The cutover could not run because no boundary is recorded. */ cutoverBlocked?: "cutover-boundary-missing"; /** STAGING rows whose upload never landed, parked for a human. */ @@ -293,6 +316,8 @@ export interface ClaimResult { rows: WorkerRow[]; shadowRetired: number; requeued: number; + /** Pre-boundary, no evidence, and no Drive identity — a human decides. */ + shadowQuarantined: number; } export async function runIntakeWorker(deps: WorkerDependencies): Promise { @@ -337,7 +362,7 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise { - const download = await deps.downloadBytes(row.storagePath); + const download = await deps.downloadBytes(row.storagePath, row.fileSha256); if (!download.ok) { // "The object is gone" and "storage was briefly unreachable" demand // opposite answers, and collapsing them to null meant a Supabase blip // parked good receipts as file-missing, permanently, for a human to // untangle. Only an AFFIRMATIVE not-found is terminal. - if (download.kind === "not-found") { + if (download.kind === "missing") { await deps.applyState(row.id, "NEEDS_REVIEW", "file-missing"); return "NEEDS_REVIEW"; } + // The stored bytes are not the ones this row was published with. + // Terminal, and loud: it means the object was replaced after + // verification, which is the exact thing sealing exists to prevent. + if (download.kind === "sha-mismatch") { + await deps.applyState(row.id, "NEEDS_REVIEW", "content-changed"); + return "NEEDS_REVIEW"; + } return retryTransient(row, deps, `storage:${download.message}`); } const bytes = download.bytes; @@ -671,7 +704,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // Routing is complete. This is the ONLY path to READ, and the only place // the claim lease is released. - await deps.finishRouting(row.id, note(null)); + await deps.finishRouting(row.id, row.claimToken, note(null)); return "READ"; } diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 1f67e4884..7fe01dfd5 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -53,6 +53,7 @@ function row(overrides: Partial = {}): BookableRow { attempts: 0, lastError: null, sendAttempted: false, + fileSha256: "s".repeat(64), ...overrides, }; } @@ -474,7 +475,7 @@ test("a MISSING receipt file refuses the booking outright", async () => { // attachment — the one failure the bookkeeper cannot fix later, because the // Purchase looks complete and nothing flags it. The receipt IS the evidence // for the expense. - const r = recorder({ downloadBytes: async () => ({ ok: false, kind: "not-found" }) }); + const r = recorder({ downloadBytes: async () => ({ ok: false, kind: "missing" }) }); const result = await bookReceipt(row(), r.deps); assert.deepEqual(result, { outcome: "needs-review", @@ -614,10 +615,30 @@ test("the SUGGESTED code is checked against the final project too", async () => }, }); await bookReceipt(row({ costCodeId: null }), r.deps); - assert.deepEqual(asked, [["proj-1", "cc-plumb"]], "asked about the FINAL project"); + // TWICE: once immediately before the QBO create, once immediately before + // the Expense write. The create is a network round trip, and a project + // reassignment that lands in between must not reach job cost. + assert.deepEqual(asked, [["proj-1", "cc-plumb"], ["proj-1", "cc-plumb"]]); assert.equal(r.expenses[0].costCodeId, null); }); +test("the phase is re-checked AFTER the create, not just before it", async () => { + // The window that matters is the one around the money write. This proves + // the second check is real: the same row, answered differently the second + // time, must produce the LATER answer. + let call = 0; + const r = recorder({ + isCostCodeAllowed: async () => { + call++; + return call === 1; // allowed before the send, revoked after it + }, + }); + await bookReceipt(row({ costCodeId: "cc-demo" }), r.deps); + assert.equal(call, 2, "asked on both sides of the create"); + assert.equal(r.expenses[0].costCodeId, null, "the post-create answer wins"); + assert.match(r.expenses[0].description, /phase cleared/); +}); + test("unassigned during READ, assigned before BOOKING: the phase is re-checked", async () => { // End to end for the exact sequence the gate named. const allowedByProject: Record = { @@ -658,3 +679,52 @@ test("a human's explicit pick is not labelled a suggestion", async () => { assert.ok(!/phase suggested/.test(r.expenses[0].description)); assert.equal(r.events[0].detail.suggestedConfidence, undefined); }); + +// ── sendAttempted is marked at the LAST possible moment (round-8 item 4) ──── + +test("a token failure leaves sendAttempted UNSET, so the key is released", async () => { + // Marking before the token refresh meant a refresh that threw left + // sendAttempted=true on a row that never reached QuickBooks — and its + // strong key was then held forever against a Purchase that does not exist. + const r = recorder({ + getTokens: async () => { throw new Error("QBNotConnectedError"); }, + }); + const result = await bookReceipt(row({ attempts: 19 }), r.deps); + assert.deepEqual(r.sendMarks, [], "never marked — nothing was sent"); + assert.equal(result.outcome, "needs-review"); + assert.equal((result as any).reason, "max-retries"); + assert.equal((result as any).releaseStrongKey, true, "so the key goes back"); +}); + +test("budget exhausted AFTER the token refresh also leaves it unset", async () => { + // The final runway check sits between the tokens and the create. A deferral + // there must not look like an attempted send. + const deadline = { startedAt: Date.now(), budgetMs: 55_000 }; + const r = recorder({ + deadline, + getTokens: async () => { + // Burn the remaining budget during the refresh. + deadline.startedAt = Date.now() - 60_000; + return { accessToken: "t", realmId: "r" } as any; + }, + }); + const result = await bookReceipt(row(), r.deps); + assert.deepEqual(result, { outcome: "deferred", reason: "out-of-budget" }); + assert.deepEqual(r.sendMarks, [], "never marked"); + assert.equal(r.purchaseCalls.length, 0); +}); + +test("a real create DOES mark it, before the call", async () => { + const order: string[] = []; + const r = recorder({ + createPurchase: async (_t, input) => { + order.push("create"); + r.purchaseCalls.push(input); + return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "attached" } as any; + }, + markSendAttempted: async id => { order.push("mark"); r.sendMarks.push(id); }, + }); + await bookReceipt(row(), r.deps); + assert.deepEqual(order, ["mark", "create"], "marked FIRST, so a mid-create death still records it"); + assert.deepEqual(r.sendMarks, ["intake-1"]); +}); diff --git a/tests/receipt-intake-claim-db.test.ts b/tests/receipt-intake-claim-db.test.ts index 8fd6bb351..55b36b3e2 100644 --- a/tests/receipt-intake-claim-db.test.ts +++ b/tests/receipt-intake-claim-db.test.ts @@ -131,3 +131,56 @@ test.after(async () => { await db.$disconnect(); } }); + +test("the finishRouting adapter is fenced on BOTH state and token", { skip }, async () => { + // The production adapter, against a real database — the mocked worker + // suites cannot see this, and the failure it prevents is a zombie worker + // publishing READ over the state its successor already produced. + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); + + const finishRouting = async (rowId: string, claimToken: string | null, stateReason: string | null) => { + const { count } = await db!.receiptIntake.updateMany({ + where: { id: rowId, state: "RECEIVED", claimToken }, + data: { state: "READ", stateReason, nextRetryAt: null, claimToken: null, claimedAt: null }, + }); + return count; + }; + + // The happy path: the holder of the current token publishes and BOTH claim + // fields are cleared. + await seed("claimdb-fence", { claimToken: "token-1", claimedAt: new Date(), nextRetryAt: new Date() }); + assert.equal(await finishRouting("claimdb-fence", "token-1", null), 1); + const published = await db!.receiptIntake.findUnique({ where: { id: "claimdb-fence" } }); + assert.equal(published?.state, "READ"); + assert.equal(published?.claimToken, null, "the token is released"); + assert.equal(published?.claimedAt, null, "and so is claimedAt"); + assert.equal(published?.nextRetryAt, null, "and the lease"); + + // The zombie: a stale token writes NOTHING, even though the row is back in + // RECEIVED and looks claimable to a state-only check. + await db!.receiptIntake.update({ + where: { id: "claimdb-fence" }, + data: { state: "RECEIVED", claimToken: "token-2", stateReason: null }, + }); + assert.equal(await finishRouting("claimdb-fence", "token-1", "zombie"), 0, "stale token is fenced out"); + const afterZombie = await db!.receiptIntake.findUnique({ where: { id: "claimdb-fence" } }); + assert.equal(afterZombie?.state, "RECEIVED", "the successor's state survives"); + assert.equal(afterZombie?.claimToken, "token-2", "and its claim is untouched"); + + // The state fence still holds independently: right token, wrong state. + await db!.receiptIntake.update({ + where: { id: "claimdb-fence" }, + data: { state: "NEEDS_REVIEW", claimToken: "token-3" }, + }); + assert.equal(await finishRouting("claimdb-fence", "token-3", null), 0, "a routed row is not re-published"); + + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); +}); + +test("SHADOW_QUARANTINE is writable — the CHECK allows it", { skip }, async () => { + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); + await seed("claimdb-quar", { state: "SHADOW_QUARANTINE", stateReason: "no-v1-evidence" }); + const row = await db!.receiptIntake.findUnique({ where: { id: "claimdb-quar" } }); + assert.equal(row?.state, "SHADOW_QUARANTINE"); + await db!.receiptIntake.deleteMany({ where: { sourceRef: { startsWith: PREFIX } } }); +}); diff --git a/tests/receipt-intake-phases.test.ts b/tests/receipt-intake-phases.test.ts new file mode 100644 index 000000000..f4963276a --- /dev/null +++ b/tests/receipt-intake-phases.test.ts @@ -0,0 +1,68 @@ +/** + * loadPhases must offer the model ONLY the phases the job actually has. + * + * The regression: it returned every active cost code company-wide, so the model + * was shown phases the project does not have, confidently suggested one, and + * booking then threw that suggestion away (isCostCodeAllowedForProject). The + * visible symptom was receipts arriving uncoded for no stated reason. The real + * cost is subtler: a plausible-but-wrong phase is exactly the kind of thing a + * reviewer accepts without checking. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolveProjectPhaseCodes, type PhaseDataSource } from "../src/lib/project-phases"; + +function source(over: Partial = {}): PhaseDataSource { + return { + getProject: async () => ({ id: "p1", status: "In Progress" }), + getEstimateCostCodes: async () => [ + { id: "cc-demo", code: "01-DEMO", name: "Demolition", isActive: true }, + ], + getSafetyCostCode: async () => null, + ...over, + } as PhaseDataSource; +} + +/** Mirrors the worker's adapter exactly. */ +async function loadPhases(projectId: string | null, ds: PhaseDataSource) { + if (!projectId) return []; + const phases = await resolveProjectPhaseCodes(ds, projectId); + return phases.map(p => ({ id: p.id, code: p.code, name: p.name })); +} + +test("a known project returns only ITS phase-eligible codes", async () => { + const phases = await loadPhases("p1", source()); + assert.deepEqual(phases.map(p => p.code), ["01-DEMO"]); +}); + +test("a project with NO eligible phases returns an empty list, not a fallback", async () => { + // This is the case the old code papered over. An empty list is a real + // answer: nothing on this job is a valid phase, so the model must suggest + // nothing rather than reach for a company-wide code that booking will + // discard. + const phases = await loadPhases("p1", source({ getEstimateCostCodes: async () => [] })); + assert.deepEqual(phases, []); +}); + +test("an unknown project returns empty rather than everything", async () => { + const phases = await loadPhases("nope", source({ getProject: async () => null })); + assert.deepEqual(phases, []); +}); + +test("a row with no project gets no phases at all", async () => { + // Suggesting one from the whole company would be a guess with nothing + // behind it — and NEEDS_JOB rows are exactly the ones a human is about to + // assign, so a stale suggestion is worse than none. + let called = false; + await loadPhases(null, source({ getProject: async () => { called = true; return null; } })); + assert.equal(called, false, "the resolver is not even consulted"); +}); + +test("the Safety phase is included when the project status allows it", async () => { + // Proof this really is the shared resolver and not a reimplementation: + // Safety is a phase no estimate lists, and only the resolver knows to add it. + const phases = await loadPhases("p1", source({ + getSafetyCostCode: async () => ({ id: "cc-safety", code: "00-SAFETY", name: "Safety Meeting", isActive: true }), + })); + assert.ok(phases.some(p => p.code === "00-SAFETY")); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 3fbf87897..57f022e79 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -8,7 +8,11 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { inspectStoredObject } from "../src/lib/receipt-intake/stored-object"; +import { + canonicalStoragePath, + downloadVerified, + inspectStoredObject, +} from "../src/lib/receipt-intake/stored-object"; import { MAX_STORED_BYTES } from "../src/lib/receipt-intake/intake-core"; import type { DocBytesResult } from "../src/lib/secure-storage"; @@ -74,3 +78,60 @@ test("missing and transient are DIFFERENT answers", async () => { })); assert.deepEqual(flaky, { ok: false, kind: "transient", message: "ECONNRESET" }); }); + +// ── Sealing and re-verification (round-8 item 2) ─────────────────────────── + +test("the canonical path is content-addressed and per-row", () => { + // The client is never given a URL for this path, and its NAME asserts the + // content — so a later comparison is against a value that cannot have been + // rewritten in place. + const sha = createHash("sha256").update(PNG).digest("hex"); + assert.equal(canonicalStoragePath("row-1", sha, "image/png"), `receipts/row-1/${sha}.png`); + assert.equal(canonicalStoragePath("row-1", sha, "application/pdf"), `receipts/row-1/${sha}.pdf`); + // Two rows with identical bytes still get separate objects — deleting one + // receipt must never remove another's evidence. + assert.notEqual(canonicalStoragePath("row-1", sha, "image/png"), canonicalStoragePath("row-2", sha, "image/png")); +}); + +test("a download whose bytes do not match the recorded sha is REFUSED", async () => { + // THE OVERWRITE ATTACK. The upload path is writable by whoever holds the + // signed URL (upsert, deliberately). If the row still pointed there, the + // verified content could be swapped afterwards and the row would keep + // asserting the old sha while storage served something else. + const realSha = createHash("sha256").update(PNG).digest("hex"); + const swapped = Buffer.from("totally different bytes"); + + const good = await downloadVerified("p.png", realSha, give({ ok: true, bytes: PNG })); + assert.deepEqual(good, { ok: true, bytes: PNG }); + + const attacked = await downloadVerified("p.png", realSha, give({ ok: true, bytes: swapped })); + assert.equal(attacked.ok, false); + assert.equal((attacked as { kind: string }).kind, "sha-mismatch"); +}); + +test("missing and transient stay distinguishable through verification", async () => { + assert.deepEqual( + await downloadVerified("p.png", "x".repeat(64), give({ ok: false, kind: "not-found" })), + { ok: false, kind: "missing" }, + ); + const flaky = await downloadVerified("p.png", "x".repeat(64), give({ + ok: false, kind: "transient", message: "ECONNRESET", + })); + assert.equal((flaky as { kind: string }).kind, "transient"); +}); + +test("a legacy row with no recorded sha is passed through, not refused", async () => { + // Rows written before sealing existed have nothing to compare against. + // Refusing them would park real receipts for a reason that is our fault. + const legacy = await downloadVerified("p.png", "", give({ ok: true, bytes: PNG })); + assert.deepEqual(legacy, { ok: true, bytes: PNG }); +}); + +test("the validator hands back the exact bytes it verified", async () => { + // The sealer copies THESE bytes rather than re-downloading, so the sealed + // object is provably the content that passed validation. + const check = await inspectStoredObject("p.png", "image/png", give({ ok: true, bytes: PNG })); + assert.ok(check.ok); + assert.ok(check.bytes.equals(PNG)); + assert.equal(createHash("sha256").update(check.bytes).digest("hex"), check.fileSha256); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 21e00dec6..23e031b8d 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -64,6 +64,8 @@ function workerRow(overrides: Partial = {}): WorkerRow { lastError: null, suggestedConfidence: null, sendAttempted: false, + claimToken: "claim-1", + fileSha256: "s".repeat(64), ...overrides, }; } @@ -92,7 +94,7 @@ interface Harness { applied: ReadPatch[]; states: { id: string; state: string; reason: string | null; patch?: Partial }[]; promoted: string[]; - finished: { id: string; stateReason: string | null }[]; + finished: { id: string; claimToken: string | null; stateReason: string | null }[]; deferred: { id: string; busyPasses: number }[]; retried: { id: string; attempts: number; reason: string }[]; claimOpts: CutoverRequest[]; @@ -113,7 +115,7 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) h.deps = { claim: async opts => { h.claimOpts.push(opts); - return { rows, shadowRetired: 0, requeued: 0, boundaryMissing: false }; + return { rows, shadowRetired: 0, requeued: 0, shadowQuarantined: 0 }; }, cutoverBoundary: async () => h.boundary, isDryRunEnabled: () => true, @@ -125,7 +127,9 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, findWeakHit: async () => null, applyState: async (id, state, reason, patch) => { h.states.push({ id, state, reason, patch }); }, - finishRouting: async (id, stateReason) => { h.finished.push({ id, stateReason }); }, + finishRouting: async (id, claimToken, stateReason) => { + h.finished.push({ id, claimToken, stateReason }); + }, companyTimeZone: async () => "America/Los_Angeles", promoteToBooking: async id => { h.promoted.push(id); return { promoted: true }; }, book: async () => { @@ -152,7 +156,7 @@ test("DRY RUN: a received row is read, deduped and routed — and never booked", // The claim leaves the row RECEIVED and holding its lease; finishRouting is // the only thing that publishes READ, after every dedup net has answered. assert.equal(h.applied[0].state, "RECEIVED"); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: null }]); assert.equal(h.applied[0].vendor, "Lowes"); assert.equal(h.applied[0].totalCents, 36498); assert.equal(h.applied[0].taxCents, 2920); @@ -245,7 +249,7 @@ test("a document the model answered on but could not read goes to a human", asyn test("a missing storage object is terminal, not an infinite read loop", async () => { const h = harness([workerRow()], { - downloadBytes: async () => ({ ok: false as const, kind: "not-found" as const }), + downloadBytes: async () => ({ ok: false as const, kind: "missing" as const }), }); await runIntakeWorker(h.deps); assert.equal(h.states[0].reason, "file-missing"); @@ -322,7 +326,7 @@ test("CUTOVER: the boundary is passed to the claim so the backlog can be split", claim: async opts => { h.claimOpts.push(opts); // Rows BEFORE the boundary were booked by v1; rows after it by nobody. - return { rows: [], shadowRetired: 7, requeued: 2, boundaryMissing: false }; + return { rows: [], shadowRetired: 7, requeued: 2, shadowQuarantined: 0 }; }, }); const summary = await runIntakeWorker(h.deps); @@ -343,7 +347,7 @@ test("CUTOVER refuses entirely when no boundary is recorded", async () => { claim: async opts => { h.claimOpts.push(opts); assert.equal(opts.boundary, null); - return { rows: [], shadowRetired: 0, requeued: 0, boundaryMissing: true }; + return { rows: [], shadowRetired: 0, requeued: 0, shadowQuarantined: 0 }; }, }); const summary = await runIntakeWorker(h.deps); @@ -586,7 +590,7 @@ test("the strong claim is attempted with the key, before any weak lookup", async // The claim writes the KEYS but leaves the row RECEIVED and holding its // lease. READ is reached only by finishRouting, once every net has spoken. assert.equal(h.applied[0].state, "RECEIVED"); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: null }]); }); test("the lease is held through routing and released only at the end", async () => { @@ -691,7 +695,7 @@ test("a plausible tax is stored and the row carries no note", async () => { await runIntakeWorker(h.deps); assert.equal(h.applied[0].taxCents, 2920, "29.20 of 364.98 is ~8%"); assert.equal(h.applied[0].stateReason, null); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: null }]); }); test("an implausible tax nulls taxCents, notes the row, and does NOT park it", async () => { @@ -705,7 +709,7 @@ test("an implausible tax nulls taxCents, notes the row, and does NOT park it", a assert.equal(h.applied[0].taxCents, null, "the bad reading is dropped, not booked"); assert.equal(h.applied[0].totalCents, 36498, "the total is untouched"); assert.deepEqual(summary.byState, { READ: 1 }, "READ, not NEEDS_REVIEW"); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: "tax-implausible" }]); }); test("the tax note survives alongside a dedup reason", async () => { @@ -730,7 +734,7 @@ test("the row stores only the tax BOOKING accepted, never a rejected reading", a await runIntakeWorker(h.deps); // buildGroups refuses to split tax on a check, so nothing was accepted. assert.equal(h.applied[0].taxCents, null); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: "tax-implausible" }]); }); test("a check with no tax reading books clean, with no note", async () => { @@ -742,7 +746,7 @@ test("a check with no tax reading books clean, with no note", async () => { }); await runIntakeWorker(h.deps); assert.equal(h.applied[0].taxCents, null); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: null }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: null }]); }); test("a tax equal to the total is refused end to end", async () => { @@ -752,7 +756,7 @@ test("a tax equal to the total is refused end to end", async () => { await runIntakeWorker(h.deps); assert.equal(h.applied[0].taxCents, null); assert.equal(h.applied[0].totalCents, 36498, "the total is untouched"); - assert.deepEqual(h.finished, [{ id: "row-1", stateReason: "tax-implausible" }]); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: "tax-implausible" }]); }); // ── Fail-closed classifier (round-5 item 4) ──────────────────────────────── @@ -909,3 +913,59 @@ test("a row that DID send keeps its key at the retry limit", async () => { assert.equal(h.states[0].reason, "max-retries"); assert.equal(h.states[0].patch, undefined, "no patch, so the key is untouched"); }); + +// ── Content changed under us (round-8 item 2) ────────────────────────────── + +test("a read whose bytes no longer match the recorded sha is TERMINAL", async () => { + // Sealing makes this nearly impossible; the check exists because "nearly" + // is not a guarantee, and reading whatever happens to be at a path is how a + // receipt for one job ends up booked against another. + const h = harness([workerRow()], { + downloadBytes: async () => ({ ok: false as const, kind: "sha-mismatch" as const, message: "x" }), + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); + assert.equal(h.states[0].reason, "content-changed"); + assert.equal(h.reads, 0, "the model never sees bytes we cannot vouch for"); +}); + +test("the recorded sha is what the download is checked against", async () => { + const asked: Array<[string, string]> = []; + const h = harness([workerRow({ fileSha256: "abc".padEnd(64, "0") })], { + downloadBytes: async (p, sha) => { + asked.push([p, sha]); + return { ok: true as const, bytes: Buffer.from("bytes") }; + }, + }); + await runIntakeWorker(h.deps); + assert.deepEqual(asked, [["receipts/intake/row-1.jpg", "abc".padEnd(64, "0")]]); +}); + +// ── SHADOW_QUARANTINE (round-8 item 1) ───────────────────────────────────── + +test("the cutover reports quarantined rows separately from retired and requeued", async () => { + // Three outcomes, because "we cannot tell" is a real answer and collapsing + // it into either of the other two either double-books or loses an expense. + const h = harness([], { + isDryRunEnabled: () => false, + claim: async opts => { + h.claimOpts.push(opts); + return { rows: [], shadowRetired: 4, requeued: 2, shadowQuarantined: 3 }; + }, + }); + const summary = await runIntakeWorker(h.deps); + assert.equal(summary.shadowRetired, 4); + assert.equal(summary.requeued, 2); + assert.equal(summary.shadowQuarantined, 3); +}); + +// ── The claim token fences the completing write (Phase 2 gate, a) ────────── + +test("finishRouting is handed the token the pass claimed with", async () => { + // A zombie worker resuming after its row was re-claimed must write nothing. + // The adapter matches on this token; the worker's job is to pass the one it + // actually holds. + const h = harness([workerRow({ claimToken: "token-abc" })]); + await runIntakeWorker(h.deps); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "token-abc", stateReason: null }]); +}); From d6e709befd73904273f2f477d99d83d0b4ce0f4a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 02:44:57 -0700 Subject: [PATCH 046/144] fix(receipts,proxy): fail-closed cron gate; machine endpoints refuse action dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 gate, two items in Phase 1 code. (a) The intake worker cron route had both problems lib/cron-auth exists to fix. `authHeader === \`Bearer ${secret}\`` is a byte-at-a-time compare that leaks the secret to anyone who can time the response. Worse, the local-dev escape hatch — no VERCEL, NODE_ENV not "production", and no CRON_SECRET — is satisfied by an UNSET environment, so any container, self-hosted build, or preview whose env drifted served this endpoint to anyone who asked. On a route that books real money into QuickBooks. Now isCronAuthorized(): constant time, required everywhere except an explicit NODE_ENV === "development", and a missing secret rejects. (b) Machine endpoints now refuse a Server Action dispatch with 403. One correction to the gate's diagnosis, because it changes the fix: the Next matcher does NOT exclude these paths from the proxy. Its FIRST entry is `{ source: "/:path*", has: [{ key: "next-action" }] }`, which routes every next-action request through regardless of path — verified by driving the proxy directly. They were reaching the code; PUBLIC_PROXY_BYPASS_PATTERN was returning next() before any action check ran. So the fix is widening MACHINE_ENDPOINT_PATTERN (cron, integrations, webhooks, twilio, health/pipeline) and the matcher is deliberately left alone: adding those paths to it would put middleware — including a DB-touching staff lookup — in front of every ordinary webhook and cron request, which is real latency and a new failure point on a hot path, for no additional protection. api/portal, api/payments, api/sub-portal and api/selections are excluded on purpose: they genuinely serve anonymous Server Actions, and 403ing them would break the client portal. Asserted in the test alongside the 403s, and the 403 assertion is mutation-verified. Co-Authored-By: Claude Fable 5.1 --- .../api/cron/receipt-intake-worker/route.ts | 23 ++-- src/proxy.ts | 33 +++++- tests/receipt-intake-auth.test.ts | 110 ++++++++++++++++++ 3 files changed, 156 insertions(+), 10 deletions(-) diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index e46baa44d..0947dccc8 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { NextResponse } from "next/server"; +import { isCronAuthorized } from "@/lib/cron-auth"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; @@ -53,9 +54,9 @@ export const maxDuration = 60; * - QBO's DocNumber/requestid idempotency means a double booking creates one * Purchase, not two. * - * Auth is the fail-closed drain-notifications pattern: whenever CRON_SECRET is - * configured it is ALWAYS required; the only unauthenticated path is a genuinely - * local dev run (not on Vercel, not production, and no secret set). + * Auth is isCronAuthorized() from lib/cron-auth: constant-time Bearer compare, + * required EVERYWHERE except an explicit NODE_ENV === "development", and a + * missing CRON_SECRET rejects rather than waving traffic through. */ const LEASE_MS = CLAIM_LEASE_MINUTES * 60_000; @@ -644,11 +645,17 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { } export async function GET(request: Request) { - const secret = process.env.CRON_SECRET; - const authHeader = request.headers.get("authorization"); - const authed = !!secret && authHeader === `Bearer ${secret}`; - const isLocalDev = !process.env.VERCEL && process.env.NODE_ENV !== "production" && !secret; - if (!authed && !isLocalDev) { + // isCronAuthorized: constant-time compare, and it fails CLOSED. + // + // The hand-rolled version this replaces had both problems the shared helper + // exists to fix. `authHeader === \`Bearer ${secret}\`` is a byte-at-a-time + // string compare that leaks the secret to anyone who can time the response. + // Worse, the `isLocalDev` escape hatch — no VERCEL, NODE_ENV not + // "production", and no CRON_SECRET — is satisfied by an unset environment, + // so any container, self-hosted build, or preview whose env drifted served + // this endpoint to anyone who asked. On a route that books real money into + // QuickBooks. + if (!isCronAuthorized(request)) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } diff --git a/src/proxy.ts b/src/proxy.ts index 46c9f2b47..f30d6d776 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -88,8 +88,37 @@ const LEGAL_PAGE_PATTERN = /^\/(?:privacy|terms|account-deletion|support)(?:\/|$ // // Exact match, mirroring the bypass entries themselves — a descendant that is // NOT bypassed still hits withAuth and needs no special case here. -const MACHINE_ENDPOINT_PATTERN = - /^\/api\/receipts\/intake(?:\/start|\/[^/]+\/(?:archived|finalize))?\/?$/; +// NOTE on the matcher (bottom of this file): its FIRST entry is +// `{ source: "/:path*", has: [{ type: "header", key: "next-action" }] }`, which +// routes EVERY next-action request through this proxy regardless of path — the +// exclusion list in the second entry does not apply to them. So these paths were +// already reaching the code below; what waved them through was +// PUBLIC_PROXY_BYPASS_PATTERN returning next() before any action check ran. +// Widening this pattern is therefore the whole fix, and the matcher is left +// alone: adding /api/cron, /api/webhook and friends to it would put middleware +// (including a DB-touching staff lookup) in front of every ordinary webhook and +// cron request, which is real latency and a new failure point on a hot path, +// for no additional protection. +// +// Deliberately NOT here: api/portal, api/payments, api/sub-portal and +// api/selections/*. Those genuinely serve anonymous Server Actions — that is the +// tradeoff their bypass exists for, and 403ing them would break the client +// portal. +const MACHINE_ENDPOINT_PATTERN = new RegExp( + "^\/api\/(?:" + [ + // Receipt Pipeline v2 — secret/Bearer in the handler, exact paths. + "receipts\/intake(?:\/start|\/[^/]+\/(?:archived|finalize))?", + // Cron: Bearer CRON_SECRET, checked in each route. + "cron\/[^?]*", + // Machine-to-machine ingest: each carries its own shared secret. + "integrations\/[^?]*", + // Stripe / Twilio: signature-verified, never session-authenticated. + "webhook(?:s)?\/[^?]*", + "twilio\/[^?]*", + // Ops probe: Bearer CRON_SECRET or a staff session, checked in-route. + "health\/pipeline", + ].join("|") + ")\/?$", +); // Test-only action dispatchers that get the proxy bypass below. Explicit, not a // prefix match: the proxy checks only the environment gates, never the route's diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 8f2159b55..d52d8c476 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -357,3 +357,113 @@ test("a secret may only declare the sources ITS key owns", async () => { assert.deepEqual(decideSource(archive, { source: "drive", sourceRef: "drive:F1" }), { ok: false, reason: "invalid-source" }); }); + +// ── The worker cron gate (Phase 2 gate, a) ───────────────────────────────── + +test("the intake worker cron route uses the shared fail-closed gate", async () => { + // Source assertion, because the hole was a BRANCH rather than a wrong + // comparison: `!VERCEL && NODE_ENV !== "production" && !CRON_SECRET` is + // satisfied by an UNSET environment, so any container or drifted preview + // served this route — which books real money into QuickBooks — to anyone. + const { readFileSync } = await import("node:fs"); + const raw = readFileSync( + new URL("../src/app/api/cron/receipt-intake-worker/route.ts", import.meta.url), + "utf8", + ); + // Comments stripped: the route DESCRIBES the hole it closed, and a naive + // scan reads that description as the hole itself. + const route = raw + .replace(/\/\*[\s\S]*?\*\//g, " ") + .split("\n") + .map(line => line.replace(/\/\/.*/, "")) + .join("\n"); + + assert.match(route, /isCronAuthorized\(request\)/, "uses the shared gate"); + assert.ok(!/isLocalDev/.test(route), "the fail-open branch is gone"); + assert.ok(!/authHeader === `Bearer/.test(route), "no plain string compare on a secret"); + assert.ok(!/process\.env\.VERCEL\b/.test(route), "no environment escape hatch"); + assert.ok(!/process\.env\.CRON_SECRET/.test(route), "the secret is read only by the shared helper"); +}); + +test("isCronAuthorized fails closed on an unset secret and is constant-time", async () => { + const { isCronAuthorized, bearerMatches } = await import("../src/lib/cron-auth"); + const env = process.env as Record; + const before = { s: env.CRON_SECRET, n: env.NODE_ENV }; + const req = (auth?: string) => + new Request("https://probuild.test/api/cron/receipt-intake-worker", { + headers: auth ? { authorization: auth } : {}, + }); + try { + env.NODE_ENV = "production"; + + // No secret configured: refuse, rather than treat "unconfigured" as open. + delete env.CRON_SECRET; + assert.equal(isCronAuthorized(req("Bearer anything")), false); + assert.equal(isCronAuthorized(req()), false); + + env.CRON_SECRET = "s3cret"; + assert.equal(isCronAuthorized(req("Bearer s3cret")), true); + assert.equal(isCronAuthorized(req("Bearer wrong")), false); + assert.equal(isCronAuthorized(req("s3cret")), false, "the scheme is part of the match"); + assert.equal(isCronAuthorized(req()), false); + + // Length is compared before the bytes, so a wrong-length header cannot + // throw out of timingSafeEqual. + assert.equal(bearerMatches("Bearer s3cre", "s3cret"), false); + assert.equal(bearerMatches("Bearer s3cretttt", "s3cret"), false); + assert.equal(bearerMatches(null, "s3cret"), false); + assert.equal(bearerMatches("Bearer s3cret", undefined), false); + } finally { + env.CRON_SECRET = before.s; + env.NODE_ENV = before.n; + } +}); + +test("machine endpoints refuse a Server Action dispatch; portal actions still work", async () => { + // The matcher's FIRST entry already routes every next-action request here + // regardless of path, so these were reaching the proxy — the bypass was + // waving them through before any action check ran. + const { default: proxy } = await loadProxy(); + const { NextRequest } = await import("next/server"); + const event = { waitUntil() {} } as any; + const env = process.env as Record; + const prod = env.NODE_ENV; + env.NODE_ENV = "production"; + + const dispatch = (p: string) => + proxy(new NextRequest(`https://probuild.test${p}`, { + method: "POST", + headers: { "next-action": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, + }), event); + + try { + for (const path of [ + "/api/cron/receipt-intake-worker", + "/api/health/pipeline", + "/api/integrations/qbo-receipts/create", + "/api/webhook/stripe", + "/api/twilio/sms", + ]) { + const res = await dispatch(path); + assert.ok(res, path); + assert.equal(res.status, 403, `${path} must refuse an action dispatch`); + assert.equal(res.headers.get("x-middleware-next"), null, path); + } + + // Routes that GENUINELY serve anonymous Server Actions are untouched — + // 403ing them would break the client portal. + for (const path of ["/api/portal/verify", "/api/payments/deposit-ingest", "/api/selections/item-comments"]) { + const res = await dispatch(path); + assert.equal(res!.headers.get("x-middleware-next"), "1", `${path} must still pass`); + } + + // And an ordinary cron call is unaffected. + const normal = await proxy( + new NextRequest("https://probuild.test/api/cron/receipt-intake-worker", { method: "GET" }), + event, + ); + assert.equal(normal!.headers.get("x-middleware-next"), "1"); + } finally { + env.NODE_ENV = prod; + } +}); From 8b332c773d185b4bb301ee591ee3218b4a26c1b5 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 03:33:11 -0700 Subject: [PATCH 047/144] fix(receipts): fence every write on the claim, seal after commit, refuse .txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 9 (1-9), Phase 3's two items, and the Phase 2 gate's claim-token item. CLAIM FENCING (Phase 2 gate). claimToken now rides on BookableRow and every worker transition is a CAS on {id, state, claimToken}: finishRouting, promoteToBooking, markSendAttempted, each book-result write, and the BOOKED commit. markSendAttempted is the last fence before QuickBooks — a superseded worker finds zero rows, aborts with outcome "stale", and sends NOTHING, so a zombie cannot post a Purchase the live worker is about to post too. A claim lost between the create and the commit rolls back the Expense as well; the successor retries, hits DocNumber idempotency, and books once under one owner. 1+3 Seal order is copy -> COMMIT the row pointer -> delete the upload object. Deleting first was unrecoverable: a failed UPDATE left the row pointing at a path whose object had just been removed. One shared sealAndPublish, fenced, used by /finalize AND the sweeper — the sweeper was publishing rows still pointing at the upload path, which stays writable by whoever holds the URL. 2 retry() reads the CURRENT send flag, not the row's copy from claim time. A failure at the attachment leg or the post-create commit was judged on stale data and released the key of a row that really does have a Purchase. 4 /start REQUIRES sha256 (400 without) and refuses to reissue an upsert URL unless the supplied hash matches the stored one — the URL is upsert:true, so handing one out without proving identity lets receipt B overwrite receipt A. 5 attachment failed:<4xx|fault> is QBO REFUSING the file: NEEDS_REVIEW on the first one, key retained. Only 5xx and thrown errors retry. 6 JSON inline limit is 3 MiB raw (base64 ~4 MiB); multipart stays 4 MiB. Both 413 with the two-step path named. 7 An ambiguous upload error records the cleanup event BEFORE deleting the row — the write may have landed, and deleting first orphans it with nothing pointing at it. 8 normalizeConfidence: Number("") is 0, so an absent confidence was being stored as "certain this phase is a poor match". Empty/whitespace/non-numeric -> null. 9 text/plain is REFUSED with a 415 naming what to send. QBO cannot attach a .txt, so accepting one meant reading it then stranding it unbookable. Porting v1's HTML->PDF conversion means a PDF generator with wrapping, pagination and WinAnsi encoding hazards — a new silent-corruption surface on a money document for the rarest input. Chose the refusal; said so in the spec. (a) /finalize reconciles late costCodeId/projectId BEFORE the alreadyFinalized early return: apply where null, 409 late-fields-conflict otherwise, same as the two-publisher path. installedAtCustomer is not a Phase 1 field. (b) covered by 4. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 32 ++++ e2e/receipt-intake.spec.ts | 129 ++++++++++++++++ .../api/cron/receipt-intake-worker/route.ts | 80 ++++++---- .../receipts/intake/[id]/finalize/route.ts | 131 +++++++++++++---- src/app/api/receipts/intake/route.ts | 60 ++++++-- src/app/api/receipts/intake/start/route.ts | 45 ++++-- src/lib/receipt-intake/book.ts | 115 +++++++++++++-- src/lib/receipt-intake/file-type.ts | 13 +- src/lib/receipt-intake/intake-core.ts | 12 ++ src/lib/receipt-intake/read.ts | 12 +- src/lib/receipt-intake/storage-cleanup.ts | 29 +++- src/lib/receipt-intake/stored-object.ts | 51 +++++++ src/lib/receipt-intake/worker.ts | 22 ++- tests/receipt-intake-auth.test.ts | 5 +- tests/receipt-intake-book.test.ts | 139 +++++++++++++++++- tests/receipt-intake-read.test.ts | 22 ++- tests/receipt-intake-stored-object.test.ts | 82 +++++++++-- tests/receipt-intake-worker.test.ts | 41 ++++++ 18 files changed, 902 insertions(+), 118 deletions(-) diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index 902a66bb9..a56bd0dcf 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -458,6 +458,38 @@ the sweeper already parked re-validates and **recovers** it rather than reportin `alreadyFinalized`, which would leave a real receipt parked while telling the caller it was fine. +### Upload limits, and text receipts + +| path | ceiling | why | +|---|---|---| +| `POST /api/receipts/intake` (JSON) | **3 MiB raw** | base64 inflates by 4/3, so 3 MiB encodes to ~4 MiB and fits the serverless body cap. 4 MiB raw would be a ~5.4 MiB request that dies at the edge with a 413 this code never sees. | +| `POST /api/receipts/intake` (multipart) | **4 MiB** | bytes are sent as-is. | +| two-step (`/start` + signed URL + `/finalize`) | **15 MiB** | the bytes never pass through this server. | + +Both inline ceilings answer with a 413 naming the two-step path. + +**`text/plain` is refused with a 415.** QuickBooks cannot attach a `.txt`, so accepting one +meant reading it with Gemini and then stranding it unbookable at +`unsupported-attachment` — worse than a clear refusal at the door. v1 converted these using +Apps Script's HTML→PDF `getAs`, which has no Node equivalent: a real port means a PDF +generator with wrapping, pagination and WinAnsi encoding (pdf-lib's standard fonts THROW on +characters they cannot encode), which is a new silent-corruption surface on a money document +for the rarest input in the pipeline. The 415 says to send a PDF or an image instead. + +### Every worker write is fenced on the claim token + +`ReceiptIntake.claimToken` is re-stamped on every claim, and each row carries it through the +pass. Every transition — `finishRouting`, `promoteToBooking`, `markSendAttempted`, each +book-result write, and the `BOOKED` commit — is a CAS on `{id, state, claimToken}`. + +The one that matters most is `markSendAttempted`: it is the **last fence before +QuickBooks**. A worker whose invocation was killed and whose row has since been re-claimed +finds zero rows there and aborts with `outcome: "stale"` **having sent nothing** — so a +zombie cannot post a Purchase the live worker is about to post as well. A claim lost later, +between the create and the commit, rolls the transaction back (Expense included); the +successor's retry hits QBO's DocNumber idempotency, gets the same Purchase, and books it +once under one owner. + ### Two machine secrets, not one They belong to different programs, so they are different keys and rotate independently. diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 323f0f224..6c6d30efb 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -894,3 +894,132 @@ test.describe("two-step upload: a reused key cannot swap the document", () => { expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.state).toBe("STAGING"); }); }); + +test.describe("round-9 intake contracts", () => { + const startPath = `${INTAKE_PATH}/start`; + const sha = (b64: string) => createHash("sha256").update(Buffer.from(b64, "base64")).digest("hex"); + const start = (request: APIRequestContext, body: Record) => + request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify(body), + maxRedirects: 0, + }); + + test("/start REFUSES without a sha256 — it is the row's only identity", async ({ request }) => { + // Without it a reused sourceRef is indistinguishable from an honest + // retry, and /start would hand out an upsert URL aimed at another + // document's object. + const res = await start(request, { + source: "drive", sourceRef: `${REF_PREFIX}nosha`, mimeType: "image/png", + }); + expect(res.status()).toBe(400); + expect((await res.json()).reason).toBe("missing-sha256"); + + const malformed = await start(request, { + source: "drive", sourceRef: `${REF_PREFIX}badsha`, mimeType: "image/png", sha256: "nope", + }); + expect(malformed.status()).toBe(400); + }); + + test("/start will not reissue an upsert URL without proving identity", async ({ request }) => { + const ref = `${REF_PREFIX}reissue`; + const first = await start(request, { + source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64), + }); + expect(first.status()).toBe(200); + minted.push((await first.json()).id); + + // Same hash: proven the same document, so the URL is reissued. + const proven = await start(request, { + source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64), + }); + expect(proven.status()).toBe(200); + expect((await proven.json()).uploadUrl).toBeTruthy(); + + // Different hash: refused BEFORE a URL exists, so receipt B can never be + // written over receipt A's object. + const unproven = await start(request, { + source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(OTHER_PNG_BASE64), + }); + expect(unproven.status()).toBe(409); + expect((await unproven.json()).error).toBe("sourceRef-conflict"); + }); + + test("a text receipt is refused with a 415 that says what to send instead", async ({ request }) => { + // QuickBooks cannot attach a .txt, so accepting one meant reading it and + // then stranding it unbookable mid-pipeline. + const res = await postIntake(request, intakeBody({ + sourceRef: `${REF_PREFIX}textfile`, + fileBase64: Buffer.from("VENDOR: Lowes\nTOTAL: 10.00").toString("base64"), + mimeType: "text/plain", + })); + expect(res.res.status()).toBe(415); + expect(res.body.error).toBe("unsupported-file-type"); + expect(res.body.reason).toMatch(/PDF/i); + expect(res.body.accepted).toContain("application/pdf"); + }); + + test("the JSON inline limit is 3 MiB raw and says so", async ({ request }) => { + // base64 inflates by 4/3, so 4 MiB raw is a ~5.4 MiB request — over the + // platform body cap, which used to reject it before this code ran. + const big = Buffer.alloc(3 * 1024 * 1024 + 1, 7).toString("base64"); + const res = await postIntake(request, intakeBody({ + sourceRef: `${REF_PREFIX}toobig`, fileBase64: big, + })); + expect(res.res.status()).toBe(413); + expect(res.body.error).toBe("payload-too-large"); + expect(res.body.maxInlineBytes).toBe(3 * 1024 * 1024); + expect(res.body.use).toMatch(/intake\/start/); + }); + + test("a sequential finalize retry still applies late fields", async ({ request }) => { + // The row already reached RECEIVED, so this takes the alreadyFinalized + // path — and answering it without applying the job assignment would drop + // that assignment while telling the caller it worked. + const ref = `${REF_PREFIX}latefields`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + const id = created.body.id; + minted.push(id); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.state).toBe("RECEIVED"); + + const applied = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ costCodeId: "e2e-mob-cc-demo" }), + maxRedirects: 0, + }); + expect(applied.status()).toBe(200); + expect((await applied.json()).alreadyFinalized).toBe(true); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.costCodeId).toBe("e2e-mob-cc-demo"); + + // Re-sending the SAME value is idempotent. + const same = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ costCodeId: "e2e-mob-cc-demo" }), + maxRedirects: 0, + }); + expect(same.status()).toBe(200); + + // A DIFFERENT value is a conflict, never a silent overwrite of what a + // human may already have set. + const conflicting = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ costCodeId: "e2e-mob-cc-dryw" }), + maxRedirects: 0, + }); + expect(conflicting.status()).toBe(409); + expect((await conflicting.json()).error).toBe("late-fields-conflict"); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.costCodeId).toBe("e2e-mob-cc-demo"); + }); + + test("a published row's object lives at the sealed, content-addressed path", async ({ request }) => { + const ref = `${REF_PREFIX}sealed-path`; + const created = await postIntake(request, intakeBody({ sourceRef: ref })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + const row = await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }); + // The single-shot path publishes directly; the two-step path seals. Either + // way the row must never be left pointing somewhere a client holds a URL for. + expect(row?.fileSha256).toHaveLength(64); + }); +}); diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 0947dccc8..e6e5259ea 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -6,8 +6,8 @@ import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; -import { downloadVerified, inspectStoredObject } from "@/lib/receipt-intake/stored-object"; -import { deleteObjectOrRecord, retryPendingCleanups } from "@/lib/receipt-intake/storage-cleanup"; +import { downloadVerified, inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; +import { deleteObjectOrRecord, retryPendingCleanups, sealObject } from "@/lib/receipt-intake/storage-cleanup"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { createRouteDeadline, type RouteDeadline } from "@/lib/quickbooks"; @@ -328,17 +328,32 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { parked++; continue; } - await prisma.receiptIntake.updateMany({ - where: { id: row.id, state: "STAGING" }, - data: { - state: "RECEIVED", - nextRetryAt: null, - mimeType: check.mimeType, - fileSize: check.fileSize, - fileSha256: check.fileSha256, + + // The SAME seal-and-publish /finalize uses. The sweep must + // never publish a row still pointing at the UPLOAD path: + // that path stays writable by whoever holds the signed URL, + // so a swept row's "verified" bytes would remain replaceable + // afterwards. + const outcome = await sealAndPublish(row.storagePath, row.id, check, { + seal: sealObject, + commit: async (canonicalPath, values) => { + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: row.id, state: "STAGING" }, + data: { + state: "RECEIVED", + nextRetryAt: null, + storagePath: canonicalPath, + mimeType: values.mimeType, + fileSize: values.fileSize, + fileSha256: values.fileSha256, + }, + }); + return count; }, + dropUpload: uploadPath => + deleteObjectOrRecord(uploadPath, "sealed").then(() => undefined), }); - published++; + if (outcome?.published) published++; continue; } if (check.kind === "transient") continue; // unknown is not a verdict @@ -479,7 +494,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { retryStorageCleanups: shouldStop => retryPendingCleanups(STAGING_SWEEP_BATCH, shouldStop), - promoteToBooking: async (rowId, weakKey) => prisma.$transaction(async tx => { + promoteToBooking: async (rowId, weakKey, claimToken) => prisma.$transaction(async tx => { // LAST weak-dedup check, taken INSIDE the transition. The check at // read time can miss a pair that arrived in the same batch window, // and READ -> BOOKING is the last instant before money moves. @@ -531,8 +546,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { orderBy: { createdAt: "asc" }, }); if (conflict) { - await tx.receiptIntake.update({ - where: { id: rowId }, + await tx.receiptIntake.updateMany({ + where: { id: rowId, state: "READ", claimToken }, data: { state: "NEEDS_REVIEW", stateReason: `weak-dup:${conflict.id}`, @@ -545,18 +560,28 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { return { promoted: false, conflictId: conflict.id }; } } - await tx.receiptIntake.update({ - where: { id: rowId }, + // CAS: only the current claim holder promotes. A superseded worker + // must not move a row into BOOKING that its successor is handling. + const { count } = await tx.receiptIntake.updateMany({ + where: { id: rowId, state: "READ", claimToken }, data: { state: "BOOKING", stateReason: null }, }); + if (count === 0) return { promoted: false, stale: true }; return { promoted: true }; }), book: row => bookReceipt(row, { db: prisma as unknown as BookPrismaClient, companyTimeZone: resolveCompanyTimeZone, - markSendAttempted: async rowId => { - await prisma.receiptIntake.update({ where: { id: rowId }, data: { sendAttempted: true } }); + markSendAttempted: async (rowId, claimToken) => { + // CAS: only the CURRENT claim holder may mark a send. A zero + // count means this worker was superseded, and bookReceipt + // aborts on it before touching QuickBooks. + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: "BOOKING", claimToken }, + data: { sendAttempted: true }, + }); + return count > 0; }, isCostCodeAllowed: (projectId, costCodeId) => isCostCodeAllowedForProject(prismaPhaseDataSource, projectId, costCodeId), @@ -575,12 +600,17 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { now: () => new Date(), }), - applyBookResult: async (rowId, result) => { + applyBookResult: async (rowId, result, claimToken) => { const now = new Date(); if (result.outcome === "booked") return; // bookReceipt already committed it + // A superseded worker writes NOTHING: the row belongs to whoever + // holds the current token, and its state is theirs to set. + if (result.outcome === "stale") return; + // Every write below is a CAS on the claim, for the same reason. + const owns = { id: rowId, claimToken } as const; if (result.outcome === "needs-review") { - await prisma.receiptIntake.update({ - where: { id: rowId }, + await prisma.receiptIntake.updateMany({ + where: owns, data: { state: "NEEDS_REVIEW", stateReason: result.reason, @@ -596,8 +626,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { if (result.outcome === "deferred") { // A switch is off: hold in BOOKING, look again in an hour, and // do NOT spend an attempt — this document did nothing wrong. - await prisma.receiptIntake.update({ - where: { id: rowId }, + await prisma.receiptIntake.updateMany({ + where: owns, data: { state: "BOOKING", stateReason: result.reason, @@ -606,8 +636,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { }); return; } - await prisma.receiptIntake.update({ - where: { id: rowId }, + await prisma.receiptIntake.updateMany({ + where: owns, data: { state: "BOOKING", attempts: result.attempts, diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index ed587c7d8..b81b06265 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -2,10 +2,56 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; -import { canonicalStoragePath, inspectStoredObject } from "@/lib/receipt-intake/stored-object"; +import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; import { deleteObjectOrRecord, sealObject } from "@/lib/receipt-intake/storage-cleanup"; export const dynamic = "force-dynamic"; + +/** + * Apply late fields where the row has none; refuse where they disagree. + * + * Returns a 409 response on conflict, or null when the row is now consistent + * with what the caller sent. + */ +async function reconcileLateFields( + id: string, + lateFields: Partial<{ costCodeId: string; projectId: string }>, +): Promise { + const entries = Object.entries(lateFields) as Array<["costCodeId" | "projectId", string]>; + if (entries.length === 0) return null; + + const current = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { costCodeId: true, projectId: true }, + }); + if (!current) return null; + + const conflicts = entries.filter(([key, value]) => current[key] !== null && current[key] !== value); + if (conflicts.length > 0) { + return NextResponse.json( + { + ok: false, + error: "late-fields-conflict", + reason: "this row already carries different values for these fields", + fields: Object.fromEntries( + conflicts.map(([key]) => [key, { stored: current[key], supplied: lateFields[key] }]), + ), + }, + { status: 409 }, + ); + } + + const toApply = Object.fromEntries(entries.filter(([key]) => current[key] === null)); + if (Object.keys(toApply).length > 0) { + // Conditional on still being null, so a concurrent writer that set it + // between the read and here wins rather than being overwritten. + await prisma.receiptIntake.updateMany({ + where: { id, ...Object.fromEntries(Object.keys(toApply).map(key => [key, null])) }, + data: toApply, + }); + } + return null; +} export const maxDuration = 30; /** @@ -26,7 +72,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const { id } = await context.params; - let body: { sha256?: unknown } = {}; + let body: { sha256?: unknown; costCodeId?: unknown; projectId?: unknown } = {}; try { body = await req.json(); } catch { @@ -34,6 +80,21 @@ export async function POST(req: Request, context: { params: Promise<{ id: string } const declaredSha = typeof body.sha256 === "string" ? body.sha256.trim().toLowerCase() : null; + // LATE FIELDS. A client that learned the job only after starting the upload + // sends them here. They are applied WHERE NULL and refused where they + // disagree — silently overwriting a value a human already set is the one + // outcome that loses information nobody can recover. + // + // NOTE: Phase 3's `installedAtCustomer` does not exist on this model; the + // same rule will apply to it when it lands. + const lateInput = { + costCodeId: typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null, + projectId: typeof body.projectId === "string" && body.projectId.trim() ? body.projectId.trim() : null, + }; + const lateFields = Object.fromEntries( + Object.entries(lateInput).filter(([, v]) => v !== null), + ) as Partial<{ costCodeId: string; projectId: string }>; + const row = await prisma.receiptIntake.findUnique({ where: { id }, select: { @@ -63,7 +124,15 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // Idempotent: finalizing an already-published row is a success, not an error // — the client's retry after a lost response must not look like a failure. + // + // But the late fields are reconciled FIRST. A sequential retry arriving + // after the row already reached RECEIVED still carries them, and answering + // alreadyFinalized without applying them drops the job assignment on the + // floor while telling the caller it worked. Same behaviour as the + // two-publisher path below, because a caller cannot tell which one it hit. if (!recoverable) { + const conflict = await reconcileLateFields(id, lateFields); + if (conflict) return conflict; return NextResponse.json({ ok: true, alreadyFinalized: true, id: row.id, state: row.state, sourceRef: row.sourceRef, projectId: row.projectId, dryRun: row.dryRun, @@ -122,36 +191,44 @@ export async function POST(req: Request, context: { params: Promise<{ id: string } } - // SEAL. Copy the verified bytes to a content-addressed path the client was - // never handed a URL for, and point the row at that. Publishing while the - // row still referenced the upload path would leave the verified content - // replaceable by anyone holding the signed URL — the row asserting one sha - // while storage served different bytes. - const canonicalPath = canonicalStoragePath(id, fileSha256, mimeType); - const sealed = await sealObject(row.storagePath, canonicalPath, check.bytes, mimeType); - if (!sealed) { - return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - } - - const published = await prisma.receiptIntake.updateMany({ - where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] } }, - data: { - state: "RECEIVED", - stateReason: null, - storagePath: sealed, - mimeType, - fileSize, - fileSha256, - nextRetryAt: null, + // ONE shared seal-and-publish, also used by the worker's stale-STAGING + // sweep, so the two publishers cannot diverge on ordering or fencing. + const outcome = await sealAndPublish(row.storagePath, id, check, { + seal: sealObject, + commit: async (canonicalPath, values) => { + const { count } = await prisma.receiptIntake.updateMany({ + // Fenced: only a row still in a publishable state moves, so a + // loser of the race writes nothing. + where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] } }, + data: { + state: "RECEIVED", + stateReason: null, + storagePath: canonicalPath, + mimeType: values.mimeType, + fileSize: values.fileSize, + fileSha256: values.fileSha256, + nextRetryAt: null, + ...lateFields, + }, + }); + return count; }, + dropUpload: uploadPath => deleteObjectOrRecord(uploadPath, "sealed").then(() => undefined), }); - if (published.count === 0) { - // Another finalize won the race and published it. Same outcome. - const now = await prisma.receiptIntake.findUnique({ + + if (!outcome) { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + if (!outcome.published) { + // Another publisher won. Same outcome for the caller — but the late + // fields still have to be reconciled against what that publisher wrote. + const reconciled = await reconcileLateFields(id, lateFields); + if (reconciled) return reconciled; + const current = await prisma.receiptIntake.findUnique({ where: { id }, select: { state: true, sourceRef: true, projectId: true, dryRun: true }, }); - return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: now?.state ?? "RECEIVED" }); + return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: current?.state ?? "RECEIVED" }); } return NextResponse.json({ diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 560da7df3..43e08f0bb 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -6,8 +6,13 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; +import { recordPendingCleanup } from "@/lib/receipt-intake/storage-cleanup"; import { EXT_BY_MIME, sniffMime } from "@/lib/receipt-intake/file-type"; -import { MAX_INLINE_UPLOAD_BYTES, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; +import { + MAX_INLINE_JSON_BYTES, + MAX_INLINE_UPLOAD_BYTES, + MAX_STORED_BYTES, +} from "@/lib/receipt-intake/intake-core"; import { ARCHIVE_READABLE_STATES, listReceiptIntakes, @@ -79,13 +84,35 @@ function bad(reason: string) { * used to die at the edge with an opaque 413 that never reached this code, so * the caller learned nothing. Say it plainly and name the path that works. */ -function tooLargeForInline() { +/** + * 415, not 400: the caller's request was well-formed, we simply will never + * accept this format. Naming what IS accepted is the difference between a + * sender who fixes it and one who retries the same file forever. + */ +function unsupportedType(declared: string) { + const essence = declared.split(";")[0].trim().toLowerCase(); + return NextResponse.json( + { + ok: false, + error: "unsupported-file-type", + reason: essence === "text/plain" + ? "text receipts are not accepted: QuickBooks cannot attach a .txt, so it would be read and then stranded unbookable. Print or export it to PDF first." + : "the stored bytes are not a format QuickBooks can attach", + accepted: ["application/pdf", "image/jpeg", "image/png", "image/heic", "image/webp", "image/gif"], + }, + { status: 415 }, + ); +} + +function tooLargeForInline(limit: number, encoding: "json" | "multipart") { return NextResponse.json( { ok: false, error: "payload-too-large", - reason: `inline uploads are limited to ${MAX_INLINE_UPLOAD_BYTES} bytes (serverless request-body cap)`, - maxInlineBytes: MAX_INLINE_UPLOAD_BYTES, + reason: encoding === "json" + ? `JSON uploads are limited to ${limit} raw bytes; base64 inflates them by 4/3 and the serverless body cap is what actually rejects a larger one` + : `multipart uploads are limited to ${limit} bytes (serverless request-body cap)`, + maxInlineBytes: limit, maxBytes: MAX_STORED_BYTES, use: "POST /api/receipts/intake/start then PUT to the signed URL then POST /api/receipts/intake/{id}/finalize", }, @@ -106,9 +133,10 @@ async function parseBody(req: Request): Promise { } const file = form.get("file"); if (!(file instanceof File)) return bad("missing-file"); - if (file.size > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); + // Multipart sends the bytes as-is, so it keeps the full 4 MiB. + if (file.size > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(MAX_INLINE_UPLOAD_BYTES, "multipart"); const bytes = Buffer.from(await file.arrayBuffer()); - if (bytes.length > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); + if (bytes.length > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(MAX_INLINE_UPLOAD_BYTES, "multipart"); return { bytes, declaredMime: file.type || "application/octet-stream", @@ -133,10 +161,14 @@ async function parseBody(req: Request): Promise { if (!base64) return bad("missing-file"); // Cap BEFORE decoding: base64 is 4/3 the byte count, so this refuses an // oversize payload without materialising it. - if (base64.length > Math.ceil(MAX_INLINE_UPLOAD_BYTES / 3) * 4 + 4) return tooLargeForInline(); + // Checked on the ENCODED length first, so an oversize payload is refused + // without materialising it. + if (base64.length > Math.ceil(MAX_INLINE_JSON_BYTES / 3) * 4 + 4) { + return tooLargeForInline(MAX_INLINE_JSON_BYTES, "json"); + } const bytes = Buffer.from(base64, "base64"); if (bytes.length === 0) return bad("missing-file"); - if (bytes.length > MAX_INLINE_UPLOAD_BYTES) return tooLargeForInline(); + if (bytes.length > MAX_INLINE_JSON_BYTES) return tooLargeForInline(MAX_INLINE_JSON_BYTES, "json"); return { bytes, declaredMime: typeof json.mimeType === "string" ? json.mimeType : "application/octet-stream", @@ -161,7 +193,7 @@ export async function POST(req: Request) { if (parsed instanceof NextResponse) return parsed; const mimeType = sniffMime(parsed.bytes, parsed.declaredMime); - if (!mimeType) return bad("unsupported-file-type"); + if (!mimeType) return unsupportedType(parsed.declaredMime); // PROVENANCE AND IDENTITY ARE NOT CALLER INPUT for a human. // @@ -301,6 +333,16 @@ export async function POST(req: Request) { uploadFailed = error instanceof Error ? `${error.name}: ${error.message}` : "upload-threw"; } if (uploadFailed) { + // AMBIGUOUS. An upload error — especially a thrown one — does not tell + // us whether bytes landed: the write may have succeeded and the + // acknowledgement been lost. So the cleanup record is written BEFORE + // the row is deleted, while `storagePath` is still known to something. + // Delete first and the object (if any) is orphaned with nothing left + // pointing at it, invisible in a private bucket forever. + // + // A no-op cleanup for an upload that genuinely never landed is free; + // the sweeper's delete simply finds nothing. + await recordPendingCleanup(storagePath, `upload-ambiguous:${uploadFailed}`.slice(0, 200)); await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); console.error("[receipts/intake] upload failed", uploadFailed); return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index 0a7504b54..a47aadda7 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -52,9 +52,23 @@ export async function POST(req: Request) { // reused sourceRef carrying a DIFFERENT document is indistinguishable from // an honest retry, and /finalize would attach one receipt's bytes to // another receipt's identity. - const expectedSha256 = typeof body.sha256 === "string" ? body.sha256.trim().toLowerCase() : null; - if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) { - return NextResponse.json({ ok: false, reason: "invalid-sha256" }, { status: 400 }); + // REQUIRED, not optional. + // + // It is the only thing that gives this row an identity before any bytes + // exist. Without it a reused sourceRef is indistinguishable from an honest + // retry, so /start would happily hand out an upsert URL pointed at another + // document's object — and the swap would only surface at /finalize, by + // which point the original bytes are gone. + const expectedSha256 = typeof body.sha256 === "string" ? body.sha256.trim().toLowerCase() : ""; + if (!/^[0-9a-f]{64}$/.test(expectedSha256)) { + return NextResponse.json( + { + ok: false, + reason: "missing-sha256", + detail: "sha256 of the bytes you are about to upload is required (64 lowercase hex chars)", + }, + { status: 400 }, + ); } const declaredSize = Number(body.fileSize); @@ -124,14 +138,25 @@ export async function POST(req: Request) { auth.user.role === "ADMIN"; if (!maySee) return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); - // SAME KEY, DIFFERENT DOCUMENT. Caught HERE, before a signed URL is - // handed out — otherwise the caller would upload receipt B over - // receipt A's object and only /finalize would notice, by which point - // A's bytes are gone. - const knownSha = existing.fileSha256 || existing.expectedSha256; - if (expectedSha256 && knownSha && knownSha.toLowerCase() !== expectedSha256) { + // IDENTITY MUST BE PROVEN BEFORE AN UPSERT URL IS REISSUED. + // + // The URL is `upsert: true` so a caller can replace its OWN partial + // upload — which is exactly why handing one out for an existing path + // requires proof this is the same document. A mismatching or + // unknown-identity request would otherwise get a URL that + // overwrites receipt A with receipt B, and only /finalize would + // notice, by which point A's bytes are gone. + const knownSha = (existing.fileSha256 || existing.expectedSha256 || "").toLowerCase(); + if (!knownSha || knownSha !== expectedSha256) { return NextResponse.json( - { ok: false, error: "sourceRef-conflict", reason: "this sourceRef already holds a different document", existingId: existing.id }, + { + ok: false, + error: "sourceRef-conflict", + reason: knownSha + ? "this sourceRef already holds a different document" + : "this sourceRef exists with no recorded hash; identity cannot be proven", + existingId: existing.id, + }, { status: 409 }, ); } diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 56d9796ff..fef6dbe6e 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -61,6 +61,11 @@ export interface BookableRow { memo: string | null; /** What finalize recorded; every download of this row is checked against it. */ fileSha256: string; + /** + * The token this pass claimed the row with. Every write is a CAS on it, so + * a worker whose claim was superseded cannot act on stale state. + */ + claimToken: string | null; attempts: number; /** Carries a previous attachment failure across a retry — see below. */ lastError: string | null; @@ -134,6 +139,11 @@ export type BookResult = * held: QBO may have created the Purchase and lost the response. */ | { outcome: "needs-review"; reason: string; releaseStrongKey: boolean } + /** + * This worker's claim was superseded. It wrote nothing and sent nothing; + * the row belongs to whoever holds the current token. + */ + | { outcome: "stale" } /** Transport-class failure: attempts+1 and a backoff. */ | { outcome: "retry"; attempts: number; nextRetryAt: Date; reason: string }; @@ -152,14 +162,22 @@ export interface BookPrismaClient { }; receiptIntake: { update(args: any): Promise; + updateMany(args: any): Promise<{ count: number }>; }; $transaction(fn: (tx: BookPrismaClient) => Promise): Promise; } export interface BookDependencies { db: BookPrismaClient; - /** Persists sendAttempted BEFORE the create — the flag must survive a crash. */ - markSendAttempted: (rowId: string) => Promise; + /** + * CAS on {id, state: BOOKING, claimToken} that persists sendAttempted. + * + * This is the LAST FENCE before QuickBooks. It returns false when the row + * has been re-claimed, and the booking then aborts having sent nothing — + * which is the point: a zombie worker resuming with a stale view must not + * create a Purchase the live worker is about to create as well. + */ + markSendAttempted: (rowId: string, claimToken: string | null) => Promise; /** The company's configured zone — Expense.date is a business calendar day. */ companyTimeZone: () => Promise; /** @@ -431,7 +449,12 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // strong key would then be held forever against a Purchase that does // not exist. Persisted rather than in-memory, because the case the flag // exists for is the process dying mid-create. - await deps.markSendAttempted(row.id); + // It is ALSO the last fence: a CAS on the claim token. If this worker + // has been superseded the write affects zero rows and we abort HERE, + // before the create — so a zombie cannot post a Purchase that the live + // worker is about to post as well. + const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); + if (!stillOurs) return { outcome: "stale" }; sent.attempted = true; result = await deps.createPurchase(tokens, input, deps.deadline); @@ -442,7 +465,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro if (terminal) return { outcome: "needs-review", reason: terminal, releaseStrongKey: !sent.attempted }; // QBTimeoutError, QBNotConnectedError, network/fetch errors, QBO // 429/5xx and DB errors are all transport-class: try again later. - return retry(row, deps, now, describe(error)); + return retry(row, deps, now, describe(error), sent.attempted); } if (!result.ok) { @@ -479,7 +502,20 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro if (result.attachment === "skipped") { return { outcome: "needs-review", reason: "unsupported-attachment:skipped", releaseStrongKey: false }; } - return retry(row, deps, now, `${ATTACHMENT_FAILED_PREFIX}${result.attachment}`); + // A `failed:<4xx>` or `failed:fault` is QBO REFUSING this file — a + // rejected format, an oversize body, a business-rule fault. Retrying it + // twenty times changes nothing except how long the Purchase sits in the + // books without its receipt, so it goes to a human on the first one. + // Only a transient class (5xx, a thrown network/abort error) is worth + // another pass. The key is retained either way: the Purchase EXISTS. + if (isTerminalAttachmentFailure(result.attachment)) { + return { + outcome: "needs-review", + reason: `attachment-refused:${result.attachment}`, + releaseStrongKey: false, + }; + } + return retry(row, deps, now, `${ATTACHMENT_FAILED_PREFIX}${result.attachment}`, sent.attempted); } // 5. One transaction: the Expense and the row's BOOKED state land together @@ -558,8 +594,14 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro }, select: { id: true }, }); - await tx.receiptIntake.update({ - where: { id: row.id }, + // CAS again inside the commit. If the row was re-claimed between + // the create and here, this transaction rolls back — including the + // Expense — and the successor retries: QBO's DocNumber idempotency + // returns the SAME Purchase, so it books once, under one owner. + // Completing a BOOKED write from a stale worker would leave two + // owners disagreeing about the row. + const claimed = await tx.receiptIntake.updateMany({ + where: { id: row.id, state: "BOOKING", claimToken: row.claimToken }, data: { state: "BOOKED", stateReason: null, @@ -570,6 +612,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro nextRetryAt: null, }, }); + if (claimed.count === 0) throw new StaleClaimError(); return expense.id; }); @@ -613,9 +656,14 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro alreadyExisted: result.alreadyExists, }; } catch (error) { + // A lost CAS is not a fault: the successor owns this row and will book + // it. Say so rather than spending an attempt on it. + if (error instanceof StaleClaimError) return { outcome: "stale" }; // The Purchase EXISTS at this point. Retrying is correct and safe: the - // DocNumber lookup will find it and return alreadyExists:true. - return retry(row, deps, now, describe(error)); + // DocNumber lookup will find it and return alreadyExists:true — and the + // key must be RETAINED, which is why this attempt's send flag is passed + // rather than the row's stale copy. + return retry(row, deps, now, describe(error), sent.attempted); } } @@ -625,9 +673,31 @@ function describe(error: unknown): string { return "UnknownError"; } +/** Thrown inside the commit transaction when the claim token no longer matches. */ +class StaleClaimError extends Error { + constructor() { + super("the claim was superseded"); + this.name = "StaleClaimError"; + } +} + /** Marks a retry as "the Purchase exists but its receipt did not attach". */ export const ATTACHMENT_FAILED_PREFIX = "attachment-failed:"; +/** + * Is this attachment failure QBO refusing the file, rather than a blip? + * + * `failed:` carries the HTTP status; `failed:fault` is an Intuit + * business-rule rejection; `failed:` comes from a thrown error and + * is transient by nature (AbortError, TypeError from fetch, ...). + */ +export function isTerminalAttachmentFailure(attachment: string): boolean { + const detail = attachment.slice("failed:".length); + if (detail === "fault") return true; + const status = Number(detail); + return Number.isFinite(status) && status >= 400 && status < 500; +} + /** * Which phase (if any) this Expense may carry, checked against the project the * row will ACTUALLY book to. @@ -672,16 +742,29 @@ function parkedBeforeSend(reason: string): BookResult { return { outcome: "needs-review", reason, releaseStrongKey: true }; } -function retry(row: BookableRow, deps: BookDependencies, now: Date, reason: string): BookResult { +function retry( + row: BookableRow, + deps: BookDependencies, + now: Date, + reason: string, + /** + * Whether THIS attempt reached the create. `row.sendAttempted` is the value + * read when the row was claimed, so it is stale the moment + * markSendAttempted runs — and a failure AFTER the create (the attachment + * leg, the Expense commit) would have been judged on it and wrongly + * released the key of a row that really does have a Purchase. + */ + sentThisAttempt = false, +): BookResult { + const sendAttempted = row.sendAttempted || sentThisAttempt; const attempts = row.attempts + 1; // `>=`, so MAX_BOOK_ATTEMPTS reads as "20 attempts in total" rather than 21. if (attempts >= MAX_BOOK_ATTEMPTS) { - // Keyed on the ROW's record of whether a send ever happened, not on the - // assumption that reaching the retry limit implies one. A row can - // exhaust its attempts entirely on storage faults, having never touched - // QuickBooks — and holding its key then quarantines the corrected - // resend against nothing. - return { outcome: "needs-review", reason: "max-retries", releaseStrongKey: !row.sendAttempted }; + // Keyed on whether a send ever happened, not on the assumption that + // reaching the retry limit implies one. A row can exhaust its attempts + // entirely on storage faults, having never touched QuickBooks — and + // holding its key then quarantines the corrected resend against nothing. + return { outcome: "needs-review", reason: "max-retries", releaseStrongKey: !sendAttempted }; } return { outcome: "retry", diff --git a/src/lib/receipt-intake/file-type.ts b/src/lib/receipt-intake/file-type.ts index 29f07c24d..414535b72 100644 --- a/src/lib/receipt-intake/file-type.ts +++ b/src/lib/receipt-intake/file-type.ts @@ -55,5 +55,16 @@ export function sniffMime(buf: Buffer, declared: string): string | null { // stored mimeType says what the file actually claims to be. if (HEIF_BRANDS.has(brand)) return "image/heif"; } - return essence === "text/plain" ? "text/plain" : null; + // text/plain is DELIBERATELY not accepted. + // + // QuickBooks cannot attach a .txt, so such a row read fine and then parked + // at booking with unsupported-attachment — stuck mid-pipeline, which is + // worse than a clear refusal at the door. v1 converted these to PDF using + // Apps Script's HTML->PDF `getAs`, which has no Node equivalent: a real + // port means a PDF generator with wrapping, pagination and WinAnsi encoding + // (pdf-lib's standard fonts THROW on characters they cannot encode). That + // is a new silent-corruption surface on a money document, for the rarest + // input in the pipeline. Refused instead — see the 415 in the intake route. + void essence; + return null; } diff --git a/src/lib/receipt-intake/intake-core.ts b/src/lib/receipt-intake/intake-core.ts index 37f55261c..472164509 100644 --- a/src/lib/receipt-intake/intake-core.ts +++ b/src/lib/receipt-intake/intake-core.ts @@ -15,6 +15,18 @@ import type { IntakeAuth } from "./intake-auth"; */ export const MAX_INLINE_UPLOAD_BYTES = 4 * 1024 * 1024; +/** + * The JSON path's raw-bytes ceiling, LOWER than the multipart one on purpose. + * + * A JSON body carries the file base64-encoded, which inflates it by 4/3. At the + * multipart limit of 4 MiB that is a ~5.4 MiB request — over the platform's + * body cap, so it died at the edge with an opaque 413 this code never saw and + * the caller learned nothing. 3 MiB raw encodes to ~4 MiB, which fits. + * + * Multipart sends the bytes as-is and keeps the full 4 MiB. + */ +export const MAX_INLINE_JSON_BYTES = 3 * 1024 * 1024; + /** The real ceiling for a stored receipt, enforced on the object itself. */ export const MAX_STORED_BYTES = 15 * 1024 * 1024; diff --git a/src/lib/receipt-intake/read.ts b/src/lib/receipt-intake/read.ts index 32ead3de2..592202550 100644 --- a/src/lib/receipt-intake/read.ts +++ b/src/lib/receipt-intake/read.ts @@ -196,7 +196,17 @@ export function normalizeDocType(value: unknown): string { * is a poor match" must stay distinguishable. */ export function normalizeConfidence(value: unknown): number | null { - const n = typeof value === "number" ? value : Number(coerce(value)); + if (typeof value === "number") { + return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : null; + } + // `Number("")` and `Number(" ")` are BOTH 0 — a real, maximally-unconfident + // reading — so coercing first turned "the model said nothing" into "the + // model is certain this phase is wrong". Those must stay distinguishable: + // the queue sorts by this, and 0 is a signal while null is an absence. + if (typeof value !== "string") return null; + const text = value.trim(); + if (!text) return null; + const n = Number(text); if (!Number.isFinite(n)) return null; return Math.min(1, Math.max(0, n)); } diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index e2c543f12..0161ea460 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -54,15 +54,32 @@ export async function sealObject( return null; } - // The upload path has served its purpose. A failure to remove it is an - // orphan, not a correctness problem — the row already points at the sealed - // copy — so it goes on the cleanup queue rather than failing the publish. - if (uploadPath !== canonicalPath) { - await deleteObjectOrRecord(uploadPath, "sealed"); - } + // NOTE: the upload object is deliberately NOT deleted here. + // + // Deleting before the row is committed is unrecoverable: if the UPDATE then + // fails, the row still points at a path whose object we just removed, and + // the receipt is gone with nothing left to retry from. The caller deletes + // only after the pointer is committed — see finalizeAndPublish. return canonicalPath; } +/** + * Queue an object for deletion WITHOUT attempting one first. + * + * For the ambiguous case: an upload that errored may still have written bytes, + * and the row that points at them is about to be deleted. Recording the path + * before that happens is the only way the orphan stays findable. + */ +export async function recordPendingCleanup(storagePath: string, reason: string): Promise { + await logAutomationEvent({ + kind: STORAGE_CLEANUP_KIND, + status: "pending", + reason, + source: "receipt-intake", + detail: { storagePath }, + }).catch(() => { /* audit only — the orphan is the lesser problem */ }); +} + /** * Delete the object. If that fails, record the path so the sweep can retry. * Never throws: the caller is already rejecting a row and must not be derailed diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index a979dc1d2..50ac64aff 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -43,6 +43,57 @@ export type VerifiedBytes = | { ok: true; bytes: Buffer } | { ok: false; kind: "missing" | "transient" | "sha-mismatch"; message?: string }; +/** + * SEAL AND PUBLISH — the one operation that moves a row out of STAGING. + * + * Shared by /intake/{id}/finalize and the worker's stale-STAGING sweep so the + * two cannot diverge: the sweeper used to publish while the row still pointed + * at the UPLOAD path, which is writable by anyone holding the signed URL, so a + * swept row's "verified" bytes stayed replaceable afterwards. + * + * Order is the whole point: + * 1. copy the verified bytes to the canonical (content-addressed) path + * 2. COMMIT the row pointer, fenced on state and claim + * 3. only then delete the upload object, best-effort + * + * A crash between 1 and 2 leaves both objects and a STAGING row: the retry + * finds the canonical copy already there, re-uploads it as a no-op, and + * commits. A failure at 3 is an orphan on the cleanup queue, not a lost + * receipt. + */ +export interface PublishOutcome { + published: boolean; + canonicalPath: string; +} + +export interface SealPublishDeps { + seal: (uploadPath: string, canonicalPath: string, bytes: Buffer, contentType: string) => Promise; + /** Fenced CAS. Returns the number of rows actually moved. */ + commit: (canonicalPath: string, check: { mimeType: string; fileSize: number; fileSha256: string }) => Promise; + /** Best-effort, AFTER the commit. Records a cleanup event on failure. */ + dropUpload: (uploadPath: string) => Promise; +} + +export async function sealAndPublish( + uploadPath: string, + rowId: string, + check: { mimeType: string; fileSize: number; fileSha256: string; bytes: Buffer }, + deps: SealPublishDeps, +): Promise { + const canonicalPath = canonicalStoragePath(rowId, check.fileSha256, check.mimeType); + const sealed = await deps.seal(uploadPath, canonicalPath, check.bytes, check.mimeType); + if (!sealed) return null; + + const moved = await deps.commit(canonicalPath, check); + // Only once the row points at the sealed copy is the upload object safe to + // remove — and only if we are the one who moved the row. A publisher that + // lost the CAS must not delete an object the winner may still be using. + if (moved > 0 && uploadPath !== canonicalPath) { + await deps.dropUpload(uploadPath); + } + return { published: moved > 0, canonicalPath }; +} + export async function downloadVerified( storagePath: string, expectedSha256: string, diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 2165ee6fb..18899cb64 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -147,10 +147,15 @@ export interface WorkerDependencies { * inside the same transaction as the transition. Returns the conflicting * row when another document with this weak key is already BOOKING/BOOKED. */ - promoteToBooking: (rowId: string, weakKey: string | null) => Promise<{ promoted: boolean; conflictId?: string }>; + promoteToBooking: ( + rowId: string, + weakKey: string | null, + claimToken: string | null, + ) => Promise<{ promoted: boolean; conflictId?: string; stale?: boolean }>; /** The pass's ONE absolute deadline — never a snapshot of "time left". */ book: (row: BookableRow) => Promise; - applyBookResult: (rowId: string, result: BookResult) => Promise; + /** CAS'd on the claim: a superseded worker's result must write nothing. */ + applyBookResult: (rowId: string, result: BookResult, claimToken: string | null) => Promise; /** AI unavailable: park for a later pass WITHOUT spending an attempt. */ deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; /** A transient fault anywhere else: spend an attempt and back off. */ @@ -393,7 +398,13 @@ export async function runIntakeWorker(deps: WorkerDependencies): Promise = {}): BookableRow { lastError: null, sendAttempted: false, fileSha256: "s".repeat(64), + claimToken: "claim-1", ...overrides, }; } @@ -88,6 +90,7 @@ function recorder(overrides: Partial = {}, opts: { estimates?: }, receiptIntake: { update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, + updateMany: async (args: any) => { intakeUpdates.push(args.data); return { count: 1 }; }, }, $transaction: async (fn: any) => fn(tx), }; @@ -106,7 +109,7 @@ function recorder(overrides: Partial = {}, opts: { estimates?: now: () => NOW, companyTimeZone: async () => "America/Los_Angeles", isCostCodeAllowed: async () => true, - markSendAttempted: async id => { sendMarks.push(id); }, + markSendAttempted: async id => { sendMarks.push(id); return true; }, ...overrides, }; return { deps, sendMarks, purchaseCalls, expenses, intakeUpdates, events }; @@ -722,9 +725,141 @@ test("a real create DOES mark it, before the call", async () => { r.purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "attached" } as any; }, - markSendAttempted: async id => { order.push("mark"); r.sendMarks.push(id); }, + markSendAttempted: async id => { order.push("mark"); r.sendMarks.push(id); return true; }, }); await bookReceipt(row(), r.deps); assert.deepEqual(order, ["mark", "create"], "marked FIRST, so a mid-create death still records it"); assert.deepEqual(r.sendMarks, ["intake-1"]); }); + +// ── An attachment QBO REFUSED is terminal (round-9 item 5) ───────────────── + +test("a 4xx or fault attachment failure goes to a human on the FIRST one", async () => { + // Retrying a file QuickBooks refused changes nothing except how long the + // Purchase sits in the books without its receipt. + for (const attachment of ["failed:400", "failed:413", "failed:415", "failed:fault"]) { + const r = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment, + }) as any, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "needs-review", attachment); + assert.equal((result as any).reason, `attachment-refused:${attachment}`); + // The Purchase EXISTS, so the key is retained either way. + assert.equal((result as any).releaseStrongKey, false, attachment); + assert.equal(r.expenses.length, 0, attachment); + } +}); + +test("a 5xx or thrown attachment failure is still retried", async () => { + for (const attachment of ["failed:500", "failed:502", "failed:AbortError", "failed:TypeError"]) { + const r = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment, + }) as any, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "retry", attachment); + } +}); + +test("isTerminalAttachmentFailure splits refusal from blip", () => { + for (const t of ["failed:400", "failed:404", "failed:413", "failed:499", "failed:fault"]) { + assert.equal(isTerminalAttachmentFailure(t), true, t); + } + for (const t of ["failed:500", "failed:503", "failed:AbortError", "failed:unknown"]) { + assert.equal(isTerminalAttachmentFailure(t), false, t); + } +}); + +// ── retry() must read the CURRENT send flag (round-9 item 2) ─────────────── + +test("attempt 20 RETAINS the key when the failure was at the create", async () => { + const r = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); + assert.equal((result as any).reason, "max-retries"); + // row.sendAttempted was false when the row was CLAIMED, but this attempt + // reached the create — so QBO may hold a Purchase and the key must stay. + assert.equal((result as any).releaseStrongKey, false, "the CURRENT send flag decides"); + assert.deepEqual(r.sendMarks, ["intake-1"]); +}); + +test("attempt 20 RETAINS the key when the failure was at the attachment leg", async () => { + const r = recorder({ + createPurchase: async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "failed:500", + }) as any, + }); + const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); + assert.equal((result as any).reason, "max-retries"); + assert.equal((result as any).releaseStrongKey, false, "the Purchase exists"); +}); + +test("attempt 20 RETAINS the key when the POST-create DB write failed", async () => { + const r = recorder(); + (r.deps.db as any).$transaction = async () => { throw new Error("connection reset"); }; + const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); + assert.equal((result as any).reason, "max-retries"); + assert.equal((result as any).releaseStrongKey, false, "the Purchase exists even though the row does not know"); +}); + +test("attempt 20 RELEASES the key only when nothing was ever sent", async () => { + const r = recorder({ + downloadBytes: async () => ({ ok: false, kind: "transient", message: "ECONNRESET" }), + }); + const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); + assert.equal((result as any).reason, "max-retries"); + assert.equal((result as any).releaseStrongKey, true); + assert.deepEqual(r.sendMarks, [], "never reached the create"); +}); + +// ── A superseded worker sends nothing (Phase 2 gate) ────────────────────── + +test("a STALE claim aborts BEFORE the QBO create", async () => { + // The zombie case: an invocation killed mid-booking resumes after its row + // has been re-claimed. markSendAttempted is a CAS on the claim, so it + // affects zero rows — and the booking stops THERE, having sent nothing. + // Posting a Purchase the live worker is also about to post is the + // double-booking this whole mechanism exists to prevent. + const r = recorder({ markSendAttempted: async () => false }); + const result = await bookReceipt(row(), r.deps); + + assert.deepEqual(result, { outcome: "stale" }); + assert.equal(r.purchaseCalls.length, 0, "QuickBooks is never called"); + assert.equal(r.expenses.length, 0, "no Expense"); + assert.deepEqual(r.intakeUpdates, [], "and no state write at all"); +}); + +test("the send fence receives the row's OWN claim token", async () => { + const seen: Array = []; + const r = recorder({ + markSendAttempted: async (_id, token) => { seen.push(token); return true; }, + }); + await bookReceipt(row({ claimToken: "token-xyz" }), r.deps); + assert.deepEqual(seen, ["token-xyz"]); +}); + +test("losing the claim DURING the commit rolls back and reports stale", async () => { + // The window between the create and the commit. The Purchase exists, so + // the successor's retry hits alreadyExists and books it once, under one + // owner — but THIS worker must not complete a BOOKED write. + const r = recorder(); + (r.deps.db as any).receiptIntake.updateMany = async () => ({ count: 0 }); + const result = await bookReceipt(row(), r.deps); + + assert.deepEqual(result, { outcome: "stale" }); + assert.equal(r.purchaseCalls.length, 1, "the create did happen"); + assert.equal(r.events.length, 0, "but nothing is logged as booked"); +}); + +test("the BOOKED write is a CAS on state AND token", async () => { + const wheres: any[] = []; + const r = recorder(); + (r.deps.db as any).receiptIntake.updateMany = async (args: any) => { + wheres.push(args.where); + return { count: 1 }; + }; + await bookReceipt(row({ claimToken: "token-abc" }), r.deps); + assert.deepEqual(wheres, [{ id: "intake-1", state: "BOOKING", claimToken: "token-abc" }]); +}); diff --git a/tests/receipt-intake-read.test.ts b/tests/receipt-intake-read.test.ts index dbbbfa85b..a0566d60b 100644 --- a/tests/receipt-intake-read.test.ts +++ b/tests/receipt-intake-read.test.ts @@ -13,7 +13,7 @@ */ import test from "node:test"; import assert from "node:assert/strict"; -import { buildReadPrompt, parseReadJson, readReceipt } from "../src/lib/receipt-intake/read"; +import { buildReadPrompt, normalizeConfidence, parseReadJson, readReceipt } from "../src/lib/receipt-intake/read"; const PHASES = [ { code: "01-DEMO", name: "Demolition" }, @@ -274,3 +274,23 @@ test("a 4xx that is not 401/403/404/429 is still DECISIVE", async () => { assert.deepEqual(outcome, { ok: false, decisive: true }, `HTTP ${status}`); } }); + +test("an absent confidence is NULL, never 0", () => { + // `Number("")` and `Number(" ")` are both 0 — a real, maximally- + // unconfident reading. Coercing first turned "the model said nothing" into + // "the model is certain this phase is a poor match", and the queue sorts on + // exactly that number. + for (const empty of ["", " ", "\t", undefined, null, {}, [], "abc", NaN, Infinity]) { + assert.equal(normalizeConfidence(empty), null, JSON.stringify(empty)); + } + // A genuine zero survives as a zero. + assert.equal(normalizeConfidence(0), 0); + assert.equal(normalizeConfidence("0"), 0); + assert.equal(normalizeConfidence("0.0"), 0); + // Normal values, and clamping at the edges. + assert.equal(normalizeConfidence(0.82), 0.82); + assert.equal(normalizeConfidence("0.82"), 0.82); + assert.equal(normalizeConfidence(1.2), 1); + assert.equal(normalizeConfidence(-3), 0); + assert.equal(normalizeConfidence(" 0.5 "), 0.5, "whitespace around a real number is fine"); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 57f022e79..08141ee47 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -12,6 +12,7 @@ import { canonicalStoragePath, downloadVerified, inspectStoredObject, + sealAndPublish, } from "../src/lib/receipt-intake/stored-object"; import { MAX_STORED_BYTES } from "../src/lib/receipt-intake/intake-core"; import type { DocBytesResult } from "../src/lib/secure-storage"; @@ -31,19 +32,6 @@ test("a real image is accepted, and its metadata comes from the BYTES", async () assert.equal(check.fileSha256, createHash("sha256").update(PNG).digest("hex")); }); -test("text/plain is the ONE type taken on its declared word", async () => { - // It has no magic bytes. Same concession the single-shot path makes. - const txt = Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"); - const ok = await inspectStoredObject("p.txt", "text/plain", give({ ok: true, bytes: txt })); - assert.ok(ok.ok); - assert.equal(ok.mimeType, "text/plain"); - - // ...and without that declaration the same bytes are unidentifiable. - const bad = await inspectStoredObject("p.bin", "", give({ ok: true, bytes: txt })); - assert.equal(bad.ok, false); - assert.equal((bad as { reason: string }).reason, "unsupported-file-type"); -}); - test("oversize, empty and unidentifiable objects are REJECTED, not published", async () => { // The signed upload URL bypassed every check the server could otherwise // make, so these are enforced on the object itself. @@ -135,3 +123,71 @@ test("the validator hands back the exact bytes it verified", async () => { assert.ok(check.bytes.equals(PNG)); assert.equal(createHash("sha256").update(check.bytes).digest("hex"), check.fileSha256); }); + +// ── Seal order: copy, COMMIT, then delete (round-9 items 1 and 3) ────────── + +/** `sealOk`/`committed` drive the outcome; the call log is always recorded. */ +function publishHarness(opts: { sealOk?: boolean; committed?: number } = {}) { + const calls: string[] = []; + const deps = { + seal: async (_u: string, canonical: string) => { + calls.push("seal"); + return opts.sealOk === false ? null : canonical; + }, + commit: async () => { calls.push("commit"); return opts.committed ?? 1; }, + dropUpload: async () => { calls.push("drop"); }, + } as never; + return { calls, deps }; +} + +const CHECK = { + mimeType: "image/png", + fileSize: PNG.length, + fileSha256: createHash("sha256").update(PNG).digest("hex"), + bytes: PNG, +}; + +test("the upload object is deleted only AFTER the row pointer is committed", async () => { + // Deleting first is unrecoverable: if the UPDATE then fails, the row still + // points at a path whose object we just removed, and the receipt is gone + // with nothing left to retry from. + const h = publishHarness(); + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, h.deps); + assert.deepEqual(h.calls, ["seal", "commit", "drop"]); + assert.equal(outcome?.published, true); + assert.equal(outcome?.canonicalPath, `receipts/row-1/${CHECK.fileSha256}.png`); +}); + +test("a FAILED commit leaves the upload object alone, so the retry can recover", async () => { + const h = publishHarness({ committed: 0 }); + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, h.deps); + assert.deepEqual(h.calls, ["seal", "commit"], "nothing was deleted"); + assert.equal(outcome?.published, false); +}); + +test("a failed SEAL never touches the row or the upload object", async () => { + const h = publishHarness({ sealOk: false }); + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, h.deps); + assert.equal(outcome, null); + assert.deepEqual(h.calls, ["seal"]); +}); + +test("a retry that finds the canonical object already there still commits", async () => { + // The crash-between-copy-and-commit case: the copy is an upsert to a + // content-addressed path, so re-sealing the same bytes is a no-op and the + // retry simply commits. + const h = publishHarness(); + const first = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, h.deps); + const second = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, h.deps); + assert.equal(first?.canonicalPath, second?.canonicalPath, "same content, same path"); + assert.equal(second?.published, true); +}); + +test("text/plain is no longer accepted at all", async () => { + // QuickBooks cannot attach a .txt, so accepting one meant reading it and + // then stranding it unbookable — worse than refusing at the door. + const txt = Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"); + const check = await inspectStoredObject("p.txt", "text/plain", give({ ok: true, bytes: txt })); + assert.equal(check.ok, false); + assert.equal((check as { reason: string }).reason, "unsupported-file-type"); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 23e031b8d..401f06a76 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -969,3 +969,44 @@ test("finishRouting is handed the token the pass claimed with", async () => { await runIntakeWorker(h.deps); assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "token-abc", stateReason: null }]); }); + +// ── A successor reclaiming mid-flight (Phase 2 gate) ─────────────────────── + +test("a predecessor superseded before promotion writes nothing and books nothing", async () => { + const h = harness([workerRow({ state: "READ", dryRun: false, claimToken: "old-token" })], { + // The CAS finds no row at {id, state: READ, claimToken: old-token} + // because the successor re-claimed and re-stamped it. + promoteToBooking: async (id, _weak, token) => { + h.promoted.push(id); + assert.equal(token, "old-token", "the predecessor offers its OWN token"); + return { promoted: false, stale: true }; + }, + }); + const summary = await runIntakeWorker(h.deps); + + assert.deepEqual(summary.byState, { STALE: 1 }); + assert.equal(h.books, 0, "no QBO call"); + assert.deepEqual(h.states, [], "no state write"); +}); + +test("a stale booking result is never written back", async () => { + const applied: unknown[] = []; + const h = harness([workerRow({ state: "BOOKING", dryRun: false })], { + book: async () => { h.books++; return { outcome: "stale" } as BookResult; }, + applyBookResult: async (_id, result) => { applied.push(result); }, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { STALE: 1 }); + // applyBookResult is still CALLED — the adapter is what refuses to write — + // and the production adapter returns early on a stale outcome. + assert.deepEqual(applied, [{ outcome: "stale" }]); +}); + +test("every book result carries the row's claim token to the writer", async () => { + const tokens: Array = []; + const h = harness([workerRow({ state: "BOOKING", dryRun: false, claimToken: "tok-9" })], { + applyBookResult: async (_id, _result, token) => { tokens.push(token); }, + }); + await runIntakeWorker(h.deps); + assert.deepEqual(tokens, ["tok-9"]); +}); From 85d6af57704756d48d8a904e6a4199b3d679aa93 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 04:12:17 -0700 Subject: [PATCH 048/144] fix(receipts): authorize late fields, CAS every mutation, make cleanup durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 10, all six. 1 /finalize now authorizes before it writes: userCanAccessProject on a supplied projectId, and the cost code validated against THAT project (a code from another job rides into the Expense and reads as overspend on a line nobody budgeted). Late fields apply only where null-or-equal and only while the row is STAGING or RECEIVED — past that the dedup keys, the phase suggestion and possibly a Purchase were all derived from the project it had at the time, and changing it after only makes the row disagree with its own history. Exact-state CAS, and the response returns PERSISTED values rather than what was asked for. 2 The cleanup record is the LAST pointer to an object once its row is gone, so recordPendingCleanup now throws if it cannot write (and reads the event back — logAutomationEvent never throws, so "no error" was not proof). If it cannot be recorded the intake route KEEPS the row as the pointer instead of orphaning the bytes; a failed row delete is surfaced too. The cleanup worker refuses to delete a path any live ReceiptIntake still references — reachable via the recovery sequence — and resolves an event only after a confirmed deletion. removeSecureDocStrict makes a missing storage client an error rather than a silent success; it is a separate export because several callers outside this feature rely on the quiet one. 3 applyRead, applyState, deferRead and retryRow all CAS on {id, state, claimToken} and return whether they owned the row; zero rows aborts the pass for it as STALE. A time-based lease cannot tell a live worker from a zombie — both hold the same id. 4 One parkTerminal() decides every terminal park, releasing the strong key whenever the PERSISTED sendAttempted is false. file-missing, content-changed, unreadable, ai-unavailable and max-retries each decided this independently before, and the ones that forgot held a key against a Purchase that never existed. 5 A sha mismatch is recoverable: the sweeper waits for the 2h upload-URL expiry before parking, and /finalize re-inspects and seals a sha-mismatch row the same way it does file-missing. A partial upload while the URL is still valid is a retry in progress, not an error state. 6 /start refuses an unsupported MIME with 415 BEFORE creating a row, and text/plain is out of the two-step map entirely. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 92 ++++++++++++ package.json | 4 +- .../api/cron/receipt-intake-worker/route.ts | 52 ++++--- .../receipts/intake/[id]/finalize/route.ts | 123 +++++++++++++-- src/app/api/receipts/intake/route.ts | 33 ++++- src/app/api/receipts/intake/start/route.ts | 25 +++- src/lib/receipt-intake/file-type.ts | 7 +- src/lib/receipt-intake/storage-cleanup.ts | 61 ++++++-- src/lib/receipt-intake/worker.ts | 140 +++++++++++++----- src/lib/secure-storage.ts | 21 +++ tests/receipt-intake-cleanup.test.ts | 79 ++++++++++ tests/receipt-intake-stored-object.test.ts | 28 ++++ tests/receipt-intake-worker.test.ts | 115 ++++++++++++-- 13 files changed, 678 insertions(+), 102 deletions(-) create mode 100644 tests/receipt-intake-cleanup.test.ts diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 6c6d30efb..e539a35ab 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -1023,3 +1023,95 @@ test.describe("round-9 intake contracts", () => { expect(row?.fileSha256).toHaveLength(64); }); }); + +test.describe("round-10 finalize authorization and recovery", () => { + const startPath = `${INTAKE_PATH}/start`; + const finalize = (request: APIRequestContext, id: string, body: Record) => + request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify(body), + maxRedirects: 0, + }); + + test("a session caller cannot attach a project it may not reach", async ({ playwright, request }) => { + // Without this any authenticated user could file a receipt against any + // project by id. + const created = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}authz` })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + + const employee = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: "e2e/.auth/contract-user.json", + }); + const res = await employee.post(`${INTAKE_PATH}/${created.body.id}/finalize`, { + headers: { "content-type": "application/json" }, + data: JSON.stringify({ projectId: "e2e-scope-oos-project" }), + maxRedirects: 0, + }); + // Either refused outright (403) or invisible to this caller (404) — what + // must NOT happen is the project landing on the row. + expect([403, 404]).toContain(res.status()); + expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.projectId) + .not.toBe("e2e-scope-oos-project"); + await employee.dispose(); + }); + + test("a cost code that is not a phase of the job is refused", async ({ request }) => { + const created = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}phasecheck` })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + + const res = await finalize(request, created.body.id, { costCodeId: "e2e-mob-cc-demo" }); + // No project on the row and none supplied, so a phase is meaningless. + expect(res.status()).toBe(400); + expect((await res.json()).error).toBe("cost-code-without-project"); + }); + + test("late fields are refused once the row has been routed", async ({ request }) => { + // Past RECEIVED the dedup keys, the phase suggestion and possibly a + // booking were all derived from the project the row had at the time. + const created = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}toolate` })); + expect(created.res.status()).toBe(200); + const id = created.body.id; + minted.push(id); + + await prisma.receiptIntake.update({ where: { id }, data: { state: "BOOKED" } }); + const res = await finalize(request, id, { projectId: "e2e-scope-oos-project" }); + expect(res.status()).toBe(409); + expect((await res.json()).error).toBe("late-fields-too-late"); + expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.projectId).toBeNull(); + }); + + test("finalize returns the PERSISTED values, not what the caller asked for", async ({ request }) => { + const created = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}persisted` })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + + const res = await finalize(request, created.body.id, {}); + expect(res.status()).toBe(200); + const body = await res.json(); + const row = await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }); + expect(body.state).toBe(row?.state); + expect(body.fileSha256).toBe(row?.fileSha256); + expect(body.costCodeId).toBe(row?.costCodeId ?? null); + }); + + test("/start refuses an unsupported type with 415 and creates NO row", async ({ request }) => { + const ref = `${REF_PREFIX}start-415`; + const res = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "drive", sourceRef: ref, mimeType: "text/plain", sha256: "a".repeat(64), + }), + maxRedirects: 0, + }); + expect(res.status()).toBe(415); + const body = await res.json(); + expect(body.error).toBe("unsupported-file-type"); + expect(body.accepted).not.toContain("text/plain"); + // The row must not exist — a STAGING row for a document we will never + // accept is something the sweeper then has to reason about. + expect(await prisma.receiptIntake.findUnique({ where: { sourceRef: ref } })).toBeNull(); + }); +}); diff --git a/package.json b/package.json index 857e2ad23..d31275293 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index e6e5259ea..81b30d28a 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -321,6 +321,14 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // to be at that path — which is the same overwrite the seal // exists to close, arriving by a different door. if (row.expectedSha256 && row.expectedSha256 !== check.fileSha256) { + // RECOVERABLE, not a park. A partial or superseded + // upload sitting at the path while the signed URL is + // still valid is exactly the state a client is about to + // fix by finishing its upload. Parking it here would + // turn a retry-in-progress into a review item, and the + // correct bytes arriving a minute later would find the + // row already gone from STAGING. + if (row.createdAt.getTime() > Date.now() - SIGNED_UPLOAD_TTL_MS) continue; await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, data: { state: "NEEDS_REVIEW", stateReason: "sha-mismatch", nextRetryAt: null }, @@ -405,18 +413,19 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { read: (bytes, mime, phases) => readReceipt(bytes, mime, phases), - applyRead: async (rowId, patch: ReadPatch) => { + applyRead: async (rowId, patch: ReadPatch, ownership) => { try { - // nextRetryAt is deliberately UNTOUCHED: the claim lease must - // survive until routing finishes. Clearing it here let an - // overlapping invocation reclaim a half-routed row and book it - // while this one was still deciding — and then this one would - // regress it. finishRouting()/applyState() release the lease. - await prisma.receiptIntake.update({ - where: { id: rowId }, + // CAS on {id, state, claimToken}. nextRetryAt is deliberately + // UNTOUCHED: the claim lease must survive until routing + // finishes. Clearing it here let an overlapping invocation + // reclaim a half-routed row and book it while this one was + // still deciding — and then this one would regress it. + // finishRouting()/applyState() release the lease. + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: ownership.state, claimToken: ownership.claimToken }, data: { ...patch, lastError: null }, }); - return { strongOwner: null }; + return { strongOwner: null, owned: count > 0 }; } catch (error) { // The partial unique index refused the claim — the DATABASE is // the lock the Apps Script did with Script Properties. Load the @@ -439,6 +448,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // write; re-throw rather than reporting a dedup hit that isn't. if (!owner) throw error; return { + owned: true, strongOwner: { id: owner.id, totalCents: owner.totalCents, @@ -458,11 +468,15 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { orderBy: { createdAt: "asc" }, }), - applyState: async (rowId, state, stateReason, patch) => { - await prisma.receiptIntake.update({ - where: { id: rowId }, + applyState: async (rowId, state, stateReason, patch, ownership) => { + const { count } = await prisma.receiptIntake.updateMany({ + where: { + id: rowId, + ...(ownership ? { state: ownership.state, claimToken: ownership.claimToken } : {}), + }, data: { ...(patch ?? {}), state, stateReason, nextRetryAt: null }, }); + return count > 0; }, // RECEIVED -> READ, and the ONLY place the routing lease is released. @@ -647,26 +661,28 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { }); }, - deferRead: async (rowId, busyPasses, reason) => { + deferRead: async (rowId, busyPasses, reason, ownership) => { // The service was unavailable; the document was never read, so this // costs no `attempts` — only a delay and one busy pass. Reuses the // booking backoff table so one outage does not hammer Gemini from // every row at once. - await prisma.receiptIntake.update({ - where: { id: rowId }, + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: ownership.state, claimToken: ownership.claimToken }, data: { busyPasses, lastError: reason, nextRetryAt: new Date(Date.now() + backoffMs(1)), }, }); + return count > 0; }, - retryRow: async (rowId, attempts, nextRetryAt, reason) => { - await prisma.receiptIntake.update({ - where: { id: rowId }, + retryRow: async (rowId, attempts, nextRetryAt, reason, ownership) => { + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: rowId, state: ownership.state, claimToken: ownership.claimToken }, data: { attempts, lastError: reason, nextRetryAt }, }); + return count > 0; }, now: () => new Date(), diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index b81b06265..0fae23e7b 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -1,6 +1,9 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; -import { authenticateIntake, STAFF_READ_ROLES } from "@/lib/receipt-intake/intake-auth"; +import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; +import { userCanAccessProject } from "@/lib/mobile-auth"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; import { deleteObjectOrRecord, sealObject } from "@/lib/receipt-intake/storage-cleanup"; @@ -13,6 +16,9 @@ export const dynamic = "force-dynamic"; * Returns a 409 response on conflict, or null when the row is now consistent * with what the caller sent. */ +/** Late fields may only land while a row is still un-routed. */ +const LATE_FIELD_STATES = ["STAGING", "RECEIVED"]; + async function reconcileLateFields( id: string, lateFields: Partial<{ costCodeId: string; projectId: string }>, @@ -22,10 +28,31 @@ async function reconcileLateFields( const current = await prisma.receiptIntake.findUnique({ where: { id }, - select: { costCodeId: true, projectId: true }, + select: { costCodeId: true, projectId: true, state: true }, }); if (!current) return null; + // NULL-OR-EQUAL only, and only BEFORE the row is routed. + // + // Past RECEIVED the read has already happened: the dedup keys, the phase + // suggestion and possibly a booking were all computed from the project this + // row had at the time. Changing it afterwards does not re-derive any of + // that — it just makes the row disagree with its own history, and after + // BOOKED it disagrees with a Purchase in the real books. + if (!LATE_FIELD_STATES.includes(current.state)) { + const differs = entries.some(([key, value]) => current[key] !== value); + if (!differs) return null; // already exactly what the caller is asking for + return NextResponse.json( + { + ok: false, + error: "late-fields-too-late", + reason: `this row is ${current.state}; its job and phase were already used to route it`, + state: current.state, + }, + { status: 409 }, + ); + } + const conflicts = entries.filter(([key, value]) => current[key] !== null && current[key] !== value); if (conflicts.length > 0) { return NextResponse.json( @@ -43,15 +70,69 @@ async function reconcileLateFields( const toApply = Object.fromEntries(entries.filter(([key]) => current[key] === null)); if (Object.keys(toApply).length > 0) { - // Conditional on still being null, so a concurrent writer that set it - // between the read and here wins rather than being overwritten. + // EXACT-state CAS, and still conditional on each field being null: a + // concurrent writer that set it — or moved the row on — wins rather + // than being overwritten. await prisma.receiptIntake.updateMany({ - where: { id, ...Object.fromEntries(Object.keys(toApply).map(key => [key, null])) }, + where: { + id, + state: current.state, + ...Object.fromEntries(Object.keys(toApply).map(key => [key, null])), + }, data: toApply, }); } return null; } + +/** + * A caller may only attach a job it can actually reach, and only a phase that + * belongs to that job. + * + * Without the first check any authenticated user could file a receipt against + * any project by id. Without the second, a cost code from another job rides + * into the Expense and every variance report reads it as overspend on a line + * nobody budgeted. + */ +async function authorizeLateFields( + auth: Extract, + rowProjectId: string | null, + lateFields: Partial<{ costCodeId: string; projectId: string }>, +): Promise { + const projectId = lateFields.projectId ?? rowProjectId; + + if (lateFields.projectId && auth.via === "session") { + if (!(await userCanAccessProject(auth.user, lateFields.projectId))) { + return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + } + } + + if (lateFields.costCodeId) { + if (!projectId) { + return NextResponse.json( + { ok: false, error: "cost-code-without-project", reason: "a phase is only meaningful against a job" }, + { status: 400 }, + ); + } + const allowed = await isCostCodeAllowedForProject( + prismaPhaseDataSource, + projectId, + lateFields.costCodeId, + ); + if (!allowed) { + return NextResponse.json( + { + ok: false, + error: "cost-code-not-a-phase", + reason: "that cost code is not a phase of this job", + projectId, + }, + { status: 400 }, + ); + } + } + return null; +} export const maxDuration = 30; /** @@ -114,13 +195,21 @@ export async function POST(req: Request, context: { params: Promise<{ id: string STAFF_READ_ROLES.includes(auth.user.role); if (!maySee) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + // Authorize the late fields BEFORE anything is written or published. + const denied = await authorizeLateFields(auth, row.projectId, lateFields); + if (denied) return denied; + // A LATE finalize on a row the sweeper already parked file-missing is a // RECOVERY, not a duplicate: the upload landed after the sweep looked. It // must re-validate and republish rather than report alreadyFinalized, which // would leave a real receipt parked forever while telling the caller it was // fine. + // Both sweeper parks are recoverable by a later, correct upload: the bytes + // arriving after the sweep looked is the normal shape of a slow client, not + // an error state a human should have to clear. const recoverable = row.state === "STAGING" - || (row.state === "NEEDS_REVIEW" && row.stateReason === "file-missing"); + || (row.state === "NEEDS_REVIEW" + && (row.stateReason === "file-missing" || row.stateReason === "sha-mismatch")); // Idempotent: finalizing an already-published row is a success, not an error // — the client's retry after a lost response must not look like a failure. @@ -133,10 +222,16 @@ export async function POST(req: Request, context: { params: Promise<{ id: string if (!recoverable) { const conflict = await reconcileLateFields(id, lateFields); if (conflict) return conflict; - return NextResponse.json({ - ok: true, alreadyFinalized: true, id: row.id, state: row.state, - sourceRef: row.sourceRef, projectId: row.projectId, dryRun: row.dryRun, + // PERSISTED values, re-read after the reconcile — the caller must be + // told what the row actually holds, not what it asked for. + const persisted = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { + state: true, sourceRef: true, projectId: true, costCodeId: true, + dryRun: true, fileSize: true, fileSha256: true, + }, }); + return NextResponse.json({ ok: true, alreadyFinalized: true, id, ...(persisted ?? {}) }); } // ONE validator, shared with the worker's stale-STAGING sweep — see @@ -231,8 +326,12 @@ export async function POST(req: Request, context: { params: Promise<{ id: string return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: current?.state ?? "RECEIVED" }); } - return NextResponse.json({ - ok: true, id, state: "RECEIVED", sourceRef: row.sourceRef, - projectId: row.projectId, dryRun: row.dryRun, fileSize, + const persisted = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { + state: true, sourceRef: true, projectId: true, costCodeId: true, + dryRun: true, fileSize: true, fileSha256: true, + }, }); + return NextResponse.json({ ok: true, id, ...(persisted ?? { state: "RECEIVED", fileSize }) }); } diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 43e08f0bb..826fe9251 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -7,7 +7,7 @@ import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; import { recordPendingCleanup } from "@/lib/receipt-intake/storage-cleanup"; -import { EXT_BY_MIME, sniffMime } from "@/lib/receipt-intake/file-type"; +import { ACCEPTED_MIME_TYPES, EXT_BY_MIME, sniffMime } from "@/lib/receipt-intake/file-type"; import { MAX_INLINE_JSON_BYTES, MAX_INLINE_UPLOAD_BYTES, @@ -98,7 +98,7 @@ function unsupportedType(declared: string) { reason: essence === "text/plain" ? "text receipts are not accepted: QuickBooks cannot attach a .txt, so it would be read and then stranded unbookable. Print or export it to PDF first." : "the stored bytes are not a format QuickBooks can attach", - accepted: ["application/pdf", "image/jpeg", "image/png", "image/heic", "image/webp", "image/gif"], + accepted: ACCEPTED_MIME_TYPES, }, { status: 415 }, ); @@ -342,8 +342,33 @@ export async function POST(req: Request) { // // A no-op cleanup for an upload that genuinely never landed is free; // the sweeper's delete simply finds nothing. - await recordPendingCleanup(storagePath, `upload-ambiguous:${uploadFailed}`.slice(0, 200)); - await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); + try { + await recordPendingCleanup(storagePath, `upload-ambiguous:${uploadFailed}`.slice(0, 200)); + } catch { + // The record is the only thing that would remember this object. If + // it cannot be written, KEEP THE ROW: a STAGING row pointing at the + // path is the remaining way to find the bytes, and the sweeper will + // resolve it. Deleting now would orphan them with nothing left + // referencing them anywhere. + console.error("[receipts/intake] cleanup unrecordable; keeping the row as the pointer", storagePath); + return NextResponse.json( + { ok: false, reason: "storage-failed", id, retained: true }, + { status: 503 }, + ); + } + + // Row deletion failure is SURFACED, not swallowed: the caller's retry + // would otherwise hit a sourceRef conflict against a row it was told + // did not exist. + try { + await prisma.receiptIntake.delete({ where: { id } }); + } catch (deleteError) { + console.error("[receipts/intake] row delete failed after an ambiguous upload", id, deleteError); + return NextResponse.json( + { ok: false, reason: "storage-failed", id, retained: true }, + { status: 503 }, + ); + } console.error("[receipts/intake] upload failed", uploadFailed); return NextResponse.json({ ok: false, reason: "storage-failed" }, { status: 503 }); } diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index a47aadda7..b18b966f9 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -6,7 +6,7 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { SECURE_BUCKET } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; -import { EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; +import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; export const dynamic = "force-dynamic"; @@ -41,11 +41,28 @@ export async function POST(req: Request) { } const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); + // REFUSED BEFORE A ROW EXISTS. A 400 after creating the row left a STAGING + // row for a document we will never accept, which the sweeper then has to + // reason about. 415, not 400: the request is well-formed, the format is + // simply one QuickBooks cannot attach. + // + // The declared mime only picks the extension; /finalize re-derives the real + // type from the STORED BYTES, so a lie costs the caller its upload. const mimeType = String(body.mimeType ?? "").split(";")[0].trim().toLowerCase(); const ext = EXT_BY_MIME[mimeType]; - // The declared mime only picks the extension here; /finalize re-derives the - // real type from the STORED BYTES, so a lie costs the caller its upload. - if (!ext) return NextResponse.json({ ok: false, reason: "unsupported-file-type" }, { status: 400 }); + if (!ext) { + return NextResponse.json( + { + ok: false, + error: "unsupported-file-type", + reason: mimeType === "text/plain" + ? "text receipts are not accepted: QuickBooks cannot attach a .txt. Print or export it to PDF first." + : "that format is not one QuickBooks can attach", + accepted: ACCEPTED_MIME_TYPES, + }, + { status: 415 }, + ); + } // The client's own hash of what it is ABOUT to upload. Persisted, because // the two-step flow hands the bytes straight to storage: without it a diff --git a/src/lib/receipt-intake/file-type.ts b/src/lib/receipt-intake/file-type.ts index 414535b72..7d3eca515 100644 --- a/src/lib/receipt-intake/file-type.ts +++ b/src/lib/receipt-intake/file-type.ts @@ -18,9 +18,14 @@ export const EXT_BY_MIME: Record = { "image/heif": "heic", "image/webp": "webp", "image/gif": "gif", - "text/plain": "txt", + // text/plain is deliberately absent: QuickBooks cannot attach a .txt, so a + // row created for one would read fine and then strand unbookable. Both + // intake paths refuse it — see the 415 in the route and /start. }; +/** The formats a caller may be told to send. Single source for the 415 body. */ +export const ACCEPTED_MIME_TYPES = Object.keys(EXT_BY_MIME); + export const MAX_INTAKE_BYTES = 15 * 1024 * 1024; /** ISO-BMFF major brands stored as image/heic (still + HEVC sequence brands). */ diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index 0161ea460..e245be749 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -14,7 +14,7 @@ */ import { logAutomationEvent } from "@/lib/automation-events"; import { prisma } from "@/lib/prisma"; -import { SECURE_BUCKET, removeSecureDoc, toSecureRef } from "@/lib/secure-storage"; +import { SECURE_BUCKET, removeSecureDocStrict, toSecureRef } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; export const STORAGE_CLEANUP_KIND = "storage-cleanup-pending"; @@ -71,13 +71,24 @@ export async function sealObject( * before that happens is the only way the orphan stays findable. */ export async function recordPendingCleanup(storagePath: string, reason: string): Promise { + // THROWS on failure, unlike an audit write. This record is not an audit + // trail — it is the ONLY thing that will remember the object once the row + // pointing at it is gone. Swallowing the failure loses the orphan silently + // and forever, so the caller must know and keep the row instead. await logAutomationEvent({ kind: STORAGE_CLEANUP_KIND, status: "pending", reason, source: "receipt-intake", detail: { storagePath }, - }).catch(() => { /* audit only — the orphan is the lesser problem */ }); + }); + const recorded = await prisma.automationEvent.findFirst({ + where: { kind: STORAGE_CLEANUP_KIND, status: "pending", detail: { contains: storagePath } }, + select: { id: true }, + }); + // logAutomationEvent is fire-and-forget by contract, so "it did not throw" + // is not proof it wrote. Read it back. + if (!recorded) throw new Error(`could not record a cleanup for ${storagePath}`); } /** @@ -87,17 +98,15 @@ export async function recordPendingCleanup(storagePath: string, reason: string): */ export async function deleteObjectOrRecord(storagePath: string, reason: string): Promise { try { - await removeSecureDoc(toSecureRef(storagePath)); + await removeSecureDocStrict(toSecureRef(storagePath)); return true; } catch (error) { console.error("[receipts/intake] object delete failed", storagePath, error instanceof Error ? error.name : "error"); - await logAutomationEvent({ - kind: STORAGE_CLEANUP_KIND, - status: "pending", - reason, - source: "receipt-intake", - detail: { storagePath }, - }).catch(() => { /* audit only — the orphan is the lesser problem */ }); + await recordPendingCleanup(storagePath, reason).catch(recordError => { + // Both the delete AND the record failed. Say so loudly: this is the + // one combination that loses an object with nothing left to find it. + console.error("[receipts/intake] ORPHANED OBJECT, no cleanup recorded", storagePath, recordError); + }); return false; } } @@ -130,13 +139,39 @@ export async function retryPendingCleanups(limit: number, shouldStop: () => bool await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "abandoned" } }); continue; } + + // NEVER delete a path a LIVE row still points at. + // + // The recovery sequence makes this reachable: an ambiguous upload + // records a cleanup, the row is deleted, the caller retries, and the + // retry's row can end up pointing at the same path — or a seal can + // publish a canonical path that an older pending event names. Deleting + // then destroys a receipt that is in active use. The event is resolved + // rather than retried forever: the object is accounted for, just not by + // us. + const referenced = await prisma.receiptIntake.findFirst({ + where: { storagePath }, + select: { id: true }, + }); + if (referenced) { + await prisma.automationEvent.update({ + where: { id: event.id }, + data: { status: "resolved", reason: `still referenced by ${referenced.id}` }, + }); + continue; + } + try { - await removeSecureDoc(toSecureRef(storagePath)); - await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "resolved" } }); - cleared++; + await removeSecureDocStrict(toSecureRef(storagePath)); } catch { // Still failing. Leave it pending for the next pass. + continue; } + // Resolved ONLY after a delete that did not throw — removeSecureDoc + // surfaces a missing storage client as an error rather than a success, + // so a misconfigured deployment cannot quietly mark the queue clean. + await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "resolved" } }); + cleared++; } return cleared; } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 18899cb64..e9ed686ee 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -138,10 +138,25 @@ export interface WorkerDependencies { * Persist the read + routing. Returns the strong-key owner when the partial * unique index rejected our claim — that rejection IS the dedup hit. */ - applyRead: (rowId: string, patch: ReadPatch) => Promise<{ strongOwner: StrongOwner | null }>; + /** + * CAS'd on {id, state, claimToken} like every other mutation. `owned:false` + * means this worker lost the row mid-pass and must abort — writing on would + * clobber whatever its successor has since decided. + */ + applyRead: ( + rowId: string, + patch: ReadPatch, + ownership: Ownership, + ) => Promise<{ strongOwner: StrongOwner | null; owned: boolean }>; findWeakHit: (rowId: string, weakKey: string) => Promise<{ id: string } | null>; /** Marks a row NEEDS_REVIEW / NON_RECEIPT / whatever routing decided, with no keys claimed. */ - applyState: (rowId: string, state: ReceiptIntakeState, stateReason: string | null, patch?: Partial) => Promise; + applyState: ( + rowId: string, + state: ReceiptIntakeState, + stateReason: string | null, + patch?: Partial, + ownership?: Ownership, + ) => Promise; /** * READ + dryRun=false -> BOOKING, and the LAST weak-dedup check, taken * inside the same transaction as the transition. Returns the conflicting @@ -157,9 +172,15 @@ export interface WorkerDependencies { /** CAS'd on the claim: a superseded worker's result must write nothing. */ applyBookResult: (rowId: string, result: BookResult, claimToken: string | null) => Promise; /** AI unavailable: park for a later pass WITHOUT spending an attempt. */ - deferRead: (rowId: string, busyPasses: number, reason: string) => Promise; + deferRead: (rowId: string, busyPasses: number, reason: string, ownership: Ownership) => Promise; /** A transient fault anywhere else: spend an attempt and back off. */ - retryRow: (rowId: string, attempts: number, nextRetryAt: Date, reason: string) => Promise; + retryRow: ( + rowId: string, + attempts: number, + nextRetryAt: Date, + reason: string, + ownership: Ownership, + ) => Promise; /** * RECEIVED -> READ, the release of the claim lease, AND the release of the * claim token. Called ONCE, after every dedup net has answered — never @@ -179,6 +200,20 @@ export interface WorkerDependencies { companyTimeZone: () => Promise; } +/** + * What a write must still be true of to be allowed. + * + * Every worker mutation is a CAS on this. `nextRetryAt` alone is a time-based + * lease and cannot distinguish a live worker from a zombie whose invocation was + * killed and whose row has since been re-claimed — both hold the same row id + * and both believe they own it. Zero rows affected means ownership was lost; + * the caller aborts rather than overwriting the successor's decisions. + */ +export interface Ownership { + state: string; + claimToken: string | null; +} + export interface StrongOwner { id: string; totalCents: number | null; @@ -460,33 +495,24 @@ export async function handleRowError( const message = error instanceof Error ? `${error.name}: ${error.message}` : "UnknownError"; if (isTerminalQboFault(error)) { - await deps.applyState(row.id, "NEEDS_REVIEW", `qbo-fault:${message}`.slice(0, 400)).catch(() => {}); - return "NEEDS_REVIEW"; + // A CLASSIFIED QBO fault means the send happened, so parkTerminal will + // (correctly) keep the key — the decision is still made in one place. + return parkTerminal(row, deps, `qbo-fault:${message}`.slice(0, 400)); } const attempts = row.attempts + 1; if (attempts >= MAX_BOOK_ATTEMPTS) { - // Same rule as booking's own ceiling: a row that exhausted its attempts - // without ever reaching QuickBooks (a weak-lookup fault, a - // finishRouting fault, a storage outage) created no Purchase, so its - // strong key must go back or a corrected resend collides with it. - await deps - .applyState( - row.id, - "NEEDS_REVIEW", - "max-retries", - row.sendAttempted ? undefined : { dedupStrongKey: null }, - ) - .catch(() => {}); - return "NEEDS_REVIEW"; + // Same rule as every other terminal park, applied in the same place. + return parkTerminal(row, deps, "max-retries"); } - await deps.retryRow( + const ownedRetry = await deps.retryRow( row.id, attempts, new Date(deps.now().getTime() + backoffMs(attempts)), `worker-error:${message}`.slice(0, 400), - ).catch(() => {}); - return "RETRY"; + ownershipOf(row), + ).catch(() => false); + return ownedRetry ? "RETRY" : "STALE"; } /** QBTimeoutError is deliberately NOT here — a timeout is transport, not a verdict. */ @@ -517,15 +543,13 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // parked good receipts as file-missing, permanently, for a human to // untangle. Only an AFFIRMATIVE not-found is terminal. if (download.kind === "missing") { - await deps.applyState(row.id, "NEEDS_REVIEW", "file-missing"); - return "NEEDS_REVIEW"; + return parkTerminal(row, deps, "file-missing"); } // The stored bytes are not the ones this row was published with. // Terminal, and loud: it means the object was replaced after // verification, which is the exact thing sealing exists to prevent. if (download.kind === "sha-mismatch") { - await deps.applyState(row.id, "NEEDS_REVIEW", "content-changed"); - return "NEEDS_REVIEW"; + return parkTerminal(row, deps, "content-changed"); } return retryTransient(row, deps, `storage:${download.message}`); } @@ -538,8 +562,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis if (!outcome.ok) { // decisive: the model answered and still could not read it -> a human. if (outcome.decisive) { - await deps.applyState(row.id, "NEEDS_REVIEW", "unreadable"); - return "NEEDS_REVIEW"; + return parkTerminal(row, deps, "unreadable"); } // The SERVICE was unavailable. That is never the document's fault, so // it costs no `attempts` — but it cannot be free forever either, or an @@ -547,11 +570,10 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // counts the busy passes separately and gives up after 20. const busyPasses = row.busyPasses + 1; if (busyPasses >= MAX_BUSY_PASSES) { - await deps.applyState(row.id, "NEEDS_REVIEW", "ai-unavailable"); - return "NEEDS_REVIEW"; + return parkTerminal(row, deps, "ai-unavailable"); } - await deps.deferRead(row.id, busyPasses, "ai-unavailable"); - return "RECEIVED"; + const owned = await deps.deferRead(row.id, busyPasses, "ai-unavailable", ownershipOf(row)); + return owned ? "RECEIVED" : "STALE"; } const read = outcome.read; @@ -647,14 +669,14 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // A multi-doc, a non-receipt, or a $0/negative misread must never hold // a dedup key — it would quarantine the real receipt that arrives next // (:531 and the v3.6 rationale). - await deps.applyRead(row.id, { + const gated = await deps.applyRead(row.id, { ...base, state: gate.state, stateReason: note(gate.stateReason), dedupStrongKey: null, duplicateOfId: gate.duplicateOfId, - }); - return gate.state; + }, ownershipOf(row)); + return gated.owned ? gate.state : "STALE"; } // The strong claim IS the partial unique index: a rejection is the hit. @@ -676,7 +698,10 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis stateReason: note(null), dedupStrongKey: keys.strong, duplicateOfId: null, - }); + }, ownershipOf(row)); + // Lost the row mid-read. Everything after this — the strong claim, the weak + // net, the publish — would be decided on a view the successor has moved past. + if (!applied.owned) return "STALE"; if (applied.strongOwner) { const second = routeState(routeInput, { strong: applied.strongOwner, weak: null }, !!row.projectId); @@ -720,6 +745,39 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis return "READ"; } +/** + * THE one place a row is parked terminally, and the one place the strong-key + * release is decided. + * + * The rule is a property of the ROW, not of the reason string: if no QBO send + * was ever attempted, no Purchase can exist, so the dedup key must go back or a + * corrected resubmission collides with a row that never became a purchase. That + * was previously re-derived at each call site, and the branches that forgot it + * (file-missing, unreadable, ai-unavailable, worker-error) each held a key + * against nothing. + * + * `sendAttempted` is the PERSISTED flag — markSendAttempted writes it before + * the create precisely so this decision survives a process that died mid-send. + */ +async function parkTerminal( + row: WorkerRow, + deps: WorkerDependencies, + reason: string, + patch?: Partial, +): Promise { + const release = row.sendAttempted ? {} : { dedupStrongKey: null }; + const owned = await deps + .applyState(row.id, "NEEDS_REVIEW", reason, { ...(patch ?? {}), ...release }, ownershipOf(row)) + .catch(() => false); + // Zero rows means a successor owns this row now; its state is theirs to set. + return owned ? "NEEDS_REVIEW" : "STALE"; +} + +/** The row as this pass claimed it — what every CAS matches on. */ +export function ownershipOf(row: WorkerRow): Ownership { + return { state: row.state, claimToken: row.claimToken }; +} + /** A transport-class fault during a row's processing: spend an attempt, back off. */ async function retryTransient(row: WorkerRow, deps: WorkerDependencies, reason: string): Promise { const attempts = row.attempts + 1; @@ -727,8 +785,14 @@ async function retryTransient(row: WorkerRow, deps: WorkerDependencies, reason: await deps.applyState(row.id, "NEEDS_REVIEW", "max-retries"); return "NEEDS_REVIEW"; } - await deps.retryRow(row.id, attempts, new Date(deps.now().getTime() + backoffMs(attempts)), reason); - return "RETRY"; + const owned = await deps.retryRow( + row.id, + attempts, + new Date(deps.now().getTime() + backoffMs(attempts)), + reason, + ownershipOf(row), + ); + return owned ? "RETRY" : "STALE"; } /** diff --git a/src/lib/secure-storage.ts b/src/lib/secure-storage.ts index 1a6badfc2..2b6ce2428 100644 --- a/src/lib/secure-storage.ts +++ b/src/lib/secure-storage.ts @@ -367,6 +367,27 @@ export async function uploadSecureDoc( } /** Best-effort removal of a secure object, for compensating a failed DB write. */ +/** + * Delete, and REFUSE to report success on anything less than a confirmed one. + * + * `removeSecureDoc` returns quietly when the ref is unusable or no storage + * client is configured, which is right for its best-effort callers (a leftover + * signature is not worth failing a contract over) but wrong for the receipt + * cleanup queue: "no client" there would mark an orphan resolved on a + * misconfigured deployment and lose it permanently. Same delete, honest result. + * + * Deliberately a separate export rather than a behaviour change to + * removeSecureDoc: several callers outside this feature do not guard it. + */ +export async function removeSecureDocStrict(ref: string): Promise { + const path = secureRefPath(ref); + if (!path) throw new Error(`not a secure ref: ${String(ref).slice(0, 80)}`); + const supabase = getSupabase(); + if (!supabase) throw new Error("secure storage is not configured"); + const { error } = await supabase.storage.from(SECURE_BUCKET).remove([path]); + if (error) throw error; +} + export async function removeSecureDoc(ref: string): Promise { const path = secureRefPath(ref); if (!path) return; diff --git a/tests/receipt-intake-cleanup.test.ts b/tests/receipt-intake-cleanup.test.ts new file mode 100644 index 000000000..00f767de8 --- /dev/null +++ b/tests/receipt-intake-cleanup.test.ts @@ -0,0 +1,79 @@ +/** + * The orphaned-object cleanup queue. + * + * This exists for one failure: a row is deleted while its object may still be + * in the bucket. After that nothing in the database references those bytes, so + * the queue record IS the last pointer to them — which makes "best effort" the + * wrong posture for writing it, and makes deleting the wrong path unrecoverable. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +const ROOT = path.resolve(__dirname, ".."); +const cleanup = readFileSync(path.join(ROOT, "src/lib/receipt-intake/storage-cleanup.ts"), "utf8"); +const intake = readFileSync(path.join(ROOT, "src/app/api/receipts/intake/route.ts"), "utf8"); +const storage = readFileSync(path.join(ROOT, "src/lib/secure-storage.ts"), "utf8"); + +test("recording a cleanup is durable, not fire-and-forget", () => { + // logAutomationEvent never throws by contract, so "it did not throw" is not + // proof it wrote. The record is read back, and the function throws if it is + // not there — the caller has to know. + const fn = cleanup.slice(cleanup.indexOf("export async function recordPendingCleanup")); + const body = fn.slice(0, fn.indexOf("\n}\n")); + assert.ok(!/\.catch\(\(\)\s*=>\s*\{/.test(body), "the write is not swallowed"); + assert.match(body, /findFirst/, "it is read back"); + assert.match(body, /throw new Error/, "and a missing record throws"); +}); + +test("an unrecordable cleanup KEEPS the row as the last pointer", () => { + // If the queue record cannot be written, deleting the row would orphan the + // bytes with nothing anywhere referencing them. The STAGING row is then the + // only way to find them, so it stays and the sweeper resolves it. + assert.match(intake, /cleanup unrecordable; keeping the row as the pointer/); + assert.match(intake, /retained: true/); +}); + +test("a failed row deletion is surfaced, not swallowed", () => { + // Otherwise the caller retries, hits a sourceRef conflict against a row it + // was just told does not exist, and has no way to interpret that. + assert.match(intake, /row delete failed after an ambiguous upload/); +}); + +test("the cleanup worker refuses to delete a path a LIVE row still points at", () => { + // Reachable through the recovery sequence: an ambiguous upload records a + // cleanup, the row goes, the caller retries, and the retry's row can point + // at the same path — or a seal publishes a canonical path an older pending + // event names. Deleting then destroys a receipt in active use. + const fn = cleanup.slice(cleanup.indexOf("export async function retryPendingCleanups")); + assert.match(fn, /receiptIntake\.findFirst\(\{\s*\n?\s*where: \{ storagePath \}/, "it checks for a referencing row"); + assert.match(fn, /still referenced by/, "and resolves rather than retrying forever"); + // The reference check must come BEFORE the delete. + assert.ok( + fn.indexOf("still referenced by") < fn.indexOf("removeSecureDocStrict"), + "the check precedes the deletion", + ); +}); + +test("an event is resolved only AFTER a confirmed deletion", () => { + const fn = cleanup.slice(cleanup.indexOf("export async function retryPendingCleanups")); + // The delete's catch continues to the next event rather than falling + // through to the resolve. + assert.match(fn, /\} catch \{[\s\S]*?continue;/, "a failed delete leaves the event pending"); + assert.ok( + fn.lastIndexOf("removeSecureDocStrict") < fn.lastIndexOf('status: "resolved" }'), + "resolve happens after the delete", + ); +}); + +test("a missing storage client is an ERROR for the cleanup path", () => { + // removeSecureDoc returns quietly with no client — right for its + // best-effort callers, catastrophic here: it would mark orphans resolved on + // a misconfigured deployment and lose them permanently. + assert.match(storage, /export async function removeSecureDocStrict/); + const strict = storage.slice(storage.indexOf("export async function removeSecureDocStrict")); + assert.match(strict.slice(0, strict.indexOf("\n}\n")), /throw new Error\("secure storage is not configured"\)/); + // ...and the cleanup queue uses the strict one, never the quiet one. + assert.ok(!/\bremoveSecureDoc\(/.test(cleanup), "cleanup never uses the quiet variant"); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 08141ee47..cbd9e1251 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -191,3 +191,31 @@ test("text/plain is no longer accepted at all", async () => { assert.equal(check.ok, false); assert.equal((check as { reason: string }).reason, "unsupported-file-type"); }); + +// ── A sha-mismatch is recoverable while the URL can still land (item 5) ──── + +test("the sweeper's two parks are the ones /finalize recovers from", () => { + // A partial upload sitting at the path while the signed URL is still valid + // is a retry in progress, not an error state. Parking it would turn the + // client's own next request into a review item — and the correct bytes + // arriving a minute later would find the row already out of STAGING. + const { readFileSync } = require("node:fs") as typeof import("node:fs"); + const path = require("node:path") as typeof import("node:path"); + const root = path.resolve(__dirname, ".."); + + const sweeper = readFileSync( + path.join(root, "src/app/api/cron/receipt-intake-worker/route.ts"), "utf8", + ); + // Both the missing-object and the sha-mismatch branches wait for expiry. + const shaBranch = sweeper.slice(sweeper.indexOf('row.expectedSha256 !== check.fileSha256')); + assert.match( + shaBranch.slice(0, shaBranch.indexOf("parked++")), + /SIGNED_UPLOAD_TTL_MS/, + "a sha mismatch waits for the upload URL to expire", + ); + + const finalize = readFileSync( + path.join(root, "src/app/api/receipts/intake/[id]/finalize/route.ts"), "utf8", + ); + assert.match(finalize, /stateReason === "file-missing" \|\| row\.stateReason === "sha-mismatch"/); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 401f06a76..57ee84f3e 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -124,9 +124,9 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), read: async () => { h.reads++; return goodRead; }, - applyRead: async (_id, patch) => { h.applied.push(patch); return { strongOwner: null }; }, + applyRead: async (_id, patch) => { h.applied.push(patch); return { owned: true, strongOwner: null }; }, findWeakHit: async () => null, - applyState: async (id, state, reason, patch) => { h.states.push({ id, state, reason, patch }); }, + applyState: async (id, state, reason, patch) => { h.states.push({ id, state, reason, patch }); return true; }, finishRouting: async (id, claimToken, stateReason) => { h.finished.push({ id, claimToken, stateReason }); }, @@ -137,8 +137,8 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) return { outcome: "booked", qbPurchaseId: "QB-1", expenseId: "e1", alreadyExisted: false } as BookResult; }, applyBookResult: async () => {}, - deferRead: async (id, busyPasses) => { h.deferred.push({ id, busyPasses }); }, - retryRow: async (id, attempts, _next, reason) => { h.retried.push({ id, attempts, reason }); }, + deferRead: async (id, busyPasses) => { h.deferred.push({ id, busyPasses }); return true; }, + retryRow: async (id, attempts, _next, reason) => { h.retried.push({ id, attempts, reason }); return true; }, now: () => NOW, monotonicMs: () => h.clock, ...overrides, @@ -191,7 +191,7 @@ test("LIVE: a READ row with dryRun=false is promoted and booked", async () => { test("a strong-key claim that loses re-routes against the owner and keeps no key", async () => { // Same total AND same canonical vendor: a confirmed duplicate. const h = harness([workerRow()], { - applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }), + applyRead: async () => ({ owned: true, strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }), }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { DUPLICATE: 1 }); @@ -201,7 +201,7 @@ test("a strong-key claim that loses re-routes against the owner and keeps no key test("a strong-key loss at a DIFFERENT total goes to a human, not to DUPLICATE", async () => { const h = harness([workerRow()], { - applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 999, canonicalVendor: "lowes" } }), + applyRead: async () => ({ owned: true, strongOwner: { id: "row-owner", totalCents: 999, canonicalVendor: "lowes" } }), }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); @@ -296,7 +296,7 @@ test("one blowing-up row does not stall the batch", async () => { test("a strong-key loss to a DIFFERENT vendor is a collision, not a duplicate", async () => { const h = harness([workerRow()], { - applyRead: async () => ({ strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "homedepot" } }), + applyRead: async () => ({ owned: true, strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "homedepot" } }), }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }); @@ -567,7 +567,7 @@ test("an EXACT duplicate becomes DUPLICATE, not NEEDS_REVIEW", async () => { applyRead: async (_id, patch) => { order.push("strong-claim"); h.applied.push(patch); - return { strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }; + return { owned: true, strongOwner: { id: "row-owner", totalCents: 36498, canonicalVendor: "lowes" } }; }, findWeakHit: async () => { order.push("weak-lookup"); return { id: "row-owner" }; }, }); @@ -581,7 +581,7 @@ test("an EXACT duplicate becomes DUPLICATE, not NEEDS_REVIEW", async () => { test("the strong claim is attempted with the key, before any weak lookup", async () => { const order: string[] = []; const h = harness([workerRow()], { - applyRead: async (_id, patch) => { order.push("strong-claim"); h.applied.push(patch); return { strongOwner: null }; }, + applyRead: async (_id, patch) => { order.push("strong-claim"); h.applied.push(patch); return { owned: true, strongOwner: null }; }, findWeakHit: async () => { order.push("weak-lookup"); return null; }, }); await runIntakeWorker(h.deps); @@ -911,7 +911,9 @@ test("a row that DID send keeps its key at the retry limit", async () => { }); await runIntakeWorker(h.deps); assert.equal(h.states[0].reason, "max-retries"); - assert.equal(h.states[0].patch, undefined, "no patch, so the key is untouched"); + // parkTerminal always sends a patch; what matters is that it does NOT carry + // a key release for a row that reached QuickBooks. + assert.ok(!("dedupStrongKey" in (h.states[0].patch ?? {})), "the key is untouched"); }); // ── Content changed under us (round-8 item 2) ────────────────────────────── @@ -1010,3 +1012,96 @@ test("every book result carries the row's claim token to the writer", async () = await runIntakeWorker(h.deps); assert.deepEqual(tokens, ["tok-9"]); }); + +// ── Ownership is CAS'd on EVERY mutation (round-10 item 3) ───────────────── + +test("losing the row aborts each mutation path instead of clobbering a successor", async () => { + // A zombie worker holds a view its successor has already moved past. Every + // write it attempts must affect zero rows and stop the pass for that row — + // a time-based lease cannot express this, because both hold the same id. + const lost = { owned: false as const }; + + // applyRead at the document-level gate. + const gate = harness([workerRow()], { + read: async () => ({ ok: true, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, + applyRead: async () => ({ ...lost, strongOwner: null }), + }); + assert.deepEqual((await runIntakeWorker(gate.deps)).byState, { STALE: 1 }); + + // applyRead at the strong claim. + const claim = harness([workerRow()], { applyRead: async () => ({ ...lost, strongOwner: null }) }); + assert.deepEqual((await runIntakeWorker(claim.deps)).byState, { STALE: 1 }); + assert.deepEqual(claim.finished, [], "never published"); + + // applyState, via a terminal park. + const park = harness([workerRow()], { + downloadBytes: async () => ({ ok: false as const, kind: "missing" as const }), + applyState: async () => false, + }); + assert.deepEqual((await runIntakeWorker(park.deps)).byState, { STALE: 1 }); + + // deferRead, via an AI outage. + const defer = harness([workerRow()], { + read: async () => ({ ok: false, decisive: false }), + deferRead: async () => false, + }); + assert.deepEqual((await runIntakeWorker(defer.deps)).byState, { STALE: 1 }); + + // retryRow, via a transient storage fault. + const retry = harness([workerRow()], { + downloadBytes: async () => ({ ok: false as const, kind: "transient" as const, message: "x" }), + retryRow: async () => false, + }); + assert.deepEqual((await runIntakeWorker(retry.deps)).byState, { STALE: 1 }); +}); + +test("every mutation is offered the row's OWN state and token", async () => { + const seen: unknown[] = []; + const h = harness([workerRow({ claimToken: "tok-7" })], { + applyRead: async (_id, patch, ownership) => { + seen.push(ownership); + h.applied.push(patch); + return { owned: true, strongOwner: null }; + }, + }); + await runIntakeWorker(h.deps); + assert.deepEqual(seen, [{ state: "RECEIVED", claimToken: "tok-7" }]); +}); + +// ── One parkTerminal decides the key release (round-10 item 4) ───────────── + +test("EVERY pre-send terminal park releases the strong key", async () => { + // Each of these used to decide independently, and the ones that forgot held + // a dedup key against a Purchase that never existed — so the corrected + // resubmission collided with nothing. + const cases: Array<[string, Partial]> = [ + ["file-missing", { downloadBytes: async () => ({ ok: false as const, kind: "missing" as const }) }], + ["content-changed", { downloadBytes: async () => ({ ok: false as const, kind: "sha-mismatch" as const, message: "x" }) }], + ["unreadable", { read: async () => ({ ok: false, decisive: true }) }], + ]; + for (const [reason, over] of cases) { + const h = harness([workerRow({ sendAttempted: false })], over); + await runIntakeWorker(h.deps); + assert.equal(h.states[0].reason, reason); + assert.equal(h.states[0].patch?.dedupStrongKey, null, `${reason} must release the key`); + } + + // ...and the AI-unavailable ceiling, which is a different code path again. + const busy = harness([workerRow({ sendAttempted: false, busyPasses: MAX_BUSY_PASSES - 1 })], { + read: async () => ({ ok: false, decisive: false }), + }); + await runIntakeWorker(busy.deps); + assert.equal(busy.states[0].reason, "ai-unavailable"); + assert.equal(busy.states[0].patch?.dedupStrongKey, null); +}); + +test("a park AFTER a send keeps the key, on every one of those paths", async () => { + for (const over of [ + { downloadBytes: async () => ({ ok: false as const, kind: "missing" as const }) }, + { read: async () => ({ ok: false as const, decisive: true }) }, + ]) { + const h = harness([workerRow({ sendAttempted: true })], over); + await runIntakeWorker(h.deps); + assert.ok(!("dedupStrongKey" in (h.states[0].patch ?? {})), "the Purchase may exist"); + } +}); From be9ae4c6fd08ff45f82a459f5bb905e04b74ef91 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:06:46 -0700 Subject: [PATCH 049/144] fix(receipts): release ownership on every transition; fence publish and reject Round-11 gate on 85d6af57, plus the Phase 2/3 follow-ups. - Every completed, deferred or terminal worker transition now clears claimToken/claimedAt in the SAME fenced write (RELEASE_CLAIM). The only exceptions are READ -> BOOKING, which hands the row straight to bookReceipt under the same token, and soft-deadline abandonment. A structural test walks every UPDATE in the dependency factory rather than naming today's set. - applyState takes ownership as a required argument; the book-result CAS pins state BOOKING so a late deferred/retry cannot overwrite a booked row. - The fenced send mark moved to immediately before the QBO create (onBeforeCreate), so account/vendor ensure faults now RELEASE the strong key instead of quarantining the corrected re-submission. - Late fields moved to src/lib/receipt-intake/late-fields.ts so their races are testable: a lost CAS re-reads, re-authorizes the phase against the CURRENT project, and only 409s when the persisted values are not what was supplied. The worker re-reads projectId right before routing, so a job assigned during the read is honoured instead of parked NEEDS_JOB. - publishStagedRow CASes on the exact state; heal recovery is restricted to file-missing and sha-mismatch, and a lost CAS deletes the object it just uploaded. - A rejected finalize deletes the row and queues its object in one transaction; an unconfirmed deletion keeps the object and answers 503. - Oversize objects are rejected from list() metadata with no body read; the matching 15 MB bucket-level limit is documented in the spec and .env.example. Co-Authored-By: Claude Fable 5.1 --- .env.example | 7 + docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 15 ++ package.json | 4 +- .../api/cron/receipt-intake-worker/route.ts | 58 +++++-- .../receipts/intake/[id]/finalize/route.ts | 147 +++++++----------- src/app/api/receipts/intake/route.ts | 63 ++++++-- src/lib/qbo-receipt-push.ts | 15 ++ src/lib/receipt-intake/book.ts | 38 +++-- src/lib/receipt-intake/late-fields.ts | 131 ++++++++++++++++ src/lib/receipt-intake/storage-cleanup.ts | 87 +++++++++++ src/lib/receipt-intake/stored-object.ts | 23 ++- src/lib/receipt-intake/worker.ts | 43 +++-- src/lib/secure-storage.ts | 31 ++++ tests/receipt-intake-book.test.ts | 101 ++++++++---- tests/receipt-intake-claim-release.test.ts | 128 +++++++++++++++ tests/receipt-intake-late-fields.test.ts | 125 +++++++++++++++ tests/receipt-intake-reject.test.ts | 146 +++++++++++++++++ tests/receipt-intake-stored-object.test.ts | 40 +++++ tests/receipt-intake-worker.test.ts | 34 ++++ 19 files changed, 1070 insertions(+), 166 deletions(-) create mode 100644 src/lib/receipt-intake/late-fields.ts create mode 100644 tests/receipt-intake-claim-release.test.ts create mode 100644 tests/receipt-intake-late-fields.test.ts create mode 100644 tests/receipt-intake-reject.test.ts diff --git a/.env.example b/.env.example index c8c1b18f0..aaeb5a200 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,13 @@ RECEIPT_INTAKE_SECRET= # (POST /api/receipts/intake/{id}/archived). Cannot create or publish a row. RECEIPT_ARCHIVE_SECRET= # +# Storage note (not an env var): the private receipts bucket must ALSO carry a +# 15 MB file-size limit in the Supabase dashboard (Storage -> bucket -> +# Settings). Two-step uploads go straight to a signed URL and never pass through +# this server, so the bucket is the only place a too-large write can actually be +# refused; MAX_STORED_BYTES in src/lib/receipt-intake/intake-core.ts only lets +# the server reject the object after the fact. +# # Shadow mode. UNSET or "true" = dry run: rows are read, deduped and routed, and # NOTHING is booked. Set to the literal "false" only at cutover. RECEIPT_INTAKE_DRYRUN= diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index a56bd0dcf..86c96412d 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -468,6 +468,21 @@ fine. Both inline ceilings answer with a 413 naming the two-step path. +**The 15 MiB ceiling is set on the Supabase bucket as well as in code.** The signed upload +URL bypasses this server entirely, so application code cannot stop the write — it can only +refuse the object afterwards, by which time the bytes are already paid for and sitting in +the bucket. Set it where the write happens: + +> Supabase dashboard → Storage → the private receipts bucket (`SECURE_BUCKET`) → Settings → +> **file size limit = 15 MB**. Supabase rejects a larger upload at the storage API with a +> 413, before any object is created. + +The server-side check stays regardless, and is checked in this order: +1. **Object metadata first** (`list({ search })` → `metadata.size`) — one small request that + costs the same whatever the object weighs. Oversize is rejected here, with no body read. +2. **Then the downloaded byte length**, because a null metadata size means "storage did not + say", not "fine". + **`text/plain` is refused with a 415.** QuickBooks cannot attach a `.txt`, so accepting one meant reading it with Gemini and then stranding it unbookable at `unsupported-attachment` — worse than a clear refusal at the door. v1 converted these using diff --git a/package.json b/package.json index d31275293..902d51cc1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 81b30d28a..fe4e17bd5 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -71,6 +71,19 @@ const WORKER_ROW_SELECT = { createdAt: true, dedupWeakKey: true, busyPasses: true, } as const; +/** + * RELEASING OWNERSHIP is part of the transition, not a follow-up write. + * + * A claim is what makes a row invisible to the next pass. Every write that + * COMPLETES, DEFERS or PARKS the work must hand it back in the same update, or + * the row stays owned by a pass that has finished: the claim query skips it, + * every fenced write misses it, and it sits until a human notices. The only + * transitions that keep it are READ -> BOOKING (the same pass books it, under + * the same token) and abandonment at the soft deadline, where the pass really + * is still holding the row. + */ +const RELEASE_CLAIM = { claimToken: null, claimedAt: null } as const; + /** * A row parked by the shadow week (dryRun=true, sitting at READ or BOOKING) is * DONE until the cutover. It is excluded from the claim rather than merely @@ -470,11 +483,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { applyState: async (rowId, state, stateReason, patch, ownership) => { const { count } = await prisma.receiptIntake.updateMany({ - where: { - id: rowId, - ...(ownership ? { state: ownership.state, claimToken: ownership.claimToken } : {}), - }, - data: { ...(patch ?? {}), state, stateReason, nextRetryAt: null }, + where: { id: rowId, state: ownership.state, claimToken: ownership.claimToken }, + data: { ...(patch ?? {}), state, stateReason, nextRetryAt: null, ...RELEASE_CLAIM }, }); return count > 0; }, @@ -569,6 +579,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // strong key goes back (same rule as book.ts). dedupStrongKey: null, nextRetryAt: null, + ...RELEASE_CLAIM, }, }); return { promoted: false, conflictId: conflict.id }; @@ -576,6 +587,11 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { } // CAS: only the current claim holder promotes. A superseded worker // must not move a row into BOOKING that its successor is handling. + // + // THE ONE TRANSITION THAT KEEPS THE CLAIM, deliberately: promotion + // hands the row straight to bookReceipt in this same pass, and both + // its send mark and its BOOKED commit CAS on this token. Releasing + // here would admit a second worker to the same booking. const { count } = await tx.receiptIntake.updateMany({ where: { id: rowId, state: "READ", claimToken }, data: { state: "BOOKING", stateReason: null }, @@ -607,8 +623,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), getTokens: deadline => getFreshQBTokens(deadline), - createPurchase: (tokens, input, deadline) => - createQBReceiptPurchase(tokens, input, {}, deadline), + createPurchase: (tokens, input, deadline, onBeforeCreate) => + createQBReceiptPurchase(tokens, input, { onBeforeCreate }, deadline), downloadBytes: (storagePath, expectedSha256) => downloadVerified(storagePath, expectedSha256), logEvent: logAutomationEvent, now: () => new Date(), @@ -620,8 +636,14 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // A superseded worker writes NOTHING: the row belongs to whoever // holds the current token, and its state is theirs to set. if (result.outcome === "stale") return; - // Every write below is a CAS on the claim, for the same reason. - const owns = { id: rowId, claimToken } as const; + // Every write below is a CAS on the claim AND on the state. + // + // The token alone is not enough here: bookReceipt own commit may + // already have moved the row to BOOKED under this same token, and a + // late deferred/retry result would then overwrite a booked row with + // "come back in an hour". Pinning BOOKING means only a row still + // waiting to book can be written by a booking result. + const owns = { id: rowId, state: "BOOKING", claimToken } as const; if (result.outcome === "needs-review") { await prisma.receiptIntake.updateMany({ where: owns, @@ -629,6 +651,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { state: "NEEDS_REVIEW", stateReason: result.reason, nextRetryAt: null, + ...RELEASE_CLAIM, // Parked before any QBO send: hand the strong key back, // or a corrected re-send of the same receipt would be // quarantined against a row that never became a purchase. @@ -646,6 +669,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { state: "BOOKING", stateReason: result.reason, nextRetryAt: new Date(now.getTime() + 60 * 60_000), + ...RELEASE_CLAIM, }, }); return; @@ -657,6 +681,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { attempts: result.attempts, lastError: result.reason.slice(0, 400), nextRetryAt: result.nextRetryAt, + ...RELEASE_CLAIM, }, }); }, @@ -672,6 +697,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { busyPasses, lastError: reason, nextRetryAt: new Date(Date.now() + backoffMs(1)), + ...RELEASE_CLAIM, }, }); return count > 0; @@ -680,11 +706,23 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { retryRow: async (rowId, attempts, nextRetryAt, reason, ownership) => { const { count } = await prisma.receiptIntake.updateMany({ where: { id: rowId, state: ownership.state, claimToken: ownership.claimToken }, - data: { attempts, lastError: reason, nextRetryAt }, + data: { attempts, lastError: reason, nextRetryAt, ...RELEASE_CLAIM }, }); return count > 0; }, + // Re-read taken RIGHT BEFORE routing, after the download and the model + // call. A late job assignment landing in that window must not be routed + // over: NEEDS_JOB for a receipt that HAS a job sends a human looking for + // a problem that no longer exists. + refreshProjectId: async rowId => { + const row = await prisma.receiptIntake.findUnique({ + where: { id: rowId }, + select: { projectId: true }, + }); + return row?.projectId ?? null; + }, + now: () => new Date(), monotonicMs: () => Date.now(), }; diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 0fae23e7b..20db352f9 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -6,83 +6,49 @@ import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; -import { deleteObjectOrRecord, sealObject } from "@/lib/receipt-intake/storage-cleanup"; +import { reconcileLateFields, type Denial, type LateFields } from "@/lib/receipt-intake/late-fields"; +import { + deleteObjectOrRecord, + rejectRowAndQueueCleanup, + sealObject, + settleQueuedCleanup, +} from "@/lib/receipt-intake/storage-cleanup"; export const dynamic = "force-dynamic"; /** - * Apply late fields where the row has none; refuse where they disagree. + * Route adapter over the late-field rules (src/lib/receipt-intake/late-fields.ts). * - * Returns a 409 response on conflict, or null when the row is now consistent - * with what the caller sent. + * The rules live in a lib because their interesting behaviour is entirely about + * races — a worker claiming the row, a state transition, a second caller + * writing a different project — and none of that is reachable from a test that + * has to stand up a route handler. */ -/** Late fields may only land while a row is still un-routed. */ -const LATE_FIELD_STATES = ["STAGING", "RECEIVED"]; - -async function reconcileLateFields( +async function applyLateFields( id: string, - lateFields: Partial<{ costCodeId: string; projectId: string }>, + lateFields: LateFields, + auth: Extract, ): Promise { - const entries = Object.entries(lateFields) as Array<["costCodeId" | "projectId", string]>; - if (entries.length === 0) return null; - - const current = await prisma.receiptIntake.findUnique({ - where: { id }, - select: { costCodeId: true, projectId: true, state: true }, + const denial = await reconcileLateFields(id, lateFields, { + read: rowId => prisma.receiptIntake.findUnique({ + where: { id: rowId }, + select: { costCodeId: true, projectId: true, state: true, claimToken: true }, + }), + applyIfNull: async (rowId, state, toApply) => { + const { count } = await prisma.receiptIntake.updateMany({ + where: { + id: rowId, + state, + claimToken: null, + ...Object.fromEntries(Object.keys(toApply).map(key => [key, null])), + }, + data: toApply, + }); + return count; + }, + authorize: projectId => authorizeLateFields(auth, projectId, lateFields), }); - if (!current) return null; - - // NULL-OR-EQUAL only, and only BEFORE the row is routed. - // - // Past RECEIVED the read has already happened: the dedup keys, the phase - // suggestion and possibly a booking were all computed from the project this - // row had at the time. Changing it afterwards does not re-derive any of - // that — it just makes the row disagree with its own history, and after - // BOOKED it disagrees with a Purchase in the real books. - if (!LATE_FIELD_STATES.includes(current.state)) { - const differs = entries.some(([key, value]) => current[key] !== value); - if (!differs) return null; // already exactly what the caller is asking for - return NextResponse.json( - { - ok: false, - error: "late-fields-too-late", - reason: `this row is ${current.state}; its job and phase were already used to route it`, - state: current.state, - }, - { status: 409 }, - ); - } - - const conflicts = entries.filter(([key, value]) => current[key] !== null && current[key] !== value); - if (conflicts.length > 0) { - return NextResponse.json( - { - ok: false, - error: "late-fields-conflict", - reason: "this row already carries different values for these fields", - fields: Object.fromEntries( - conflicts.map(([key]) => [key, { stored: current[key], supplied: lateFields[key] }]), - ), - }, - { status: 409 }, - ); - } - - const toApply = Object.fromEntries(entries.filter(([key]) => current[key] === null)); - if (Object.keys(toApply).length > 0) { - // EXACT-state CAS, and still conditional on each field being null: a - // concurrent writer that set it — or moved the row on — wins rather - // than being overwritten. - await prisma.receiptIntake.updateMany({ - where: { - id, - state: current.state, - ...Object.fromEntries(Object.keys(toApply).map(key => [key, null])), - }, - data: toApply, - }); - } - return null; + return denial ? NextResponse.json(denial.body, { status: denial.status }) : null; } /** @@ -97,22 +63,22 @@ async function reconcileLateFields( async function authorizeLateFields( auth: Extract, rowProjectId: string | null, - lateFields: Partial<{ costCodeId: string; projectId: string }>, -): Promise { + lateFields: LateFields, +): Promise { const projectId = lateFields.projectId ?? rowProjectId; if (lateFields.projectId && auth.via === "session") { if (!(await userCanAccessProject(auth.user, lateFields.projectId))) { - return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); + return { status: 403, body: { ok: false, reason: "forbidden" } }; } } if (lateFields.costCodeId) { if (!projectId) { - return NextResponse.json( - { ok: false, error: "cost-code-without-project", reason: "a phase is only meaningful against a job" }, - { status: 400 }, - ); + return { + status: 400, + body: { ok: false, error: "cost-code-without-project", reason: "a phase is only meaningful against a job" }, + }; } const allowed = await isCostCodeAllowedForProject( prismaPhaseDataSource, @@ -120,15 +86,15 @@ async function authorizeLateFields( lateFields.costCodeId, ); if (!allowed) { - return NextResponse.json( - { + return { + status: 400, + body: { ok: false, error: "cost-code-not-a-phase", reason: "that cost code is not a phase of this job", projectId, }, - { status: 400 }, - ); + }; } } return null; @@ -197,7 +163,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // Authorize the late fields BEFORE anything is written or published. const denied = await authorizeLateFields(auth, row.projectId, lateFields); - if (denied) return denied; + if (denied) return NextResponse.json(denied.body, { status: denied.status }); // A LATE finalize on a row the sweeper already parked file-missing is a // RECOVERY, not a duplicate: the upload landed after the sweep looked. It @@ -220,7 +186,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // floor while telling the caller it worked. Same behaviour as the // two-publisher path below, because a caller cannot tell which one it hit. if (!recoverable) { - const conflict = await reconcileLateFields(id, lateFields); + const conflict = await applyLateFields(id, lateFields, auth); if (conflict) return conflict; // PERSISTED values, re-read after the reconcile — the caller must be // told what the row actually holds, not what it asked for. @@ -250,11 +216,18 @@ export async function POST(req: Request, context: { params: Promise<{ id: string if (check.kind === "transient") { return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); } - // REJECTED. The row goes, and so must the object — nothing references it - // once the row is gone, so a failed delete is recorded for the sweep to - // retry rather than shrugged off. - await prisma.receiptIntake.delete({ where: { id } }).catch(() => { /* best effort */ }); - await deleteObjectOrRecord(row.storagePath, check.reason); + // REJECTED. The row goes, and so must the object — but the two writes + // are ONE transaction. A best-effort delete followed by a best-effort + // cleanup could drop the row and lose the object with nothing left + // referencing or remembering it. + const rejected = await rejectRowAndQueueCleanup(id, row.storagePath, check.reason); + if (!rejected.ok) { + // The row's deletion is not confirmed, so it may still point at + // these bytes. Keep the object and answer retryably; an identical + // retry re-validates and rejects again. + return NextResponse.json({ ok: false, reason: "reject-failed", retryable: true }, { status: 503 }); + } + await settleQueuedCleanup(rejected.eventId, row.storagePath); const status = check.reason.startsWith("file-too-large") ? 413 : 400; return NextResponse.json( { ok: false, reason: check.reason, maxBytes: MAX_STORED_BYTES }, @@ -317,7 +290,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string if (!outcome.published) { // Another publisher won. Same outcome for the caller — but the late // fields still have to be reconciled against what that publisher wrote. - const reconciled = await reconcileLateFields(id, lateFields); + const reconciled = await applyLateFields(id, lateFields, auth); if (reconciled) return reconciled; const current = await prisma.receiptIntake.findUnique({ where: { id }, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 826fe9251..df40af188 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -6,7 +6,7 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; -import { recordPendingCleanup } from "@/lib/receipt-intake/storage-cleanup"; +import { deleteObjectOrRecord, recordPendingCleanup } from "@/lib/receipt-intake/storage-cleanup"; import { ACCEPTED_MIME_TYPES, EXT_BY_MIME, sniffMime } from "@/lib/receipt-intake/file-type"; import { MAX_INLINE_JSON_BYTES, @@ -403,14 +403,37 @@ async function storeObject(storagePath: string, bytes: Buffer, mimeType: string) } } -async function publishStagedRow(id: string): Promise { +/** The only two parked reasons a later, correct upload may recover from. */ +export const RECOVERABLE_REASONS = ["file-missing", "sha-mismatch"]; + +async function publishStagedRow(id: string, expectState = "STAGING"): Promise { try { - const published = await prisma.receiptIntake.update({ - where: { id }, + // EXACT-state CAS. `update` by id alone would publish a row that had + // since moved on — a booked row dragged back to RECEIVED and re-read, + // which is a second Purchase waiting to happen. + const { count } = await prisma.receiptIntake.updateMany({ + where: { id, state: expectState }, data: { state: "RECEIVED" }, + }); + if (count === 0) { + const current = await prisma.receiptIntake.findUnique({ + where: { id }, + select: { state: true }, + }); + // Somebody else published it; that is the outcome the caller wanted. + if (current?.state === "RECEIVED") { + return NextResponse.json({ ok: true, id, state: "RECEIVED", alreadyPublished: true }); + } + return NextResponse.json( + { ok: false, error: "publish-conflict", id, state: current?.state ?? "gone" }, + { status: 409 }, + ); + } + const published = await prisma.receiptIntake.findUnique({ + where: { id }, select: { id: true, state: true, sourceRef: true, projectId: true, dryRun: true }, }); - return NextResponse.json({ ok: true, ...published }); + return NextResponse.json({ ok: true, ...(published ?? { id, state: "RECEIVED" }) }); } catch (error) { // The bytes ARE stored; only the publish failed. Leave the row in // STAGING — 503 tells the caller to retry, and the retry resumes. @@ -445,7 +468,7 @@ async function respondToSourceRefConflict( where: { sourceRef }, select: { id: true, state: true, source: true, sourceRef: true, projectId: true, - dryRun: true, fileSha256: true, createdById: true, storagePath: true, + dryRun: true, fileSha256: true, createdById: true, storagePath: true, stateReason: true, }, }); // The row vanished between the failed insert and this read (a delete @@ -492,16 +515,38 @@ async function respondToSourceRefConflict( // The caller just handed us the bytes again, so the orphan is fixable: // store them and republish. This is the retry HEALING the row rather // than merely reporting on it. - const healable = existing.state === "STAGING" || existing.state === "NEEDS_REVIEW"; + // Recovery is restricted to the two reasons a later, correct upload can + // actually fix. "Any NEEDS_REVIEW row" was far too broad: a row parked + // for a vendor mismatch, a zero total, or a QBO fault would be dragged + // back to RECEIVED and re-read, discarding the decision a human had + // already made about it. + const healable = existing.state === "STAGING" + || (existing.state === "NEEDS_REVIEW" && RECOVERABLE_REASONS.includes(existing.stateReason ?? "")); if (healable) { const healed = await storeObject(payload.storagePath, payload.bytes, payload.mimeType); if (!healed) { return NextResponse.json({ ok: false, error: "storage-failed" }, { status: 503 }); } - await prisma.receiptIntake.update({ - where: { id: existing.id }, + // EXACT state AND reason. Losing this race means somebody moved the + // row while we were uploading, so the object we just wrote is + // unreferenced — clean it up rather than orphan it. + const { count } = await prisma.receiptIntake.updateMany({ + where: { + id: existing.id, + state: existing.state, + ...(existing.state === "NEEDS_REVIEW" ? { stateReason: existing.stateReason } : {}), + }, data: { storagePath: payload.storagePath, state: "RECEIVED", stateReason: null, nextRetryAt: null }, }); + if (count === 0) { + if (payload.storagePath !== existing.storagePath) { + await deleteObjectOrRecord(payload.storagePath, "heal-lost-race"); + } + return NextResponse.json( + { ok: false, error: "publish-conflict", id: existing.id }, + { status: 409 }, + ); + } return NextResponse.json({ ok: true, recovered: true, id: existing.id, state: "RECEIVED", sourceRef: existing.sourceRef, projectId: existing.projectId, dryRun: existing.dryRun, diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 49369a48c..9ef2262b4 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -211,6 +211,20 @@ export interface QboReceiptProjectCandidate { export interface QboReceiptPushDependencies { qbQueryFn: (tokens: QBTokens, query: string) => Promise; qbCreateFn: (tokens: QBTokens, payload: Record, requestId: string) => Promise<{ id: string }>; + /** + * Invoked IMMEDIATELY before qbCreateFn, and nowhere else. + * + * Callers that record "a Purchase may now exist" need that record to happen + * at the last possible instant. Everything above this line — the DocNumber + * query, the project match, the vendor/customer ensures, the account + * verification, the money validation — can fail without any Purchase being + * created, and a caller that marked earlier would treat those as + * "might have booked" forever. + * + * Throwing from this hook aborts the create, which is the point: it is also + * the caller's last chance to check it still owns the row. + */ + onBeforeCreate?: () => Promise; ensureVendorFn: (tokens: QBTokens, name: string) => Promise; // Injectable (unlike the plain re-export of ensureQBCustomer) because the // customer is now resolved on EVERY create — there is no more per-client @@ -810,6 +824,7 @@ export async function createQBReceiptPurchase( if (isBudgetExhausted(deadline)) { throw new QBBudgetExhaustedError("Route budget exhausted before the QBO Purchase create"); } + await deps.onBeforeCreate?.(); const created = await qbCreateFn(tokens, payload, requestId); let attachment: ReceiptAttachmentStatus = "skipped"; diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index fef6dbe6e..598a2677c 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -193,7 +193,9 @@ export interface BookDependencies { createPurchase: ( tokens: QBTokens, input: CreateQBReceiptPurchaseInput, - deadline?: RouteDeadline, + deadline: RouteDeadline | undefined, + /** Invoked by the QBO core immediately before the create. */ + onBeforeCreate: () => Promise, ) => Promise; /** * The invocation's ONE absolute deadline. Undefined = unbounded (tests). @@ -449,16 +451,28 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // strong key would then be held forever against a Purchase that does // not exist. Persisted rather than in-memory, because the case the flag // exists for is the process dying mid-create. - // It is ALSO the last fence: a CAS on the claim token. If this worker - // has been superseded the write affects zero rows and we abort HERE, - // before the create — so a zombie cannot post a Purchase that the live - // worker is about to post as well. - const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); - if (!stillOurs) return { outcome: "stale" }; - sent.attempted = true; - - result = await deps.createPurchase(tokens, input, deps.deadline); + // The mark happens INSIDE createQBReceiptPurchase, immediately before + // the create — not here. + // + // Everything the QBO core does first can fail without any Purchase + // existing: the DocNumber query, the project match, ensureVendor, + // ensureCustomer, the account verification, the money validation. + // Marking before all of that meant a vendor-duplicate or an + // account-config fault left sendAttempted=true, and the row then held + // its dedup key forever against a Purchase that was never created. + // + // The hook is also the last ownership fence: a CAS on the claim token + // that THROWS when this worker has been superseded, which aborts the + // create so a zombie cannot post a Purchase the live worker is about to + // post as well. + result = await deps.createPurchase(tokens, input, deps.deadline, async () => { + const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); + if (!stillOurs) throw new StaleClaimError(); + sent.attempted = true; + }); } catch (error) { + // A lost CAS from inside the create hook: nothing was sent. + if (error instanceof StaleClaimError) return { outcome: "stale" }; const terminal = terminalReasonFor(error); // A send WAS attempted: QBO may hold a Purchase whose response we lost, // so the key stays claimed even though the row is parked. @@ -610,6 +624,10 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro bookedAt: now, lastError: null, nextRetryAt: null, + // Ownership is released by the write that completes the + // transition — a booked row is nobody's to hold. + claimToken: null, + claimedAt: null, }, }); if (claimed.count === 0) throw new StaleClaimError(); diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts new file mode 100644 index 000000000..68b0f1bbf --- /dev/null +++ b/src/lib/receipt-intake/late-fields.ts @@ -0,0 +1,131 @@ +/** + * Late job/phase assignment on an intake row. + * + * A forwarder often knows the bytes before it knows the job: Drive files land + * in a project folder, mobile captures pick a job on the next screen, and a + * replayed finalize carries the fields the first attempt could not. So the + * fields may arrive AFTER the row does — but only under rules, because every + * one of them is a way to change money after the fact. + * + * Extracted from the route so the races are testable: the whole point of this + * module is what happens when a worker, a second caller, or a state transition + * moves the row between the read and the write. + */ + +export interface LateFields { + costCodeId?: string; + projectId?: string; +} + +export interface LateFieldRow { + costCodeId: string | null; + projectId: string | null; + state: string; + claimToken?: string | null; +} + +/** A refusal, shaped so the route can hand it straight to NextResponse.json. */ +export interface Denial { + status: number; + body: Record; +} + +export interface LateFieldsDeps { + read(id: string): Promise; + /** updateMany fenced on {id, state, claimToken: null, : null}; returns the count. */ + applyIfNull(id: string, state: string, toApply: Record): Promise; + /** Re-runs the caller's authorization against a given project. */ + authorize(projectId: string | null): Promise; +} + +/** Late fields may only land while a row is still un-routed. */ +export const LATE_FIELD_STATES = ["STAGING", "RECEIVED"]; + +export async function reconcileLateFields( + id: string, + lateFields: LateFields, + deps: LateFieldsDeps, +): Promise { + const entries = Object.entries(lateFields).filter(([, value]) => value !== undefined) as Array< + ["costCodeId" | "projectId", string] + >; + if (entries.length === 0) return null; + + const current = await deps.read(id); + if (!current) return null; + + // NULL-OR-EQUAL only, and only BEFORE the row is routed. + // + // Past RECEIVED the read has already happened: the dedup keys, the phase + // suggestion and possibly a booking were all computed from the project this + // row had at the time. Changing it afterwards does not re-derive any of + // that — it just makes the row disagree with its own history, and after + // BOOKED it disagrees with a Purchase in the real books. + if (!LATE_FIELD_STATES.includes(current.state)) { + const differs = entries.some(([key, value]) => current[key] !== value); + if (!differs) return null; // already exactly what the caller is asking for + return { + status: 409, + body: { + ok: false, + error: "late-fields-too-late", + reason: `this row is ${current.state}; its job and phase were already used to route it`, + state: current.state, + }, + }; + } + + const conflicts = entries.filter(([key, value]) => current[key] !== null && current[key] !== value); + if (conflicts.length > 0) { + return { + status: 409, + body: { + ok: false, + error: "late-fields-conflict", + reason: "this row already carries different values for these fields", + fields: Object.fromEntries( + conflicts.map(([key]) => [key, { stored: current[key], supplied: lateFields[key] }]), + ), + }, + }; + } + + const toApply = Object.fromEntries(entries.filter(([key]) => current[key] === null)); + if (Object.keys(toApply).length === 0) return null; + + // THE ROW MUST BE UNCLAIMED. + // + // A worker that claimed this row read its projectId at claim time and is + // routing on that value right now. Writing a project underneath it does not + // change what it decided — it just makes the row disagree with the routing + // it is about to publish (a receipt that now HAS a job, parked NEEDS_JOB). + // The fence is applied by the caller's `applyIfNull`. + const count = await deps.applyIfNull(id, current.state, toApply as Record); + if (count > 0) return null; + + // The CAS lost. That is NOT automatically "busy": the same zero comes back + // when a concurrent caller already wrote exactly these values, when the + // state moved, and when a DIFFERENT project was written underneath us. + // Re-read and decide from what is persisted, not from the stale read. + const after = await deps.read(id); + const settled = after !== null && entries.every(([key, value]) => after[key] === value); + if (!settled) { + return { + status: 409, + body: { + ok: false, + error: "late-fields-busy", + reason: "this row changed underneath the write; retry in a moment", + retryable: after?.claimToken != null, + state: after?.state ?? "gone", + }, + }; + } + + // The values match — but a concurrent write may have moved the PROJECT, and + // the phase we were asked to accept was authorized against the project this + // row had BEFORE that. Re-authorize against the project the row carries + // now; otherwise losing a race is a way to attach a cost code from another + // job, which is exactly what the first authorization existed to prevent. + return await deps.authorize(after.projectId); +} diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index e245be749..b0cf5c448 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -91,6 +91,93 @@ export async function recordPendingCleanup(storagePath: string, reason: string): if (!recorded) throw new Error(`could not record a cleanup for ${storagePath}`); } +/** + * Reject a row and queue its object for deletion IN ONE TRANSACTION. + * + * The two writes cannot be separate. Delete-then-record loses the object + * whenever the record fails (nothing references the bytes any more, and nothing + * remembers them). Record-then-delete leaves a cleanup event naming a path a + * live row still points at, and the sweep would delete a receipt in use — the + * sweep's own "still referenced" guard papers over that, but only until the row + * is re-pointed. Both in one transaction means the queue entry exists if and + * only if the row is gone. + * + * Returns false when the row's deletion is NOT confirmed. The caller must then + * keep the object and fail retryably: an object with no queue entry and a row + * that still exists is a state we can resume from; the reverse is not. + */ +/** The two writes the reject transaction needs — injectable so it is testable. */ +export interface RejectTxClient { + automationEvent: { create(args: { data: Record; select: { id: true } }): Promise<{ id: string }> }; + receiptIntake: { + deleteMany(args: { where: { id: string } }): Promise<{ count: number }>; + findUnique(args: { where: { id: string }; select: { id: true } }): Promise<{ id: string } | null>; + }; +} +export interface RejectClient { + $transaction(fn: (tx: RejectTxClient) => Promise): Promise; +} + +export async function rejectRowAndQueueCleanup( + rowId: string, + storagePath: string, + reason: string, + db: RejectClient = prisma as unknown as RejectClient, +): Promise<{ ok: true; eventId: string } | { ok: false }> { + try { + const eventId = await db.$transaction(async tx => { + const event = await tx.automationEvent.create({ + data: { + kind: STORAGE_CLEANUP_KIND, + status: "pending", + reason: reason.slice(0, 500), + source: "receipt-intake", + detail: JSON.stringify({ storagePath, rowId }), + }, + select: { id: true }, + }); + await tx.receiptIntake.deleteMany({ where: { id: rowId } }); + // deleteMany's count is 0 both when somebody else already deleted + // the row (fine — it is gone, which is all we need) and when the id + // never matched. Only the row's ABSENCE is the condition worth + // committing on, so assert that directly. + const survivor = await tx.receiptIntake.findUnique({ where: { id: rowId }, select: { id: true } }); + if (survivor) throw new Error(`row ${rowId} still exists after delete`); + return event.id; + }); + return { ok: true, eventId }; + } catch (error) { + console.error( + "[receipts/intake] reject transaction failed", + rowId, + error instanceof Error ? error.name : "error", + ); + return { ok: false }; + } +} + +/** + * Try the queued deletion now. A failure is not an error for the caller — the + * event stays pending and the worker's sweep retries it. + */ +export async function settleQueuedCleanup(eventId: string, storagePath: string): Promise { + try { + await removeSecureDocStrict(toSecureRef(storagePath)); + } catch (error) { + console.error( + "[receipts/intake] queued delete failed, left pending", + storagePath, + error instanceof Error ? error.name : "error", + ); + return false; + } + // Resolve only AFTER a delete that did not throw, same rule as the sweep. + await prisma.automationEvent + .update({ where: { id: eventId }, data: { status: "resolved" } }) + .catch(() => { /* the sweep will find it still pending and re-check */ }); + return true; +} + /** * Delete the object. If that fails, record the path so the sweep can retry. * Never throws: the caller is already rejecting a row and must not be derailed diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index 50ac64aff..2f3b84bcf 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -13,7 +13,12 @@ * straight to Supabase, so nothing it declared about the file is evidence. */ import { createHash } from "node:crypto"; -import { downloadDocBytesResult, toSecureRef, type DocBytesResult } from "@/lib/secure-storage"; +import { + downloadDocBytesResult, + secureObjectSize, + toSecureRef, + type DocBytesResult, +} from "@/lib/secure-storage"; import { EXT_BY_MIME, sniffMime } from "./file-type"; import { MAX_STORED_BYTES } from "./intake-core"; @@ -140,7 +145,23 @@ export async function inspectStoredObject( */ declaredMime: string, download: (ref: string) => Promise = downloadDocBytesResult, + /** Metadata-only size lookup; injected so the "no body read" test is provable. */ + sizeOf: (storagePath: string) => Promise = secureObjectSize, ): Promise { + // SIZE FIRST, FROM METADATA — before a single byte is read. + // + // The signed upload URL bypasses this server, so nothing has seen this + // object yet. Downloading it to discover it is 400 MB is how one upload + // takes the worker's whole invocation (and its memory) with it. `list` + // returns the metadata row in one small request whatever the object's size. + // + // A null size is "unknown", not "fine": the download below still enforces + // the limit on the bytes it actually got. + const declaredSize = await sizeOf(storagePath); + if (declaredSize !== null && declaredSize > MAX_STORED_BYTES) { + return { ok: false, kind: "rejected", reason: `file-too-large:${declaredSize}` }; + } + const result = await download(toSecureRef(storagePath)); if (!result.ok) { return result.kind === "not-found" diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index e9ed686ee..63f0cce39 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -127,6 +127,16 @@ export interface WorkerDependencies { /** Retry storage deletes that failed when a row was rejected. */ retryStorageCleanups: (shouldStop: () => boolean) => Promise; loadPhases: (projectId: string | null) => Promise<{ id: string; code: string; name: string }[]>; + /** + * Re-read the row's projectId immediately before routing. + * + * The claim snapshot can be stale by seconds: /finalize accepts a late job + * assignment while a row is unclaimed, and the Gemini read that runs in + * between takes 25 seconds. Routing on the snapshot published NEEDS_JOB for + * a receipt that HAS a job by then — and NEEDS_JOB is where a human goes + * looking for exactly that problem. + */ + refreshProjectId: (rowId: string) => Promise; /** * Tagged, and VERIFIED: the bytes must hash to what the row recorded at * finalize. A sha stored once and never re-checked proves nothing about @@ -154,8 +164,9 @@ export interface WorkerDependencies { rowId: string, state: ReceiptIntakeState, stateReason: string | null, - patch?: Partial, - ownership?: Ownership, + patch: Partial | undefined, + /** REQUIRED. An unowned write clobbers whatever the successor decided. */ + ownership: Ownership, ) => Promise; /** * READ + dryRun=false -> BOOKING, and the LAST weak-dedup check, taken @@ -633,6 +644,11 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis suggestedConfidence: read.suggestedConfidence, }; + // Re-read RIGHT BEFORE routing. Everything above — the download, a 25s + // model call — is time in which a late job assignment can have landed. + const projectId = await deps.refreshProjectId(row.id).catch(() => row.projectId); + const hasProject = !!projectId; + const routeInput = { docType: read.docType, amount: keys.amount, @@ -664,7 +680,7 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis return reason ? `${reason};tax-implausible` : "tax-implausible"; }; - const gate = routeState(routeInput, { strong: null, weak: null }, !!row.projectId); + const gate = routeState(routeInput, { strong: null, weak: null }, hasProject); if (gate.state !== "READ") { // A multi-doc, a non-receipt, or a $0/negative misread must never hold // a dedup key — it would quarantine the real receipt that arrives next @@ -704,13 +720,13 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis if (!applied.owned) return "STALE"; if (applied.strongOwner) { - const second = routeState(routeInput, { strong: applied.strongOwner, weak: null }, !!row.projectId); - await deps.applyState(row.id, second.state, note(second.stateReason), { + const second = routeState(routeInput, { strong: applied.strongOwner, weak: null }, hasProject); + const owned = await deps.applyState(row.id, second.state, note(second.stateReason), { ...base, dedupStrongKey: null, duplicateOfId: second.duplicateOfId, - }); - return second.state; + }, ownershipOf(row)); + return owned ? second.state : "STALE"; } // No strong hit (or no strong key at all — a placeholder ref). The weak net @@ -724,19 +740,19 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // re-checks the weak net. const weak = await deps.findWeakHit(row.id, keys.weak); if (weak) { - const third = routeState(routeInput, { strong: null, weak }, !!row.projectId); + const third = routeState(routeInput, { strong: null, weak }, hasProject); // RELEASE the strong key. Nothing was sent to QuickBooks, so this row // is parked pre-send and the documented rule applies to it like any // other. Holding the key made a CORRECTED resend of the same receipt // collide with a row that was never booked — the review queue then had // two rows and neither could proceed. The weak pair is still visible to // a human through duplicateOfId and the reason. - await deps.applyState(row.id, third.state, note(third.stateReason), { + const owned = await deps.applyState(row.id, third.state, note(third.stateReason), { ...base, dedupStrongKey: null, duplicateOfId: third.duplicateOfId, - }); - return third.state; + }, ownershipOf(row)); + return owned ? third.state : "STALE"; } // Routing is complete. This is the ONLY path to READ, and the only place @@ -782,8 +798,9 @@ export function ownershipOf(row: WorkerRow): Ownership { async function retryTransient(row: WorkerRow, deps: WorkerDependencies, reason: string): Promise { const attempts = row.attempts + 1; if (attempts >= MAX_BOOK_ATTEMPTS) { - await deps.applyState(row.id, "NEEDS_REVIEW", "max-retries"); - return "NEEDS_REVIEW"; + // Through parkTerminal like every other terminal park, so the + // strong-key release is decided in exactly one place. + return parkTerminal(row, deps, "max-retries"); } const owned = await deps.retryRow( row.id, diff --git a/src/lib/secure-storage.ts b/src/lib/secure-storage.ts index 2b6ce2428..85d98bc62 100644 --- a/src/lib/secure-storage.ts +++ b/src/lib/secure-storage.ts @@ -379,6 +379,37 @@ export async function uploadSecureDoc( * Deliberately a separate export rather than a behaviour change to * removeSecureDoc: several callers outside this feature do not guard it. */ +/** + * Byte size of a stored object WITHOUT downloading it. + * + * The signed upload URL bypasses this server entirely, so the first time we see + * a two-step object is when something reads it — and reading it is exactly what + * must not happen for a 400 MB file. `list` with a search returns the metadata + * row, which is one small request regardless of the object's size. + * + * Returns null when the size cannot be determined (missing object, no client, + * an older storage API without metadata): callers treat that as "unknown" and + * fall through to their normal path rather than refusing on an absence. + */ +export async function secureObjectSize(storagePath: string): Promise { + const supabase = getSupabase(); + if (!supabase) return null; + const slash = storagePath.lastIndexOf("/"); + const dir = slash > 0 ? storagePath.slice(0, slash) : ""; + const name = slash > 0 ? storagePath.slice(slash + 1) : storagePath; + try { + const { data, error } = await supabase.storage + .from(SECURE_BUCKET) + .list(dir, { search: name, limit: 100 }); + if (error || !data) return null; + const match = data.find(entry => entry.name === name); + const size = (match?.metadata as { size?: unknown } | undefined)?.size; + return typeof size === "number" && Number.isFinite(size) ? size : null; + } catch { + return null; + } +} + export async function removeSecureDocStrict(ref: string): Promise { const path = secureRefPath(ref); if (!path) throw new Error(`not a secure ref: ${String(ref).slice(0, 80)}`); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index d54cfcb99..5d4c8e448 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -29,6 +29,23 @@ import { QboVendorDuplicateError, } from "../src/lib/qbo-receipt-push"; +/** + * A createPurchase stub that stands for a call which REACHED the create. + * + * createQBReceiptPurchase fires `onBeforeCreate` immediately before the HTTP + * create and nowhere else, so whether a stub invokes it is the whole difference + * between "QuickBooks may hold a Purchase" and "nothing was ever sent" — which + * is what decides whether a parked row keeps its strong dedup key. Stubs + * standing for a PRE-create refusal (an account or vendor ensure, an ok:false + * decision) deliberately do not use this. + */ +function atCreate(fn: (...args: any[]) => Promise) { + return async (tokens: any, input: any, deadline: any, onBeforeCreate?: () => Promise) => { + await onBeforeCreate?.(); + return fn(tokens, input, deadline); + }; +} + const NOW = new Date("2026-09-01T12:00:00.000Z"); function row(overrides: Partial = {}): BookableRow { @@ -100,10 +117,10 @@ function recorder(overrides: Partial = {}, opts: { estimates?: isPushEnabled: () => true, isPushPaused: async () => false, getTokens: async () => ({ accessToken: "t", realmId: "r" }) as any, - createPurchase: async (_tokens, input) => { + createPurchase: atCreate(async (_tokens: any, input: any) => { purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: input.fileId.slice(0, 21), alreadyExists: false, attachment: "attached" }; - }, + }) as any, downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), logEvent: async (event) => { events.push(event); }, now: () => NOW, @@ -235,7 +252,7 @@ test("every PRE-send refusal releases the key; every POST-send one holds it", as // Post-send: QBO may hold a Purchase whose response we lost, so the key // stays claimed even though the row is parked. const faulted = recorder({ - createPurchase: async () => { throw new QboPurchaseFaultError(400, "closed period", "6210"); }, + createPurchase: atCreate(async () => { throw new QboPurchaseFaultError(400, "closed period", "6210"); }) as any, }); const result = await bookReceipt(row(), faulted.deps); assert.equal((result as any).releaseStrongKey, false); @@ -263,15 +280,31 @@ test("a dryRun row can never reach QuickBooks, even called directly", async () = }); test("QBO business-rule faults are TERMINAL, never retried", async () => { - const cases: [unknown, string][] = [ - [new QboPurchaseFaultError(400, "closed period", "6210"), "qbo-fault:6210"], + // A fault raised by the PURCHASE create may have created one: QuickBooks + // answered, and a lost response looks exactly like this. Key retained. + const posted = recorder({ + createPurchase: atCreate(async () => { + throw new QboPurchaseFaultError(400, "closed period", "6210"); + }) as any, + }); + assert.deepEqual(await bookReceipt(row(), posted.deps), { + outcome: "needs-review", reason: "qbo-fault:6210", releaseStrongKey: false, + }); + + // The ENSURES — resolving the expense account, creating the vendor — run + // BEFORE the create, so nothing was posted and the strong key goes back. + // This is what moving the fenced send mark to the create bought: these two + // used to quarantine the corrected re-submission against a booking that + // never happened. + const preCreate: [unknown, string][] = [ [new QboAccountConfigError("bad account"), "qbo-fault:account-config"], [new QboVendorDuplicateError("Lowes"), "qbo-fault:vendor-duplicate"], ]; - for (const [error, reason] of cases) { + for (const [error, reason] of preCreate) { const r = recorder({ createPurchase: async () => { throw error; } }); const result = await bookReceipt(row(), r.deps); - assert.deepEqual(result, { outcome: "needs-review", reason, releaseStrongKey: false }, reason); + assert.deepEqual(result, { outcome: "needs-review", reason, releaseStrongKey: true }, reason); + assert.deepEqual(r.sendMarks, [], "the create was never reached"); assert.equal(r.expenses.length, 0); } }); @@ -338,9 +371,9 @@ test("attachmentBlocker mirrors QBO's own ceilings", () => { test("an attachment upload that FAILED is retried, never reported as booked", async () => { const r = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "failed:500", - }) as any, + })) as any, }); const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "retry"); @@ -353,18 +386,18 @@ test("an EXISTING purchase is held to the SAME attachment standard", async () => // response — exactly when a Purchase is most likely to be sitting in the // books without its image. It was the one path exempt from the check. const failing = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "failed:500", - }) as any, + })) as any, }); const failed = await bookReceipt(row(), failing.deps); assert.equal(failed.outcome, "retry", "an upload fault on an existing Purchase is recoverable"); assert.equal(failing.expenses.length, 0, "and it is NOT booked meanwhile"); const skipped = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "skipped", - }) as any, + })) as any, }); const skippedResult = await bookReceipt(row(), skipped.deps); assert.equal(skippedResult.outcome, "needs-review"); @@ -391,15 +424,15 @@ test("a previous attachment failure does NOT block the recovery attempt", async test("already-attached counts as attached on the fresh-create path too", async () => { const r = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "already-attached", - }) as any, + })) as any, }); assert.equal((await bookReceipt(row(), r.deps)).outcome, "booked"); }); test("a QBTimeoutError retries on the backoff schedule", async () => { - const r = recorder({ createPurchase: async () => { throw new QBTimeoutError("timed out"); } }); + const r = recorder({ createPurchase: atCreate(async () => { throw new QBTimeoutError("timed out"); }) as any }); const first = await bookReceipt(row({ attempts: 0 }), r.deps); assert.equal(first.outcome, "retry"); assert.equal((first as any).attempts, 1); @@ -407,22 +440,22 @@ test("a QBTimeoutError retries on the backoff schedule", async () => { assert.equal((first as any).reason, "QBTimeoutError"); const third = await bookReceipt(row({ attempts: 2 }), recorder({ - createPurchase: async () => { throw new QBTimeoutError("timed out"); }, + createPurchase: atCreate(async () => { throw new QBTimeoutError("timed out"); }) as any, }).deps); assert.equal((third as any).nextRetryAt.getTime(), NOW.getTime() + 60 * 60_000); }); test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTAL", async () => { - const transient = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + const transient = recorder({ createPurchase: atCreate(async () => { throw new TypeError("fetch failed"); }) as any }); assert.equal((await bookReceipt(row({ attempts: 5 }), transient.deps)).outcome, "retry"); // row.attempts 18 -> this is attempt 19: still retryable. - const nearly = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + const nearly = recorder({ createPurchase: atCreate(async () => { throw new TypeError("fetch failed"); }) as any }); assert.equal((await bookReceipt(row({ attempts: 18 }), nearly.deps)).outcome, "retry"); // row.attempts 19 -> this is attempt 20, the last one the constant allows. // sendAttempted is what decides the key, not the fact of reaching the limit. - const exhausted = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + const exhausted = recorder({ createPurchase: atCreate(async () => { throw new TypeError("fetch failed"); }) as any }); assert.deepEqual(await bookReceipt(row({ attempts: 19, sendAttempted: true }), exhausted.deps), { outcome: "needs-review", reason: "max-retries", @@ -444,10 +477,10 @@ test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTA test("alreadyExists books identically — the lost-response retry", async () => { const r = recorder({ - createPurchase: async (_t, input) => ({ + createPurchase: atCreate(async (_t: any, input: any) => ({ ok: true, qbPurchaseId: "QB-7", docNumber: input.fileId.slice(0, 21), alreadyExists: true, attachment: "already-attached", - }) as any, + })) as any, }); const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "booked"); @@ -573,11 +606,11 @@ test("ample runway books normally, and threads ONE deadline into both QBO calls" const r = recorder({ deadline, getTokens: async d => { seen.push(d); return { accessToken: "t", realmId: "r" } as any; }, - createPurchase: async (_t, input, d) => { + createPurchase: atCreate(async (_t: any, input: any, d: any) => { seen.push(d); r.purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "attached" } as any; - }, + }) as any, }); const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "booked"); @@ -720,12 +753,12 @@ test("budget exhausted AFTER the token refresh also leaves it unset", async () = test("a real create DOES mark it, before the call", async () => { const order: string[] = []; const r = recorder({ - createPurchase: async (_t, input) => { + createPurchase: atCreate(async (_t: any, input: any) => { order.push("create"); r.purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "attached" } as any; - }, - markSendAttempted: async id => { order.push("mark"); r.sendMarks.push(id); return true; }, + }) as any, + markSendAttempted: async (id: string) => { order.push("mark"); r.sendMarks.push(id); return true; }, }); await bookReceipt(row(), r.deps); assert.deepEqual(order, ["mark", "create"], "marked FIRST, so a mid-create death still records it"); @@ -739,9 +772,9 @@ test("a 4xx or fault attachment failure goes to a human on the FIRST one", async // Purchase sits in the books without its receipt. for (const attachment of ["failed:400", "failed:413", "failed:415", "failed:fault"]) { const r = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment, - }) as any, + })) as any, }); const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "needs-review", attachment); @@ -755,9 +788,9 @@ test("a 4xx or fault attachment failure goes to a human on the FIRST one", async test("a 5xx or thrown attachment failure is still retried", async () => { for (const attachment of ["failed:500", "failed:502", "failed:AbortError", "failed:TypeError"]) { const r = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment, - }) as any, + })) as any, }); const result = await bookReceipt(row(), r.deps); assert.equal(result.outcome, "retry", attachment); @@ -776,7 +809,7 @@ test("isTerminalAttachmentFailure splits refusal from blip", () => { // ── retry() must read the CURRENT send flag (round-9 item 2) ─────────────── test("attempt 20 RETAINS the key when the failure was at the create", async () => { - const r = recorder({ createPurchase: async () => { throw new TypeError("fetch failed"); } }); + const r = recorder({ createPurchase: atCreate(async () => { throw new TypeError("fetch failed"); }) as any }); const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); assert.equal((result as any).reason, "max-retries"); // row.sendAttempted was false when the row was CLAIMED, but this attempt @@ -787,9 +820,9 @@ test("attempt 20 RETAINS the key when the failure was at the create", async () = test("attempt 20 RETAINS the key when the failure was at the attachment leg", async () => { const r = recorder({ - createPurchase: async () => ({ + createPurchase: atCreate(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: false, attachment: "failed:500", - }) as any, + })) as any, }); const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); assert.equal((result as any).reason, "max-retries"); diff --git a/tests/receipt-intake-claim-release.test.ts b/tests/receipt-intake-claim-release.test.ts new file mode 100644 index 000000000..63b733849 --- /dev/null +++ b/tests/receipt-intake-claim-release.test.ts @@ -0,0 +1,128 @@ +/** + * Ownership release. + * + * The worker claims a row by writing a claim token, and every write it makes + * afterwards is fenced on {id, state, claimToken}. That fence is only half the + * mechanism: a transition that COMPLETES the work must also hand the row back, + * in the SAME write. Leave the token behind and the row is owned by a pass that + * has finished — the next pass's CAS matches nothing, no other write can move + * it, and it sits until a human notices. (The claim query skips rows that carry + * a live token, which is what makes the leak permanent rather than a delay.) + * + * This walks every UPDATE in the worker's dependency factory rather than + * naming the ones that exist today, so a transition added later is covered by + * default instead of by remembering to add a case here. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +const ROOT = path.resolve(__dirname, ".."); +const route = readFileSync(path.join(ROOT, "src/app/api/cron/receipt-intake-worker/route.ts"), "utf8"); + +/** Every `data: { ... }` of every receiptIntake update in the file, with context. */ +function updateBlocks(source: string): { data: string; where: string }[] { + const blocks: { data: string; where: string }[] = []; + const needle = "receiptIntake.update"; + let at = source.indexOf(needle); + while (at !== -1) { + // Balanced scan from the call's opening brace to its close. + const open = source.indexOf("{", at); + let depth = 0; + let end = open; + for (; end < source.length; end++) { + if (source[end] === "{") depth++; + else if (source[end] === "}") { depth--; if (depth === 0) break; } + } + const call = source.slice(open, end + 1); + const dataAt = call.indexOf("data: {"); + if (dataAt !== -1) { + let d = 0; + const i = call.indexOf("{", dataAt); + let stop = i; + for (; stop < call.length; stop++) { + if (call[stop] === "{") d++; + else if (call[stop] === "}") { d--; if (d === 0) break; } + } + // Everything before `data:` is the where clause (and, for the + // aliased ones, the `owns` object it was built from). + blocks.push({ data: call.slice(i, stop + 1), where: call.slice(0, dataAt) }); + } + at = source.indexOf(needle, end); + } + return blocks; +} + +/** + * Only the CLAIM HOLDER's writes. The cutover sweep, the STAGING sweep and the + * claim query itself all write rows nobody owns — releasing a claim there is + * meaningless, and the claim query is the one write that TAKES ownership. + */ +const heldSection = route.slice(route.indexOf("applyState: async")); +const blocks = updateBlocks(heldSection); + +test("the walker actually found the worker's updates", () => { + // A structural test that matches nothing passes vacuously forever. + assert.ok(blocks.length >= 7, `expected the worker's updates, found ${blocks.length}`); +}); + +test("EVERY completed, deferred or terminal transition releases the claim", () => { + const leaked: string[] = []; + for (const block of blocks) { + // Shorthand counts: `{ attempts, nextRetryAt }` is every bit as much a + // transition as `{ nextRetryAt: x }`, and a filter that only saw the + // long form let a real leak through when this guard was first written. + const setsState = /\bstate\b/.test(block.data); + const setsRetry = /\bnextRetryAt\b/.test(block.data); + if (!setsState && !setsRetry) continue; // not a transition (e.g. the send mark) + + // THE ONE EXCEPTION: READ -> BOOKING hands the row straight to + // bookReceipt in the same pass, and both its send mark and its BOOKED + // commit CAS on this same token. Releasing here would admit a second + // worker to the same booking. + const isPromotion = /state: "BOOKING", stateReason: null/.test(block.data) && !setsRetry; + if (isPromotion) continue; + + const releases = /claimToken: null/.test(block.data) || /RELEASE_CLAIM/.test(block.data); + if (!releases) leaked.push(block.data.replace(/\s+/g, " ").slice(0, 120)); + } + assert.deepEqual(leaked, [], "these transitions keep a claim nobody will ever release"); +}); + +test("releasing clears BOTH claim fields, never just the token", () => { + // claimedAt is what the stuck-row health probe reads. A token cleared + // without its timestamp leaves a row that looks claimed to everything + // except the CAS. + assert.match(route, /const RELEASE_CLAIM = \{ claimToken: null, claimedAt: null \}/); + const halfReleased = blocks + .filter(b => /claimToken: null/.test(b.data) && !/claimedAt: null/.test(b.data)) + .map(b => b.data.replace(/\s+/g, " ").slice(0, 90)); + assert.deepEqual(halfReleased, [], "these rows still look claimed to the health probe"); +}); + +test("a routed row keeps no claim: finishRouting clears both fields under its fence", () => { + const fn = route.slice(route.indexOf("finishRouting: async")); + const body = fn.slice(0, fn.indexOf("\n },")); + assert.match(body, /where: \{ id: rowId, state: "RECEIVED", claimToken \}/, "fenced on state AND token"); + assert.match(body, /state: "READ"/); + assert.match(body, /claimToken: null/); + assert.match(body, /claimedAt: null/); +}); + +test("every fenced write CASes on the OWNERSHIP it was handed, not on the id alone", () => { + // `where: { id }` on its own is how a superseded pass overwrites the work + // of the pass that replaced it. + const fenced = blocks.filter(b => /\bstate\b/.test(b.data) || /\bnextRetryAt\b/.test(b.data)); + assert.ok(fenced.length >= 6, `expected the worker's transitions, found ${fenced.length}`); + for (const block of fenced) { + // Either the token is named in the where clause, or it arrives via the + // `owns` alias — which is itself built from {state, claimToken}. + assert.match( + block.where, + /claimToken|where: owns/, + `unfenced write: ${block.data.replace(/\s+/g, " ").slice(0, 90)}`, + ); + } + assert.match(route, /const owns = \{ id: rowId, state: "BOOKING", claimToken \} as const;/); +}); diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts new file mode 100644 index 000000000..c2534e436 --- /dev/null +++ b/tests/receipt-intake-late-fields.test.ts @@ -0,0 +1,125 @@ +/** + * Late job/phase assignment, and the races around it. + * + * A late field is a write to a row somebody else may be holding. The read that + * decides whether the write is allowed and the write itself are two round trips + * to Postgres, and every interesting bug lives in the gap: the worker claims, + * the state moves, a second caller writes a DIFFERENT project. A CAS that + * simply reports "busy" on a lost race is not enough — the same lost CAS also + * means "somebody already wrote exactly this", and answering 409 there makes a + * correct retry loop forever. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + reconcileLateFields, + type Denial, + type LateFieldRow, + type LateFieldsDeps, +} from "../src/lib/receipt-intake/late-fields"; + +function row(over: Partial = {}): LateFieldRow { + return { costCodeId: null, projectId: null, state: "RECEIVED", claimToken: null, ...over }; +} + +interface Trace { + deps: LateFieldsDeps; + applied: Record[]; + authorized: (string | null)[]; +} + +/** `reads` is consumed one per call, so a race can be scripted precisely. */ +function deps(reads: LateFieldRow[], count: number, denial: Denial | null = null): Trace { + const t: Trace = { applied: [], authorized: [], deps: null as unknown as LateFieldsDeps }; + let i = 0; + t.deps = { + read: async () => reads[Math.min(i++, reads.length - 1)] ?? null, + applyIfNull: async (_id, _state, toApply) => { t.applied.push(toApply); return count; }, + authorize: async projectId => { t.authorized.push(projectId); return denial; }, + }; + return t; +} + +test("an un-routed row with empty fields takes the values", async () => { + const t = deps([row()], 1); + assert.equal(await reconcileLateFields("r1", { projectId: "p1", costCodeId: "c1" }, t.deps), null); + assert.deepEqual(t.applied, [{ projectId: "p1", costCodeId: "c1" }]); +}); + +test("a routed row refuses a DIFFERENT project and never writes", async () => { + // Past RECEIVED the dedup keys, the phase suggestion and possibly a Purchase + // were all derived from the project the row had. Changing it now does not + // re-derive any of that. + const t = deps([row({ state: "READ", projectId: "p1" })], 1); + const denial = await reconcileLateFields("r1", { projectId: "p2" }, t.deps); + assert.equal(denial?.status, 409); + assert.equal(denial?.body.error, "late-fields-too-late"); + assert.deepEqual(t.applied, [], "no write was attempted"); +}); + +test("a routed row accepts a repeat of what it already holds", async () => { + // The client's retry after a lost response carries the same fields. That is + // not a conflict, and answering 409 would make a correct client give up. + const t = deps([row({ state: "BOOKED", projectId: "p1" })], 1); + assert.equal(await reconcileLateFields("r1", { projectId: "p1" }, t.deps), null); +}); + +// ── the races ─────────────────────────────────────────────────────────────── + +test("STATE-TRANSITION RACE: the row moves between the read and the write", async () => { + // Read says RECEIVED/unclaimed, so the write is allowed. By the time it + // runs a worker has claimed and read the row, and the CAS matches nothing. + // The persisted project is NOT what was supplied, so this is a real 409 — + // and a retryable one, because a claim is transient. + const t = deps( + [row(), row({ state: "READ", projectId: null, claimToken: "tok-9" })], + 0, + ); + const denial = await reconcileLateFields("r1", { projectId: "p1" }, t.deps); + assert.equal(denial?.status, 409); + assert.equal(denial?.body.error, "late-fields-busy"); + assert.equal(denial?.body.state, "READ"); + assert.equal(denial?.body.retryable, true, "a claim clears on its own; the client should retry"); + assert.deepEqual(t.authorized, [], "a row that does not hold our values is not re-authorized"); +}); + +test("a lost CAS whose row holds EXACTLY what was supplied is a success", async () => { + // Two callers finalized the same row with the same fields. One won. The + // loser must not be told 409 — nothing is wrong and nothing is left to do. + const t = deps([row(), row({ projectId: "p1" })], 0); + assert.equal(await reconcileLateFields("r1", { projectId: "p1" }, t.deps), null); + assert.deepEqual(t.authorized, ["p1"], "still re-authorized against the persisted project"); +}); + +test("CONCURRENT PROJECT CHANGE: the phase is re-authorized against the NEW project", async () => { + // The phase was authorized against the project the row had at read time. A + // concurrent write moved the row to a different job, and our cost code is + // not one of ITS phases. Accepting it here would file the receipt against a + // phase of another job — the exact thing the first check exists to stop. + const denied: Denial = { status: 400, body: { ok: false, error: "cost-code-not-a-phase" } }; + const t = deps([row({ projectId: "p1" }), row({ projectId: "p2", costCodeId: "c1" })], 0, denied); + const result = await reconcileLateFields("r1", { costCodeId: "c1" }, t.deps); + assert.deepEqual(t.authorized, ["p2"], "against the project the row carries NOW, not p1"); + assert.equal(result?.status, 400); + assert.equal(result?.body.error, "cost-code-not-a-phase"); +}); + +test("a row that vanished under the write is a non-retryable conflict", async () => { + const t = deps([row(), null as unknown as LateFieldRow], 0); + const denial = await reconcileLateFields("r1", { projectId: "p1" }, t.deps); + assert.equal(denial?.body.state, "gone"); + assert.equal(denial?.body.retryable, false); +}); + +test("a conflicting stored value is refused before any write", async () => { + const t = deps([row({ projectId: "p1" })], 1); + const denial = await reconcileLateFields("r1", { projectId: "p2" }, t.deps); + assert.equal(denial?.body.error, "late-fields-conflict"); + assert.deepEqual(t.applied, []); +}); + +test("no late fields means no reads and no writes at all", async () => { + const t = deps([row()], 1); + assert.equal(await reconcileLateFields("r1", {}, t.deps), null); + assert.deepEqual(t.applied, []); +}); diff --git a/tests/receipt-intake-reject.test.ts b/tests/receipt-intake-reject.test.ts new file mode 100644 index 000000000..6fcd6f820 --- /dev/null +++ b/tests/receipt-intake-reject.test.ts @@ -0,0 +1,146 @@ +/** + * Rejecting a row, and publishing one. + * + * Both are two writes that must agree. A reject deletes the row AND queues its + * object for deletion: do them separately and either the bytes are orphaned + * with nothing left to remember them (delete first, record fails) or the queue + * names a path a live row still points at (record first, delete fails). A + * publish moves STAGING -> RECEIVED: do it by id alone and a row that moved on + * gets dragged back to RECEIVED and re-read, which for a BOOKED row is a second + * Purchase. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { + rejectRowAndQueueCleanup, + type RejectClient, + type RejectTxClient, +} from "../src/lib/receipt-intake/storage-cleanup"; + +const ROOT = path.resolve(__dirname, ".."); +const intake = readFileSync(path.join(ROOT, "src/app/api/receipts/intake/route.ts"), "utf8"); +const finalize = readFileSync( + path.join(ROOT, "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", +); + +interface Store { + rows: Set; + events: { id: string; data: Record }[]; + committed: boolean; +} + +/** A $transaction that really rolls back: the fake state is only kept on commit. */ +function client(rows: string[], opts: { undeletable?: boolean } = {}): { db: RejectClient; store: Store } { + const store: Store = { rows: new Set(rows), events: [], committed: false }; + const db: RejectClient = { + $transaction: async fn => { + const stagedRows = new Set(store.rows); + const stagedEvents: Store["events"] = []; + let seq = 0; + const tx: RejectTxClient = { + automationEvent: { + create: async ({ data }) => { + const id = `ev-${++seq}`; + stagedEvents.push({ id, data }); + return { id }; + }, + }, + receiptIntake: { + deleteMany: async ({ where }) => { + if (opts.undeletable) return { count: 0 }; + return { count: stagedRows.delete(where.id) ? 1 : 0 }; + }, + findUnique: async ({ where }) => (stagedRows.has(where.id) ? { id: where.id } : null), + }, + }; + const out = await fn(tx); + store.rows = stagedRows; + store.events.push(...stagedEvents); + store.committed = true; + return out; + }, + }; + return { db, store }; +} + +test("a reject deletes the row and queues the object in ONE transaction", async () => { + const { db, store } = client(["row-1"]); + const injected = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "unsupported-type", db); + assert.equal(injected.ok, true); + assert.equal(store.rows.has("row-1"), false, "the row is gone"); + assert.equal(store.events.length, 1, "and exactly one cleanup is queued"); + assert.equal(store.events[0].data.status, "pending"); + assert.match(String(store.events[0].data.detail), /receipts\/intake\/row-1\.bin/); +}); + +test("a row that survives the delete rolls the queue entry back and reports failure", async () => { + // The caller must then KEEP the object: a row that still exists may still + // point at those bytes, so deleting them would destroy a live receipt. + const { db, store } = client(["row-1"], { undeletable: true }); + const result = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "empty-file", db); + assert.equal(result.ok, false); + assert.equal(store.committed, false, "nothing committed"); + assert.deepEqual(store.events, [], "no orphan record for a row that is still there"); + assert.equal(store.rows.has("row-1"), true); +}); + +test("rejecting an already-deleted row still queues the cleanup", async () => { + // The retry of a reject whose response was lost. deleteMany counts zero, + // but the row is ABSENT, which is the condition that matters — and its + // object is unreferenced, so it still has to be queued. + const { db, store } = client([], {}); + const result = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "unsupported-type", db); + assert.equal(result.ok, true); + assert.equal(store.events.length, 1); +}); + +test("an unconfirmed reject answers 503 and keeps the object", () => { + const branch = finalize.slice(finalize.indexOf("const rejected = await rejectRowAndQueueCleanup")); + const head = branch.slice(0, branch.indexOf("settleQueuedCleanup")); + assert.match(head, /reject-failed/); + assert.match(head, /status: 503/); + assert.ok( + !/deleteObjectOrRecord|removeSecureDoc/.test(head), + "no object deletion on the unconfirmed path", + ); +}); + +test("publishing STAGING -> RECEIVED is fenced on the exact state", () => { + const fn = intake.slice(intake.indexOf("async function publishStagedRow")); + const body = fn.slice(0, fn.indexOf("\n/**")); + assert.match(body, /updateMany/, "not a bare update by id"); + assert.match(body, /where: \{ id, state: expectState \}/); + assert.match(body, /alreadyPublished: true/, "an already-RECEIVED row is the outcome we wanted"); + assert.match(body, /publish-conflict/); +}); + +test("recovery is restricted to the two reasons a re-upload can actually fix", () => { + // "Any NEEDS_REVIEW row" would drag a row parked for a vendor mismatch, a + // zero total, or a QBO fault back to RECEIVED and re-read it, discarding a + // decision a human had already made. + assert.match(intake, /RECOVERABLE_REASONS = \["file-missing", "sha-mismatch"\]/); + assert.match(intake, /RECOVERABLE_REASONS\.includes\(existing\.stateReason \?\? ""\)/); + assert.match( + finalize, + /row\.stateReason === "file-missing" \|\| row\.stateReason === "sha-mismatch"/, + "the finalize route uses the same two reasons", + ); +}); + +test("a heal that loses its CAS deletes the object it just uploaded", () => { + // The upload happened before the CAS. Losing the race means nothing + // references those bytes, and the row we were healing belongs to somebody + // else now. + const heal = intake.slice(intake.indexOf("const healed = await storeObject")); + const body = heal.slice(0, heal.indexOf("return NextResponse.json({\n ok: true, recovered: true")); + assert.match(body, /if \(count === 0\)/); + assert.match(body, /deleteObjectOrRecord\(payload\.storagePath, "heal-lost-race"\)/); + assert.match(body, /publish-conflict/); + assert.ok( + body.indexOf("payload.storagePath !== existing.storagePath") < body.indexOf("heal-lost-race"), + "and never deletes the path the surviving row still points at", + ); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index cbd9e1251..92ef08648 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -48,6 +48,46 @@ test("oversize, empty and unidentifiable objects are REJECTED, not published", a assert.equal((exe as { reason: string }).reason, "unsupported-file-type"); }); +test("an oversize object is rejected from METADATA, with no body read at all", async () => { + // The signed upload URL bypasses this server, so the first time anything + // here sees the object is now. Downloading it to discover it is 400 MB is + // how one upload takes the whole invocation — and its memory — with it. + let downloads = 0; + const check = await inspectStoredObject( + "p.bin", + "image/jpeg", + async () => { downloads++; throw new Error("the body must never be fetched"); }, + async () => MAX_STORED_BYTES + 1, + ); + assert.equal(check.ok, false); + assert.equal((check as { reason: string }).reason, `file-too-large:${MAX_STORED_BYTES + 1}`); + assert.equal(downloads, 0, "not one byte was read"); +}); + +test("an UNKNOWN metadata size still downloads, and the bytes decide", async () => { + // A null size is "storage did not say", not "fine". The byte-length check + // below it is what actually enforces the limit. + const check = await inspectStoredObject( + "p.png", + "image/png", + give({ ok: true, bytes: PNG }), + async () => null, + ); + assert.ok(check.ok); +}); + +test("a metadata size AT the ceiling is not rejected before the download", async () => { + let downloads = 0; + const check = await inspectStoredObject( + "p.png", + "image/png", + async () => { downloads++; return { ok: true as const, bytes: PNG }; }, + async () => MAX_STORED_BYTES, + ); + assert.ok(check.ok); + assert.equal(downloads, 1); +}); + test("exactly at the ceiling is allowed", async () => { const atLimit = Buffer.concat([PNG, Buffer.alloc(MAX_STORED_BYTES - PNG.length, 0)]); assert.equal(atLimit.length, MAX_STORED_BYTES); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 57ee84f3e..9020659fd 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -122,6 +122,9 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) sweepStaleStaging: async () => { h.sweepCalls++; return 0; }, retryStorageCleanups: async () => { h.cleanupCalls++; return 0; }, loadPhases: async () => [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing" }], + // Defaults to what the row already carries: the interesting case is the + // one that overrides it, where a late assignment landed mid-pass. + refreshProjectId: async rowId => rows.find(r => r.id === rowId)?.projectId ?? null, downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), read: async () => { h.reads++; return goodRead; }, applyRead: async (_id, patch) => { h.applied.push(patch); return { owned: true, strongOwner: null }; }, @@ -811,6 +814,37 @@ test("an unreadable date falls back to the COMPANY's calendar day, not UTC's", a // ── The sweep lives inside the run's budget (round-5 item 7) ─────────────── +test("INTERLEAVING: a job assigned after the claim is honoured, not parked NEEDS_JOB", async () => { + // The pass claims a row with no project, spends ~25s in the reader, and a + // finalize writes the project in the meantime. Routing on the value read at + // claim time would publish NEEDS_JOB for a receipt that HAS a job — and + // NEEDS_JOB is exactly where a human goes looking for that problem, so the + // row would sit in the one queue that means the opposite of its state. + const h = harness([workerRow({ projectId: null })], { + refreshProjectId: async () => "proj-late", + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { READ: 1 }, "routed, not parked"); + assert.deepEqual(h.states, [], "no NEEDS_JOB park was written"); + assert.deepEqual(h.finished, [{ id: "row-1", claimToken: "claim-1", stateReason: null }]); +}); + +test("a row with no job at claim time AND none at routing time still parks", async () => { + // The control for the test above: the re-read is a re-read, not a way to + // pretend every row has a job. + const h = harness([workerRow({ projectId: null })], { refreshProjectId: async () => null }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { NEEDS_JOB: 1 }); +}); + +test("a failing re-read falls back to the claimed value instead of losing the row", async () => { + const h = harness([workerRow({ projectId: "proj-1" })], { + refreshProjectId: async () => { throw new Error("pool exhausted"); }, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { READ: 1 }); +}); + test("the deadline starts at invocation entry, so a slow sweep cannot overrun it", async () => { // The sweep downloads objects. Timing it OUT of the budget meant it could // eat the platform timeout and the worker would still go on to start a 25s From 8c21abdfe07f34702bcc6ea328c8a610f2b97aa8 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:14:33 -0700 Subject: [PATCH 050/144] fix(receipts): validate the phase against the job at intake, not just at finalize /start stored a caller-supplied costCodeId unchecked, and nothing downstream re-checked it: /finalize only authorizes the fields the finalize CALL carries, so a cross-project phase survived by simply being omitted at finalize and rode into the Expense. One shared gate, authorizePhase() in late-fields.ts, now used by /start, the single-shot POST /api/receipts/intake (same gap) and /finalize, always AFTER the project authorization. 400 cost-code-not-a-phase on mismatch, before any row exists. Tests: the start->finalize regression proves finalize runs zero authorizations when the field is omitted (which is why /start must be the gate), plus a source guard that every entry point calls AND returns the denial. Mutation-tested by disabling the gate. Co-Authored-By: Claude Fable 5.1 --- .../receipts/intake/[id]/finalize/route.ts | 36 +++----- src/app/api/receipts/intake/route.ts | 11 +++ src/app/api/receipts/intake/start/route.ts | 22 ++++- src/lib/receipt-intake/late-fields.ts | 40 +++++++++ tests/receipt-intake-late-fields.test.ts | 86 +++++++++++++++++++ 5 files changed, 168 insertions(+), 27 deletions(-) diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 20db352f9..a2a027278 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -6,7 +6,12 @@ import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; -import { reconcileLateFields, type Denial, type LateFields } from "@/lib/receipt-intake/late-fields"; +import { + authorizePhase, + reconcileLateFields, + type Denial, + type LateFields, +} from "@/lib/receipt-intake/late-fields"; import { deleteObjectOrRecord, rejectRowAndQueueCleanup, @@ -73,31 +78,10 @@ async function authorizeLateFields( } } - if (lateFields.costCodeId) { - if (!projectId) { - return { - status: 400, - body: { ok: false, error: "cost-code-without-project", reason: "a phase is only meaningful against a job" }, - }; - } - const allowed = await isCostCodeAllowedForProject( - prismaPhaseDataSource, - projectId, - lateFields.costCodeId, - ); - if (!allowed) { - return { - status: 400, - body: { - ok: false, - error: "cost-code-not-a-phase", - reason: "that cost code is not a phase of this job", - projectId, - }, - }; - } - } - return null; + // Same rule, same implementation, as /start applies to a phase supplied + // there — the two must never be able to disagree. + return await authorizePhase(projectId, lateFields.costCodeId ?? null, (project, code) => + isCostCodeAllowedForProject(prismaPhaseDataSource, project, code)); } export const maxDuration = 30; diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index df40af188..52445414c 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -3,6 +3,9 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; +import { authorizePhase } from "@/lib/receipt-intake/late-fields"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; @@ -245,6 +248,14 @@ export async function POST(req: Request) { if (!allowed) return NextResponse.json({ ok: false, reason: "forbidden" }, { status: 403 }); } + // A phase belongs to a job. Same gate as /start, AFTER the project + // authorization above: nothing downstream re-checks a cost code that was + // supplied when the row was created, so this is the only place to catch one + // that belongs to another job. + const badPhase = await authorizePhase(parsed.projectId, parsed.costCodeId, (project, code) => + isCostCodeAllowedForProject(prismaPhaseDataSource, project, code)); + if (badPhase) return NextResponse.json(badPhase.body, { status: badPhase.status }); + const id = randomUUID(); const ext = EXT_BY_MIME[mimeType] ?? "bin"; const storagePath = `receipts/intake/${id}.${ext}`; diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index b18b966f9..a8729edd0 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -8,6 +8,9 @@ import { getSupabase } from "@/lib/supabase"; import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; +import { authorizePhase } from "@/lib/receipt-intake/late-fields"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -107,6 +110,23 @@ export async function POST(req: Request) { } } + // THE PHASE IS CHECKED HERE OR NOWHERE. + // + // A costCodeId supplied at /start used to be stored unchecked, and nothing + // downstream re-checks it: /finalize only authorizes the fields the + // FINALIZE call carries, so a client could smuggle a phase from another job + // past every gate simply by omitting it at finalize. The FK gives a 400 for + // a cost code that does not exist at all, which is a different question + // from whether it belongs to this job. + // + // AFTER the project authorization above, never before: validating against a + // project the caller cannot reach would answer questions about somebody + // else's job. + const costCodeId = str(body.costCodeId); + const badPhase = await authorizePhase(projectId, costCodeId, (project, code) => + isCostCodeAllowedForProject(prismaPhaseDataSource, project, code)); + if (badPhase) return NextResponse.json(badPhase.body, { status: badPhase.status }); + const id = randomUUID(); const storagePath = `receipts/intake/${id}.${ext}`; @@ -120,7 +140,7 @@ export async function POST(req: Request) { state: "STAGING", dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId, - costCodeId: str(body.costCodeId), + costCodeId, createdById: auth.via === "session" ? auth.user.id : null, // Forwarder-only, same as the single-shot path: this is the // claim that v1 already booked the document. diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index 68b0f1bbf..b253ed571 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -129,3 +129,43 @@ export async function reconcileLateFields( // job, which is exactly what the first authorization existed to prevent. return await deps.authorize(after.projectId); } + +/** + * A phase is only valid against the job it belongs to. + * + * Shared by /start and /finalize deliberately. /start used to store a + * caller-supplied `costCodeId` unchecked, and nothing downstream re-checked it: + * /finalize only authorizes the fields the finalize CALL carries, so omitting + * the field there let a cross-project phase survive all the way into the + * Expense — where every variance report reads it as overspend on a line nobody + * budgeted, on a job that never bought it. + */ +export async function authorizePhase( + projectId: string | null, + costCodeId: string | null, + isCostCodeAllowed: (projectId: string, costCodeId: string) => Promise, +): Promise { + if (!costCodeId) return null; + if (!projectId) { + return { + status: 400, + body: { + ok: false, + error: "cost-code-without-project", + reason: "a phase is only meaningful against a job", + }, + }; + } + if (!(await isCostCodeAllowed(projectId, costCodeId))) { + return { + status: 400, + body: { + ok: false, + error: "cost-code-not-a-phase", + reason: "that cost code is not a phase of this job", + projectId, + }, + }; + } + return null; +} diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts index c2534e436..301239451 100644 --- a/tests/receipt-intake-late-fields.test.ts +++ b/tests/receipt-intake-late-fields.test.ts @@ -11,7 +11,10 @@ */ import test from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { + authorizePhase, reconcileLateFields, type Denial, type LateFieldRow, @@ -123,3 +126,86 @@ test("no late fields means no reads and no writes at all", async () => { assert.equal(await reconcileLateFields("r1", {}, t.deps), null); assert.deepEqual(t.applied, []); }); + +// ── The phase gate: /start is the only place a start-time phase is checked ── + +/** A job with exactly one phase of its own. Anything else belongs elsewhere. */ +const phasesOf: Record = { "proj-a": ["cc-a"], "proj-b": ["cc-b"] }; +const isCostCodeAllowed = async (projectId: string, costCodeId: string) => + (phasesOf[projectId] ?? []).includes(costCodeId); + +test("a phase from ANOTHER job is refused at /start, before a row exists", async () => { + const denial = await authorizePhase("proj-a", "cc-b", isCostCodeAllowed); + assert.equal(denial?.status, 400); + assert.equal(denial?.body.error, "cost-code-not-a-phase"); + assert.equal(denial?.body.projectId, "proj-a"); +}); + +test("a phase with no job at all is refused: it is not meaningful", async () => { + const denial = await authorizePhase(null, "cc-a", isCostCodeAllowed); + assert.equal(denial?.status, 400); + assert.equal(denial?.body.error, "cost-code-without-project"); +}); + +test("the job's own phase passes, and no phase at all is not a lookup", async () => { + assert.equal(await authorizePhase("proj-a", "cc-a", isCostCodeAllowed), null); + let looked = 0; + assert.equal( + await authorizePhase("proj-a", null, async () => { looked++; return true; }), + null, + ); + assert.equal(looked, 0); +}); + +test("REGRESSION: a cross-project phase cannot survive by being OMITTED at finalize", async () => { + // The hole this closes. /start stored a caller-supplied costCodeId + // unchecked, and /finalize only authorizes the fields the FINALIZE call + // carries — so a client that sent the phase at /start and then finalized + // with an empty body got a published row holding a phase from another job, + // having passed no check anywhere. The Expense inherits it and the other + // job's variance report reads overspend on a line nobody budgeted. + // + // Step 1: finalize with no late fields runs NO authorization. That is + // correct — there is nothing to authorize — and it is exactly why /start + // has to be the gate. + let authorizations = 0; + const finalize = deps([row({ projectId: "proj-a", costCodeId: "cc-b" })], 1); + const counted: LateFieldsDeps = { + ...finalize.deps, + authorize: async projectId => { authorizations++; return finalize.deps.authorize(projectId); }, + }; + assert.equal(await reconcileLateFields("r1", {}, counted), null); + assert.equal(authorizations, 0, "finalize never re-checks a phase it was not sent"); + assert.deepEqual(finalize.applied, [], "and writes nothing"); + + // Step 2: so the same payload must be refused at /start, where the row + // would otherwise be created holding it. + const atStart = await authorizePhase("proj-a", "cc-b", isCostCodeAllowed); + assert.equal(atStart?.status, 400, "the row is never created"); + assert.equal(atStart?.body.error, "cost-code-not-a-phase"); +}); + +test("both intake entry points call the SAME phase gate", () => { + // Two copies of this rule is how they drift, and a route that skips it is + // this whole regression again. + const routes = [ + "src/app/api/receipts/intake/start/route.ts", + "src/app/api/receipts/intake/route.ts", + "src/app/api/receipts/intake/[id]/finalize/route.ts", + ]; + for (const rel of routes) { + const source = readFileSync(path.join(__dirname, "..", rel), "utf8"); + assert.match(source, /authorizePhase\(/, `${rel} does not use the shared gate`); + // CALLING it is not the same as OBEYING it: the denial has to end the + // request, or the row is created with the phase anyway. + assert.match( + source, + /if \(badPhase\) return NextResponse\.json\(badPhase\.body, \{ status: badPhase\.status \}\);|return await authorizePhase\(/, + `${rel} does not return the denial`, + ); + assert.ok( + !/error: "cost-code-not-a-phase"/.test(source), + `${rel} carries its own copy of the rule`, + ); + } +}); From 6f8ecc30d77cb4981633a7df4602c71477979693 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:30:03 -0700 Subject: [PATCH 051/144] fix(receipts): initial publication obeys null-or-equal and validates the tuple MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing spread the finalize's late fields straight over the row, which made initial publication the one path that could silently REPLACE a job, a phase or a tax answer captured at /start. - mergeCapturedFields() applies null-or-equal to every captured field (including installedAtCustomer, which Phase 3 adds — the rule is written over the keys handed in, so it covers the column the day it lands). A disagreement is a 409. - The RESULTING project/phase tuple is authorized, not the supplied half: a late projectId=B against a phase captured for job A is a 409 captured-phase-conflict unless the phase is also valid for B. A caller's own bad pair stays a 400. - The publishing UPDATE CASes over every original captured value, so a concurrent writer that filled one in invalidates the tuple this publish validated. Losing that CAS on a row still in STAGING now answers 409 publish-conflict rather than claiming alreadyFinalized — nothing was published, and the forwarders drop their copy of anything we say we hold. Tests cover the A/B case, an existing tax answer, the guard covering unwritten fields, and the route wiring. Mutation-tested by restoring the blind spread. Co-Authored-By: Claude Fable 5.1 --- .../receipts/intake/[id]/finalize/route.ts | 87 ++++++++++++-- src/lib/receipt-intake/late-fields.ts | 94 +++++++++++++++ tests/receipt-intake-late-fields.test.ts | 108 ++++++++++++++++++ 3 files changed, 279 insertions(+), 10 deletions(-) diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index a2a027278..2444c77c0 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -8,6 +8,7 @@ import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; import { authorizePhase, + mergeCapturedFields, reconcileLateFields, type Denial, type LateFields, @@ -130,7 +131,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string where: { id }, select: { id: true, state: true, stateReason: true, sourceRef: true, storagePath: true, - mimeType: true, projectId: true, dryRun: true, createdById: true, + mimeType: true, projectId: true, costCodeId: true, dryRun: true, createdById: true, fileSha256: true, expectedSha256: true, }, }); @@ -243,15 +244,63 @@ export async function POST(req: Request, context: { params: Promise<{ id: string } } + // THE PUBLISH IS SUBJECT TO THE SAME NULL-OR-EQUAL RULE as every other + // late-field write. + // + // This used to spread `lateFields` straight into the publishing update, + // which made initial publication the one path that could silently REPLACE a + // job, a phase or a tax answer captured at /start. A client that captured + // the job at /start and sent a different one at finalize simply won, and + // nothing recorded that the first answer had ever existed. + const merged = mergeCapturedFields( + { projectId: row.projectId, costCodeId: row.costCodeId }, + lateFields, + ); + if ("status" in merged) return NextResponse.json(merged.body, { status: merged.status }); + + // AND THE RESULTING TUPLE IS VALIDATED, not the supplied half of it. + // + // authorizeLateFields above only saw what this REQUEST carried. A finalize + // that sends projectId=B against a phase captured for job A supplies a + // project that is fine on its own and a phase it never mentions — and the + // row ends up filed under B's job with A's phase. The check has to be on + // the pair the row will actually hold. + const mixed = merged.from.projectId !== merged.from.costCodeId; + const badTuple = await authorizePhase( + merged.resulting.projectId, + merged.resulting.costCodeId, + (project, code) => isCostCodeAllowedForProject(prismaPhaseDataSource, project, code), + ); + if (badTuple) { + // A caller's OWN bad pair is a 400 (fix the request). A pair only this + // MERGE created — half captured, half late — is a 409: the request is + // well-formed, it just disagrees with what the row already holds. + return mixed + ? NextResponse.json( + { + ...badTuple.body, + error: "captured-phase-conflict", + reason: "the phase already on this row is not a phase of the job you sent", + captured: { projectId: row.projectId, costCodeId: row.costCodeId }, + resulting: merged.resulting, + }, + { status: 409 }, + ) + : NextResponse.json(badTuple.body, { status: badTuple.status }); + } + // ONE shared seal-and-publish, also used by the worker's stale-STAGING // sweep, so the two publishers cannot diverge on ordering or fencing. const outcome = await sealAndPublish(row.storagePath, id, check, { seal: sealObject, commit: async (canonicalPath, values) => { const { count } = await prisma.receiptIntake.updateMany({ - // Fenced: only a row still in a publishable state moves, so a - // loser of the race writes nothing. - where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] } }, + // Fenced on the state AND on every captured value this publish + // was validated against. A concurrent writer that filled in the + // job or the phase underneath us invalidates the tuple checked + // above, so this publish must lose rather than write a row it + // never authorized. + where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] }, ...merged.guard }, data: { state: "RECEIVED", stateReason: null, @@ -260,7 +309,9 @@ export async function POST(req: Request, context: { params: Promise<{ id: string fileSize: values.fileSize, fileSha256: values.fileSha256, nextRetryAt: null, - ...lateFields, + // NULL-OR-EQUAL: only the fields the row does not already + // answer. Never a blind spread of what the caller sent. + ...merged.apply, }, }); return count; @@ -272,15 +323,31 @@ export async function POST(req: Request, context: { params: Promise<{ id: string return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); } if (!outcome.published) { - // Another publisher won. Same outcome for the caller — but the late - // fields still have to be reconciled against what that publisher wrote. - const reconciled = await applyLateFields(id, lateFields, auth); - if (reconciled) return reconciled; + // The CAS lost — which is now TWO different things. Either another + // publisher moved the row (the caller's answer is "already finalized"), + // or a captured value changed underneath a row that is still waiting to + // publish, in which case nothing was published and saying otherwise + // would be a lie the client acts on. const current = await prisma.receiptIntake.findUnique({ where: { id }, select: { state: true, sourceRef: true, projectId: true, dryRun: true }, }); - return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: current?.state ?? "RECEIVED" }); + if (!current || current.state === "STAGING") { + return NextResponse.json( + { + ok: false, + error: "publish-conflict", + reason: "this row changed while it was being published; retry", + retryable: true, + }, + { status: 409 }, + ); + } + // Another publisher won. Same outcome for the caller — but the late + // fields still have to be reconciled against what that publisher wrote. + const reconciled = await applyLateFields(id, lateFields, auth); + if (reconciled) return reconciled; + return NextResponse.json({ ok: true, alreadyFinalized: true, id, state: current.state }); } const persisted = await prisma.receiptIntake.findUnique({ diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index b253ed571..bda8b2668 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -169,3 +169,97 @@ export async function authorizePhase( } return null; } + +/** + * Fields CAPTURED when the row was created, which a later finalize may fill in + * but never overwrite. + * + * `installedAtCustomer` is listed deliberately even though the column does not + * exist yet (it lands in Phase 3): the rule is written over whatever keys the + * caller hands in, so the field is covered the day it is added rather than + * needing somebody to remember this file. It is a TAX answer — whether the + * material was installed at the customer's site decides how the purchase is + * taxed — so an overwrite there is a wrong number in the books, not a mislabel. + */ +export const CAPTURED_FIELDS = ["projectId", "costCodeId", "installedAtCustomer"] as const; + +export type CapturedValues = Record; + +export interface CapturedMerge { + /** Only the fields that are actually changing. */ + apply: Record; + /** + * The ORIGINAL captured values, for a CAS on the publishing update: if any + * of them moved between the read and the write, the publish must lose. + */ + guard: Record; + /** What the row will hold once `apply` lands. */ + resulting: { projectId: string | null; costCodeId: string | null }; + /** Where each half of the resulting phase tuple came from. */ + from: { projectId: "captured" | "late" | "none"; costCodeId: "captured" | "late" | "none" }; +} + +/** + * NULL-OR-EQUAL at publish time too. + * + * Initial publication used to spread the finalize's late fields straight over + * the row, which made /finalize the one path that could silently REPLACE a job, + * phase or tax answer captured at /start — the exact overwrite every other path + * refuses. A client that captured the job at /start and sent a different one at + * finalize simply won, and nothing recorded that the first answer ever existed. + */ +export function mergeCapturedFields( + captured: CapturedValues, + lateFields: LateFields, +): Denial | CapturedMerge { + const apply: Record = {}; + const guard: Record = {}; + const conflicts: Record = {}; + + for (const [key, value] of Object.entries(captured)) { + // The CAS covers EVERY captured field, not just the ones being written: + // a concurrent writer that filled in the phase we are about to leave + // alone still invalidates the tuple this publish validated. + guard[key] = value ?? null; + } + + for (const [key, supplied] of Object.entries(lateFields)) { + if (supplied === undefined) continue; + const stored = captured[key] ?? null; + if (stored === null) { + apply[key] = supplied; + continue; + } + if (stored !== supplied) conflicts[key] = { stored, supplied }; + } + + if (Object.keys(conflicts).length > 0) { + return { + status: 409, + body: { + ok: false, + error: "late-fields-conflict", + reason: "this row already carries different values for these fields", + fields: conflicts, + }, + }; + } + + const pick = (key: "projectId" | "costCodeId") => { + const stored = (captured[key] ?? null) as string | null; + if (stored !== null) return { value: stored, from: "captured" as const }; + const late = lateFields[key] ?? null; + return late !== null + ? { value: late, from: "late" as const } + : { value: null, from: "none" as const }; + }; + const project = pick("projectId"); + const phase = pick("costCodeId"); + + return { + apply, + guard, + resulting: { projectId: project.value, costCodeId: phase.value }, + from: { projectId: project.from, costCodeId: phase.from }, + }; +} diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts index 301239451..37da14aee 100644 --- a/tests/receipt-intake-late-fields.test.ts +++ b/tests/receipt-intake-late-fields.test.ts @@ -15,6 +15,7 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { authorizePhase, + mergeCapturedFields, reconcileLateFields, type Denial, type LateFieldRow, @@ -209,3 +210,110 @@ test("both intake entry points call the SAME phase gate", () => { ); } }); + +// ── Initial publication is bound by the same rules (Phase 3 gate) ─────────── + +const FINALIZE = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", +); + +test("publishing FILLS IN a captured field that is null", () => { + const merged = mergeCapturedFields({ projectId: null, costCodeId: null }, { projectId: "proj-a" }); + assert.ok(!("status" in merged)); + assert.deepEqual(merged.apply, { projectId: "proj-a" }); + assert.deepEqual(merged.resulting, { projectId: "proj-a", costCodeId: null }); +}); + +test("publishing NEVER overwrites a captured job", () => { + // The hole: initial publication spread the finalize's fields straight over + // the row, so a different job sent at finalize simply won. + const merged = mergeCapturedFields({ projectId: "proj-a", costCodeId: null }, { projectId: "proj-b" }); + assert.ok("status" in merged); + assert.equal(merged.status, 409); + assert.equal(merged.body.error, "late-fields-conflict"); +}); + +test("publishing NEVER overwrites an existing TAX answer", () => { + // installedAtCustomer decides how the purchase is taxed. Silently replacing + // the answer captured at /start is a wrong number in the books, not a + // mislabel — and the field is covered here before the column even exists. + const captured = { projectId: "proj-a", costCodeId: null, installedAtCustomer: true }; + const flipped = mergeCapturedFields(captured, { installedAtCustomer: false } as never); + assert.ok("status" in flipped); + assert.equal(flipped.status, 409); + assert.deepEqual((flipped.body.fields as Record).installedAtCustomer, { + stored: true, supplied: false, + }); + + // The same answer again is a retry, not a conflict. + const same = mergeCapturedFields(captured, { installedAtCustomer: true } as never); + assert.ok(!("status" in same)); + assert.deepEqual(same.apply, {}, "nothing to write"); + + // And an UNANSWERED one is still filled in. + const first = mergeCapturedFields( + { projectId: "proj-a", costCodeId: null, installedAtCustomer: null }, + { installedAtCustomer: true } as never, + ); + assert.ok(!("status" in first)); + assert.deepEqual(first.apply, { installedAtCustomer: true } as never); +}); + +test("A/B: a late job with a phase captured for ANOTHER job is refused", async () => { + // The pair the row would END UP holding is what has to be valid. Neither + // half is wrong on its own: proj-b is a job the caller may reach, and cc-a + // was authorized when it was captured — against job A. + const merged = mergeCapturedFields({ projectId: null, costCodeId: "cc-a" }, { projectId: "proj-b" }); + assert.ok(!("status" in merged)); + assert.deepEqual(merged.resulting, { projectId: "proj-b", costCodeId: "cc-a" }); + assert.deepEqual(merged.from, { projectId: "late", costCodeId: "captured" }, + "half captured, half late — a pair only the merge created"); + + const denial = await authorizePhase(merged.resulting.projectId, merged.resulting.costCodeId, isCostCodeAllowed); + assert.equal(denial?.body.error, "cost-code-not-a-phase"); + + // A phase that IS valid for the late job goes through: the rule is about + // the tuple, not about mixing sources. + const ok = mergeCapturedFields({ projectId: null, costCodeId: "cc-b" }, { projectId: "proj-b" }); + assert.ok(!("status" in ok)); + assert.equal(await authorizePhase(ok.resulting.projectId, ok.resulting.costCodeId, isCostCodeAllowed), null); +}); + +test("the publish CAS covers EVERY captured field, not just the written ones", () => { + // A concurrent writer that filled in the phase we were going to leave alone + // invalidates the tuple this publish validated, so it must lose. + const merged = mergeCapturedFields( + { projectId: null, costCodeId: null, installedAtCustomer: null }, + { projectId: "proj-a" }, + ); + assert.ok(!("status" in merged)); + assert.deepEqual(merged.guard, { projectId: null, costCodeId: null, installedAtCustomer: null }); + assert.deepEqual(merged.apply, { projectId: "proj-a" }, "but only one field is written"); +}); + +test("the publishing UPDATE spreads the merge, never the raw late fields", () => { + const commit = FINALIZE.slice(FINALIZE.indexOf("const outcome = await sealAndPublish")); + const body = commit.slice(0, commit.indexOf("dropUpload:")); + assert.match(body, /\.\.\.merged\.guard/, "CAS over the captured values"); + assert.match(body, /\.\.\.merged\.apply/, "and only the fields that change"); + assert.ok(!/\.\.\.lateFields/.test(body), "the blind spread is gone"); +}); + +test("a mixed tuple is a 409, and a caller's own bad pair stays a 400", () => { + const branch = FINALIZE.slice(FINALIZE.indexOf("const badTuple = await authorizePhase")); + const body = branch.slice(0, branch.indexOf("// ONE shared seal-and-publish")); + assert.match(body, /captured-phase-conflict/); + assert.match(body, /status: 409/); + assert.match(body, /status: badTuple\.status/, "the un-mixed case keeps its own status"); +}); + +test("losing the publish CAS on a STAGING row is a conflict, NOT alreadyFinalized", () => { + // Nothing was published. Telling the client it was finalized is a lie it + // acts on: the forwarders drop their copy of a receipt they are told we hold. + const branch = FINALIZE.slice(FINALIZE.indexOf("if (!outcome.published)")); + const body = branch.slice(0, branch.indexOf("alreadyFinalized")); + assert.match(body, /current\.state === "STAGING"/); + assert.match(body, /publish-conflict/); + assert.match(body, /status: 409/); +}); From 1cb66bba4774fc22ab87bafd3be69b5d1095fffc Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:45:46 -0700 Subject: [PATCH 052/144] test(receipts): make the cleanup-test function slices EOL-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `indexOf("\n}\n")` is -1 on a CRLF checkout (the bytes are "\r\n}\r\n"), and `slice(0, -1)` then hands back the rest of the file — so an assertion scoped to one function silently read every function after it, and passed or failed for the wrong reason depending on who cloned the repo. One bodyOf() helper matching /\r?\n\}\r?\n/, asserting it found both the declaration and a closing brace, used by all four slices. Its own control test runs the same fixture with LF and with CRLF. Verified by CRLF-ifying the sources under test: green with the helper, and the whole file fails with the old slicing restored. Co-Authored-By: Claude Fable 5.1 --- tests/receipt-intake-cleanup.test.ts | 40 +++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/receipt-intake-cleanup.test.ts b/tests/receipt-intake-cleanup.test.ts index 00f767de8..fa1bdc920 100644 --- a/tests/receipt-intake-cleanup.test.ts +++ b/tests/receipt-intake-cleanup.test.ts @@ -16,12 +16,40 @@ const cleanup = readFileSync(path.join(ROOT, "src/lib/receipt-intake/storage-cle const intake = readFileSync(path.join(ROOT, "src/app/api/receipts/intake/route.ts"), "utf8"); const storage = readFileSync(path.join(ROOT, "src/lib/secure-storage.ts"), "utf8"); +/** + * The body of one top-level function, EOL-agnostic. + * + * `indexOf("\n}\n")` returns -1 on a CRLF checkout (the bytes there are + * "\r\n}\r\n"), and `slice(0, -1)` then quietly hands back the REST OF THE + * FILE — so an assertion scoped to one function silently starts reading every + * function after it, and these tests pass or fail for the wrong reason. Git's + * autocrlf makes that a property of who cloned the repo, not of the code. + */ +function bodyOf(source: string, declaration: string): string { + const from = source.indexOf(declaration); + assert.notEqual(from, -1, `not found: ${declaration}`); + const rest = source.slice(from); + const end = rest.search(/\r?\n\}\r?\n/); + assert.notEqual(end, -1, `no closing brace found for ${declaration}`); + return rest.slice(0, end); +} + +test("bodyOf stops at the function it was given, on either line ending", () => { + // The control. Without it the helper could go back to returning the whole + // file and every assertion below would still pass. + const lf = "function a() {\n inA();\n}\n\nfunction b() {\n inB();\n}\n"; + for (const text of [lf, lf.replace(/\n/g, "\r\n")]) { + const body = bodyOf(text, "function a()"); + assert.match(body, /inA\(\)/); + assert.ok(!body.includes("inB()"), "it did not run on into the next function"); + } +}); + test("recording a cleanup is durable, not fire-and-forget", () => { // logAutomationEvent never throws by contract, so "it did not throw" is not // proof it wrote. The record is read back, and the function throws if it is // not there — the caller has to know. - const fn = cleanup.slice(cleanup.indexOf("export async function recordPendingCleanup")); - const body = fn.slice(0, fn.indexOf("\n}\n")); + const body = bodyOf(cleanup, "export async function recordPendingCleanup"); assert.ok(!/\.catch\(\(\)\s*=>\s*\{/.test(body), "the write is not swallowed"); assert.match(body, /findFirst/, "it is read back"); assert.match(body, /throw new Error/, "and a missing record throws"); @@ -46,7 +74,7 @@ test("the cleanup worker refuses to delete a path a LIVE row still points at", ( // cleanup, the row goes, the caller retries, and the retry's row can point // at the same path — or a seal publishes a canonical path an older pending // event names. Deleting then destroys a receipt in active use. - const fn = cleanup.slice(cleanup.indexOf("export async function retryPendingCleanups")); + const fn = bodyOf(cleanup, "export async function retryPendingCleanups"); assert.match(fn, /receiptIntake\.findFirst\(\{\s*\n?\s*where: \{ storagePath \}/, "it checks for a referencing row"); assert.match(fn, /still referenced by/, "and resolves rather than retrying forever"); // The reference check must come BEFORE the delete. @@ -57,7 +85,7 @@ test("the cleanup worker refuses to delete a path a LIVE row still points at", ( }); test("an event is resolved only AFTER a confirmed deletion", () => { - const fn = cleanup.slice(cleanup.indexOf("export async function retryPendingCleanups")); + const fn = bodyOf(cleanup, "export async function retryPendingCleanups"); // The delete's catch continues to the next event rather than falling // through to the resolve. assert.match(fn, /\} catch \{[\s\S]*?continue;/, "a failed delete leaves the event pending"); @@ -72,8 +100,8 @@ test("a missing storage client is an ERROR for the cleanup path", () => { // best-effort callers, catastrophic here: it would mark orphans resolved on // a misconfigured deployment and lose them permanently. assert.match(storage, /export async function removeSecureDocStrict/); - const strict = storage.slice(storage.indexOf("export async function removeSecureDocStrict")); - assert.match(strict.slice(0, strict.indexOf("\n}\n")), /throw new Error\("secure storage is not configured"\)/); + const strict = bodyOf(storage, "export async function removeSecureDocStrict"); + assert.match(strict, /throw new Error\("secure storage is not configured"\)/); // ...and the cleanup queue uses the strict one, never the quiet one. assert.ok(!/\bremoveSecureDoc\(/.test(cleanup), "cleanup never uses the quiet variant"); }); From 7050cbe176fff74bc629224f639ce6363fbafed4 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:13:41 -0700 Subject: [PATCH 053/144] fix(receipts): fence the publish on the exact observed park, not the state set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finalize published any NEEDS_REVIEW row and fenced only on `state: { in: [...] }`. Inspecting and sealing takes seconds, so a row re-parked in that window (or claimed by the worker) was reset to RECEIVED by the stale finalizer, discarding the newer decision and re-reading a row somebody else owned. - publishFence(row) pins the EXACT state, the EXACT stateReason and claimToken: null. Both publishers use it — /finalize and the single-shot heal. - finalizeDisposition() is the one list of recoverable parks (file-missing, sha-mismatch); the duplicated RECOVERABLE_REASONS in the intake route is gone. Any other NEEDS_REVIEW reason now answers 409 not-recoverable instead of being laundered into RECEIVED. - Race regressions: the reason changes during sealing, and a claim is taken during sealing — both update zero rows, leave the row untouched and keep the upload object. Plus an unchanged-row control. Mutation-tested by weakening the fence back to state-only. Co-Authored-By: Claude Fable 5.1 --- .../receipts/intake/[id]/finalize/route.ts | 45 ++++-- src/app/api/receipts/intake/route.ts | 21 ++- src/lib/receipt-intake/stored-object.ts | 46 ++++++ tests/receipt-intake-reject.test.ts | 16 +- tests/receipt-intake-stored-object.test.ts | 148 +++++++++++++++++- 5 files changed, 240 insertions(+), 36 deletions(-) diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 2444c77c0..9f312ceb1 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -5,7 +5,12 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; -import { inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; +import { + finalizeDisposition, + inspectStoredObject, + publishFence, + sealAndPublish, +} from "@/lib/receipt-intake/stored-object"; import { authorizePhase, mergeCapturedFields, @@ -154,13 +159,24 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // RECOVERY, not a duplicate: the upload landed after the sweep looked. It // must re-validate and republish rather than report alreadyFinalized, which // would leave a real receipt parked forever while telling the caller it was - // fine. - // Both sweeper parks are recoverable by a later, correct upload: the bytes - // arriving after the sweep looked is the normal shape of a slow client, not - // an error state a human should have to clear. - const recoverable = row.state === "STAGING" - || (row.state === "NEEDS_REVIEW" - && (row.stateReason === "file-missing" || row.stateReason === "sha-mismatch")); + // fine. Both sweeper parks are recoverable that way — bytes arriving after + // the sweep looked is the normal shape of a slow client, not an error state + // a human has to clear. Every OTHER park is a human's decision, and this + // path must not launder it into RECEIVED. + const disposition = finalizeDisposition(row); + if (disposition === "not-recoverable") { + return NextResponse.json( + { + ok: false, + error: "not-recoverable", + reason: "this row is parked for review; a re-upload does not clear it", + state: row.state, + stateReason: row.stateReason, + }, + { status: 409 }, + ); + } + const recoverable = disposition === "publish"; // Idempotent: finalizing an already-published row is a success, not an error // — the client's retry after a lost response must not look like a failure. @@ -295,12 +311,13 @@ export async function POST(req: Request, context: { params: Promise<{ id: string seal: sealObject, commit: async (canonicalPath, values) => { const { count } = await prisma.receiptIntake.updateMany({ - // Fenced on the state AND on every captured value this publish - // was validated against. A concurrent writer that filled in the - // job or the phase underneath us invalidates the tuple checked - // above, so this publish must lose rather than write a row it - // never authorized. - where: { id, state: { in: ["STAGING", "NEEDS_REVIEW"] }, ...merged.guard }, + // Fenced on the EXACT state and reason observed, on the row + // being unclaimed, and on every captured value this publish was + // validated against. Anything that moved between the read and + // this write — a re-park under a different reason, a worker + // claim, a job filled in — invalidates what was checked above, + // so the publish must lose rather than overwrite it. + where: { id, ...publishFence(row), ...merged.guard }, data: { state: "RECEIVED", stateReason: null, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 52445414c..bd77603de 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -16,6 +16,7 @@ import { MAX_INLINE_UPLOAD_BYTES, MAX_STORED_BYTES, } from "@/lib/receipt-intake/intake-core"; +import { finalizeDisposition, publishFence } from "@/lib/receipt-intake/stored-object"; import { ARCHIVE_READABLE_STATES, listReceiptIntakes, @@ -414,9 +415,6 @@ async function storeObject(storagePath: string, bytes: Buffer, mimeType: string) } } -/** The only two parked reasons a later, correct upload may recover from. */ -export const RECOVERABLE_REASONS = ["file-missing", "sha-mismatch"]; - async function publishStagedRow(id: string, expectState = "STAGING"): Promise { try { // EXACT-state CAS. `update` by id alone would publish a row that had @@ -531,22 +529,21 @@ async function respondToSourceRefConflict( // for a vendor mismatch, a zero total, or a QBO fault would be dragged // back to RECEIVED and re-read, discarding the decision a human had // already made about it. - const healable = existing.state === "STAGING" - || (existing.state === "NEEDS_REVIEW" && RECOVERABLE_REASONS.includes(existing.stateReason ?? "")); + // ONE list, shared with /finalize (stored-object.ts). Two copies of + // "which parks a re-upload may clear" is how the two publishers come to + // disagree about whether a human's decision can be overwritten. + const healable = finalizeDisposition(existing) === "publish"; if (healable) { const healed = await storeObject(payload.storagePath, payload.bytes, payload.mimeType); if (!healed) { return NextResponse.json({ ok: false, error: "storage-failed" }, { status: 503 }); } - // EXACT state AND reason. Losing this race means somebody moved the - // row while we were uploading, so the object we just wrote is + // The SAME fence /finalize publishes under: exact state, exact + // reason, unclaimed. Losing this race means somebody moved the row + // while we were uploading, so the object we just wrote is // unreferenced — clean it up rather than orphan it. const { count } = await prisma.receiptIntake.updateMany({ - where: { - id: existing.id, - state: existing.state, - ...(existing.state === "NEEDS_REVIEW" ? { stateReason: existing.stateReason } : {}), - }, + where: { id: existing.id, ...publishFence(existing) }, data: { storagePath: payload.storagePath, state: "RECEIVED", stateReason: null, nextRetryAt: null }, }); if (count === 0) { diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index 2f3b84bcf..eb7aea520 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -190,3 +190,49 @@ export async function inspectStoredObject( bytes, }; } + +/** + * The only two parked reasons a later, correct upload may recover from. + * + * "Any NEEDS_REVIEW row" is far too broad: a row parked for a vendor mismatch, + * a zero total or a QBO fault would be dragged back to RECEIVED and re-read, + * discarding a decision a human already made about it — and, past BOOKING, that + * re-read is a second Purchase waiting to happen. + */ +export const RECOVERABLE_PARK_REASONS = ["file-missing", "sha-mismatch"]; + +export interface ObservedRow { + state: string; + stateReason: string | null; +} + +/** What a finalize may do with the row it just read. */ +export type FinalizeDisposition = "publish" | "not-recoverable" | "settled"; + +export function finalizeDisposition(row: ObservedRow): FinalizeDisposition { + if (row.state === "STAGING") return "publish"; + if (row.state === "NEEDS_REVIEW") { + return RECOVERABLE_PARK_REASONS.includes(row.stateReason ?? "") ? "publish" : "not-recoverable"; + } + return "settled"; +} + +/** + * The CAS a publish must carry: the EXACT state and reason that were observed, + * and an unclaimed row. + * + * `state: { in: [...] }` was not enough. Inspecting the object and sealing it + * takes seconds, and in that window the reason can change — a row parked + * `file-missing` can be re-parked `vendor-mismatch`, or the worker can claim it. + * A publish fenced only on the state SET would then reset a reason it never + * looked at back to RECEIVED, discarding the newer decision and republishing a + * row somebody else now owns. Pinning the reason makes that update match zero + * rows, which is a 409 the client can retry rather than a silent overwrite. + */ +export function publishFence(row: ObservedRow): { + state: string; + stateReason: string | null; + claimToken: null; +} { + return { state: row.state, stateReason: row.stateReason, claimToken: null }; +} diff --git a/tests/receipt-intake-reject.test.ts b/tests/receipt-intake-reject.test.ts index 6fcd6f820..039b79a1d 100644 --- a/tests/receipt-intake-reject.test.ts +++ b/tests/receipt-intake-reject.test.ts @@ -13,6 +13,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; +import { RECOVERABLE_PARK_REASONS } from "../src/lib/receipt-intake/stored-object"; import { rejectRowAndQueueCleanup, type RejectClient, @@ -121,13 +122,14 @@ test("recovery is restricted to the two reasons a re-upload can actually fix", ( // "Any NEEDS_REVIEW row" would drag a row parked for a vendor mismatch, a // zero total, or a QBO fault back to RECEIVED and re-read it, discarding a // decision a human had already made. - assert.match(intake, /RECOVERABLE_REASONS = \["file-missing", "sha-mismatch"\]/); - assert.match(intake, /RECOVERABLE_REASONS\.includes\(existing\.stateReason \?\? ""\)/); - assert.match( - finalize, - /row\.stateReason === "file-missing" \|\| row\.stateReason === "sha-mismatch"/, - "the finalize route uses the same two reasons", - ); + // ONE list, in the lib, asked by both publishers — two copies is how they + // come to disagree about whether a human's decision can be overwritten. + assert.deepEqual(RECOVERABLE_PARK_REASONS, ["file-missing", "sha-mismatch"]); + assert.match(intake, /finalizeDisposition\(existing\) === "publish"/); + assert.match(finalize, /finalizeDisposition\(row\)/, "the finalize route asks the same rule"); + for (const source of [intake, finalize]) { + assert.ok(!/"file-missing" \|\| /.test(source), "no hand-rolled copy of the list"); + } }); test("a heal that loses its CAS deletes the object it just uploaded", () => { diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 92ef08648..05f3c254b 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -8,10 +8,15 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { canonicalStoragePath, downloadVerified, + finalizeDisposition, inspectStoredObject, + publishFence, + RECOVERABLE_PARK_REASONS, sealAndPublish, } from "../src/lib/receipt-intake/stored-object"; import { MAX_STORED_BYTES } from "../src/lib/receipt-intake/intake-core"; @@ -239,8 +244,7 @@ test("the sweeper's two parks are the ones /finalize recovers from", () => { // is a retry in progress, not an error state. Parking it would turn the // client's own next request into a review item — and the correct bytes // arriving a minute later would find the row already out of STAGING. - const { readFileSync } = require("node:fs") as typeof import("node:fs"); - const path = require("node:path") as typeof import("node:path"); + // node:fs and node:path are imported at the top of this file now. const root = path.resolve(__dirname, ".."); const sweeper = readFileSync( @@ -257,5 +261,143 @@ test("the sweeper's two parks are the ones /finalize recovers from", () => { const finalize = readFileSync( path.join(root, "src/app/api/receipts/intake/[id]/finalize/route.ts"), "utf8", ); - assert.match(finalize, /stateReason === "file-missing" \|\| row\.stateReason === "sha-mismatch"/); + // Both sweeper parks are the ones /finalize may recover from, and it asks + // the shared rule rather than carrying its own copy of the list. + assert.match(finalize, /finalizeDisposition\(row\)/); + assert.deepEqual(RECOVERABLE_PARK_REASONS, ["file-missing", "sha-mismatch"]); +}); + +// ── Which parks a re-upload may clear, and the fence it publishes under ───── + +test("only the two SWEEPER parks are recoverable; a human's park is not", () => { + assert.equal(finalizeDisposition({ state: "STAGING", stateReason: null }), "publish"); + for (const reason of RECOVERABLE_PARK_REASONS) { + assert.equal(finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason }), "publish", reason); + } + // Everything else parked for review is somebody's decision. Republishing it + // drags the row back to RECEIVED and re-reads it, discarding that decision. + for (const reason of ["vendor-mismatch", "weak-dup:row-9", "qbo-fault:6210", "amount-mismatch", null]) { + assert.equal( + finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason }), + "not-recoverable", + String(reason), + ); + } + // And a row that already moved on is simply settled — not an error. + for (const state of ["RECEIVED", "READ", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE"]) { + assert.equal(finalizeDisposition({ state, stateReason: null }), "settled", state); + } +}); + +test("the publish fence pins the exact state, the exact reason and an unclaimed row", () => { + assert.deepEqual(publishFence({ state: "NEEDS_REVIEW", stateReason: "file-missing" }), { + state: "NEEDS_REVIEW", + stateReason: "file-missing", + claimToken: null, + }); + assert.deepEqual(publishFence({ state: "STAGING", stateReason: null }), { + state: "STAGING", + stateReason: null, + claimToken: null, + }); +}); + +/** Enough of Prisma's updateMany semantics to run a CAS against one row. */ +function rowStore(row: Record) { + const store = { ...row }; + return { + get: () => store, + set: (patch: Record) => Object.assign(store, patch), + updateMany: (where: Record, data: Record) => { + const matches = Object.entries(where).every(([k, v]) => store[k] === v); + if (!matches) return 0; + Object.assign(store, data); + return 1; + }, + }; +} + +test("RACE: a reason that changes during sealing loses the publish, and writes nothing", async () => { + // The window is real: inspecting the object and sealing it takes seconds, + // and the worker can re-park the row in that time. Fenced only on the state + // SET (`state: { in: ["STAGING", "NEEDS_REVIEW"] }`) the stale finalizer + // would reset a reason it never looked at back to RECEIVED — discarding the + // newer decision and republishing a row somebody else now owns. + const store = rowStore({ + id: "row-1", state: "NEEDS_REVIEW", stateReason: "file-missing", claimToken: null, + }); + const observed = { state: store.get().state as string, stateReason: store.get().stateReason as string }; + assert.equal(finalizeDisposition(observed), "publish", "it was recoverable when we read it"); + const fence = publishFence(observed); + + let dropped = false; + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, { + seal: async (_u: string, canonical: string) => { + // THE RACE, in the exact window it happens: the worker re-parks the + // row while we are copying the bytes. + store.set({ stateReason: "vendor-mismatch" }); + return canonical; + }, + commit: async (canonicalPath: string) => + store.updateMany( + { id: "row-1", ...fence }, + { state: "RECEIVED", stateReason: null, storagePath: canonicalPath }, + ), + dropUpload: async () => { dropped = true; }, + } as never); + + assert.equal(outcome?.published, false, "zero rows updated"); + assert.equal(store.get().state, "NEEDS_REVIEW", "the row is untouched"); + assert.equal(store.get().stateReason, "vendor-mismatch", "the newer decision survives"); + assert.equal(dropped, false, "and the upload object is kept for the retry"); +}); + +test("RACE: a worker claim taken during sealing also loses the publish", async () => { + const store = rowStore({ + id: "row-1", state: "STAGING", stateReason: null, claimToken: null, + }); + const fence = publishFence({ state: "STAGING", stateReason: null }); + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, { + seal: async (_u: string, canonical: string) => { + store.set({ claimToken: "sweeper-1" }); + return canonical; + }, + commit: async () => store.updateMany({ id: "row-1", ...fence }, { state: "RECEIVED" }), + dropUpload: async () => {}, + } as never); + assert.equal(outcome?.published, false); + assert.equal(store.get().state, "STAGING", "the sweeper's row is left alone"); +}); + +test("an unchanged row still publishes — the control", async () => { + const store = rowStore({ + id: "row-1", state: "NEEDS_REVIEW", stateReason: "sha-mismatch", claimToken: null, + }); + const fence = publishFence({ state: "NEEDS_REVIEW", stateReason: "sha-mismatch" }); + const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, { + seal: async (_u: string, canonical: string) => canonical, + commit: async () => store.updateMany({ id: "row-1", ...fence }, { state: "RECEIVED", stateReason: null }), + dropUpload: async () => {}, + } as never); + assert.equal(outcome?.published, true); + assert.equal(store.get().state, "RECEIVED"); +}); + +test("both publishers use the shared fence, and finalize refuses the other parks", () => { + const finalize = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", + ); + const intake = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/route.ts"), + "utf8", + ); + assert.match(finalize, /where: \{ id, \.\.\.publishFence\(row\), \.\.\.merged\.guard \}/); + assert.match(intake, /where: \{ id: existing\.id, \.\.\.publishFence\(existing\) \}/); + assert.ok( + !/state: \{ in: \["STAGING", "NEEDS_REVIEW"\] \}/.test(finalize), + "the state-SET fence is gone", + ); + assert.match(finalize, /error: "not-recoverable"/); + assert.match(finalize, /disposition === "not-recoverable"/); }); From 93bb7853417a5b3dd6ca851152edb920d4c4dae1 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:20:20 -0700 Subject: [PATCH 054/144] test(e2e): bring the intake spec up to the route's contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both failures were STALE SPEC, not route bugs. :420 — an unsupported FORMAT has been a 415 since the text/plain decision (the request is well-formed; the file is one QuickBooks cannot attach, and the body names what to send instead). The case list now carries the expected status per case and checks `error` for the 415 and `reason` for the 400s. :975 — a phase is only valid against its own job, so a finalize sending `costCodeId` with NO project on the row is now a 400 cost-code-without-project. The row is created with PROJECT_ID, whose approved mobile estimate is what makes `e2e-mob-cc-demo` one of its phases. Added the two cases that pair with it: a cost code that is not a phase of THIS job (400 cost-code-not-a-phase, nothing written) and the control that the job's own phase is accepted — without which both refusals would pass against a gate that refused everything. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 67 +++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index e539a35ab..8071500ec 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -34,6 +34,11 @@ const SECRET = process.env.RECEIPT_INTAKE_SECRET || ""; // report what it archived, and nothing else. Cross-use is a 403. const ARCHIVE_SECRET = process.env.RECEIPT_ARCHIVE_SECRET || ""; +// The e2e test project from data.setup.ts, and the phase that belongs to it +// (via the approved mobile estimate). A phase is only valid against its own job, +// so any spec sending a costCodeId has to send this project too. +const PROJECT_ID = "cmml6vt3y000lpwrh0p9p3k12"; + // One prefix for everything this file creates, so teardown can be exact. const REF_PREFIX = "drive:e2e-intake-"; const FILE_ID = `${Date.now()}-a`; @@ -417,17 +422,25 @@ test.describe("intake POST", () => { expect(body.reason).toBe("invalid-source"); }); - test("deterministic bad input is a 400, not a 500 the forwarder retries forever", async ({ request }) => { - const cases: [string, string][] = [ - [intakeBody({ source: "carrier-pigeon", sourceRef: `${REF_PREFIX}src` }), "invalid-source"], - [JSON.stringify({ source: "drive", sourceRef: `${REF_PREFIX}nofile` }), "missing-file"], + test("deterministic bad input is terminal, not a 500 the forwarder retries forever", async ({ request }) => { + // A malformed REQUEST is a 400. An unsupported FORMAT is a 415 — the + // request is well-formed, the file is simply one QuickBooks cannot + // attach, and the body names what to send instead. Both are terminal: + // what must never happen is a 5xx the forwarder retries forever. + const cases: [string, number, string][] = [ + [intakeBody({ source: "carrier-pigeon", sourceRef: `${REF_PREFIX}src` }), 400, "invalid-source"], + [JSON.stringify({ source: "drive", sourceRef: `${REF_PREFIX}nofile` }), 400, "missing-file"], // Base64 of "hello" — not a document format we can read. - [intakeBody({ sourceRef: `${REF_PREFIX}junk`, fileBase64: "aGVsbG8=", mimeType: "image/png" }), "unsupported-file-type"], + [intakeBody({ sourceRef: `${REF_PREFIX}junk`, fileBase64: "aGVsbG8=", mimeType: "image/png" }), 415, "unsupported-file-type"], ]; - for (const [data, reason] of cases) { + for (const [data, status, name] of cases) { const { res, body } = await postIntake(request, data); - expect(res.status(), reason).toBe(400); - expect(body.reason).toBe(reason); + expect(res.status(), name).toBe(status); + // 400s carry `reason`; the 415 carries `error` plus a human `reason` + // and the accepted list. + expect(body.reason ?? body.error, name).toBeTruthy(); + expect([body.reason, body.error], name).toContain(name); + if (status === 415) expect(body.accepted, name).toContain("application/pdf"); } }); @@ -977,7 +990,11 @@ test.describe("round-9 intake contracts", () => { // path — and answering it without applying the job assignment would drop // that assignment while telling the caller it worked. const ref = `${REF_PREFIX}latefields`; - const created = await postIntake(request, intakeBody({ sourceRef: ref })); + // The row carries the JOB, because a phase is only valid against one: + // `e2e-mob-cc-demo` is a phase of PROJECT_ID via the approved mobile + // estimate (data.setup.ts). Without the project this is a 400 + // cost-code-without-project, which is a different test (below). + const created = await postIntake(request, intakeBody({ sourceRef: ref, projectId: PROJECT_ID })); expect(created.res.status()).toBe(200); const id = created.body.id; minted.push(id); @@ -1068,6 +1085,38 @@ test.describe("round-10 finalize authorization and recovery", () => { expect((await res.json()).error).toBe("cost-code-without-project"); }); + test("a cost code that is not a phase of THIS job is refused", async ({ request }) => { + // The row has a job, and the phase is not one of its phases. Neither + // half is malformed — the PAIR is wrong, and letting it through files + // the receipt against a line this job never budgeted. + const created = await postIntake(request, intakeBody({ + sourceRef: `${REF_PREFIX}phasejob`, projectId: PROJECT_ID, + })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + + const res = await finalize(request, created.body.id, { costCodeId: "e2e-not-a-phase-of-anything" }); + expect(res.status()).toBe(400); + expect((await res.json()).error).toBe("cost-code-not-a-phase"); + expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.costCodeId) + .toBeNull(); + }); + + test("the job's OWN phase is accepted", async ({ request }) => { + // The control: without it the two refusals above would pass just as + // well against a gate that refused everything. + const created = await postIntake(request, intakeBody({ + sourceRef: `${REF_PREFIX}phaseok`, projectId: PROJECT_ID, + })); + expect(created.res.status()).toBe(200); + minted.push(created.body.id); + + const res = await finalize(request, created.body.id, { costCodeId: "e2e-mob-cc-demo" }); + expect(res.status()).toBe(200); + expect((await prisma.receiptIntake.findUnique({ where: { id: created.body.id } }))?.costCodeId) + .toBe("e2e-mob-cc-demo"); + }); + test("late fields are refused once the row has been routed", async ({ request }) => { // Past RECEIVED the dedup keys, the phase suggestion and possibly a // booking were all derived from the project the row had at the time. From 0c7112dc8e68c94c58eba6ea8a3fd1099f68052e Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:41:05 -0700 Subject: [PATCH 055/144] fix(receipts): authorize the EFFECTIVE project on every session finalize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project access was only re-checked when the request supplied a projectId, so a user whose access to a job had been revoked could still finalize — publish — their existing row on that job, and still attach a phase to it, simply by not mentioning the project the row already had. authorizeEffectiveProject(stored, late, canAccess) now runs for every session caller before any write: it authorizes `late ?? stored`, 403 project-forbidden when revoked, and does no lookup at all when there is no project either way. authorizeLateFields is renamed authorizeFinalization, because that is what it now gates. Tests: unit cases for stored-only, supplied-wins, still-allowed and no-project, a source guard that the check is unconditional and precedes every write path, and an e2e where the revoked owner finalizes their own row on a job they cannot reach — 403, and the phase they sent is not written. Mutation-tested by restoring the supplied-project-only condition. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 39 +++++++++++ .../receipts/intake/[id]/finalize/route.ts | 34 ++++++---- src/lib/receipt-intake/late-fields.ts | 33 ++++++++++ tests/receipt-intake-late-fields.test.ts | 64 +++++++++++++++++++ 4 files changed, 157 insertions(+), 13 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 8071500ec..61773b7eb 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -1085,6 +1085,45 @@ test.describe("round-10 finalize authorization and recovery", () => { expect((await res.json()).error).toBe("cost-code-without-project"); }); + test("a user with no access to the row's OWN job cannot finalize it", async ({ playwright, request }) => { + // Revocation has to bite on the project the ROW holds, not only on one + // the request supplies. Before this, a user whose access was revoked + // could still publish their existing row on that job — and attach a + // phase to it — simply by not mentioning the project. + const created = await postIntake(request, intakeBody({ + sourceRef: `${REF_PREFIX}revoked`, projectId: "e2e-scope-oos-project", + })); + expect(created.res.status()).toBe(200); + const id = created.body.id; + minted.push(id); + + // The row is THEIRS (so the ownership check passes and this test is + // about project access, not about a guessed id), on a job + // contract-staff has no ProjectAccess for. + const owner = await prisma.user.findUnique({ where: { email: "contract-staff@test.local" } }); + expect(owner, "contract-staff fixture must exist").toBeTruthy(); + await prisma.receiptIntake.update({ where: { id }, data: { createdById: owner!.id } }); + + const employee = await playwright.request.newContext({ + baseURL: "http://localhost:3000", + storageState: "e2e/.auth/contract-user.json", + }); + const res = await employee.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json" }, + // No projectId in the body: the row already has one, and that is + // the whole point of the case. + data: JSON.stringify({ costCodeId: "e2e-mob-cc-demo" }), + maxRedirects: 0, + }); + expect(res.status()).toBe(403); + expect((await res.json()).error).toBe("project-forbidden"); + // NOTHING written: the gate runs before any late field is applied. + const after = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(after?.costCodeId).toBeNull(); + expect(after?.projectId).toBe("e2e-scope-oos-project"); + await employee.dispose(); + }); + test("a cost code that is not a phase of THIS job is refused", async ({ request }) => { // The row has a job, and the phase is not one of its phases. Neither // half is malformed — the PAIR is wrong, and letting it through files diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 9f312ceb1..baec34d94 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -12,6 +12,7 @@ import { sealAndPublish, } from "@/lib/receipt-intake/stored-object"; import { + authorizeEffectiveProject, authorizePhase, mergeCapturedFields, reconcileLateFields, @@ -57,31 +58,38 @@ async function applyLateFields( }); return count; }, - authorize: projectId => authorizeLateFields(auth, projectId, lateFields), + authorize: projectId => authorizeFinalization(auth, projectId, lateFields), }); return denial ? NextResponse.json(denial.body, { status: denial.status }) : null; } /** - * A caller may only attach a job it can actually reach, and only a phase that - * belongs to that job. + * A caller may only finalize against a job it can actually reach, and may only + * attach a phase that belongs to that job. * * Without the first check any authenticated user could file a receipt against - * any project by id. Without the second, a cost code from another job rides - * into the Expense and every variance report reads it as overspend on a line - * nobody budgeted. + * any project by id — and, before this authorized the EFFECTIVE project rather + * than only a supplied one, a user whose access had been revoked could still + * publish and phase their existing row on that job simply by not mentioning it. + * Without the second, a cost code from another job rides into the Expense and + * every variance report reads it as overspend on a line nobody budgeted. */ -async function authorizeLateFields( +async function authorizeFinalization( auth: Extract, rowProjectId: string | null, lateFields: LateFields, ): Promise { const projectId = lateFields.projectId ?? rowProjectId; - if (lateFields.projectId && auth.via === "session") { - if (!(await userCanAccessProject(auth.user, lateFields.projectId))) { - return { status: 403, body: { ok: false, reason: "forbidden" } }; - } + // EVERY session call, not just the ones carrying a project. The shared + // secret is a trusted forwarder with no user to scope by. + if (auth.via === "session") { + const forbidden = await authorizeEffectiveProject( + rowProjectId, + lateFields.projectId ?? null, + candidate => userCanAccessProject(auth.user, candidate), + ); + if (forbidden) return forbidden; } // Same rule, same implementation, as /start applies to a phase supplied @@ -152,7 +160,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string if (!maySee) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); // Authorize the late fields BEFORE anything is written or published. - const denied = await authorizeLateFields(auth, row.projectId, lateFields); + const denied = await authorizeFinalization(auth, row.projectId, lateFields); if (denied) return NextResponse.json(denied.body, { status: denied.status }); // A LATE finalize on a row the sweeper already parked file-missing is a @@ -276,7 +284,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // AND THE RESULTING TUPLE IS VALIDATED, not the supplied half of it. // - // authorizeLateFields above only saw what this REQUEST carried. A finalize + // authorizeFinalization above only saw what this REQUEST carried. A finalize // that sends projectId=B against a phase captured for job A supplies a // project that is fine on its own and a phase it never mentions — and the // row ends up filed under B's job with A's phase. The check has to be on diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index bda8b2668..202eed5d9 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -263,3 +263,36 @@ export function mergeCapturedFields( from: { projectId: project.from, costCodeId: phase.from }, }; } + +/** + * The project this finalization actually touches — checked on EVERY session + * call, not only when the request supplies a new one. + * + * The old rule authorized `lateFields.projectId` and nothing else, so access was + * only ever re-checked when the caller happened to send a project. A user whose + * access to a job had been revoked could still finalize (publish) their existing + * row on that job, and still attach a phase to it, because the request named no + * project at all — the row already had one. Revocation has to bite on the + * EFFECTIVE project: what the row will hold when this call is done. + * + * A row with no project either way is nothing to authorize: there is no job to + * be revoked from. Ownership of the row itself is a separate check. + */ +export async function authorizeEffectiveProject( + storedProjectId: string | null, + lateProjectId: string | null, + canAccessProject: (projectId: string) => Promise, +): Promise { + const effective = lateProjectId ?? storedProjectId; + if (!effective) return null; + if (await canAccessProject(effective)) return null; + return { + status: 403, + body: { + ok: false, + reason: "forbidden", + error: "project-forbidden", + projectId: effective, + }, + }; +} diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts index 37da14aee..08fcd76a6 100644 --- a/tests/receipt-intake-late-fields.test.ts +++ b/tests/receipt-intake-late-fields.test.ts @@ -14,6 +14,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; import { + authorizeEffectiveProject, authorizePhase, mergeCapturedFields, reconcileLateFields, @@ -317,3 +318,66 @@ test("losing the publish CAS on a STAGING row is a conflict, NOT alreadyFinalize assert.match(body, /publish-conflict/); assert.match(body, /status: 409/); }); + +// ── Revocation has to bite on the row's OWN project (Phase 3 gate) ────────── + +test("a revoked user is refused on the project the ROW already holds", async () => { + // The hole: access was only re-checked when the request supplied a project. + // A user whose access to a job was revoked could still finalize — publish — + // their existing row on that job, and still attach a phase to it, simply by + // not mentioning the project the row already had. + const looked: string[] = []; + const denial = await authorizeEffectiveProject("proj-a", null, async id => { + looked.push(id); + return false; + }); + assert.deepEqual(looked, ["proj-a"], "the STORED project is what gets checked"); + assert.equal(denial?.status, 403); + assert.equal(denial?.body.error, "project-forbidden"); + assert.equal(denial?.body.projectId, "proj-a"); +}); + +test("a user who still has access passes, and a supplied project wins", async () => { + assert.equal(await authorizeEffectiveProject("proj-a", null, async () => true), null); + + // The effective project is what the row will HOLD: a supplied one replaces + // the stored one, so that is the one to authorize. + const looked: string[] = []; + assert.equal( + await authorizeEffectiveProject("proj-a", "proj-b", async id => { looked.push(id); return true; }), + null, + ); + assert.deepEqual(looked, ["proj-b"]); +}); + +test("no project either way is nothing to authorize — and no lookup", async () => { + let looked = 0; + assert.equal( + await authorizeEffectiveProject(null, null, async () => { looked++; return false; }), + null, + ); + assert.equal(looked, 0, "there is no job to be revoked from"); +}); + +test("finalize authorizes the effective project on EVERY session call, before any write", () => { + const finalize = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", + ); + // Unconditional for a session caller — NOT gated on the request carrying a + // project, which is exactly what let a revoked user through. + assert.match( + finalize, + /if \(auth\.via === "session"\) \{\s*\r?\n\s*const forbidden = await authorizeEffectiveProject\(/, + ); + assert.ok( + !/if \(lateFields\.projectId && auth\.via === "session"\)/.test(finalize), + "the supplied-project-only check is gone", + ); + // And it runs before anything can be written or published. + const gate = finalize.indexOf("const denied = await authorizeFinalization(auth, row.projectId"); + assert.notEqual(gate, -1); + for (const write of ["await applyLateFields(", "await sealAndPublish(", "rejectRowAndQueueCleanup("]) { + assert.ok(gate < finalize.indexOf(write), `the gate precedes ${write}`); + } +}); From 856517b669d40b4bae4f8fdaf753fdec350c404b Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 07:14:34 -0700 Subject: [PATCH 056/144] fix(receipts): fence the reject, trust v1 evidence, re-arm swept uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (1) rejectRowAndQueueCleanup deletes under the FULL fence — id, observed state, stateReason, claimToken null, storagePath — inside the same transaction as the cleanup event, and requires exactly one row. A publish that wins mid-reject rolls the event back with the delete: nothing deleted, nothing queued, 409 publish-conflict. "Already gone" is no longer success, because queueing an absent row's path is how a live object gets swept. (2) Cutover: evidence outranks the timestamp. Candidates are now every parked row, so a file v1 had already booked but the forwarder handed over AFTER the flip is retired instead of being requeued and booked a second time — which for an email/chat row no idempotency can collapse. The boundary now only decides what to do with evidence-FREE rows. Triage extracted to cutover.ts so it is testable. (3) /start on a row the sweeper parked file-missing or sha-mismatch issues a NEW signed URL and re-arms the expected sha, fenced on the exact park, instead of answering alreadyReceived — which told the forwarder we held a receipt we did not hold, and it deletes its only copy on that answer. e2e covers sweep -> /start -> upload -> /finalize, plus a control that a non-recoverable park still answers alreadyReceived. (4) The already-exists branch fires a fenced onExistingPurchase hook before it touches the attachment, so a row that exhausts its retries there RETAINS its strong key: a Purchase found by the idempotency query is still a Purchase. The test fake now mirrors the real control flow and does NOT call onBeforeCreate. Mutation-tested: (1) delete-by-id, (2) evidence gated on createdAt, (4) purchaseKnownToExist dropped — each fails the tests that cover it. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 97 +++++++++++ .../api/cron/receipt-intake-worker/route.ts | 67 ++++---- .../receipts/intake/[id]/finalize/route.ts | 28 +++- src/app/api/receipts/intake/start/route.ts | 76 ++++++++- src/lib/qbo-receipt-push.ts | 16 ++ src/lib/receipt-intake/book.ts | 71 ++++++-- src/lib/receipt-intake/cutover.ts | 65 ++++++++ src/lib/receipt-intake/storage-cleanup.ts | 65 ++++++-- tests/receipt-intake-book.test.ts | 111 ++++++++++++- tests/receipt-intake-cutover.test.ts | 87 +++++++++- tests/receipt-intake-reject.test.ts | 151 ++++++++++++++---- 11 files changed, 728 insertions(+), 106 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 61773b7eb..1a597b171 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -906,6 +906,103 @@ test.describe("two-step upload: a reused key cannot swap the document", () => { expect((await finalized.json()).error).toBe("sha-mismatch"); expect((await prisma.receiptIntake.findUnique({ where: { id } }))?.state).toBe("STAGING"); }); + + test("SWEPT then re-uploaded: /start re-arms the row and /finalize publishes it", async ({ request }) => { + // End to end for the recovery the sweeper leaves behind. The old + // behaviour answered `alreadyReceived` for any non-STAGING row, which + // told the forwarder we held a receipt we did not hold — and it deletes + // its only copy on that answer — leaving the row parked forever with + // nothing to recover from. + const ref = `${REF_PREFIX}swept-restart`; + const first = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(OTHER_PNG_BASE64) }), + maxRedirects: 0, + }); + expect(first.status()).toBe(200); + const { id } = await first.json(); + minted.push(id); + + // Exactly what the stale-STAGING sweep leaves behind when the upload + // never landed. + await prisma.receiptIntake.update({ + where: { id }, + data: { state: "NEEDS_REVIEW", stateReason: "file-missing" }, + }); + + // The client comes back with the correct document — a DIFFERENT hash + // from the one it first announced, which for a STAGING row would be a + // sourceRef-conflict. Here there are no verified bytes to protect. + const rearmed = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64) }), + maxRedirects: 0, + }); + expect(rearmed.status()).toBe(200); + const rearmedBody = await rearmed.json(); + expect(rearmedBody.id).toBe(id); + expect(rearmedBody.recovered).toBe(true, "a new URL, not alreadyReceived"); + expect(rearmedBody.uploadUrl).toBeTruthy(); + const armed = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(armed?.expectedSha256).toBe(sha(PNG_BASE64), "the new hash is what finalize will verify"); + expect(armed?.state).toBe("NEEDS_REVIEW", "still parked until the bytes actually land"); + + // "Upload": the spec cannot PUT to Supabase, so real bytes are put at a + // path by the single-shot route and the row is pointed at them — the + // same seeding trick the sha-mismatch case above uses. + const seeded = await request.post(INTAKE_PATH, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: intakeBody({ sourceRef: `${REF_PREFIX}swept-restart-src` }), + maxRedirects: 0, + }); + expect(seeded.status()).toBe(200); + const seededRow = await prisma.receiptIntake.findUnique({ where: { id: (await seeded.json()).id } }); + minted.push(seededRow!.id); + await prisma.receiptIntake.update({ where: { id }, data: { storagePath: seededRow!.storagePath } }); + + const finalized = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: "{}", + maxRedirects: 0, + }); + expect(finalized.status()).toBe(200); + const done = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(done?.state).toBe("RECEIVED", "the swept row recovered all the way to published"); + expect(done?.stateReason).toBeNull(); + expect(done?.fileSha256).toBe(sha(PNG_BASE64)); + }); + + test("a park a re-upload CANNOT fix still answers alreadyReceived", async ({ request }) => { + // The control. Only file-missing and sha-mismatch are recoverable; a + // row parked on a human's decision must not be handed a fresh URL that + // would let a client overwrite the document under review. + const ref = `${REF_PREFIX}swept-notrecoverable`; + const first = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64) }), + maxRedirects: 0, + }); + expect(first.status()).toBe(200); + const { id, storagePath } = await first.json(); + minted.push(id); + await prisma.receiptIntake.update({ + where: { id }, + data: { state: "NEEDS_REVIEW", stateReason: "vendor-mismatch" }, + }); + + const again = await request.post(startPath, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ source: "drive", sourceRef: ref, mimeType: "image/png", sha256: sha(PNG_BASE64) }), + maxRedirects: 0, + }); + expect(again.status()).toBe(200); + const body = await again.json(); + expect(body.alreadyReceived).toBe(true); + expect(body.uploadUrl).toBeUndefined(); + const row = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(row?.storagePath).toBe(storagePath, "nothing was re-armed"); + expect(row?.stateReason).toBe("vendor-mismatch"); + }); }); test.describe("round-9 intake contracts", () => { diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index fe4e17bd5..02128bdd8 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -13,7 +13,9 @@ import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { createRouteDeadline, type RouteDeadline } from "@/lib/quickbooks"; import { readReceipt } from "@/lib/receipt-intake/read"; import { canonicalVendor } from "@/lib/receipt-intake/keys"; -import { resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; +import { + driveFileIdOf, + triageCutoverRows, resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; import { resolveCompanyTimeZone } from "@/lib/company-timezone"; import { isCostCodeAllowedForProject, resolveProjectPhaseCodes } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; @@ -152,15 +154,26 @@ async function claim(opts: CutoverRequest): Promise { // rows this applies to: they book under the DRIVE FILE ID, so // QBO's DocNumber/requestid idempotency collapses a v1/v2 // overlap into one Purchase. + // EVERY parked row, not just the ones older than the + // boundary. Evidence outranks the timestamp: the forwarder can + // hand over a file v1 had ALREADY booked minutes after the + // flip (a queued send, a retry, a slow archive step), and + // filtering by createdAt first meant those rows never reached + // the evidence check at all — they went straight into the + // requeue and v2 booked a second Purchase for an email or chat + // receipt, where there is no shared identity to collapse it. + // The boundary is only used to decide what to do with rows that + // have NO evidence either way. const candidates = await tx.receiptIntake.findMany({ - where: { ...parked, createdAt: { lt: opts.boundary } }, - select: { id: true, source: true, sourceRef: true, archivedByV1: true }, + where: parked, + select: { + id: true, source: true, sourceRef: true, + archivedByV1: true, createdAt: true, + }, }); const driveIds = candidates - .map(r => (r.source === "drive" && r.sourceRef.startsWith("drive:") - ? r.sourceRef.slice("drive:".length) - : null)) + .map(driveFileIdOf) .filter((v): v is string => !!v); const bookedByV1 = driveIds.length @@ -194,17 +207,10 @@ async function claim(opts: CutoverRequest): Promise { // double-paying; retiring risks losing a // real expense. A human checks QBO and uses // "book anyway". - const evidenced: string[] = []; - const unevidenced: string[] = []; - const quarantined: string[] = []; - for (const row of candidates) { - const driveId = row.source === "drive" && row.sourceRef.startsWith("drive:") - ? row.sourceRef.slice("drive:".length) - : null; - if (row.archivedByV1 || (driveId && bookedByV1.has(driveId))) evidenced.push(row.id); - else if (driveId) unevidenced.push(row.id); - else quarantined.push(row.id); - } + // ONE implementation of the three-way split, in the lib, so + // it is testable without standing up a cron route. + const { evidenced, unevidenced, quarantined } = + triageCutoverRows(candidates, opts.boundary, bookedByV1); if (evidenced.length) { const retired = await tx.receiptIntake.updateMany({ @@ -229,18 +235,17 @@ async function claim(opts: CutoverRequest): Promise { shadowQuarantined = held.count; } - // Everything else — after the boundary, or before it with no - // evidence — is v2's to book. - const handed = await tx.receiptIntake.updateMany({ - where: { - OR: [ - { ...parked, createdAt: { gte: opts.boundary } }, - ...(unevidenced.length ? [{ id: { in: unevidenced } }] : []), - ], - }, - data: { dryRun: false, nextRetryAt: null }, - }); - requeued = handed.count; + // Everything else is v2's to book. The list is built row by + // row above rather than re-derived from a createdAt predicate + // here, so the two can never disagree about which rows the + // evidence check already claimed. + if (unevidenced.length) { + const handed = await tx.receiptIntake.updateMany({ + where: { id: { in: unevidenced } }, + data: { dryRun: false, nextRetryAt: null }, + }); + requeued = handed.count; + } if (shadowRetired > 0 || requeued > 0 || shadowQuarantined > 0) { console.log("[cron/receipt-intake-worker] cutover", JSON.stringify({ @@ -623,8 +628,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { isPushEnabled: () => process.env.QBO_RECEIPT_PUSH_ENABLED === "true", isPushPaused: () => isPaused(PAUSE_KEYS.receiptPush), getTokens: deadline => getFreshQBTokens(deadline), - createPurchase: (tokens, input, deadline, onBeforeCreate) => - createQBReceiptPurchase(tokens, input, { onBeforeCreate }, deadline), + createPurchase: (tokens, input, deadline, onBeforeCreate, onExistingPurchase) => + createQBReceiptPurchase(tokens, input, { onBeforeCreate, onExistingPurchase }, deadline), downloadBytes: (storagePath, expectedSha256) => downloadVerified(storagePath, expectedSha256), logEvent: logAutomationEvent, now: () => new Date(), diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index baec34d94..3f7676df2 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -229,12 +229,30 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // are ONE transaction. A best-effort delete followed by a best-effort // cleanup could drop the row and lose the object with nothing left // referencing or remembering it. - const rejected = await rejectRowAndQueueCleanup(id, row.storagePath, check.reason); + const rejected = await rejectRowAndQueueCleanup( + { + id, + state: row.state, + stateReason: row.stateReason, + storagePath: row.storagePath, + }, + check.reason, + ); if (!rejected.ok) { - // The row's deletion is not confirmed, so it may still point at - // these bytes. Keep the object and answer retryably; an identical - // retry re-validates and rejects again. - return NextResponse.json({ ok: false, reason: "reject-failed", retryable: true }, { status: 503 }); + // The fence lost, so this row is not ours to reject: a publisher + // moved it (or claimed it) while we were inspecting the object. + // NOTHING is deleted — not the row, not the object, not even a + // cleanup record — because those bytes may now belong to a + // published receipt. + return NextResponse.json( + { + ok: false, + error: "publish-conflict", + reason: "this row changed while it was being rejected; retry", + retryable: true, + }, + { status: 409 }, + ); } await settleQueuedCleanup(rejected.eventId, row.storagePath); const status = check.reason.startsWith("file-too-large") ? 413 : 400; diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index a8729edd0..a56ae3d8f 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -9,6 +9,8 @@ import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; +import { finalizeDisposition, publishFence } from "@/lib/receipt-intake/stored-object"; +import { deleteObjectOrRecord } from "@/lib/receipt-intake/storage-cleanup"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; @@ -164,7 +166,7 @@ export async function POST(req: Request) { const existing = await prisma.receiptIntake.findUnique({ where: { sourceRef: decided.sourceRef }, select: { - id: true, sourceRef: true, state: true, storagePath: true, + id: true, sourceRef: true, state: true, stateReason: true, storagePath: true, createdById: true, expectedSha256: true, fileSha256: true, }, }); @@ -175,6 +177,74 @@ export async function POST(req: Request) { auth.user.role === "ADMIN"; if (!maySee) return NextResponse.json({ ok: false, error: "sourceRef-conflict" }, { status: 409 }); + // A ROW THE SWEEPER PARKED AS RECOVERABLE GETS A NEW URL, NOT + // "alreadyReceived". + // + // file-missing and sha-mismatch both mean the bytes we hold are not + // the document (or are not there at all), and the row never + // published. Answering alreadyReceived told the forwarder we had a + // receipt we did not have — and it deletes its only copy on that + // answer — leaving the row parked forever with nothing to recover + // from. So the upload is re-armed instead: a fresh signed URL, and + // the sha the caller is about to upload becomes the expected one. + // + // The identity check below is deliberately skipped for these two: + // it exists to stop receipt B overwriting receipt A's VERIFIED + // bytes, and here there are none to protect. + const recoverable = existing.state !== "STAGING" + && finalizeDisposition(existing) === "publish"; + if (recoverable) { + const retryPath = `receipts/intake/${existing.id}.${ext}`; + const rearmed = await signUpload(retryPath); + if (!rearmed) { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + // Fenced on the EXACT park, same rule as every other publish + // path: losing it means somebody moved the row while we were + // signing, and re-arming it then would point a live row at an + // empty path. + const { count } = await prisma.receiptIntake.updateMany({ + where: { id: existing.id, ...publishFence(existing) }, + data: { + storagePath: retryPath, + expectedSha256, + // The stored hash is what /finalize verifies against. + // Whatever was recorded describes bytes that are gone + // or were never right. + fileSha256: "", + mimeType, + fileSize: 0, + nextRetryAt: null, + }, + }); + if (count === 0) { + return NextResponse.json( + { + ok: false, + error: "publish-conflict", + reason: "this row changed while a new upload URL was being issued; retry", + retryable: true, + existingId: existing.id, + }, + { status: 409 }, + ); + } + // A different declared type means a different extension, so the + // old object (if any) is now unreferenced. + if (retryPath !== existing.storagePath) { + await deleteObjectOrRecord(existing.storagePath, "start-rearmed-repath"); + } + return NextResponse.json({ + ok: true, + resumed: true, + recovered: true, + id: existing.id, + state: existing.state, + maxBytes: MAX_STORED_BYTES, + ...rearmed, + }); + } + // IDENTITY MUST BE PROVEN BEFORE AN UPSERT URL IS REISSUED. // // The URL is `upsert: true` so a caller can replace its OWN partial @@ -205,7 +275,9 @@ export async function POST(req: Request) { } const resumed = await signUpload(existing.storagePath); if (!resumed) return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - return NextResponse.json({ ok: true, resumed: true, id: existing.id, ...resumed }); + return NextResponse.json({ + ok: true, resumed: true, id: existing.id, maxBytes: MAX_STORED_BYTES, ...resumed, + }); } if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2003") { return NextResponse.json({ ok: false, reason: "unknown-project-or-cost-code" }, { status: 400 }); diff --git a/src/lib/qbo-receipt-push.ts b/src/lib/qbo-receipt-push.ts index 9ef2262b4..c7722a0d2 100644 --- a/src/lib/qbo-receipt-push.ts +++ b/src/lib/qbo-receipt-push.ts @@ -225,6 +225,18 @@ export interface QboReceiptPushDependencies { * the caller's last chance to check it still owns the row. */ onBeforeCreate?: () => Promise; + /** + * Invoked when the idempotency query finds THIS file's Purchase already in + * QuickBooks, immediately before anything else is done with it. + * + * The alreadyExists branch returns WITHOUT ever reaching qbCreateFn, so + * `onBeforeCreate` never fires on it. That left the caller unable to tell + * "a Purchase exists for this row" from "nothing was ever sent" — and a row + * that exhausted its retries on this path would hand back its dedup key + * while a real Purchase sat in the books, so a resubmission would book the + * same receipt twice. This hook is that signal. + */ + onExistingPurchase?: () => Promise; ensureVendorFn: (tokens: QBTokens, name: string) => Promise; // Injectable (unlike the plain re-export of ensureQBCustomer) because the // customer is now resolved on EVERY create — there is no more per-client @@ -671,6 +683,10 @@ export async function createQBReceiptPurchase( if (existing.length > 1 || !(existing[0].PrivateNote ?? "").includes(marker)) { return { ok: false, reason: "docnumber-conflict", docNumber }; } + // THE PURCHASE EXISTS. Say so before doing anything else with it: the + // attachment re-check below is a QBO round trip that can fail, and the + // caller still has to know a Purchase is there. + await deps.onExistingPurchase?.(); // The Purchase exists, but that does NOT mean the receipt file made it // across. The common way to reach this branch is a first attempt whose // Purchase response was lost (timeout/kill) AFTER QBO committed it — diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 598a2677c..e69a3a708 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -196,6 +196,12 @@ export interface BookDependencies { deadline: RouteDeadline | undefined, /** Invoked by the QBO core immediately before the create. */ onBeforeCreate: () => Promise, + /** + * Invoked by the QBO core when it finds this file's Purchase ALREADY in + * QuickBooks — a path that never reaches the create, so `onBeforeCreate` + * does not fire on it. + */ + onExistingPurchase: () => Promise, ) => Promise; /** * The invocation's ONE absolute deadline. Undefined = unbounded (tests). @@ -430,7 +436,11 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro fileContentType: row.mimeType, }; - const sent = { attempted: false }; + // TWO WAYS a Purchase can exist for this row by the time we are done: + // this attempt posted one, or the idempotency query found one an earlier + // attempt posted. Both mean the strong dedup key must be RETAINED when the + // row parks — releasing it lets a resubmission book the same receipt twice. + const sent = { attempted: false, purchaseKnownToExist: false }; let result: CreateQBReceiptPurchaseResult; try { @@ -465,21 +475,36 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // that THROWS when this worker has been superseded, which aborts the // create so a zombie cannot post a Purchase the live worker is about to // post as well. - result = await deps.createPurchase(tokens, input, deps.deadline, async () => { - const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); - if (!stillOurs) throw new StaleClaimError(); - sent.attempted = true; - }); + result = await deps.createPurchase( + tokens, + input, + deps.deadline, + async () => { + const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); + if (!stillOurs) throw new StaleClaimError(); + sent.attempted = true; + }, + // FENCED THE SAME WAY, for the same reason: the persisted flag is + // what a later pass reads, and a superseded worker must not write it + // (or carry on) at all. + async () => { + const stillOurs = await deps.markSendAttempted(row.id, row.claimToken); + if (!stillOurs) throw new StaleClaimError(); + sent.purchaseKnownToExist = true; + }, + ); } catch (error) { // A lost CAS from inside the create hook: nothing was sent. if (error instanceof StaleClaimError) return { outcome: "stale" }; const terminal = terminalReasonFor(error); // A send WAS attempted: QBO may hold a Purchase whose response we lost, // so the key stays claimed even though the row is parked. - if (terminal) return { outcome: "needs-review", reason: terminal, releaseStrongKey: !sent.attempted }; + if (terminal) { + return { outcome: "needs-review", reason: terminal, releaseStrongKey: !purchaseMayExist(sent) }; + } // QBTimeoutError, QBNotConnectedError, network/fetch errors, QBO // 429/5xx and DB errors are all transport-class: try again later. - return retry(row, deps, now, describe(error), sent.attempted); + return retry(row, deps, now, describe(error), purchaseMayExist(sent)); } if (!result.ok) { @@ -529,7 +554,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro releaseStrongKey: false, }; } - return retry(row, deps, now, `${ATTACHMENT_FAILED_PREFIX}${result.attachment}`, sent.attempted); + return retry(row, deps, now, `${ATTACHMENT_FAILED_PREFIX}${result.attachment}`, purchaseMayExist(sent)); } // 5. One transaction: the Expense and the row's BOOKED state land together @@ -681,10 +706,21 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // DocNumber lookup will find it and return alreadyExists:true — and the // key must be RETAINED, which is why this attempt's send flag is passed // rather than the row's stale copy. - return retry(row, deps, now, describe(error), sent.attempted); + return retry(row, deps, now, describe(error), purchaseMayExist(sent)); } } +/** + * Does QuickBooks (possibly) hold a Purchase for this row after this attempt? + * + * `attempted` covers a create we issued — including one whose response we lost. + * `purchaseKnownToExist` covers the idempotency query finding one an earlier + * attempt posted, which is the path that never reaches the create at all. + */ +function purchaseMayExist(sent: { attempted: boolean; purchaseKnownToExist: boolean }): boolean { + return sent.attempted || sent.purchaseKnownToExist; +} + function describe(error: unknown): string { if (error instanceof QBTimeoutError) return "QBTimeoutError"; if (error instanceof Error) return `${error.name}: ${error.message}`.slice(0, 400); @@ -766,15 +802,16 @@ function retry( now: Date, reason: string, /** - * Whether THIS attempt reached the create. `row.sendAttempted` is the value - * read when the row was claimed, so it is stale the moment - * markSendAttempted runs — and a failure AFTER the create (the attachment - * leg, the Expense commit) would have been judged on it and wrongly - * released the key of a row that really does have a Purchase. + * Whether THIS attempt learned that a Purchase may exist — either because it + * reached the create, or because the idempotency query found one already + * there. `row.sendAttempted` is the value read when the row was CLAIMED, so + * it is stale the moment the fenced mark runs, and a failure after that + * point (the attachment leg, the Expense commit) judged on it alone would + * wrongly release the key of a row that really does have a Purchase. */ - sentThisAttempt = false, + purchaseMayExistNow = false, ): BookResult { - const sendAttempted = row.sendAttempted || sentThisAttempt; + const sendAttempted = row.sendAttempted || purchaseMayExistNow; const attempts = row.attempts + 1; // `>=`, so MAX_BOOK_ATTEMPTS reads as "20 attempts in total" rather than 21. if (attempts >= MAX_BOOK_ATTEMPTS) { diff --git a/src/lib/receipt-intake/cutover.ts b/src/lib/receipt-intake/cutover.ts index 717a8358b..01b512bc0 100644 --- a/src/lib/receipt-intake/cutover.ts +++ b/src/lib/receipt-intake/cutover.ts @@ -49,3 +49,68 @@ export function parseCutoverBoundary(value: string | null | undefined): Date | n if (!Number.isFinite(at.getTime())) return null; return at; } + +/** One parked shadow row, as the cutover sees it. */ +export interface CutoverCandidate { + id: string; + source: string; + sourceRef: string; + archivedByV1: boolean; + createdAt: Date; +} + +export interface CutoverTriage { + /** v1 booked it. Retire — SHADOW_DONE, never booked here. */ + evidenced: string[]; + /** Nobody booked it (or we can collapse a double). Hand to v2. */ + unevidenced: string[]; + /** Cannot be settled from data. A human checks QuickBooks. */ + quarantined: string[]; +} + +/** The Drive file id a row books under, or null when it has no shared identity. */ +export function driveFileIdOf(row: { source: string; sourceRef: string }): string | null { + return row.source === "drive" && row.sourceRef.startsWith("drive:") + ? row.sourceRef.slice("drive:".length) + : null; +} + +/** + * Split the shadow backlog three ways. + * + * EVIDENCE OUTRANKS THE TIMESTAMP. The old rule looked only at rows older than + * the boundary, so a file v1 had already booked but the forwarder handed over + * AFTER the flip (a queued send, a retry, a slow archive step) never reached + * the evidence check at all: it went straight into the requeue and v2 booked a + * second Purchase. For an email or chat row that is unrecoverable by + * idempotency — v2 books under the intake UUID, which v1 never saw — so it is + * a real duplicate in the real books. + * + * The boundary only decides what to do with rows carrying NO evidence: + * - after it -> v1 was not booking; v2 takes it. + * - before it, Drive row -> v2 takes it. Safe because it books under the + * Drive file id, so a v1/v2 overlap collapses into one Purchase. + * - before it, anything else -> quarantine. Booking risks double-paying and + * retiring risks losing a real expense, so a human decides. + */ +export function triageCutoverRows( + candidates: CutoverCandidate[], + boundary: Date, + bookedByV1: ReadonlySet, +): CutoverTriage { + const triage: CutoverTriage = { evidenced: [], unevidenced: [], quarantined: [] }; + for (const row of candidates) { + const driveId = driveFileIdOf(row); + if (row.archivedByV1 || (driveId && bookedByV1.has(driveId))) { + triage.evidenced.push(row.id); + continue; + } + if (row.createdAt >= boundary) { + triage.unevidenced.push(row.id); + continue; + } + if (driveId) triage.unevidenced.push(row.id); + else triage.quarantined.push(row.id); + } + return triage; +} diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index b0cf5c448..f668b31de 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -110,17 +110,29 @@ export async function recordPendingCleanup(storagePath: string, reason: string): export interface RejectTxClient { automationEvent: { create(args: { data: Record; select: { id: true } }): Promise<{ id: string }> }; receiptIntake: { - deleteMany(args: { where: { id: string } }): Promise<{ count: number }>; - findUnique(args: { where: { id: string }; select: { id: true } }): Promise<{ id: string } | null>; + deleteMany(args: { where: Record }): Promise<{ count: number }>; }; } + +/** + * The row as it was OBSERVED, which is what the delete is fenced on. + * + * Same shape and same reason as publishFence: a reject and a publish race for + * the same row, and each must lose cleanly rather than act on a row the other + * has already moved. + */ +export interface RejectFence { + id: string; + state: string; + stateReason: string | null; + storagePath: string; +} export interface RejectClient { $transaction(fn: (tx: RejectTxClient) => Promise): Promise; } export async function rejectRowAndQueueCleanup( - rowId: string, - storagePath: string, + row: RejectFence, reason: string, db: RejectClient = prisma as unknown as RejectClient, ): Promise<{ ok: true; eventId: string } | { ok: false }> { @@ -132,30 +144,57 @@ export async function rejectRowAndQueueCleanup( status: "pending", reason: reason.slice(0, 500), source: "receipt-intake", - detail: JSON.stringify({ storagePath, rowId }), + detail: JSON.stringify({ storagePath: row.storagePath, rowId: row.id }), }, select: { id: true }, }); - await tx.receiptIntake.deleteMany({ where: { id: rowId } }); - // deleteMany's count is 0 both when somebody else already deleted - // the row (fine — it is gone, which is all we need) and when the id - // never matched. Only the row's ABSENCE is the condition worth - // committing on, so assert that directly. - const survivor = await tx.receiptIntake.findUnique({ where: { id: rowId }, select: { id: true } }); - if (survivor) throw new Error(`row ${rowId} still exists after delete`); + // THE FULL FENCE, and EXACTLY ONE ROW. + // + // A reject races a publish for the same row: a concurrent /finalize + // (or the sweeper) can move it to RECEIVED, re-park it under a + // different reason, or seal its object to a new path in the time + // this call spent inspecting the bytes. Deleting by id alone + // destroys that row and, worse, queues ITS object for deletion — + // the published receipt's own bytes. Pinning the observed state, + // reason and storagePath means the loser deletes nothing. + // + // "Already gone" is deliberately NOT treated as success: an absent + // row is a row somebody else accounted for, and queueing its path + // for deletion here is how a live object gets swept. + const { count } = await tx.receiptIntake.deleteMany({ + where: { + id: row.id, + state: row.state, + stateReason: row.stateReason, + claimToken: null, + storagePath: row.storagePath, + }, + }); + if (count !== 1) throw new RejectFenceLost(row.id); return event.id; }); return { ok: true, eventId }; } catch (error) { + // Either way NOTHING is committed: the queue entry rolls back with the + // delete, so there is no cleanup record naming a path a live row still + // points at. console.error( "[receipts/intake] reject transaction failed", - rowId, + row.id, error instanceof Error ? error.name : "error", ); return { ok: false }; } } +/** The delete matched no row: somebody else moved it. Never a partial commit. */ +class RejectFenceLost extends Error { + constructor(rowId: string) { + super(`reject fence lost for ${rowId}`); + this.name = "RejectFenceLost"; + } +} + /** * Try the queued deletion now. A failure is not an error for the caller — the * event stays pending and the worker's sweep retries it. diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 5d4c8e448..008deb744 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -10,6 +10,8 @@ */ import test from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { appliedTaxCents, attachmentBlocker, @@ -46,6 +48,28 @@ function atCreate(fn: (...args: any[]) => Promise) { }; } +/** + * A stub for the ALREADY-EXISTS branch, mirroring the real control flow. + * + * createQBReceiptPurchase returns there from the idempotency query and never + * reaches qbCreateFn, so `onBeforeCreate` does NOT fire — only + * `onExistingPurchase` does. A fake that called onBeforeCreate anyway would + * mark the row "sent" and hide the very bug this branch had: a Purchase that + * exists while the row believes nothing was ever sent. + */ +function atExisting(fn: (...args: any[]) => Promise) { + return async ( + tokens: any, + input: any, + deadline: any, + _onBeforeCreate?: () => Promise, + onExistingPurchase?: () => Promise, + ) => { + await onExistingPurchase?.(); + return fn(tokens, input, deadline); + }; +} + const NOW = new Date("2026-09-01T12:00:00.000Z"); function row(overrides: Partial = {}): BookableRow { @@ -386,7 +410,7 @@ test("an EXISTING purchase is held to the SAME attachment standard", async () => // response — exactly when a Purchase is most likely to be sitting in the // books without its image. It was the one path exempt from the check. const failing = recorder({ - createPurchase: atCreate(async () => ({ + createPurchase: atExisting(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "failed:500", })) as any, }); @@ -395,7 +419,7 @@ test("an EXISTING purchase is held to the SAME attachment standard", async () => assert.equal(failing.expenses.length, 0, "and it is NOT booked meanwhile"); const skipped = recorder({ - createPurchase: atCreate(async () => ({ + createPurchase: atExisting(async () => ({ ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "skipped", })) as any, }); @@ -411,10 +435,10 @@ test("a previous attachment failure does NOT block the recovery attempt", async // Short-circuiting on lastError made the stranded-receipt case permanent — // the opposite of what the guard was for. const r = recorder({ - createPurchase: async (_t, input) => { + createPurchase: atExisting(async (_t: any, input: any) => { r.purchaseCalls.push(input); return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "already-attached" } as any; - }, + }) as any, }); const result = await bookReceipt(row({ lastError: "attachment-failed:failed:500" }), r.deps); assert.equal(result.outcome, "booked", "the recovery succeeded and the row books"); @@ -477,7 +501,7 @@ test("a plain network error retries; MAX_BOOK_ATTEMPTS means 20 attempts in TOTA test("alreadyExists books identically — the lost-response retry", async () => { const r = recorder({ - createPurchase: atCreate(async (_t: any, input: any) => ({ + createPurchase: atExisting(async (_t: any, input: any) => ({ ok: true, qbPurchaseId: "QB-7", docNumber: input.fileId.slice(0, 21), alreadyExists: true, attachment: "already-attached", })) as any, @@ -896,3 +920,80 @@ test("the BOOKED write is a CAS on state AND token", async () => { await bookReceipt(row({ claimToken: "token-abc" }), r.deps); assert.deepEqual(wheres, [{ id: "intake-1", state: "BOOKING", claimToken: "token-abc" }]); }); + +// ── A Purchase found by the idempotency query is still a Purchase ─────────── + +test("the already-exists branch does NOT go through onBeforeCreate", async () => { + // The control for every test below: if the fake called onBeforeCreate here + // the row would look "sent" for the wrong reason and the bug would be + // invisible. The real core returns from the idempotency query. + const seen: string[] = []; + const r = recorder({ + createPurchase: async (_t: any, _i: any, _d: any, onBeforeCreate: any, onExisting: any) => { + seen.push("create-called"); + await onExisting?.(); + void onBeforeCreate; + return { ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "already-attached" } as any; + }, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal(result.outcome, "booked"); + assert.deepEqual(seen, ["create-called"]); + assert.deepEqual(r.sendMarks, ["intake-1"], "the EXISTING-purchase hook marked it, fenced the same way"); +}); + +test("attempt 20 on an ALREADY-EXISTING purchase retains the strong key", async () => { + // The hole: this path never reaches the create, so `sendAttempted` stayed + // false — and a row that exhausted its retries here handed its dedup key + // back while a real Purchase sat in the books. The next submission of the + // same receipt would then book it a second time. + const r = recorder({ + createPurchase: atExisting(async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "failed:500", + })) as any, + }); + const result = await bookReceipt(row({ attempts: 19, sendAttempted: false }), r.deps); + assert.equal((result as any).reason, "max-retries"); + assert.equal((result as any).releaseStrongKey, false, "the Purchase EXISTS"); + assert.deepEqual(r.sendMarks, ["intake-1"], "and the flag was persisted, not just held in memory"); +}); + +test("a terminal attachment refusal on an existing purchase also retains the key", async () => { + const r = recorder({ + createPurchase: atExisting(async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "failed:415", + })) as any, + }); + const result = await bookReceipt(row(), r.deps); + assert.equal((result as any).reason, "attachment-refused:failed:415"); + assert.equal((result as any).releaseStrongKey, false); +}); + +test("a STALE claim on the existing-purchase hook aborts, exactly like the create hook", async () => { + const r = recorder({ + markSendAttempted: async () => false, + createPurchase: atExisting(async () => ({ + ok: true, qbPurchaseId: "QB-1", docNumber: "d", alreadyExists: true, attachment: "already-attached", + })) as any, + }); + assert.deepEqual(await bookReceipt(row(), r.deps), { outcome: "stale" }); + assert.equal(r.expenses.length, 0, "a superseded worker books nothing"); +}); + +test("the QBO core fires onExistingPurchase before it touches the attachment", () => { + // Ordering matters: the attachment re-check is a round trip that can fail, + // and the caller still has to know a Purchase is there. + const source = readFileSync( + path.join(__dirname, "..", "src/lib/qbo-receipt-push.ts"), + "utf8", + ); + const branch = source.slice(source.indexOf("if (existing.length > 0) {")); + const body = branch.slice(0, branch.indexOf("alreadyExists: true")); + assert.match(body, /await deps\.onExistingPurchase\?\.\(\);/); + assert.ok( + body.indexOf("onExistingPurchase") < body.indexOf("ensureAttachmentOnExistingPurchase"), + "the signal precedes the attachment work", + ); + // And the create hook is NOT fired on this path. + assert.ok(!body.includes("onBeforeCreate"), "onBeforeCreate belongs to the create path only"); +}); diff --git a/tests/receipt-intake-cutover.test.ts b/tests/receipt-intake-cutover.test.ts index 5b3bd967c..72f094d47 100644 --- a/tests/receipt-intake-cutover.test.ts +++ b/tests/receipt-intake-cutover.test.ts @@ -8,7 +8,13 @@ */ import test from "node:test"; import assert from "node:assert/strict"; -import { parseCutoverBoundary, CUTOVER_SETTING_KEY } from "../src/lib/receipt-intake/cutover"; +import { + parseCutoverBoundary, + CUTOVER_SETTING_KEY, + driveFileIdOf, + triageCutoverRows, + type CutoverCandidate, +} from "../src/lib/receipt-intake/cutover"; test("a missing or malformed boundary is null — never epoch", () => { // The dangerous failure: `new Date(undefined)` style coercion yielding a @@ -31,3 +37,82 @@ test("the setting key is stable — an operator writes this row at the flip", () // Renaming it silently would make every future cutover refuse. assert.equal(CUTOVER_SETTING_KEY, "cutoverV1StoppedAt"); }); + +// ── The three-way split: evidence outranks the timestamp ─────────────────── + +const BOUNDARY = new Date("2026-08-25T00:00:00.000Z"); +const before = new Date("2026-08-24T12:00:00.000Z"); +const after = new Date("2026-08-26T12:00:00.000Z"); + +function candidate(over: Partial = {}): CutoverCandidate { + return { + id: "row-1", + source: "drive", + sourceRef: "drive:FILE1", + archivedByV1: false, + createdAt: before, + ...over, + }; +} + +test("an AFTER-boundary row the forwarder says v1 archived is RETIRED, never booked", async () => { + // The hole: the candidate query filtered on createdAt first, so a file v1 + // had ALREADY booked but handed over after the flip (a queued send, a + // retry, a slow archive step) never reached the evidence check — it went + // into the requeue and v2 booked a SECOND Purchase. For an email or chat + // row there is no shared identity to collapse that: v2 books under the + // intake UUID, which v1 never saw. + for (const source of ["email", "chat"]) { + const triage = triageCutoverRows( + [candidate({ source, sourceRef: `${source}:msg-1`, archivedByV1: true, createdAt: after })], + BOUNDARY, + new Set(), + ); + assert.deepEqual(triage.evidenced, ["row-1"], source); + assert.deepEqual(triage.unevidenced, [], `${source}: never handed to v2`); + assert.deepEqual(triage.quarantined, [], source); + } +}); + +test("a v1 booked marker also retires an after-boundary row", async () => { + const triage = triageCutoverRows( + [candidate({ createdAt: after })], + BOUNDARY, + new Set(["FILE1"]), + ); + assert.deepEqual(triage.evidenced, ["row-1"]); +}); + +test("only EVIDENCE-FREE rows are judged by the boundary", async () => { + const rows = [ + candidate({ id: "after-nothing", source: "email", sourceRef: "email:m2", createdAt: after }), + candidate({ id: "before-drive", sourceRef: "drive:F2", createdAt: before }), + candidate({ id: "before-email", source: "email", sourceRef: "email:m3", createdAt: before }), + ]; + const triage = triageCutoverRows(rows, BOUNDARY, new Set()); + // After the flip nothing but v2 could have booked it. + assert.deepEqual(triage.unevidenced, ["after-nothing", "before-drive"]); + // Shadow-window, no evidence, no shared identity: a human decides. + assert.deepEqual(triage.quarantined, ["before-email"]); + assert.deepEqual(triage.evidenced, []); +}); + +test("the boundary instant itself counts as AFTER, and every row lands in exactly one bucket", async () => { + const rows = [ + candidate({ id: "at-boundary", source: "chat", sourceRef: "chat:m1", createdAt: BOUNDARY }), + candidate({ id: "evidenced", archivedByV1: true }), + candidate({ id: "quarantine", source: "web", sourceRef: "web:u1" }), + ]; + const triage = triageCutoverRows(rows, BOUNDARY, new Set()); + assert.deepEqual(triage.unevidenced, ["at-boundary"]); + assert.deepEqual(triage.evidenced, ["evidenced"]); + assert.deepEqual(triage.quarantined, ["quarantine"]); + const all = [...triage.evidenced, ...triage.unevidenced, ...triage.quarantined]; + assert.equal(all.length, rows.length, "no row is dropped or counted twice"); +}); + +test("driveFileIdOf only claims a shared identity for a real drive ref", () => { + assert.equal(driveFileIdOf({ source: "drive", sourceRef: "drive:ABC" }), "ABC"); + assert.equal(driveFileIdOf({ source: "email", sourceRef: "drive:ABC" }), null); + assert.equal(driveFileIdOf({ source: "drive", sourceRef: "web:ABC" }), null); +}); diff --git a/tests/receipt-intake-reject.test.ts b/tests/receipt-intake-reject.test.ts index 039b79a1d..342eb2b58 100644 --- a/tests/receipt-intake-reject.test.ts +++ b/tests/receipt-intake-reject.test.ts @@ -27,18 +27,37 @@ const finalize = readFileSync( "utf8", ); +type Row = Record; + interface Store { - rows: Set; + rows: Row[]; events: { id: string; data: Record }[]; committed: boolean; } -/** A $transaction that really rolls back: the fake state is only kept on commit. */ -function client(rows: string[], opts: { undeletable?: boolean } = {}): { db: RejectClient; store: Store } { - const store: Store = { rows: new Set(rows), events: [], committed: false }; +const parked = (over: Row = {}): Row => ({ + id: "row-1", + state: "STAGING", + stateReason: null, + claimToken: null, + storagePath: "receipts/intake/row-1.bin", + ...over, +}); + +/** + * A $transaction that really rolls back, over rows that really match a where. + * + * The fence is the whole subject here, so the fake has to evaluate it rather + * than match on the id like the code used to. + */ +function client(rows: Row[], onTx?: (store: Store) => void): { db: RejectClient; store: Store } { + const store: Store = { rows: rows.map(r => ({ ...r })), events: [], committed: false }; const db: RejectClient = { $transaction: async fn => { - const stagedRows = new Set(store.rows); + // A concurrent writer, running between the caller's read and this + // transaction — the race this fence exists for. + onTx?.(store); + let staged = store.rows.map(r => ({ ...r })); const stagedEvents: Store["events"] = []; let seq = 0; const tx: RejectTxClient = { @@ -51,14 +70,15 @@ function client(rows: string[], opts: { undeletable?: boolean } = {}): { db: Rej }, receiptIntake: { deleteMany: async ({ where }) => { - if (opts.undeletable) return { count: 0 }; - return { count: stagedRows.delete(where.id) ? 1 : 0 }; + const matches = staged.filter(row => + Object.entries(where).every(([k, v]) => row[k] === v)); + staged = staged.filter(row => !matches.includes(row)); + return { count: matches.length }; }, - findUnique: async ({ where }) => (stagedRows.has(where.id) ? { id: where.id } : null), }, }; const out = await fn(tx); - store.rows = stagedRows; + store.rows = staged; store.events.push(...stagedEvents); store.committed = true; return out; @@ -68,44 +88,71 @@ function client(rows: string[], opts: { undeletable?: boolean } = {}): { db: Rej } test("a reject deletes the row and queues the object in ONE transaction", async () => { - const { db, store } = client(["row-1"]); - const injected = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "unsupported-type", db); + const { db, store } = client([parked()]); + const injected = await rejectRowAndQueueCleanup(parked() as never, "unsupported-type", db); assert.equal(injected.ok, true); - assert.equal(store.rows.has("row-1"), false, "the row is gone"); + assert.deepEqual(store.rows, [], "the row is gone"); assert.equal(store.events.length, 1, "and exactly one cleanup is queued"); assert.equal(store.events[0].data.status, "pending"); assert.match(String(store.events[0].data.detail), /receipts\/intake\/row-1\.bin/); }); -test("a row that survives the delete rolls the queue entry back and reports failure", async () => { - // The caller must then KEEP the object: a row that still exists may still - // point at those bytes, so deleting them would destroy a live receipt. - const { db, store } = client(["row-1"], { undeletable: true }); - const result = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "empty-file", db); +test("PUBLISH vs REJECT: a row published mid-reject is not deleted, and nothing is queued", async () => { + // The race: /finalize inspects the object, decides it is unsupported, and + // in that window a concurrent publisher (another finalize, or the sweeper) + // moves the row to RECEIVED and seals its object to the canonical path. + // Deleting by id would destroy a PUBLISHED receipt and queue its live + // bytes for deletion. + const { db, store } = client([parked()], s => { + s.rows = [parked({ state: "RECEIVED", storagePath: "receipts/row-1/abc.png" })]; + }); + const result = await rejectRowAndQueueCleanup(parked() as never, "unsupported-file-type", db); assert.equal(result.ok, false); - assert.equal(store.committed, false, "nothing committed"); - assert.deepEqual(store.events, [], "no orphan record for a row that is still there"); - assert.equal(store.rows.has("row-1"), true); + assert.equal(store.committed, false, "the whole transaction rolled back"); + assert.deepEqual(store.events, [], "no cleanup naming a path a live row points at"); + assert.equal(store.rows.length, 1, "the published row survives"); + assert.equal(store.rows[0].state, "RECEIVED"); }); -test("rejecting an already-deleted row still queues the cleanup", async () => { - // The retry of a reject whose response was lost. deleteMany counts zero, - // but the row is ABSENT, which is the condition that matters — and its - // object is unreferenced, so it still has to be queued. - const { db, store } = client([], {}); - const result = await rejectRowAndQueueCleanup("row-1", "receipts/intake/row-1.bin", "unsupported-type", db); - assert.equal(result.ok, true); - assert.equal(store.events.length, 1); +test("a row re-parked or claimed mid-reject also loses the fence", async () => { + for (const moved of [ + parked({ stateReason: "file-missing", state: "NEEDS_REVIEW" }), + parked({ claimToken: "worker-1" }), + parked({ storagePath: "receipts/intake/row-1-other.bin" }), + ]) { + const { db, store } = client([parked()], s => { s.rows = [moved]; }); + const result = await rejectRowAndQueueCleanup(parked() as never, "empty-file", db); + assert.equal(result.ok, false, JSON.stringify(moved)); + assert.deepEqual(store.events, [], "nothing queued"); + assert.equal(store.rows.length, 1, "nothing deleted"); + } +}); + +test("a row that is already GONE is not treated as a successful reject", async () => { + // An absent row is a row somebody else accounted for. Queueing its path for + // deletion here is exactly how a live object gets swept: the retry of a + // reject can arrive after the id was reused by a re-created row, or after + // the object was sealed under a path a new row points at. + const { db, store } = client([]); + const result = await rejectRowAndQueueCleanup(parked() as never, "unsupported-type", db); + assert.equal(result.ok, false); + assert.deepEqual(store.events, []); }); -test("an unconfirmed reject answers 503 and keeps the object", () => { +test("a lost reject fence answers 409 publish-conflict and keeps the object", () => { const branch = finalize.slice(finalize.indexOf("const rejected = await rejectRowAndQueueCleanup")); const head = branch.slice(0, branch.indexOf("settleQueuedCleanup")); - assert.match(head, /reject-failed/); - assert.match(head, /status: 503/); + // The reject is fenced on what was OBSERVED, so losing it means the row is + // not ours to reject — a 409 the caller can retry, never a 2xx and never a + // deletion. + assert.match(head, /state: row\.state/); + assert.match(head, /stateReason: row\.stateReason/); + assert.match(head, /storagePath: row\.storagePath/); + assert.match(head, /publish-conflict/); + assert.match(head, /status: 409/); assert.ok( !/deleteObjectOrRecord|removeSecureDoc/.test(head), - "no object deletion on the unconfirmed path", + "no object deletion on the lost-fence path", ); }); @@ -146,3 +193,43 @@ test("a heal that loses its CAS deletes the object it just uploaded", () => { "and never deletes the path the surviving row still points at", ); }); + +// ── /start re-arms a recoverable park instead of claiming we hold it ──────── + +const start = readFileSync( + path.join(ROOT, "src/app/api/receipts/intake/start/route.ts"), + "utf8", +); + +test("/start hands a recoverable park a NEW url, and asks the shared rule which parks those are", () => { + // Answering alreadyReceived here told the forwarder we held a receipt we + // did not hold — and it deletes its only copy on that answer. + assert.match(start, /finalizeDisposition\(existing\) === "publish"/); + const branch = start.slice(start.indexOf("if (recoverable) {")); + const body = branch.slice(0, branch.indexOf("// IDENTITY MUST BE PROVEN")); + assert.match(body, /recovered: true/); + assert.match(body, /expectedSha256,/, "the sha the client is about to upload is re-armed"); + assert.match(body, /fileSha256: "",/, "and the stale stored hash is cleared"); + // Fenced like every other publish-path write, and a lost fence writes + // nothing rather than pointing a live row at an empty path. + assert.match(body, /where: \{ id: existing\.id, \.\.\.publishFence\(existing\) \}/); + assert.match(body, /publish-conflict/); +}); + +test("the re-arm branch runs BEFORE the identity check, and only for a parked row", () => { + // For a STAGING row the sha check still stands: it is what stops receipt B + // from being handed a URL over receipt A's verified bytes. + assert.match(start, /existing\.state !== "STAGING"\s*\n\s*&& finalizeDisposition\(existing\) === "publish"/); + assert.ok( + start.indexOf("if (recoverable) {") < start.indexOf("const knownSha ="), + "the recoverable branch is taken before the identity check", + ); + assert.match(start, /alreadyReceived: true/, "everything else still answers alreadyReceived"); +}); + +test("a re-arm that changes the extension does not orphan the old object", () => { + const branch = start.slice(start.indexOf("if (recoverable) {")); + const body = branch.slice(0, branch.indexOf("// IDENTITY MUST BE PROVEN")); + assert.match(body, /retryPath !== existing\.storagePath/); + assert.match(body, /deleteObjectOrRecord\(existing\.storagePath, "start-rearmed-repath"\)/); +}); From c2e640888d088d6c5aa09e5d89799e80355c4faf Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 08:07:34 -0700 Subject: [PATCH 057/144] fix(receipts): fenced sweeper reject, upload leases, own bucket, sourceRef shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (1) The stale-STAGING sweeper rejects through rejectRowAndQueueCleanup(), checks the outcome, and touches no bytes when the fence loses. Its old unfenced delete-by-id-and-state plus bare object delete could destroy a row a concurrent /finalize had just published, or the bytes that published row pointed at. (2) New `ReceiptIntake.uploadUrlExpiresAt` (schema + migration + apply script + verifier). /start stamps it on every URL it issues — new row, resumed upload and re-armed park — and the sweeper judges by the LEASE, never row age: a re-issued URL makes a row younger than itself. Nothing destructive happens while the lease is live; those passes are skipped and counted as `upload-lease-active`. (Interpreted as a skip, not a state change: parking a row whose upload is still in flight is the same mistake, one queue further along.) (3) Receipts move to their own private bucket, `receipt-intake`, with a 15 MiB limit and a MIME allow-list — the only place a signed-URL write can actually be refused, and one a path bug cannot use to reach the contract store. All intake storage goes through src/lib/receipt-intake/bucket.ts. scripts/apply-receipt-intake.mjs creates or VERIFIES it over the Storage API and exits nonzero on a different limit, MIME list or public flag. An unknown object size is now transient rather than permission to download. (4) sourceRef shape is validated per source in decideSource (so both entry points get it): nonempty tail, no control characters, 512 bytes, and a real shape for drive / email / chat. `drive:` with an empty tail used to be a valid PERMANENT idempotency key that every later empty-tail forward collided with. Also: refreshed the markSendAttempted contract comment now that both QBO-core hooks call it. .env.example and the spec's deploy notes updated. Mutation-tested: unfenced reject, lease ignored on reject, unknown size falling through to a download. Co-Authored-By: Claude Fable 5.1 --- .env.example | 20 +- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 57 ++++- .../migration.sql | 2 + prisma/schema.prisma | 6 + scripts/apply-receipt-intake.mjs | 139 +++++++++++- .../api/cron/receipt-intake-worker/route.ts | 67 ++++-- src/app/api/receipts/intake/route.ts | 36 ++- src/app/api/receipts/intake/start/route.ts | 45 ++-- src/lib/receipt-intake/book.ts | 16 +- src/lib/receipt-intake/bucket.ts | 208 ++++++++++++++++++ src/lib/receipt-intake/intake-core.ts | 60 ++++- src/lib/receipt-intake/queries.ts | 6 +- src/lib/receipt-intake/storage-cleanup.ts | 31 +-- src/lib/receipt-intake/stored-object.ts | 37 ++-- src/lib/receipt-intake/worker.ts | 25 +++ tests/apply-receipt-intake.test.ts | 106 ++++++++- tests/receipt-intake-archive-contract.test.ts | 6 +- tests/receipt-intake-auth.test.ts | 73 +++++- tests/receipt-intake-book.test.ts | 2 +- tests/receipt-intake-cleanup.test.ts | 24 +- tests/receipt-intake-reject.test.ts | 64 ++++++ tests/receipt-intake-stored-object.test.ts | 83 +++++-- tests/receipt-intake-worker.test.ts | 52 +++++ 23 files changed, 1003 insertions(+), 162 deletions(-) create mode 100644 src/lib/receipt-intake/bucket.ts diff --git a/.env.example b/.env.example index aaeb5a200..bab7d9454 100644 --- a/.env.example +++ b/.env.example @@ -45,12 +45,20 @@ RECEIPT_INTAKE_SECRET= # (POST /api/receipts/intake/{id}/archived). Cannot create or publish a row. RECEIPT_ARCHIVE_SECRET= # -# Storage note (not an env var): the private receipts bucket must ALSO carry a -# 15 MB file-size limit in the Supabase dashboard (Storage -> bucket -> -# Settings). Two-step uploads go straight to a signed URL and never pass through -# this server, so the bucket is the only place a too-large write can actually be -# refused; MAX_STORED_BYTES in src/lib/receipt-intake/intake-core.ts only lets -# the server reject the object after the fact. +# Storage note (not an env var): receipts live in their OWN private bucket, +# `receipt-intake`, carrying a 15 MiB file-size limit and an allow-list of the +# six formats QuickBooks can attach. Two-step uploads go straight to a signed +# URL and never pass through this server, so the bucket is the only place a +# too-large or wrong-type write can actually be refused; MAX_STORED_BYTES in +# src/lib/receipt-intake/intake-core.ts only lets the server reject the object +# after the fact. It is separate from `secure-docs` because those limits are +# per-bucket, and because a signed upload URL is a write capability that must +# not point at the bucket holding signed contracts. +# +# Do NOT create it by hand: scripts/apply-receipt-intake.mjs creates it when +# missing and VERIFIES it when present (it needs SUPABASE_URL and +# SUPABASE_SERVICE_KEY in the environment, and exits nonzero if the bucket +# exists with a different limit, MIME list, or public flag). # # Shadow mode. UNSET or "true" = dry run: rows are read, deduped and routed, and # NOTHING is booked. Set to the literal "false" only at cutover. diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index 86c96412d..cd3f19297 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -99,7 +99,12 @@ model ReceiptIntake { createdBy User? @relation(fields: [createdById], references: [id], onDelete: SetNull) // file (Supabase secure-docs, private) - storagePath String // receipts/intake/. in SECURE_BUCKET + storagePath String // receipts/intake/. in the `receipt-intake` bucket + // When the signed upload URL /intake/start last issued stops working. The + // sweeper uses THIS, not createdAt: a row whose URL was re-issued is older + // than its lease, and judging it on row age parked receipts whose own upload + // link was still live. Null on rows that never had a signed URL. + uploadUrlExpiresAt DateTime? fileName String? mimeType String fileSize Int @@ -158,6 +163,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "suggestedConfidence" DOUBLE PRECISION, "createdById" TEXT, "storagePath" TEXT NOT NULL, "fileName" TEXT, "mimeType" TEXT NOT NULL, "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, + "expectedSha256" TEXT, "uploadUrlExpiresAt" TIMESTAMP(3), "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, "taxCents" INTEGER, "docType" TEXT, "refNumber" TEXT, "memo" TEXT, "readJson" TEXT, "readAt" TIMESTAMP(3), "dedupStrongKey" TEXT, "dedupWeakKey" TEXT, "duplicateOfId" TEXT, @@ -184,7 +190,24 @@ CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("cre -- expenseId -> "Expense"(id) ON DELETE SET NULL ``` -Run the apply script against prod BEFORE merging (CLAUDE.md pre-deploy rule #2). +Run the apply script against prod BEFORE merging (CLAUDE.md pre-deploy rule #2): + +```bash +SUPABASE_URL=... SUPABASE_SERVICE_KEY=... node scripts/apply-receipt-intake.mjs --yes --expect-db postgres --expect-host +``` + +It does BOTH halves of the rollout and verifies each: + +1. **Schema** — additive, idempotent DDL, then a shape check (every column, the CHECK + constraint, the FKs, and the partial unique index verified by its DEFINITION, not its + name). +2. **Storage** — creates the private `receipt-intake` bucket, or verifies the existing one. + It exits nonzero on a different file-size limit, a different MIME allow-list, or a public + bucket, and never rewrites one. + +Both halves are safe to re-run. The bucket step needs `SUPABASE_URL` and +`SUPABASE_SERVICE_KEY`; without them the script refuses rather than skipping it, because a +missing bucket policy is invisible until a 400 MB object is already stored. ## 3. Endpoint contracts @@ -473,15 +496,35 @@ URL bypasses this server entirely, so application code cannot stop the write — refuse the object afterwards, by which time the bytes are already paid for and sitting in the bucket. Set it where the write happens: -> Supabase dashboard → Storage → the private receipts bucket (`SECURE_BUCKET`) → Settings → -> **file size limit = 15 MB**. Supabase rejects a larger upload at the storage API with a -> 413, before any object is created. +> `node scripts/apply-receipt-intake.mjs --yes --expect-db … --expect-host …`, with +> `SUPABASE_URL` and `SUPABASE_SERVICE_KEY` in the environment. It creates the bucket when +> missing and VERIFIES it when present, and exits **nonzero** if it exists with a different +> file-size limit, a different MIME allow-list, or as a public bucket. It never silently +> "corrects" one: overwriting a limit somebody set deliberately is how a 400 MB upload +> becomes possible again next quarter. + +Receipts live in their **own** bucket, `receipt-intake` — not `secure-docs`: + +* Those limits are **per bucket**, so `secure-docs` cannot carry a receipt policy without + imposing it on contracts, e-signatures and invoice PDFs. +* A signed upload URL is a **write capability**. Issuing one against the bucket that also + holds countersigned contracts means a path-handling bug in intake is a write into the + contract store. +* The orphan sweep **deletes** objects, unattended, from paths read out of an event log. It + must not be able to reach anything but receipts. + +All intake reads and writes go through `src/lib/receipt-intake/bucket.ts`, which is the one +place that names the bucket. The server-side check stays regardless, and is checked in this order: 1. **Object metadata first** (`list({ search })` → `metadata.size`) — one small request that costs the same whatever the object weighs. Oversize is rejected here, with no body read. -2. **Then the downloaded byte length**, because a null metadata size means "storage did not - say", not "fine". +2. **Then the downloaded byte length**, as a second line for anything the metadata missed. + +An **unknown** size is `transient`, not permission to proceed: a storage hiccup, a missing +client or an API without metadata all used to fall through to the download — which is the +read this check exists to avoid, taken on exactly the objects we know least about. Both +callers retry a transient answer. **`text/plain` is refused with a 415.** QuickBooks cannot attach a `.txt`, so accepting one meant reading it with Gemini and then stranding it unbookable at diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 312b2c1ea..14ee2afed 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, "expectedSha256" TEXT, + "uploadUrlExpiresAt" TIMESTAMP(3), "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -64,6 +65,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( -- existing table, so a column added to the CREATE above would never reach it. ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadUrlExpiresAt" TIMESTAMP(3); ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 09b663a72..c9cf9bc96 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3053,6 +3053,12 @@ model ReceiptIntake { /// The two-step flow hands the bytes straight to storage, so this is the only /// way to notice that a reused sourceRef is carrying a DIFFERENT document. expectedSha256 String? + /// When the signed upload URL /intake/start last issued stops working. + /// The sweeper uses THIS, not createdAt, to decide whether an object could + /// still arrive: a row whose URL was re-issued is younger than its row, and + /// parking it on row age declared a receipt missing while its own upload link + /// was live. Null on rows that never had a signed URL (the single-shot path). + uploadUrlExpiresAt DateTime? // read results (cents, like AutomationEvent) vendor String? diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index a50e4c95e..fd3905fab 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -98,6 +98,7 @@ export const statements = [ "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, "expectedSha256" TEXT, + "uploadUrlExpiresAt" TIMESTAMP(3), "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -133,6 +134,7 @@ export const statements = [ // it. This is the whole reason the script is re-runnable. `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadUrlExpiresAt" TIMESTAMP(3)`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT`, @@ -270,7 +272,8 @@ const expectedColumns = { "id", "source", "sourceRef", "state", "dryRun", "stateReason", "projectId", "costCodeId", "suggestedCostCodeId", "suggestedConfidence", "createdById", "storagePath", "fileName", "mimeType", "fileSize", - "fileSha256", "expectedSha256", "sendAttempted", "archivedByV1", + "fileSha256", "expectedSha256", "uploadUrlExpiresAt", + "sendAttempted", "archivedByV1", "vendor", "txnDate", "totalCents", "taxCents", "docType", "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", "dedupWeakKey", "duplicateOfId", "qbPurchaseId", "expenseId", @@ -304,6 +307,136 @@ const expectedPartialIndexes = [{ ], }]; + +// ── The receipts bucket ──────────────────────────────────────────────────── +// +// Provisioned HERE rather than by hand in the dashboard, because two of its +// settings are load-bearing and invisible from the application: +// +// * fileSizeLimit — the two-step upload goes straight to a signed URL that +// never passes through the server, so the BUCKET is the only place a 400 MB +// write can actually be refused. Application code can only reject the +// object afterwards, once the bytes are already stored and paid for. +// * allowedMimeTypes — same reason, for a format QuickBooks cannot attach. +// +// It is its own bucket, not `secure-docs`: those limits are per-bucket and +// cannot be imposed on contracts and invoice PDFs, and a signed upload URL is a +// write capability that must not point anywhere near the contract store. +// +// Idempotent: create if missing, otherwise VERIFY. A bucket that exists with +// the wrong limit is a hard failure — silently "fixing" a limit somebody set +// deliberately is how a 400 MB upload becomes possible again next quarter. +export const RECEIPT_BUCKET = "receipt-intake"; +export const RECEIPT_BUCKET_FILE_SIZE_LIMIT = 15 * 1024 * 1024; +// EXACTLY the list src/lib/receipt-intake/file-type.ts accepts (asserted by +// tests/apply-receipt-intake.test.ts). A bucket that allows more than the code +// does lets an unreadable file be stored; one that allows less rejects uploads +// the code promised were fine, at a signed URL where the caller sees only a +// storage error. +export const RECEIPT_BUCKET_MIME_TYPES = [ + "application/pdf", + "image/jpeg", + "image/png", + "image/heic", + "image/heif", + "image/webp", + "image/gif", +]; + +/** Normalizes Supabase's file_size_limit, which comes back as bytes or "15MB". */ +export function parseSizeLimit(value) { + if (value === null || value === undefined) return null; + if (typeof value === "number") return value; + const text = String(value).trim(); + if (/^\d+$/.test(text)) return Number(text); + const match = text.match(/^(\d+(?:\.\d+)?)\s*(b|kb|mb|gb)$/i); + if (!match) return null; + const scale = { b: 1, kb: 1024, mb: 1024 * 1024, gb: 1024 * 1024 * 1024 }[match[2].toLowerCase()]; + return Math.round(Number(match[1]) * scale); +} + +async function storageRequest(baseUrl, key, path, init = {}) { + const res = await fetch(`${baseUrl.replace(/\/$/, "")}/storage/v1${path}`, { + ...init, + headers: { + authorization: `Bearer ${key}`, + apikey: key, + "content-type": "application/json", + ...(init.headers ?? {}), + }, + }); + const text = await res.text(); + let body = null; + try { + body = text ? JSON.parse(text) : null; + } catch { + body = { raw: text.slice(0, 200) }; + } + return { status: res.status, ok: res.ok, body }; +} + +/** + * Create or verify the bucket. Returns "created" | "verified"; THROWS when it + * exists with a different policy, because that is a fact the operator has to + * see rather than a state to overwrite. + */ +export async function ensureReceiptBucket(baseUrl, key, request = storageRequest) { + const existing = await request(baseUrl, key, `/bucket/${RECEIPT_BUCKET}`, { method: "GET" }); + + if (existing.status === 404) { + const created = await request(baseUrl, key, "/bucket", { + method: "POST", + body: JSON.stringify({ + id: RECEIPT_BUCKET, + name: RECEIPT_BUCKET, + public: false, + file_size_limit: RECEIPT_BUCKET_FILE_SIZE_LIMIT, + allowed_mime_types: RECEIPT_BUCKET_MIME_TYPES, + }), + }); + if (!created.ok) { + throw new Error(`could not create bucket ${RECEIPT_BUCKET}: ${created.status} ${JSON.stringify(created.body)}`); + } + return "created"; + } + if (!existing.ok) { + throw new Error(`could not read bucket ${RECEIPT_BUCKET}: ${existing.status} ${JSON.stringify(existing.body)}`); + } + + const bucket = existing.body ?? {}; + const problems = []; + if (bucket.public === true) problems.push("bucket is PUBLIC; receipts must be private"); + const limit = parseSizeLimit(bucket.file_size_limit); + if (limit !== RECEIPT_BUCKET_FILE_SIZE_LIMIT) { + problems.push(`file_size_limit is ${bucket.file_size_limit} (${limit} bytes), expected ${RECEIPT_BUCKET_FILE_SIZE_LIMIT}`); + } + const allowed = bucket.allowed_mime_types ?? null; + if (!Array.isArray(allowed)) { + problems.push("allowed_mime_types is unset; any file type could be uploaded"); + } else { + const missing = RECEIPT_BUCKET_MIME_TYPES.filter(m => !allowed.includes(m)); + const extra = allowed.filter(m => !RECEIPT_BUCKET_MIME_TYPES.includes(m)); + if (missing.length) problems.push(`allowed_mime_types is missing ${missing.join(", ")}`); + if (extra.length) problems.push(`allowed_mime_types carries unexpected ${extra.join(", ")}`); + } + if (problems.length) { + throw new Error(`bucket ${RECEIPT_BUCKET} exists with the wrong policy:\n - ${problems.join("\n - ")}`); + } + return "verified"; +} + +async function applyBucket() { + const baseUrl = process.env.SUPABASE_URL; + const key = process.env.SUPABASE_SERVICE_KEY; + if (!baseUrl || !key) { + console.error("REFUSING: SUPABASE_URL and SUPABASE_SERVICE_KEY are required to provision the receipts bucket."); + console.error(" The bucket carries the 15 MiB and MIME limits that the signed-upload path cannot enforce anywhere else."); + process.exit(1); + } + const outcome = await ensureReceiptBucket(baseUrl, key); + console.log(`bucket ${RECEIPT_BUCKET}: ${outcome} (private, ${RECEIPT_BUCKET_FILE_SIZE_LIMIT} bytes, ${RECEIPT_BUCKET_MIME_TYPES.length} mime types)`); +} + async function main() { if (!process.argv.includes("--yes")) { console.error("Refusing to run without --yes (and --expect-db / --expect-host)."); @@ -397,6 +530,10 @@ async function main() { console.log(`verified partial index ${name}: ${row.def}`); } + // The bucket last: a failure here must not leave the table half-made, + // and the schema is useless without somewhere to put the bytes anyway. + await applyBucket(); + console.log("\nReceiptIntake migration applied and verified."); } finally { await prisma.$disconnect(); diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 02128bdd8..ae68e470c 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -5,9 +5,14 @@ import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; -import { downloadDocBytesResult, toSecureRef } from "@/lib/secure-storage"; import { downloadVerified, inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; -import { deleteObjectOrRecord, retryPendingCleanups, sealObject } from "@/lib/receipt-intake/storage-cleanup"; +import { + deleteObjectOrRecord, + rejectRowAndQueueCleanup, + retryPendingCleanups, + sealObject, + settleQueuedCleanup, +} from "@/lib/receipt-intake/storage-cleanup"; import { getFreshQBTokens } from "@/lib/quickbooks-payments"; import { createQBReceiptPurchase } from "@/lib/qbo-receipt-push"; import { createRouteDeadline, type RouteDeadline } from "@/lib/quickbooks"; @@ -26,13 +31,13 @@ import { CLAIM_LEASE_MINUTES, CLAIM_LOCK_KEY, RUN_HARD_BUDGET_MS, - SIGNED_UPLOAD_TTL_MS, STAGING_SWEEP_BATCH, STAGING_SWEEP_MINUTES, type ClaimResult, type CutoverRequest, isUniqueViolation, runIntakeWorker, + uploadLeaseActive, type ReadPatch, type WorkerDependencies, type WorkerRow, @@ -310,8 +315,8 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { const stale = await prisma.receiptIntake.findMany({ where: { state: "STAGING", createdAt: { lt: cutoff } }, select: { - id: true, storagePath: true, mimeType: true, - createdAt: true, expectedSha256: true, + id: true, storagePath: true, mimeType: true, stateReason: true, + createdAt: true, expectedSha256: true, uploadUrlExpiresAt: true, }, // Small on purpose: each row costs a storage round trip, and the // sweep runs BEFORE any receipt is processed. A big batch here @@ -322,10 +327,23 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { let published = 0; let parked = 0; let rejected = 0; + let leaseActive = 0; for (const row of stale) { // The sweep is inside the run's deadline, not outside it. if (shouldStop()) break; + // NOTHING DESTRUCTIVE WHILE THE UPLOAD LEASE IS LIVE. + // + // `uploadUrlExpiresAt` is the promise /start made to this + // client. Until it passes, whatever is (or is not) at the path + // is provisional: an empty path is an upload in flight, and a + // half-written or superseded object is one the client is about + // to replace. Publishing is still allowed — a complete, correct + // object is a complete, correct object — but parking it as + // file-missing, or DELETING it as unacceptable, would destroy a + // receipt whose own upload link is still working. + const leaseLive = uploadLeaseActive(row); + // THE SAME validator /finalize uses. Publishing on "the object // exists" alone would wave through a 40 MB video, an executable, // or a truncated upload that /finalize would have refused — and @@ -346,7 +364,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // turn a retry-in-progress into a review item, and the // correct bytes arriving a minute later would find the // row already gone from STAGING. - if (row.createdAt.getTime() > Date.now() - SIGNED_UPLOAD_TTL_MS) continue; + if (leaseLive) { leaseActive++; continue; } await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, data: { state: "NEEDS_REVIEW", stateReason: "sha-mismatch", nextRetryAt: null }, @@ -389,7 +407,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // link was still perfectly usable — a slow phone on a bad // connection came back to find its row already in the review // queue. Wait until the URL cannot possibly land any more. - if (row.createdAt.getTime() > Date.now() - SIGNED_UPLOAD_TTL_MS) continue; + if (leaseLive) { leaseActive++; continue; } await prisma.receiptIntake.updateMany({ where: { id: row.id, state: "STAGING" }, data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, @@ -397,14 +415,37 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { parked++; continue; } - // Rejected: the object exists and is not acceptable. Same - // outcome as /finalize — the row goes and so does the object. - await prisma.receiptIntake.deleteMany({ where: { id: row.id, state: "STAGING" } }); - await deleteObjectOrRecord(row.storagePath, check.reason); + // Rejected: the object exists and is not acceptable. + // + // Not while the lease is live: what is at the path may be a + // partial write the client is still finishing, and deleting the + // row destroys the only record of an inbound receipt. + if (leaseLive) { leaseActive++; continue; } + + // The SAME fenced transaction /finalize rejects with: the row + // and its cleanup record commit together, under the exact state + // and path we just inspected. The unfenced delete this replaces + // could destroy a row a concurrent /finalize had just published + // — and then delete the bytes that published row pointed at. + const dropped = await rejectRowAndQueueCleanup( + { + id: row.id, + state: "STAGING", + stateReason: row.stateReason, + storagePath: row.storagePath, + }, + check.reason, + ); + // FENCE LOST: somebody else owns this row now. Touch NOTHING — + // above all not the object, which the winner may be using. + if (!dropped.ok) continue; + await settleQueuedCleanup(dropped.eventId, row.storagePath); rejected++; } - if (published || parked || rejected) { - console.log("[cron/receipt-intake-worker] STAGING sweep", JSON.stringify({ published, parked, rejected })); + if (published || parked || rejected || leaseActive) { + console.log("[cron/receipt-intake-worker] STAGING sweep", JSON.stringify({ + published, parked, rejected, "upload-lease-active": leaseActive, + })); } return published + parked + rejected; }, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index bd77603de..1602e5968 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -6,7 +6,7 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; -import { SECURE_BUCKET, secureObjectExists } from "@/lib/secure-storage"; +import { receiptObjectSize, uploadReceiptObject } from "@/lib/receipt-intake/bucket"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; import { deleteObjectOrRecord, recordPendingCleanup } from "@/lib/receipt-intake/storage-cleanup"; @@ -337,10 +337,8 @@ export async function POST(req: Request) { // then a clean insert rather than a conflict against a half-written row. let uploadFailed: string | null = null; try { - const upload = await supabase.storage - .from(SECURE_BUCKET) - .upload(storagePath, parsed.bytes, { contentType: mimeType, upsert: false }); - if (upload.error) uploadFailed = upload.error.message; + const stored = await uploadReceiptObject(storagePath, parsed.bytes, mimeType); + if (!stored) uploadFailed = "upload-failed"; } catch (error) { uploadFailed = error instanceof Error ? `${error.name}: ${error.message}` : "upload-threw"; } @@ -399,21 +397,9 @@ export async function POST(req: Request) { * retry finds the STAGING row, confirms the object really is there, and * finishes the job. */ -/** Upload bytes to the private bucket. Returns false on any failure. */ -async function storeObject(storagePath: string, bytes: Buffer, mimeType: string): Promise { - const supabase = getSupabase(); - if (!supabase) return false; - try { - const { error } = await supabase.storage - .from(SECURE_BUCKET) - .upload(storagePath, bytes, { contentType: mimeType, upsert: true }); - if (error) console.error("[receipts/intake] heal upload failed", error.message); - return !error; - } catch (error) { - console.error("[receipts/intake] heal upload threw", error instanceof Error ? error.name : "error"); - return false; - } -} +/** Upload bytes to the receipts bucket. Returns false on any failure. */ +const storeObject = (storagePath: string, bytes: Buffer, mimeType: string) => + uploadReceiptObject(storagePath, bytes, mimeType, { upsert: true }); async function publishStagedRow(id: string, expectState = "STAGING"): Promise { try { @@ -520,7 +506,15 @@ async function respondToSourceRefConflict( // and the forwarder could delete its only copy of a receipt we did not // have. The state a row happens to be parked in says nothing about whether // its bytes exist. - if (!(await secureObjectExists(existing.storagePath))) { + // Metadata, not a download: this runs on every replay, and the object may + // be 15 MB. A TRANSIENT answer is not evidence of absence — healing on it + // would overwrite a document that is really there — so it is answered 503 + // and the forwarder retries with its copy intact. + const present = await receiptObjectSize(existing.storagePath); + if (!present.ok && present.kind === "transient") { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + if (!present.ok) { // The caller just handed us the bytes again, so the orphan is fixable: // store them and republish. This is the retry HEALING the row rather // than merely reporting on it. diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index a56ae3d8f..48633b65a 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -3,13 +3,13 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; -import { SECURE_BUCKET } from "@/lib/secure-storage"; -import { getSupabase } from "@/lib/supabase"; import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; +import { uploadLeaseExpiry } from "@/lib/receipt-intake/worker"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; import { finalizeDisposition, publishFence } from "@/lib/receipt-intake/stored-object"; +import { createReceiptUploadUrl } from "@/lib/receipt-intake/bucket"; import { deleteObjectOrRecord } from "@/lib/receipt-intake/storage-cleanup"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; @@ -156,6 +156,10 @@ export async function POST(req: Request) { // never trusted as the stored one — only checked against it. fileSha256: "", expectedSha256, + // The promise this response makes: until then the client's URL + // works, so nothing may declare the object missing or reject + // the row for what is at that path. + uploadUrlExpiresAt: uploadLeaseExpiry(), }, select: { id: true, sourceRef: true, state: true }, }); @@ -208,6 +212,7 @@ export async function POST(req: Request) { data: { storagePath: retryPath, expectedSha256, + uploadUrlExpiresAt: uploadLeaseExpiry(), // The stored hash is what /finalize verifies against. // Whatever was recorded describes bytes that are gone // or were never right. @@ -275,6 +280,13 @@ export async function POST(req: Request) { } const resumed = await signUpload(existing.storagePath); if (!resumed) return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + // A RESUMED url is a new lease. Without this the sweeper still + // judged the row by its original createdAt and could park (or + // reject) it while the URL just handed out was live. + await prisma.receiptIntake.updateMany({ + where: { id: existing.id, state: "STAGING" }, + data: { uploadUrlExpiresAt: uploadLeaseExpiry() }, + }); return NextResponse.json({ ok: true, resumed: true, id: existing.id, maxBytes: MAX_STORED_BYTES, ...resumed, }); @@ -301,25 +313,10 @@ export async function POST(req: Request) { }); } -async function signUpload(storagePath: string): Promise<{ uploadUrl: string; token: string; storagePath: string } | null> { - const supabase = getSupabase(); - if (!supabase) return null; - try { - // upsert: true — a resumed /start for the SAME row must be able to - // overwrite a partial or failed upload at the same path. Without it the - // second attempt fails on "already exists" and the row can never be - // finalized. The sha checks above are what stop this from overwriting a - // DIFFERENT document. - const { data, error } = await supabase.storage - .from(SECURE_BUCKET) - .createSignedUploadUrl(storagePath, { upsert: true }); - if (error || !data) { - console.error("[receipts/intake/start] sign failed", error?.message); - return null; - } - return { uploadUrl: data.signedUrl, token: data.token, storagePath }; - } catch (error) { - console.error("[receipts/intake/start] sign threw", error instanceof Error ? error.name : "error"); - return null; - } -} +/** + * The URL is `upsert: true` (see bucket.ts) so a resumed /start for the SAME + * row can replace its own partial upload — without that the second attempt + * fails "already exists" and the row can never be finalized. The sha checks + * above are what stop it from overwriting a DIFFERENT document. + */ +const signUpload = createReceiptUploadUrl; diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index e69a3a708..556c4d878 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -18,7 +18,7 @@ * (tests/receipt-intake-book.test.ts). No module mocking — CI is Node 20. */ import { matchCostCode } from "@/lib/project-match"; -import { toSecureRef } from "@/lib/secure-storage"; +import { receiptObjectRef } from "./bucket"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { QBTimeoutError, @@ -172,10 +172,16 @@ export interface BookDependencies { /** * CAS on {id, state: BOOKING, claimToken} that persists sendAttempted. * - * This is the LAST FENCE before QuickBooks. It returns false when the row - * has been re-claimed, and the booking then aborts having sent nothing — - * which is the point: a zombie worker resuming with a stale view must not + * This is the LAST FENCE before QuickBooks, and the ONLY state check that + * matters at this point: it returns false when the row has been re-claimed + * or has moved on, and the booking then aborts having sent nothing — which + * is the point, because a zombie worker resuming with a stale view must not * create a Purchase the live worker is about to create as well. + * + * Called from BOTH QBO-core hooks: immediately before the create, and when + * the idempotency query finds a Purchase already there. The second is not + * a send, but it is the same fact about the row (QuickBooks holds a + * Purchase for it), written under the same fence. */ markSendAttempted: (rowId: string, claimToken: string | null) => Promise; /** The company's configured zone — Expense.date is a business calendar day. */ @@ -594,7 +600,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro const driveFileId = driveFileIdOf(row); const receiptUrl = driveFileId ? `https://drive.google.com/file/d/${driveFileId}/view` - : toSecureRef(row.storagePath); + : receiptObjectRef(row.storagePath); const docRef = isCheck ? `Check #${(row.refNumber ?? "").replace(/^Check/, "") || "?"}${row.memo ? ` — "${row.memo}"` : ""}` : (row.refNumber && row.refNumber !== "NoInv" ? `Invoice ${row.refNumber}` : "Receipt"); diff --git a/src/lib/receipt-intake/bucket.ts b/src/lib/receipt-intake/bucket.ts new file mode 100644 index 000000000..0b29861d6 --- /dev/null +++ b/src/lib/receipt-intake/bucket.ts @@ -0,0 +1,208 @@ +/** + * The intake feature's OWN private bucket. + * + * Intake objects used to live in `secure-docs` alongside signed contracts, + * e-signatures and invoice PDFs. Three reasons that was wrong, and all of them + * are about blast radius rather than tidiness: + * + * 1. The size and MIME ceilings are set PER BUCKET in Supabase, and the + * two-step upload goes straight to a signed URL that never passes through + * this server — so the bucket is the only place a 400 MB write or an + * executable can actually be refused. `secure-docs` cannot carry a receipt + * policy without imposing it on every other document type. + * 2. A signed upload URL is a write capability. Issuing one against the bucket + * that also holds countersigned contracts means a path-handling bug in the + * intake code is a write into the contract store. + * 3. Cleanup deletes objects. The orphan sweep runs unattended against paths + * read out of an event log; it must not be able to reach anything but + * receipts. + * + * Everything intake does with storage goes through this module, so there is one + * place that names the bucket and one place to audit. + */ +import { getSupabase } from "@/lib/supabase"; +import { isNotFoundError, type DocBytesResult } from "@/lib/secure-storage"; +import { ACCEPTED_MIME_TYPES } from "./file-type"; +import { MAX_STORED_BYTES } from "./intake-core"; + +export const RECEIPT_BUCKET = "receipt-intake"; + +/** + * The bucket policy, exported so scripts/apply-receipt-intake.mjs and this code + * cannot disagree about what was provisioned. + */ +export const RECEIPT_BUCKET_POLICY = { + name: RECEIPT_BUCKET, + public: false, + fileSizeLimit: MAX_STORED_BYTES, + allowedMimeTypes: ACCEPTED_MIME_TYPES, +} as const; + +/** A human-readable reference for logs and QBO memos. Never dereferenced. */ +export function receiptObjectRef(storagePath: string): string { + return `${RECEIPT_BUCKET}:${storagePath}`; +} + +/** A path we are willing to touch: inside the bucket, no traversal, no absolutes. */ +function safePath(storagePath: string): string | null { + if (!storagePath || storagePath.startsWith("/") || storagePath.includes("..")) return null; + return storagePath; +} + +export type SizeResult = + | { ok: true; size: number } + | { ok: false; kind: "missing" | "transient"; message?: string }; + +/** + * Byte size from METADATA — never a download. + * + * `list` with a search returns the metadata row in one small request whatever + * the object weighs, which is the only way to refuse a 400 MB upload without + * first pulling it into this process. + * + * TAGGED, and an unknown size is TRANSIENT rather than "fine, carry on". The + * previous null-means-unknown contract meant a storage hiccup, a missing + * client, or an API without metadata all fell through to the download — which + * is precisely the thing this call exists to avoid, on precisely the objects we + * know least about. + */ +export async function receiptObjectSize(storagePath: string): Promise { + const path = safePath(storagePath); + if (!path) return { ok: false, kind: "missing" }; + const supabase = getSupabase(); + if (!supabase) return { ok: false, kind: "transient", message: "storage-not-configured" }; + const slash = path.lastIndexOf("/"); + const dir = slash > 0 ? path.slice(0, slash) : ""; + const name = slash > 0 ? path.slice(slash + 1) : path; + try { + const { data, error } = await supabase.storage + .from(RECEIPT_BUCKET) + .list(dir, { search: name, limit: 100 }); + if (error) { + return isNotFoundError(error as { message?: string; status?: number }) + ? { ok: false, kind: "missing" } + : { ok: false, kind: "transient", message: String(error.message ?? "list-failed").slice(0, 200) }; + } + const match = data?.find(entry => entry.name === name); + // An empty listing IS an answer: the object is not there. + if (!match) return { ok: false, kind: "missing" }; + const size = (match.metadata as { size?: unknown } | undefined)?.size; + return typeof size === "number" && Number.isFinite(size) + ? { ok: true, size } + // Present but sizeless: the one case where we genuinely do not know, + // and it must not become permission to download. + : { ok: false, kind: "transient", message: "size-unavailable" }; + } catch (error) { + return { + ok: false, + kind: "transient", + message: error instanceof Error ? `${error.name}: ${error.message}`.slice(0, 200) : "list-threw", + }; + } +} + +/** Tagged download, so a confirmed 404 and a storage blip cannot book the same. */ +export async function downloadReceiptObject(storagePath: string): Promise { + const path = safePath(storagePath); + if (!path) return { ok: false, kind: "not-found" }; + const supabase = getSupabase(); + if (!supabase) return { ok: false, kind: "transient", message: "storage-not-configured" }; + try { + const { data, error } = await supabase.storage.from(RECEIPT_BUCKET).download(path); + if (error) { + return isNotFoundError(error as { message?: string; status?: number }) + ? { ok: false, kind: "not-found" } + : { ok: false, kind: "transient", message: String(error.message ?? "download-failed").slice(0, 200) }; + } + if (!data) return { ok: false, kind: "not-found" }; + return { ok: true, bytes: Buffer.from(await data.arrayBuffer()) }; + } catch (error) { + return { + ok: false, + kind: "transient", + message: error instanceof Error ? `${error.name}: ${error.message}`.slice(0, 200) : "download-threw", + }; + } +} + +/** Write bytes we have already validated. Returns false on any storage fault. */ +export async function uploadReceiptObject( + storagePath: string, + bytes: Buffer, + contentType: string, + opts: { upsert?: boolean } = {}, +): Promise { + const path = safePath(storagePath); + if (!path) return false; + const supabase = getSupabase(); + if (!supabase) return false; + try { + const { error } = await supabase.storage + .from(RECEIPT_BUCKET) + .upload(path, bytes, { contentType, upsert: opts.upsert ?? false }); + if (error) { + console.error("[receipts/intake] upload failed", error.message); + return false; + } + return true; + } catch (error) { + console.error("[receipts/intake] upload threw", error instanceof Error ? error.name : "error"); + return false; + } +} + +/** Delete, and THROW on anything short of a confirmed removal. */ +export async function removeReceiptObject(storagePath: string): Promise { + const path = safePath(storagePath); + if (!path) throw new Error(`not a receipt object path: ${String(storagePath).slice(0, 80)}`); + const supabase = getSupabase(); + // Never a silent success: the cleanup queue would mark an orphan resolved on + // a misconfigured deployment and lose it permanently. + if (!supabase) throw new Error("receipt storage is not configured"); + const { error } = await supabase.storage.from(RECEIPT_BUCKET).remove([path]); + if (error) throw error; +} + +/** The signed URL a client PUTs its bytes to. Scoped to ONE path, by design. */ +export async function createReceiptUploadUrl( + storagePath: string, +): Promise<{ uploadUrl: string; token: string; storagePath: string } | null> { + const path = safePath(storagePath); + if (!path) return null; + const supabase = getSupabase(); + if (!supabase) return null; + try { + // upsert: a resumed /start for the SAME row must be able to replace a + // partial upload at the same path. + const { data, error } = await supabase.storage + .from(RECEIPT_BUCKET) + .createSignedUploadUrl(path, { upsert: true }); + if (error || !data) { + console.error("[receipts/intake] sign failed", error?.message); + return null; + } + return { uploadUrl: data.signedUrl, token: data.token, storagePath: path }; + } catch (error) { + console.error("[receipts/intake] sign threw", error instanceof Error ? error.name : "error"); + return null; + } +} + +/** A time-limited read URL, for the archive mirror. */ +export async function signReceiptDownloadUrl( + storagePath: string, + ttlSeconds: number, +): Promise { + const path = safePath(storagePath); + if (!path) return null; + const supabase = getSupabase(); + if (!supabase) return null; + try { + const { data, error } = await supabase.storage + .from(RECEIPT_BUCKET) + .createSignedUrl(path, ttlSeconds); + return error || !data ? null : data.signedUrl; + } catch { + return null; + } +} diff --git a/src/lib/receipt-intake/intake-core.ts b/src/lib/receipt-intake/intake-core.ts index 472164509..7b733cfc8 100644 --- a/src/lib/receipt-intake/intake-core.ts +++ b/src/lib/receipt-intake/intake-core.ts @@ -63,9 +63,12 @@ export function decideSource( if (!auth.allowedSources.has(source)) return { ok: false, reason: "invalid-source" }; if (!MACHINE_SOURCES.has(source)) return { ok: false, reason: "invalid-source" }; if (!body.sourceRef) return { ok: false, reason: "missing-sourceRef" }; - if (!body.sourceRef.startsWith(`${source}:`)) { - return { ok: false, reason: "sourceRef-namespace-mismatch" }; - } + // SHAPE, not just namespace. `drive:` with an empty tail used to be a + // valid permanent idempotency key that every later empty-tail forward + // collided with — and for Drive the tail is also the QuickBooks + // DocNumber seed. + const shape = validateSourceRef(source, body.sourceRef); + if (!shape.ok) return { ok: false, reason: shape.reason }; return { ok: true, source, sourceRef: body.sourceRef }; } @@ -84,3 +87,54 @@ export function decideSource( } return { ok: true, source, sourceRef: `${source}:${randomUUID()}` }; } + +/** + * The longest sourceRef we will store. Long enough for any real Chat resource + * name, short enough that it cannot be used as a payload: it lands in a UNIQUE + * index, in QuickBooks-facing identity (Drive rows book under this id) and in + * every log line about the row. + */ +export const MAX_SOURCE_REF_BYTES = 512; + +/** + * Per-source shape for the part AFTER `:`. + * + * The namespace prefix alone was the only check, so `drive:` with nothing after + * it was a valid, unique, permanent idempotency key — and every subsequent + * empty-tail forward collided with it and was answered "already received", + * silently dropping real receipts. Worse for Drive specifically: the tail IS + * the QuickBooks DocNumber seed, so a junk tail becomes a junk DocNumber and + * two junk tails sharing a 21-character prefix collide in the books. + * + * Each pattern describes the id the forwarder actually holds: + * drive — a Drive file id. + * email — `/`: one message can carry several + * receipts, and each attachment is its own document. + * chat — a Chat message resource name, optionally naming the attachment. + */ +export const SOURCE_REF_PATTERNS: Record = { + drive: /^[A-Za-z0-9_-]{10,128}$/, + email: /^[A-Za-z0-9_.+=~-]{1,256}\/\d{1,4}$/, + chat: /^spaces\/[A-Za-z0-9_-]{1,128}\/messages\/[A-Za-z0-9_.=-]{1,256}(?:\/attachments\/[A-Za-z0-9_.=-]{1,256})?$/, +}; + +export type SourceRefCheck = { ok: true } | { ok: false; reason: string }; + +/** Shared by the single-shot POST and /start — one shape rule, checked once. */ +export function validateSourceRef(source: string, sourceRef: string): SourceRefCheck { + if (Buffer.byteLength(sourceRef, "utf8") > MAX_SOURCE_REF_BYTES) { + return { ok: false, reason: "sourceRef-too-long" }; + } + const prefix = `${source}:`; + if (!sourceRef.startsWith(prefix)) return { ok: false, reason: "sourceRef-namespace-mismatch" }; + const tail = sourceRef.slice(prefix.length); + if (!tail) return { ok: false, reason: "invalid-sourceRef" }; + // No control characters or whitespace anywhere, whatever the source: this + // value is echoed into logs and compared for equality. + if (/[\u0000-\u001f\u007f\s]/.test(sourceRef)) return { ok: false, reason: "invalid-sourceRef" }; + const pattern = SOURCE_REF_PATTERNS[source]; + // An unknown source never reaches here (decideSource checks the list first), + // and if one ever did, "no pattern" must not mean "anything goes". + if (!pattern) return { ok: false, reason: "invalid-source" }; + return pattern.test(tail) ? { ok: true } : { ok: false, reason: "invalid-sourceRef" }; +} diff --git a/src/lib/receipt-intake/queries.ts b/src/lib/receipt-intake/queries.ts index cc67b1a0e..3390842f0 100644 --- a/src/lib/receipt-intake/queries.ts +++ b/src/lib/receipt-intake/queries.ts @@ -7,7 +7,7 @@ * outside the worker should read from it. */ import { prisma } from "@/lib/prisma"; -import { resolveDocUrl, toSecureRef } from "@/lib/secure-storage"; +import { signReceiptDownloadUrl } from "./bucket"; export const RECEIPT_INTAKE_LIST_SELECT = { id: true, @@ -138,7 +138,7 @@ export async function listReceiptIntakes(args: ListReceiptIntakesArgs) { export async function withArchiveDownloadUrls( rows: T[], /** Injectable so the contract is testable without Supabase. */ - sign: (ref: string, ttlSeconds: number) => Promise = resolveDocUrl, + sign: (storagePath: string, ttlSeconds: number) => Promise = signReceiptDownloadUrl, ): Promise & { projectName: string | null; downloadUrl: string | null }>> { return Promise.all( rows.map(async row => { @@ -146,7 +146,7 @@ export async function withArchiveDownloadUrls), projectName: project?.name ?? null, - downloadUrl: await sign(toSecureRef(row.storagePath), ARCHIVE_SIGNED_URL_TTL_SECONDS), + downloadUrl: await sign(row.storagePath, ARCHIVE_SIGNED_URL_TTL_SECONDS), }; }), ); diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index f668b31de..cc8395d93 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -14,8 +14,7 @@ */ import { logAutomationEvent } from "@/lib/automation-events"; import { prisma } from "@/lib/prisma"; -import { SECURE_BUCKET, removeSecureDocStrict, toSecureRef } from "@/lib/secure-storage"; -import { getSupabase } from "@/lib/supabase"; +import { removeReceiptObject, uploadReceiptObject } from "./bucket"; export const STORAGE_CLEANUP_KIND = "storage-cleanup-pending"; @@ -37,22 +36,10 @@ export async function sealObject( bytes: Buffer, contentType: string, ): Promise { - const supabase = getSupabase(); - if (!supabase) return null; - try { - // upsert: the canonical path is content-addressed, so a re-seal of the - // SAME bytes is a no-op by construction and must not fail. - const { error } = await supabase.storage - .from(SECURE_BUCKET) - .upload(canonicalPath, bytes, { contentType, upsert: true }); - if (error) { - console.error("[receipts/intake] seal failed", error.message); - return null; - } - } catch (error) { - console.error("[receipts/intake] seal threw", error instanceof Error ? error.name : "error"); - return null; - } + // upsert: the canonical path is content-addressed, so a re-seal of the + // SAME bytes is a no-op by construction and must not fail. + const copied = await uploadReceiptObject(canonicalPath, bytes, contentType, { upsert: true }); + if (!copied) return null; // NOTE: the upload object is deliberately NOT deleted here. // @@ -201,7 +188,7 @@ class RejectFenceLost extends Error { */ export async function settleQueuedCleanup(eventId: string, storagePath: string): Promise { try { - await removeSecureDocStrict(toSecureRef(storagePath)); + await removeReceiptObject(storagePath); } catch (error) { console.error( "[receipts/intake] queued delete failed, left pending", @@ -224,7 +211,7 @@ export async function settleQueuedCleanup(eventId: string, storagePath: string): */ export async function deleteObjectOrRecord(storagePath: string, reason: string): Promise { try { - await removeSecureDocStrict(toSecureRef(storagePath)); + await removeReceiptObject(storagePath); return true; } catch (error) { console.error("[receipts/intake] object delete failed", storagePath, error instanceof Error ? error.name : "error"); @@ -288,12 +275,12 @@ export async function retryPendingCleanups(limit: number, shouldStop: () => bool } try { - await removeSecureDocStrict(toSecureRef(storagePath)); + await removeReceiptObject(storagePath); } catch { // Still failing. Leave it pending for the next pass. continue; } - // Resolved ONLY after a delete that did not throw — removeSecureDoc + // Resolved ONLY after a delete that did not throw — removeReceiptObject // surfaces a missing storage client as an error rather than a success, // so a misconfigured deployment cannot quietly mark the queue clean. await prisma.automationEvent.update({ where: { id: event.id }, data: { status: "resolved" } }); diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index eb7aea520..21510e6a5 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -13,12 +13,8 @@ * straight to Supabase, so nothing it declared about the file is evidence. */ import { createHash } from "node:crypto"; -import { - downloadDocBytesResult, - secureObjectSize, - toSecureRef, - type DocBytesResult, -} from "@/lib/secure-storage"; +import type { DocBytesResult } from "@/lib/secure-storage"; +import { downloadReceiptObject, receiptObjectSize, type SizeResult } from "./bucket"; import { EXT_BY_MIME, sniffMime } from "./file-type"; import { MAX_STORED_BYTES } from "./intake-core"; @@ -102,9 +98,9 @@ export async function sealAndPublish( export async function downloadVerified( storagePath: string, expectedSha256: string, - download: (ref: string) => Promise = downloadDocBytesResult, + download: (storagePath: string) => Promise = downloadReceiptObject, ): Promise { - const result = await download(toSecureRef(storagePath)); + const result = await download(storagePath); if (!result.ok) { return result.kind === "not-found" ? { ok: false, kind: "missing" } @@ -144,9 +140,9 @@ export async function inspectStoredObject( * format that CAN be identified is identified from the bytes. */ declaredMime: string, - download: (ref: string) => Promise = downloadDocBytesResult, + download: (storagePath: string) => Promise = downloadReceiptObject, /** Metadata-only size lookup; injected so the "no body read" test is provable. */ - sizeOf: (storagePath: string) => Promise = secureObjectSize, + sizeOf: (storagePath: string) => Promise = receiptObjectSize, ): Promise { // SIZE FIRST, FROM METADATA — before a single byte is read. // @@ -155,14 +151,23 @@ export async function inspectStoredObject( // takes the worker's whole invocation (and its memory) with it. `list` // returns the metadata row in one small request whatever the object's size. // - // A null size is "unknown", not "fine": the download below still enforces - // the limit on the bytes it actually got. - const declaredSize = await sizeOf(storagePath); - if (declaredSize !== null && declaredSize > MAX_STORED_BYTES) { - return { ok: false, kind: "rejected", reason: `file-too-large:${declaredSize}` }; + // AN UNKNOWN SIZE IS TRANSIENT, not permission to proceed. It used to mean + // "carry on and let the byte-length check catch it" — which is the download + // this call exists to avoid, taken on exactly the objects we know least + // about (a storage hiccup, a missing client, an API with no metadata). The + // sweep and the client both retry a transient answer; neither can be hurt + // by waiting, and both can be hurt by a 400 MB read. + const declared = await sizeOf(storagePath); + if (!declared.ok) { + return declared.kind === "missing" + ? { ok: false, kind: "missing" } + : { ok: false, kind: "transient", message: declared.message ?? "size-unavailable" }; + } + if (declared.size > MAX_STORED_BYTES) { + return { ok: false, kind: "rejected", reason: `file-too-large:${declared.size}` }; } - const result = await download(toSecureRef(storagePath)); + const result = await download(storagePath); if (!result.ok) { return result.kind === "not-found" ? { ok: false, kind: "missing" } diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 63f0cce39..660722f9e 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -74,6 +74,31 @@ export const STAGING_SWEEP_BATCH = 10; * own upload link was still usable. */ export const SIGNED_UPLOAD_TTL_MS = 2 * 60 * 60_000; + +/** When a URL issued now stops working. Written to the row by /intake/start. */ +export function uploadLeaseExpiry(now: Date = new Date()): Date { + return new Date(now.getTime() + SIGNED_UPLOAD_TTL_MS); +} + +/** + * Could the object still arrive under a live upload URL? + * + * ROW AGE IS THE WRONG QUESTION. A row whose URL was re-issued (a resumed + * /start, or a re-arm after the sweeper parked it) is older than its lease, and + * judging it on createdAt declared a receipt missing — or destroyed one it + * called unacceptable — while the client's own upload link was still live and + * about to land. `uploadUrlExpiresAt` is what /start actually promised. + * + * Null means no signed URL was ever issued for this row (the single-shot path + * writes its bytes through the server), so the row's own age is all there is. + */ +export function uploadLeaseActive( + row: { uploadUrlExpiresAt?: Date | null; createdAt: Date }, + now: Date = new Date(), +): boolean { + if (row.uploadUrlExpiresAt) return row.uploadUrlExpiresAt.getTime() > now.getTime(); + return row.createdAt.getTime() > now.getTime() - SIGNED_UPLOAD_TTL_MS; +} /** * Consecutive AI-unavailable passes before a row is parked for a human. Ported * from v3.4: an outage that never ends still has to end somewhere, and 20 diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts index 82b06c8fd..23c09d113 100644 --- a/tests/apply-receipt-intake.test.ts +++ b/tests/apply-receipt-intake.test.ts @@ -15,7 +15,16 @@ import assert from "node:assert/strict"; import test from "node:test"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { RECEIPT_INTAKE_STATES, statements, targetMatches } from "../scripts/apply-receipt-intake.mjs"; +import { + RECEIPT_BUCKET, + RECEIPT_BUCKET_FILE_SIZE_LIMIT, + RECEIPT_BUCKET_MIME_TYPES, + RECEIPT_INTAKE_STATES, + ensureReceiptBucket, + parseSizeLimit, + statements, + targetMatches, +} from "../scripts/apply-receipt-intake.mjs"; import { RECEIPT_INTAKE_STATES as RUNTIME_STATES } from "../src/lib/receipt-intake/route-state"; const migrationSql = readFileSync( @@ -248,3 +257,98 @@ test("verification asserts the CHECK ALLOWS every state, not just that it exists assert.match(source, /pg_get_constraintdef\(oid\) AS def FROM pg_constraint/, "verify reads the definition"); assert.match(source, /does not allow/, "and fails loudly naming what is missing"); }); + +// ── The receipts bucket is provisioned, not assumed (round-13 item 3) ─────── + +test("the bucket policy in the script matches the one the code writes through", async () => { + // Two places name the same limits: the provisioner and the runtime module. + // If they drift, the runtime happily writes objects the bucket refuses (or, + // worse, accepts objects the runtime thinks are impossible). + const { RECEIPT_BUCKET_POLICY } = await import("../src/lib/receipt-intake/bucket"); + assert.equal(RECEIPT_BUCKET, RECEIPT_BUCKET_POLICY.name); + assert.equal(RECEIPT_BUCKET_FILE_SIZE_LIMIT, RECEIPT_BUCKET_POLICY.fileSizeLimit); + assert.deepEqual( + [...RECEIPT_BUCKET_MIME_TYPES].sort(), + [...RECEIPT_BUCKET_POLICY.allowedMimeTypes].sort(), + "the accepted formats and the bucket's allow-list are the same list", + ); + assert.equal(RECEIPT_BUCKET_POLICY.public, false); + assert.equal(RECEIPT_BUCKET_FILE_SIZE_LIMIT, 15 * 1024 * 1024); +}); + +test("a missing bucket is CREATED private, with both limits", async () => { + const calls: Array<{ path: string; method: string; body: any }> = []; + const outcome = await ensureReceiptBucket("https://x.supabase.co", "key", async (_u: string, _k: string, path: string, init: any = {}) => { + calls.push({ path, method: init.method, body: init.body ? JSON.parse(init.body) : null }); + if (init.method === "GET") return { status: 404, ok: false, body: { error: "not found" } }; + return { status: 200, ok: true, body: { name: RECEIPT_BUCKET } }; + }); + assert.equal(outcome, "created"); + assert.equal(calls[0].method, "GET", "it looks before it creates"); + assert.equal(calls[1].body.public, false); + assert.equal(calls[1].body.file_size_limit, RECEIPT_BUCKET_FILE_SIZE_LIMIT); + assert.deepEqual(calls[1].body.allowed_mime_types, RECEIPT_BUCKET_MIME_TYPES); +}); + +test("an existing bucket with the right policy is VERIFIED, and nothing is written", async () => { + let writes = 0; + const outcome = await ensureReceiptBucket("https://x.supabase.co", "key", async (_u: string, _k: string, _p: string, init: any = {}) => { + if (init.method !== "GET") { writes++; return { status: 200, ok: true, body: {} }; } + return { + status: 200, ok: true, + body: { + name: RECEIPT_BUCKET, public: false, + file_size_limit: RECEIPT_BUCKET_FILE_SIZE_LIMIT, + allowed_mime_types: RECEIPT_BUCKET_MIME_TYPES, + }, + }; + }); + assert.equal(outcome, "verified"); + assert.equal(writes, 0, "re-running provisions nothing"); +}); + +test("a DIFFERENT limit is a hard failure, never a silent correction", async () => { + // Overwriting a limit somebody set deliberately is how a 400 MB upload + // becomes possible again next quarter. The operator has to see it. + const cases: Array<[Record, RegExp]> = [ + [{ file_size_limit: 50 * 1024 * 1024 }, /file_size_limit/], + [{ file_size_limit: "50MB" }, /file_size_limit/], + [{ public: true }, /PUBLIC/], + [{ allowed_mime_types: null }, /allowed_mime_types is unset/], + [{ allowed_mime_types: ["image/png"] }, /missing/], + [{ allowed_mime_types: [...RECEIPT_BUCKET_MIME_TYPES, "application/zip"] }, /unexpected/], + ]; + for (const [override, expected] of cases) { + const body = { + name: RECEIPT_BUCKET, public: false, + file_size_limit: RECEIPT_BUCKET_FILE_SIZE_LIMIT, + allowed_mime_types: RECEIPT_BUCKET_MIME_TYPES, + ...override, + }; + await assert.rejects( + () => ensureReceiptBucket("https://x.supabase.co", "key", async () => ({ status: 200, ok: true, body })), + expected, + JSON.stringify(override), + ); + } +}); + +test("Supabase's file_size_limit is read in either shape", () => { + // It comes back as a byte count from some API versions and as "15MB" from + // others; reading only one of those would fail a correct bucket. + assert.equal(parseSizeLimit(15728640), 15728640); + assert.equal(parseSizeLimit("15728640"), 15728640); + assert.equal(parseSizeLimit("15MB"), 15728640); + assert.equal(parseSizeLimit("15 mb"), 15728640); + assert.equal(parseSizeLimit(null), null); + assert.equal(parseSizeLimit("enormous"), null); +}); + +test("a storage read failure stops the run rather than assuming the bucket is fine", async () => { + await assert.rejects( + () => ensureReceiptBucket("https://x.supabase.co", "key", async () => ({ + status: 500, ok: false, body: { error: "boom" }, + })), + /could not read bucket/, + ); +}); diff --git a/tests/receipt-intake-archive-contract.test.ts b/tests/receipt-intake-archive-contract.test.ts index 62f0dd6ab..68d76abbe 100644 --- a/tests/receipt-intake-archive-contract.test.ts +++ b/tests/receipt-intake-archive-contract.test.ts @@ -60,7 +60,7 @@ test("each row gets a short-lived signed URL and a flat project name", async () ); assert.equal(rows[0].projectName, "Berg ADU"); - assert.equal(rows[0].downloadUrl, "https://signed.test/secure:receipts/intake/a.jpg"); + assert.equal(rows[0].downloadUrl, "https://signed.test/receipts/intake/a.jpg"); assert.equal(rows[1].projectName, null, "a project-less row is still archivable"); // The nested relation is flattened away — the script gets `projectName`. assert.ok(!("project" in rows[0])); @@ -69,7 +69,9 @@ test("each row gets a short-lived signed URL and a flat project name", async () // and a URL captured from a log is useless by morning. assert.equal(ARCHIVE_SIGNED_URL_TTL_SECONDS, 600); assert.deepEqual(signed.map(s => s.ttl), [600, 600]); - assert.deepEqual(signed.map(s => s.ref), ["secure:receipts/intake/a.jpg", "secure:receipts/intake/b.pdf"]); + // The receipts bucket is named by the signer, so what it is handed is the + // PATH — an object in another bucket cannot be reached from here at all. + assert.deepEqual(signed.map(s => s.ref), ["receipts/intake/a.jpg", "receipts/intake/b.pdf"]); }); test("a row whose URL cannot be signed is returned with null, never dropped", async () => { diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 103f2c1e5..95b8067b7 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -235,7 +235,9 @@ test("provenance rules are shared by BOTH upload paths", async () => { { ok: false, reason: "sourceRef-namespace-mismatch" }); assert.deepEqual(decideSource(secret, { source: "web", sourceRef: "web:x" }), { ok: false, reason: "invalid-source" }); - assert.ok(decideSource(secret, { source: "drive", sourceRef: "drive:FILE1" }).ok); + // A REAL Drive file id shape. The old rule accepted any tail at all, + // including none. + assert.ok(decideSource(secret, { source: "drive", sourceRef: "drive:1AbCdEfGhIjKlMnOp_qR" }).ok); // The inline body cap is well under the stored cap, which is the whole // reason the two-step path exists. @@ -352,10 +354,10 @@ test("a secret may only declare the sources ITS key owns", async () => { capability: "archive", allowedSources: new Set(), } as any; - assert.ok(decideSource(ingest, { source: "drive", sourceRef: "drive:F1" }).ok); + assert.ok(decideSource(ingest, { source: "drive", sourceRef: "drive:1AbCdEfGhIjKlMnOp_qR" }).ok); // The archive key owns no sources, so it can never mint an intake row even // if it somehow reached this code. - assert.deepEqual(decideSource(archive, { source: "drive", sourceRef: "drive:F1" }), + assert.deepEqual(decideSource(archive, { source: "drive", sourceRef: "drive:1AbCdEfGhIjKlMnOp_qR" }), { ok: false, reason: "invalid-source" }); }); @@ -468,3 +470,68 @@ test("machine endpoints refuse a Server Action dispatch; portal actions still wo env.NODE_ENV = prod; } }); + +// ── sourceRef shape, per source (round-13 item 4) ────────────────────────── + +test("a sourceRef must carry a real id for its source, not just the prefix", async () => { + const { validateSourceRef, decideSource, MAX_SOURCE_REF_BYTES } = + await import("../src/lib/receipt-intake/intake-core"); + + // THE REGRESSION: `drive:` with an empty tail was a valid, unique, + // PERMANENT idempotency key. Every later empty-tail forward collided with + // it and was answered "already received", so real receipts were dropped — + // and for Drive the tail is also the QuickBooks DocNumber seed. + for (const source of ["drive", "email", "chat"]) { + assert.deepEqual( + validateSourceRef(source, `${source}:`), + { ok: false, reason: "invalid-sourceRef" }, + source, + ); + } + + // Oversized: this value lands in a unique index, in logs, and in + // QuickBooks-facing identity. + const long = `drive:${"a".repeat(MAX_SOURCE_REF_BYTES)}`; + assert.deepEqual(validateSourceRef("drive", long), { ok: false, reason: "sourceRef-too-long" }); + assert.equal(MAX_SOURCE_REF_BYTES, 512); + + // Shape, per source. + assert.deepEqual(validateSourceRef("drive", "drive:1AbCdEfGhIjKlMnOp_qR"), { ok: true }); + assert.deepEqual(validateSourceRef("drive", "drive:short"), { ok: false, reason: "invalid-sourceRef" }); + assert.deepEqual(validateSourceRef("drive", "drive:has spaces here"), { ok: false, reason: "invalid-sourceRef" }); + assert.deepEqual(validateSourceRef("email", "email:CADnq=abc123def/0"), { ok: true }); + assert.deepEqual( + validateSourceRef("email", "email:CADnq=abc123def"), + { ok: false, reason: "invalid-sourceRef" }, + "one message can carry several receipts; the attachment index is part of the identity", + ); + assert.deepEqual( + validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def"), + { ok: true }, + ); + assert.deepEqual( + validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def/attachments/ATT.1"), + { ok: true }, + ); + assert.deepEqual( + validateSourceRef("chat", "chat:AAQANF47osY"), + { ok: false, reason: "invalid-sourceRef" }, + "a bare space id is not a message", + ); + + // A control character is never part of an id, whatever the source. + assert.deepEqual( + validateSourceRef("drive", "drive:1AbCdEfGhIjKlMnOp\u0000qR"), + { ok: false, reason: "invalid-sourceRef" }, + ); + + // And BOTH entry points get it, because decideSource is where it is applied. + const secret = { + ok: true, via: "secret", user: null, userVia: null, + capability: "ingest", allowedSources: new Set(["drive", "email", "chat"]), + } as any; + assert.deepEqual(decideSource(secret, { source: "drive", sourceRef: "drive:" }), + { ok: false, reason: "invalid-sourceRef" }); + assert.deepEqual(decideSource(secret, { source: "drive", sourceRef: `drive:${"a".repeat(600)}` }), + { ok: false, reason: "sourceRef-too-long" }); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 008deb744..98071fb0a 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -245,7 +245,7 @@ test("a non-drive row books under its intake id and stores the secure ref", asyn const r = recorder(); await bookReceipt(row({ source: "mobile", sourceRef: "mobile:abc", id: "intake-9" }), r.deps); assert.equal(r.purchaseCalls[0].fileId, "intake-9"); - assert.equal(r.expenses[0].receiptUrl, "secure:receipts/intake/intake-1.jpg"); + assert.equal(r.expenses[0].receiptUrl, "receipt-intake:receipts/intake/intake-1.jpg"); }); test("a project with no estimate is terminal, spends NO attempt, and RELEASES the strong key", async () => { diff --git a/tests/receipt-intake-cleanup.test.ts b/tests/receipt-intake-cleanup.test.ts index fa1bdc920..c7778cc56 100644 --- a/tests/receipt-intake-cleanup.test.ts +++ b/tests/receipt-intake-cleanup.test.ts @@ -14,7 +14,7 @@ import path from "node:path"; const ROOT = path.resolve(__dirname, ".."); const cleanup = readFileSync(path.join(ROOT, "src/lib/receipt-intake/storage-cleanup.ts"), "utf8"); const intake = readFileSync(path.join(ROOT, "src/app/api/receipts/intake/route.ts"), "utf8"); -const storage = readFileSync(path.join(ROOT, "src/lib/secure-storage.ts"), "utf8"); +const bucket = readFileSync(path.join(ROOT, "src/lib/receipt-intake/bucket.ts"), "utf8"); /** * The body of one top-level function, EOL-agnostic. @@ -79,7 +79,7 @@ test("the cleanup worker refuses to delete a path a LIVE row still points at", ( assert.match(fn, /still referenced by/, "and resolves rather than retrying forever"); // The reference check must come BEFORE the delete. assert.ok( - fn.indexOf("still referenced by") < fn.indexOf("removeSecureDocStrict"), + fn.indexOf("still referenced by") < fn.indexOf("removeReceiptObject"), "the check precedes the deletion", ); }); @@ -90,18 +90,20 @@ test("an event is resolved only AFTER a confirmed deletion", () => { // through to the resolve. assert.match(fn, /\} catch \{[\s\S]*?continue;/, "a failed delete leaves the event pending"); assert.ok( - fn.lastIndexOf("removeSecureDocStrict") < fn.lastIndexOf('status: "resolved" }'), + fn.lastIndexOf("removeReceiptObject") < fn.lastIndexOf('status: "resolved" }'), "resolve happens after the delete", ); }); test("a missing storage client is an ERROR for the cleanup path", () => { - // removeSecureDoc returns quietly with no client — right for its - // best-effort callers, catastrophic here: it would mark orphans resolved on - // a misconfigured deployment and lose them permanently. - assert.match(storage, /export async function removeSecureDocStrict/); - const strict = bodyOf(storage, "export async function removeSecureDocStrict"); - assert.match(strict, /throw new Error\("secure storage is not configured"\)/); - // ...and the cleanup queue uses the strict one, never the quiet one. - assert.ok(!/\bremoveSecureDoc\(/.test(cleanup), "cleanup never uses the quiet variant"); + // A deleter that returns quietly with no client is right for best-effort + // callers and catastrophic here: it would mark orphans resolved on a + // misconfigured deployment and lose them permanently. + assert.match(bucket, /export async function removeReceiptObject/); + const strict = bodyOf(bucket, "export async function removeReceiptObject"); + assert.match(strict, /throw new Error\("receipt storage is not configured"\)/); + // ...and cleanup never reaches for a best-effort variant, or for any bucket + // but the receipts one. + assert.ok(!/removeSecureDoc/.test(cleanup), "cleanup never uses the quiet variant"); + assert.ok(!/SECURE_BUCKET/.test(cleanup), "and never touches the shared document bucket"); }); diff --git a/tests/receipt-intake-reject.test.ts b/tests/receipt-intake-reject.test.ts index 342eb2b58..15a427f98 100644 --- a/tests/receipt-intake-reject.test.ts +++ b/tests/receipt-intake-reject.test.ts @@ -233,3 +233,67 @@ test("a re-arm that changes the extension does not orphan the old object", () => assert.match(body, /retryPath !== existing\.storagePath/); assert.match(body, /deleteObjectOrRecord\(existing\.storagePath, "start-rearmed-repath"\)/); }); + +// ── The sweeper rejects through the same fenced transaction ──────────────── + +const sweeper = readFileSync( + path.join(ROOT, "src/app/api/cron/receipt-intake-worker/route.ts"), + "utf8", +); + +test("SWEEPER RACE: a publish that wins mid-sweep leaves the row and the bytes alone", async () => { + // The sweep reads a batch, then spends a storage round trip per row + // deciding what to do with it. A /finalize arriving in that window can + // publish the row and seal its object — and the old unfenced + // `deleteMany({ id, state: "STAGING" })` plus a bare object delete would + // then destroy a published receipt's row OR its bytes, depending on + // timing. The reject transaction is what makes that a no-op. + const { db, store } = client([parked()], s => { + s.rows = [parked({ state: "RECEIVED", storagePath: "receipts/row-1/sealed.png" })]; + }); + let deletedBytes = 0; + const dropped = await rejectRowAndQueueCleanup(parked() as never, "unsupported-file-type", db); + if (dropped.ok) deletedBytes++; // the caller only touches storage on success + assert.equal(dropped.ok, false, "the fence lost"); + assert.equal(deletedBytes, 0, "so no object was deleted"); + assert.deepEqual(store.events, [], "and nothing was queued for deletion"); + assert.equal(store.rows[0].state, "RECEIVED", "the published row is untouched"); +}); + +test("the sweeper uses the fenced reject, and touches no bytes when it loses", () => { + const fn = sweeper.slice(sweeper.indexOf("sweepStaleStaging: async")); + const body = fn.slice(0, fn.indexOf("loadPhases:")); + assert.match(body, /const dropped = await rejectRowAndQueueCleanup\(/); + assert.match(body, /if \(!dropped\.ok\) continue;/); + assert.match(body, /settleQueuedCleanup\(dropped\.eventId, row\.storagePath\)/); + // The unfenced pair this replaces. + assert.ok( + !/deleteMany\(\{ where: \{ id: row\.id, state: "STAGING" \} \}\)/.test(body), + "no delete-by-id-and-state", + ); + assert.ok( + body.indexOf("if (!dropped.ok) continue;") < body.indexOf("settleQueuedCleanup"), + "the object is only touched after the row is provably gone", + ); +}); + +test("nothing destructive happens while the upload lease is live", () => { + const fn = sweeper.slice(sweeper.indexOf("sweepStaleStaging: async")); + const body = fn.slice(0, fn.indexOf("loadPhases:")); + // Three destructive outcomes, three lease checks: sha-mismatch park, + // file-missing park, and the reject. + assert.equal( + (body.match(/if \(leaseLive\) \{ leaseActive\+\+; continue; \}/g) ?? []).length, + 3, + "every destructive branch waits for the lease", + ); + // Publishing is NOT gated on it: a complete, correct object is a complete, + // correct object whether or not the URL is still live. + const publishBranch = body.slice(body.indexOf("if (check.ok) {"), body.indexOf("if (check.kind === \"transient\")")); + assert.ok(publishBranch.includes("sealAndPublish")); + assert.equal( + (publishBranch.match(/if \(leaseLive\)/g) ?? []).length, + 1, + "only the sha-mismatch park inside the ok branch waits", + ); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 05f3c254b..c1fd9f5c2 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -27,9 +27,23 @@ const PNG = Buffer.from( "base64", ); const give = (r: DocBytesResult) => async () => r; +/** A metadata size that says "small, and definitely there". */ +const sized = (size: number) => async () => ({ ok: true as const, size }); +const SMALL = sized(1); +/** + * inspectStoredObject reads the size from METADATA before it reads a body, and + * its default lookup talks to Supabase. Every case here supplies both stubs, so + * a test can never accidentally exercise the real bucket. + */ +const inspect = ( + path: string, + mime: string, + download: Parameters[2], + size: Parameters[3] = SMALL, +) => inspectStoredObject(path, mime, download, size); test("a real image is accepted, and its metadata comes from the BYTES", async () => { - const check = await inspectStoredObject("p.jpg", "application/pdf", give({ ok: true, bytes: PNG })); + const check = await inspect("p.jpg", "application/pdf", give({ ok: true, bytes: PNG })); assert.ok(check.ok); // The declared type said PDF. The bytes say PNG, and the bytes win. assert.equal(check.mimeType, "image/png"); @@ -40,16 +54,16 @@ test("a real image is accepted, and its metadata comes from the BYTES", async () test("oversize, empty and unidentifiable objects are REJECTED, not published", async () => { // The signed upload URL bypassed every check the server could otherwise // make, so these are enforced on the object itself. - const big = await inspectStoredObject("p.jpg", "image/jpeg", give({ + const big = await inspect("p.jpg", "image/jpeg", give({ ok: true, bytes: Buffer.alloc(MAX_STORED_BYTES + 1, 1), })); assert.equal(big.ok, false); assert.match((big as { reason: string }).reason, /^file-too-large:/); - const empty = await inspectStoredObject("p.jpg", "image/jpeg", give({ ok: true, bytes: Buffer.alloc(0) })); + const empty = await inspect("p.jpg", "image/jpeg", give({ ok: true, bytes: Buffer.alloc(0) })); assert.equal((empty as { reason: string }).reason, "empty-file"); - const exe = await inspectStoredObject("p.exe", "image/jpeg", give({ ok: true, bytes: Buffer.from("MZ\x90\x00") })); + const exe = await inspect("p.exe", "image/jpeg", give({ ok: true, bytes: Buffer.from("MZ\x90\x00") })); assert.equal((exe as { reason: string }).reason, "unsupported-file-type"); }); @@ -58,36 +72,56 @@ test("an oversize object is rejected from METADATA, with no body read at all", a // here sees the object is now. Downloading it to discover it is 400 MB is // how one upload takes the whole invocation — and its memory — with it. let downloads = 0; - const check = await inspectStoredObject( + const check = await inspect( "p.bin", "image/jpeg", async () => { downloads++; throw new Error("the body must never be fetched"); }, - async () => MAX_STORED_BYTES + 1, + sized(MAX_STORED_BYTES + 1), ); assert.equal(check.ok, false); assert.equal((check as { reason: string }).reason, `file-too-large:${MAX_STORED_BYTES + 1}`); assert.equal(downloads, 0, "not one byte was read"); }); -test("an UNKNOWN metadata size still downloads, and the bytes decide", async () => { - // A null size is "storage did not say", not "fine". The byte-length check - // below it is what actually enforces the limit. - const check = await inspectStoredObject( +test("an UNKNOWN metadata size is TRANSIENT, and still reads no body", async () => { + // "Storage did not say" used to mean "carry on and let the byte-length + // check catch it" — which is the download this call exists to avoid, taken + // on exactly the objects we know least about. Both callers retry a + // transient answer; neither is harmed by waiting, and both are harmed by a + // 400 MB read. + let downloads = 0; + const check = await inspect( "p.png", "image/png", - give({ ok: true, bytes: PNG }), - async () => null, + async () => { downloads++; throw new Error("the body must never be fetched"); }, + async () => ({ ok: false as const, kind: "transient" as const, message: "size-unavailable" }), ); - assert.ok(check.ok); + assert.equal(check.ok, false); + assert.equal((check as { kind: string }).kind, "transient"); + assert.equal(downloads, 0, "not one byte was read"); +}); + +test("a size lookup that says MISSING is missing, and reads no body either", async () => { + // An empty listing is an answer. Downloading to rediscover a 404 is a + // round trip that can only produce the same verdict. + let downloads = 0; + const check = await inspect( + "p.png", + "image/png", + async () => { downloads++; throw new Error("the body must never be fetched"); }, + async () => ({ ok: false as const, kind: "missing" as const }), + ); + assert.equal((check as { kind: string }).kind, "missing"); + assert.equal(downloads, 0); }); test("a metadata size AT the ceiling is not rejected before the download", async () => { let downloads = 0; - const check = await inspectStoredObject( + const check = await inspect( "p.png", "image/png", async () => { downloads++; return { ok: true as const, bytes: PNG }; }, - async () => MAX_STORED_BYTES, + sized(MAX_STORED_BYTES), ); assert.ok(check.ok); assert.equal(downloads, 1); @@ -96,17 +130,17 @@ test("a metadata size AT the ceiling is not rejected before the download", async test("exactly at the ceiling is allowed", async () => { const atLimit = Buffer.concat([PNG, Buffer.alloc(MAX_STORED_BYTES - PNG.length, 0)]); assert.equal(atLimit.length, MAX_STORED_BYTES); - const check = await inspectStoredObject("p.png", "image/png", give({ ok: true, bytes: atLimit })); + const check = await inspect("p.png", "image/png", give({ ok: true, bytes: atLimit })); assert.ok(check.ok, "the boundary itself is not oversize"); }); test("missing and transient are DIFFERENT answers", async () => { // A confirmed 404 is terminal for the sweep; a storage blip must come back // next pass rather than park a good receipt as file-missing. - const missing = await inspectStoredObject("p.jpg", "image/jpeg", give({ ok: false, kind: "not-found" })); + const missing = await inspect("p.jpg", "image/jpeg", give({ ok: false, kind: "not-found" })); assert.deepEqual(missing, { ok: false, kind: "missing" }); - const flaky = await inspectStoredObject("p.jpg", "image/jpeg", give({ + const flaky = await inspect("p.jpg", "image/jpeg", give({ ok: false, kind: "transient", message: "ECONNRESET", })); assert.deepEqual(flaky, { ok: false, kind: "transient", message: "ECONNRESET" }); @@ -163,7 +197,7 @@ test("a legacy row with no recorded sha is passed through, not refused", async ( test("the validator hands back the exact bytes it verified", async () => { // The sealer copies THESE bytes rather than re-downloading, so the sealed // object is provably the content that passed validation. - const check = await inspectStoredObject("p.png", "image/png", give({ ok: true, bytes: PNG })); + const check = await inspect("p.png", "image/png", give({ ok: true, bytes: PNG })); assert.ok(check.ok); assert.ok(check.bytes.equals(PNG)); assert.equal(createHash("sha256").update(check.bytes).digest("hex"), check.fileSha256); @@ -232,7 +266,7 @@ test("text/plain is no longer accepted at all", async () => { // QuickBooks cannot attach a .txt, so accepting one meant reading it and // then stranding it unbookable — worse than refusing at the door. const txt = Buffer.from("VENDOR: Lowes\nTOTAL: 10.00"); - const check = await inspectStoredObject("p.txt", "text/plain", give({ ok: true, bytes: txt })); + const check = await inspect("p.txt", "text/plain", give({ ok: true, bytes: txt })); assert.equal(check.ok, false); assert.equal((check as { reason: string }).reason, "unsupported-file-type"); }); @@ -250,13 +284,16 @@ test("the sweeper's two parks are the ones /finalize recovers from", () => { const sweeper = readFileSync( path.join(root, "src/app/api/cron/receipt-intake-worker/route.ts"), "utf8", ); - // Both the missing-object and the sha-mismatch branches wait for expiry. + // Both the missing-object and the sha-mismatch branches wait for the + // UPLOAD LEASE to expire — the promise /start actually made, not the row's + // age, which is older than the lease on any re-issued URL. const shaBranch = sweeper.slice(sweeper.indexOf('row.expectedSha256 !== check.fileSha256')); assert.match( shaBranch.slice(0, shaBranch.indexOf("parked++")), - /SIGNED_UPLOAD_TTL_MS/, - "a sha mismatch waits for the upload URL to expire", + /if \(leaseLive\) \{ leaseActive\+\+; continue; \}/, + "a sha mismatch waits for the upload lease to expire", ); + assert.match(sweeper, /const leaseLive = uploadLeaseActive\(row\);/); const finalize = readFileSync( path.join(root, "src/app/api/receipts/intake/[id]/finalize/route.ts"), "utf8", diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index 9020659fd..ddf2470be 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -10,6 +10,8 @@ */ import test from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { runIntakeWorker, dateOnly, @@ -23,6 +25,9 @@ import { type ReadPatch, type WorkerDependencies, type WorkerRow, + uploadLeaseActive, + uploadLeaseExpiry, + SIGNED_UPLOAD_TTL_MS, } from "../src/lib/receipt-intake/worker"; import { normalizeDocType, type ReadOutcome } from "../src/lib/receipt-intake/read"; import type { BookResult } from "../src/lib/receipt-intake/book"; @@ -1139,3 +1144,50 @@ test("a park AFTER a send keeps the key, on every one of those paths", async () assert.ok(!("dedupStrongKey" in (h.states[0].patch ?? {})), "the Purchase may exist"); } }); + +// ── The upload lease, not the row's age (round-13 item 2) ────────────────── + +test("a re-issued upload URL keeps the row safe from the sweeper", () => { + // The row is old; its LEASE is not. Judging it on createdAt declared a + // receipt missing — or destroyed one it called unacceptable — while the + // client's own upload link was live and about to land. + const old = new Date(NOW.getTime() - 6 * 60 * 60_000); + assert.equal( + uploadLeaseActive({ createdAt: old, uploadUrlExpiresAt: new Date(NOW.getTime() + 60_000) }, NOW), + true, + "a fresh lease on an old row", + ); + assert.equal( + uploadLeaseActive({ createdAt: old, uploadUrlExpiresAt: new Date(NOW.getTime() - 60_000) }, NOW), + false, + "an expired lease is expired, however recently the row was touched", + ); + // A row with no lease at all (the single-shot path writes its bytes through + // the server) falls back to its own age. + assert.equal(uploadLeaseActive({ createdAt: old, uploadUrlExpiresAt: null }, NOW), false); + assert.equal( + uploadLeaseActive({ createdAt: new Date(NOW.getTime() - 60_000), uploadUrlExpiresAt: null }, NOW), + true, + ); +}); + +test("the lease a URL is issued under is exactly the signed-URL TTL", () => { + assert.equal(uploadLeaseExpiry(NOW).getTime() - NOW.getTime(), SIGNED_UPLOAD_TTL_MS); + assert.equal(SIGNED_UPLOAD_TTL_MS, 2 * 60 * 60_000); +}); + +test("/start stamps a lease on EVERY url it issues", () => { + const start = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/start/route.ts"), + "utf8", + ); + // Three: the new row, the re-armed park, and the resumed STAGING upload. + // A URL handed out without a lease is one the sweeper cannot see coming. + assert.equal( + (start.match(/uploadUrlExpiresAt: uploadLeaseExpiry\(\)/g) ?? []).length, + 3, + "create, re-arm and resume all stamp the lease", + ); + const signed = (start.match(/await signUpload\(/g) ?? []).length; + assert.equal(signed, 3, "and those are all the places a URL is issued"); +}); From 281d14bfcfe00b1475c95d3f01378f0465117916 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 08:34:48 -0700 Subject: [PATCH 058/144] fix(receipts): tagged presence on the replay path; a secret owns SOURCES not rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (A) The boolean `secureObjectExists` collapsed "confirmed 404" and "storage is unhappy" into false, and the replay path re-uploads and re-points on a false — so a transient fault orphaned the object that was really there and left the row pointing at a second copy. The route already reads the tagged `receiptObjectSize` (c2e64088) and answers 503 on transient; this deletes the collapsing helper so nothing can reach for it again, and adds the classification tests (empty listing and 404 are missing; 5xx/401/429/throw/sizeless are transient) plus a guard that the fault branch precedes the healing one. (B) finalize took `via === "secret"` as blanket authority over any id, so the Apps Script key could publish, re-point and attach a job to a mobile capture or web upload that belongs to a person. It now selects the row's `source` and requires it in `auth.allowedSources` — the same list that scopes creation — answering 403 source-not-owned before any detail is returned or written. Unit guard plus an e2e that seeds a mobile row and proves nothing is disclosed or changed. Co-Authored-By: Claude Fable 5.1 --- e2e/receipt-intake.spec.ts | 32 +++++++++ .../receipts/intake/[id]/finalize/route.ts | 24 ++++++- src/lib/receipt-intake/bucket.ts | 27 +++++-- src/lib/secure-storage.ts | 6 -- tests/receipt-intake-auth.test.ts | 33 +++++++++ tests/receipt-intake-stored-object.test.ts | 71 +++++++++++++++++++ 6 files changed, 180 insertions(+), 13 deletions(-) diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 1a597b171..012835b73 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -1182,6 +1182,38 @@ test.describe("round-10 finalize authorization and recovery", () => { expect((await res.json()).error).toBe("cost-code-without-project"); }); + test("the ingest secret cannot finalize a row it does not own the SOURCE of", async ({ request }) => { + // decideSource already stops a forwarder CREATING a row outside its + // namespace, but finalize took `via === "secret"` as blanket authority + // over any id — so the Apps Script key could publish, re-point and + // attach a job to somebody's mobile capture or web upload. + const created = await postIntake(request, intakeBody({ sourceRef: `${REF_PREFIX}notmysource` })); + expect(created.res.status()).toBe(200); + const id = created.body.id; + minted.push(id); + + // Exactly the shape a phone or the web uploader leaves behind. Seeded + // directly because the secret can no longer create one. + await prisma.receiptIntake.update({ + where: { id }, + data: { source: "mobile", sourceRef: `mobile:${id}` }, + }); + + const res = await request.post(`${INTAKE_PATH}/${id}/finalize`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ projectId: PROJECT_ID }), + maxRedirects: 0, + }); + expect(res.status()).toBe(403); + const body = await res.json(); + expect(body.error).toBe("source-not-owned"); + // 403 before ANY detail is returned or written. + expect(body.storagePath).toBeUndefined(); + expect(body.state).toBeUndefined(); + const after = await prisma.receiptIntake.findUnique({ where: { id } }); + expect(after?.projectId).toBeNull(); + }); + test("a user with no access to the row's OWN job cannot finalize it", async ({ playwright, request }) => { // Revocation has to bite on the project the ROW holds, not only on one // the request supplies. Before this, a user whose access was revoked diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 3f7676df2..396c69ce8 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -143,13 +143,35 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const row = await prisma.receiptIntake.findUnique({ where: { id }, select: { - id: true, state: true, stateReason: true, sourceRef: true, storagePath: true, + id: true, source: true, state: true, stateReason: true, sourceRef: true, storagePath: true, mimeType: true, projectId: true, costCodeId: true, dryRun: true, createdById: true, fileSha256: true, expectedSha256: true, }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); + // A SECRET OWNS SOURCES, NOT ROWS. + // + // decideSource already refuses to let a forwarder CREATE a row outside its + // own namespace, but finalize took `via === "secret"` as blanket authority + // over any id — so the Apps Script key could publish, re-point and attach a + // job to a mobile capture or a web upload that belongs to a person. The + // same list that scopes creation scopes this: an ingest key owns + // drive/email/chat, and nothing else. + // + // 403, not 404: the caller is authenticated and the row is real; what it + // lacks is authority. Checked BEFORE any detail is returned or written. + if (auth.via === "secret" && !auth.allowedSources.has(row.source)) { + return NextResponse.json( + { + ok: false, + error: "source-not-owned", + reason: `this key does not own ${row.source} rows`, + }, + { status: 403 }, + ); + } + // Same rule as the conflict path: a session caller may only finalize its // OWN row (or hold a bookkeeping role). Otherwise a guessed id would let one // user publish another's upload. diff --git a/src/lib/receipt-intake/bucket.ts b/src/lib/receipt-intake/bucket.ts index 0b29861d6..441bff5c8 100644 --- a/src/lib/receipt-intake/bucket.ts +++ b/src/lib/receipt-intake/bucket.ts @@ -66,18 +66,33 @@ export type SizeResult = * is precisely the thing this call exists to avoid, on precisely the objects we * know least about. */ -export async function receiptObjectSize(storagePath: string): Promise { +export interface BucketLister { + list( + dir: string, + opts: { search: string; limit: number }, + ): Promise<{ + data: Array<{ name: string; metadata?: unknown }> | null; + error: { message?: string; status?: number; statusCode?: string | number; error?: string } | null; + }>; +} + +export async function receiptObjectSize( + storagePath: string, + /** Injected only by tests: the classification is the whole subject here. */ + lister: BucketLister | null = null, +): Promise { const path = safePath(storagePath); if (!path) return { ok: false, kind: "missing" }; - const supabase = getSupabase(); - if (!supabase) return { ok: false, kind: "transient", message: "storage-not-configured" }; + const from = lister ?? getSupabase()?.storage.from(RECEIPT_BUCKET) ?? null; + // No client is a CONFIGURATION fault, not an absent object. Saying "missing" + // here is what let a misconfigured deployment re-upload over a document that + // was really there. + if (!from) return { ok: false, kind: "transient", message: "storage-not-configured" }; const slash = path.lastIndexOf("/"); const dir = slash > 0 ? path.slice(0, slash) : ""; const name = slash > 0 ? path.slice(slash + 1) : path; try { - const { data, error } = await supabase.storage - .from(RECEIPT_BUCKET) - .list(dir, { search: name, limit: 100 }); + const { data, error } = await from.list(dir, { search: name, limit: 100 }); if (error) { return isNotFoundError(error as { message?: string; status?: number }) ? { ok: false, kind: "missing" } diff --git a/src/lib/secure-storage.ts b/src/lib/secure-storage.ts index 85d98bc62..15aca5a94 100644 --- a/src/lib/secure-storage.ts +++ b/src/lib/secure-storage.ts @@ -278,12 +278,6 @@ export async function downloadDocBytesResult( } } -/** True only when storage affirmatively confirms the object is there. */ -export async function secureObjectExists(storagePath: string): Promise { - const result = await downloadDocBytesResult(toSecureRef(storagePath)); - return result.ok; -} - /** * Read a stored document's bytes server-side using the service key. * diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 95b8067b7..484e4226c 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -12,6 +12,8 @@ */ import test from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; process.env.NEXTAUTH_SECRET ??= "test-secret"; process.env.DATABASE_URL ??= "postgresql://test:test@localhost:5432/test"; @@ -535,3 +537,34 @@ test("a sourceRef must carry a real id for its source, not just the prefix", asy assert.deepEqual(decideSource(secret, { source: "drive", sourceRef: `drive:${"a".repeat(600)}` }), { ok: false, reason: "sourceRef-too-long" }); }); + +// ── A secret owns SOURCES, not rows (Phase 2 gate, B) ───────────────────── + +test("finalize scopes a secret caller to the sources its key owns", () => { + const finalize = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", + ); + // The row's OWN source is selected and checked against the same list that + // scopes creation — otherwise the Apps Script key is authority over a + // mobile capture that belongs to a person. + assert.match(finalize, /source: true,/, "the source is selected"); + assert.match(finalize, /auth\.via === "secret" && !auth\.allowedSources\.has\(row\.source\)/); + assert.match(finalize, /error: "source-not-owned"/); + assert.match(finalize, /status: 403/); + // BEFORE any detail is returned or any late field applied. + const gate = finalize.indexOf("source-not-owned"); + for (const later of ["const maySee", "authorizeFinalization(auth, row.projectId", "await sealAndPublish("]) { + assert.ok(gate < finalize.indexOf(later), `the source gate precedes ${later}`); + } +}); + +test("the ingest key's source list is exactly the machine sources", async () => { + const { INGEST_ALLOWED_SOURCES } = await loadAuth(); + const { MACHINE_SOURCES } = await import("../src/lib/receipt-intake/intake-core"); + assert.deepEqual([...INGEST_ALLOWED_SOURCES].sort(), [...MACHINE_SOURCES].sort()); + // So a mobile or web row is owned by NO secret, which is the point. + for (const source of ["mobile", "web"]) { + assert.ok(!INGEST_ALLOWED_SOURCES.has(source), source); + } +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index c1fd9f5c2..0d6135ebf 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -20,6 +20,7 @@ import { sealAndPublish, } from "../src/lib/receipt-intake/stored-object"; import { MAX_STORED_BYTES } from "../src/lib/receipt-intake/intake-core"; +import { receiptObjectSize } from "../src/lib/receipt-intake/bucket"; import type { DocBytesResult } from "../src/lib/secure-storage"; const PNG = Buffer.from( @@ -438,3 +439,73 @@ test("both publishers use the shared fence, and finalize refuses the other parks assert.match(finalize, /error: "not-recoverable"/); assert.match(finalize, /disposition === "not-recoverable"/); }); + +// ── Presence is TAGGED: 404 and "storage is unhappy" are different answers ── + +test("the size lookup separates a real absence from a storage fault", async () => { + // The bug this closes: a helper that collapsed both into `false`. The intake + // replay path reads it, and on a false it RE-UPLOADS and re-points the row — + // so a transient fault orphaned the object that was really there and left + // the row pointing at a second copy. + const lister = (result: unknown) => ({ list: async () => result as never }); + + const missing = await receiptObjectSize("receipts/intake/a.png", lister({ data: [], error: null })); + assert.deepEqual(missing, { ok: false, kind: "missing" }, "an empty listing IS an answer"); + + const notFound = await receiptObjectSize( + "receipts/intake/a.png", + lister({ data: null, error: { status: 404, message: "Object not found" } }), + ); + assert.equal((notFound as { kind: string }).kind, "missing"); + + for (const error of [ + { status: 500, message: "boom" }, + { status: 401, message: "invalid jwt" }, + { status: 429, message: "slow down" }, + { message: "fetch failed" }, + ]) { + const fault = await receiptObjectSize("receipts/intake/a.png", lister({ data: null, error })); + assert.equal((fault as { kind: string }).kind, "transient", JSON.stringify(error)); + } + + const found = await receiptObjectSize( + "receipts/intake/a.png", + lister({ data: [{ name: "a.png", metadata: { size: 1234 } }], error: null }), + ); + assert.deepEqual(found, { ok: true, size: 1234 }); + + // Present but sizeless is the one genuinely unknown case, and it must not + // become permission to download. + const sizeless = await receiptObjectSize( + "receipts/intake/a.png", + lister({ data: [{ name: "a.png", metadata: {} }], error: null }), + ); + assert.equal((sizeless as { kind: string }).kind, "transient"); + + // A throwing client is a transport fault, never evidence of absence. + const threw = await receiptObjectSize("receipts/intake/a.png", { + list: async () => { throw new TypeError("fetch failed"); }, + }); + assert.equal((threw as { kind: string }).kind, "transient"); +}); + +test("the replay path heals only on an AFFIRMATIVE absence, and 503s on a fault", () => { + const intake = readFileSync( + path.join(__dirname, "..", "src/app/api/receipts/intake/route.ts"), + "utf8", + ); + const branch = intake.slice(intake.indexOf("const present = await receiptObjectSize(")); + const head = branch.slice(0, branch.indexOf("const healable")); + assert.match(head, /present\.kind === "transient"/); + assert.match(head, /status: 503/); + // The transient answer is handled BEFORE the not-ok branch that heals, so a + // storage fault can never reach storeObject. + assert.ok( + head.indexOf('present.kind === "transient"') < head.indexOf("if (!present.ok) {"), + "the fault check comes first", + ); + assert.ok(!/storeObject/.test(head), "nothing is written on the fault path"); + // And the collapsing helper is gone, so nothing can reintroduce it. + const storage = readFileSync(path.join(__dirname, "..", "src/lib/secure-storage.ts"), "utf8"); + assert.ok(!/secureObjectExists/.test(storage), "no boolean exists-check to reach for"); +}); From 8418a67c5930ea0a1bea08568674642c1102cebd Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 09:06:00 -0700 Subject: [PATCH 059/144] fix(receipts): one ceiling, production ref shapes, stable receipt refs, atomic terminal release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-14 items 2-5 plus the three interim ones. Item 1 (uploadLeaseVersion) is NOT in this commit — it is next. (2) The sourceRef validator now matches what the Apps Script actually sends: `drive:`, `email::`, `chat::`. One implementation in decideSource, so both endpoints get it; e2e drives the production formats and the `drive:` / oversized rejections through both doors. (3) Expense.receiptUrl holds `receipt-intake:///` — a signed URL written into that column is dead ten minutes later, and a bare path does not say which bucket. resolveReceiptUrl() mints a short-lived URL and follows an object that moved (sealed, archived) via the intake row. Wired into resolveDocUrl (so every existing reader gets it), the expenses tab loader, and ai-review, whose SSRF check now names the `/sign/` prefix and runs on the RESOLVED url. (4) One ceiling everywhere: QBO_ATTACHMENT_MAX_BYTES = 8 MiB, used by the bucket policy, /start's declared-size check, inspectStoredObject and the booking preflight. 15 MiB at the door and 8 MiB at the books meant everything in between was stored, read, and then stranded after we had told the sender we had it. (5) Early terminal outcomes (multi-doc, non-receipt, zero/refund, no-job) now go through applyState, which releases claimToken/claimedAt/nextRetryAt in the same fenced write. applyRead is the ONE lease-keeping write and its type pins it to "RECEIVED", so a terminal state cannot be routed back through it. (A) Every post-send step (the post-create phase check, the Expense commit) is inside the protected block, and parkTerminal re-reads the PERSISTED send flag instead of the claim-time snapshot — an unreadable flag RETAINS the key, because retaining costs a review and releasing wrongly costs a second Purchase. (B) migration.sql converges on the state CHECK exactly like the apply script (drop-if-different + add). The token-presence test is replaced by a semantic parity one comparing state order, convergence, scoping and the wanted_def literal. (C) `detail.fileId` is a DRIVE id or absent — it is dual-written into the driveFileId column the cutover queries. Non-Drive rows carry `intakeId`, which is now a first-class identity in journey grouping and keying (two v2 receipts sharing a DocNumber prefix no longer merge into one journey). Mutation-tested: the claim-snapshot park, the non-converging migration. Co-Authored-By: Claude Fable 5.1 --- .env.example | 2 +- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 8 +- e2e/receipt-intake.spec.ts | 57 +++++++ package.json | 4 +- .../migration.sql | 27 +++- scripts/apply-receipt-intake.mjs | 8 +- src/app/api/automation/ai-review/route.ts | 22 ++- .../api/cron/receipt-intake-worker/route.ts | 11 ++ src/app/api/receipts/intake/route.ts | 2 +- src/lib/automation-events.ts | 63 +++++++- src/lib/receipt-intake/book.ts | 69 ++++++--- src/lib/receipt-intake/file-type.ts | 3 +- src/lib/receipt-intake/intake-core.ts | 41 ++++- src/lib/receipt-intake/receipt-url.ts | 90 +++++++++++ src/lib/receipt-intake/worker.ts | 51 +++++- src/lib/secure-storage.ts | 7 + src/lib/time-expense-actions.ts | 86 ++++++----- tests/apply-receipt-intake.test.ts | 70 ++++++++- tests/automation-events-grouping.test.ts | 52 +++++++ tests/receipt-intake-auth.test.ts | 24 ++- tests/receipt-intake-book.test.ts | 24 ++- tests/receipt-intake-claim-release.test.ts | 22 +++ tests/receipt-intake-worker.test.ts | 146 ++++++++++++++++-- tests/receipt-url.test.ts | 130 ++++++++++++++++ 24 files changed, 897 insertions(+), 122 deletions(-) create mode 100644 src/lib/receipt-intake/receipt-url.ts create mode 100644 tests/receipt-url.test.ts diff --git a/.env.example b/.env.example index bab7d9454..ab868a790 100644 --- a/.env.example +++ b/.env.example @@ -46,7 +46,7 @@ RECEIPT_INTAKE_SECRET= RECEIPT_ARCHIVE_SECRET= # # Storage note (not an env var): receipts live in their OWN private bucket, -# `receipt-intake`, carrying a 15 MiB file-size limit and an allow-list of the +# `receipt-intake`, carrying an 8 MiB file-size limit and an allow-list of the # six formats QuickBooks can attach. Two-step uploads go straight to a signed # URL and never pass through this server, so the bucket is the only place a # too-large or wrong-type write can actually be refused; MAX_STORED_BYTES in diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index cd3f19297..df03c4a92 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -224,7 +224,7 @@ missing bucket policy is invisible until a 400 MB object is already stored. `threadName?`) or JSON `{fileBase64, mimeType, fileName?, source, sourceRef?, projectId?, costCodeId?, threadName?}`. `source` in mobile|email|drive|chat|web. Machine callers MUST send `sourceRef`; session/Bearer callers get `web:` / `mobile:` minted - server-side. Max 15 MB. Accept pdf/jpeg/png/heic/webp/gif/txt; sniff magic bytes for + server-side. Max 8 MiB (QuickBooks' attachment ceiling). Accept pdf/jpeg/png/heic/webp/gif; sniff magic bytes for images the way `receipts/parse` does (route.ts:37). - **Behavior**: sha256 the bytes; create the row (catch P2002 on `sourceRef` and return the existing row with `{ok:true, alreadyReceived:true}`); upload to `SECURE_BUCKET` at @@ -487,11 +487,11 @@ fine. |---|---|---| | `POST /api/receipts/intake` (JSON) | **3 MiB raw** | base64 inflates by 4/3, so 3 MiB encodes to ~4 MiB and fits the serverless body cap. 4 MiB raw would be a ~5.4 MiB request that dies at the edge with a 413 this code never sees. | | `POST /api/receipts/intake` (multipart) | **4 MiB** | bytes are sent as-is. | -| two-step (`/start` + signed URL + `/finalize`) | **15 MiB** | the bytes never pass through this server. | +| two-step (`/start` + signed URL + `/finalize`) | **8 MiB** | the bytes never pass through this server. The ceiling is QuickBooks' own attachment limit: anything larger is a receipt that would be stored, read, and then stranded `unsupported-attachment:size` after we had already told the sender we had it. One constant, `QBO_ATTACHMENT_MAX_BYTES` in `intake-core.ts`. | Both inline ceilings answer with a 413 naming the two-step path. -**The 15 MiB ceiling is set on the Supabase bucket as well as in code.** The signed upload +**The 8 MiB ceiling is set on the Supabase bucket as well as in code.** The signed upload URL bypasses this server entirely, so application code cannot stop the write — it can only refuse the object afterwards, by which time the bytes are already paid for and sitting in the bucket. Set it where the write happens: @@ -599,7 +599,7 @@ through this server at all: 3. `POST /api/receipts/intake/{id}/finalize` with `{sha256?}` -> publishes `STAGING` -> `RECEIVED`. The server re-reads the object and derives the mime, the size and the sha FROM STORAGE; a declared `sha256` is checked against that and a mismatch is a 409. Over - 15 MB or an unreadable format deletes the row and refuses. + 8 MiB or an unreadable format deletes the row and refuses. Both paths share `decideSource` (provenance and idempotency), so a session/Bearer caller can never choose `source` or `sourceRef` on either, and `uploadId` is scoped to the authenticated diff --git a/e2e/receipt-intake.spec.ts b/e2e/receipt-intake.spec.ts index 012835b73..8b8835196 100644 --- a/e2e/receipt-intake.spec.ts +++ b/e2e/receipt-intake.spec.ts @@ -1055,6 +1055,63 @@ test.describe("round-9 intake contracts", () => { expect((await unproven.json()).error).toBe("sourceRef-conflict"); }); + test("the PRODUCTION sourceRef formats are accepted by both endpoints", async ({ request }) => { + // Exactly what the Apps Script forwarder sends. A validator that + // accepted more than production sends would have accepted the bug it + // exists to stop; one that accepts LESS breaks the forwarder silently, + // so both shapes are driven through both doors. + const stamp = Date.now(); + const emailRef = `email:1993f0a3c9c4d0${stamp % 100}:0f1e2d3c4b5a6978`; + const chatRef = `chat:spaces/AAQANF47osY/messages/e2e.${stamp}:0`; + + const inline = await postIntake(request, JSON.stringify({ + source: "email", sourceRef: emailRef, + fileBase64: PNG_BASE64, mimeType: "image/png", fileName: "e.png", + })); + expect(inline.res.status()).toBe(200); + minted.push(inline.body.id); + + const started = await request.post(`${INTAKE_PATH}/start`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "chat", sourceRef: chatRef, mimeType: "image/png", + sha256: createHash("sha256").update(Buffer.from(PNG_BASE64, "base64")).digest("hex"), + }), + maxRedirects: 0, + }); + expect(started.status()).toBe(200); + minted.push((await started.json()).id); + }); + + test("a namespace with no id, and an oversized one, are refused at both doors", async ({ request }) => { + // `drive:` with an empty tail was a valid, unique, PERMANENT idempotency + // key: every later empty-tail forward collided with it and was told + // "already received", so real receipts were dropped. + const bad: Array<[string, string]> = [ + ["drive:", "invalid-sourceRef"], + [`drive:${"a".repeat(600)}`, "sourceRef-too-long"], + ["drive:short", "invalid-sourceRef"], + ]; + for (const [sourceRef, reason] of bad) { + const inline = await postIntake(request, intakeBody({ sourceRef })); + expect(inline.res.status(), sourceRef).toBe(400); + expect(inline.body.reason, sourceRef).toBe(reason); + + const started = await request.post(`${INTAKE_PATH}/start`, { + headers: { "content-type": "application/json", "x-receipt-intake-secret": SECRET }, + data: JSON.stringify({ + source: "drive", sourceRef, mimeType: "image/png", sha256: "a".repeat(64), + }), + maxRedirects: 0, + }); + expect(started.status(), sourceRef).toBe(400); + expect((await started.json()).reason, sourceRef).toBe(reason); + } + // And nothing was created for any of them. + const rows = await prisma.receiptIntake.findMany({ where: { sourceRef: { startsWith: "drive:a" } } }); + expect(rows).toHaveLength(0); + }); + test("a text receipt is refused with a 415 that says what to send instead", async ({ request }) => { // QuickBooks cannot attach a .txt, so accepting one meant reading it and // then stranding it unbookable mid-pipeline. diff --git a/package.json b/package.json index 902d51cc1..1e5540628 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:qbo-token-refresh": "tsx --test tests/qbo-token-refresh-timeout.test.ts", "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", - "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts", + "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +28,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 14ee2afed..8ba0187e7 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -83,13 +83,30 @@ CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdAt_idx" ON "ReceiptIntake"("cre CREATE INDEX IF NOT EXISTS "ReceiptIntake_costCodeId_idx" ON "ReceiptIntake"("costCodeId"); CREATE INDEX IF NOT EXISTS "ReceiptIntake_createdById_idx" ON "ReceiptIntake"("createdById"); +-- CONVERGENT, exactly like scripts/apply-receipt-intake.mjs: a constraint that +-- exists with a STALE definition is replaced, not left alone. "Create only when +-- absent" is what let the two diverge — a database that already carried an +-- older state list (one without SHADOW_QUARANTINE, say) kept it forever here +-- while the apply script corrected it in production, so the same repo described +-- two different tables depending on which path built them. DO $$ +DECLARE current_def TEXT; + wanted_def TEXT := 'CHECK ((state = ANY (ARRAY[''STAGING''::text, ''RECEIVED''::text, ''READ''::text, ''NEEDS_JOB''::text, ''NEEDS_REVIEW''::text, ''BOOKING''::text, ''BOOKED''::text, ''ARCHIVED''::text, ''DUPLICATE''::text, ''VOID''::text, ''NON_RECEIPT''::text, ''SHADOW_DONE''::text, ''SHADOW_QUARANTINE''::text])))'; BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conname = 'ReceiptIntake_state_check' - AND conrelid = '"ReceiptIntake"'::regclass - ) THEN + SELECT pg_get_constraintdef(oid) INTO current_def + FROM pg_constraint + WHERE conname = 'ReceiptIntake_state_check' + AND conrelid = '"ReceiptIntake"'::regclass; + + IF current_def IS NULL THEN + ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" + CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', + 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', + 'SHADOW_DONE', 'SHADOW_QUARANTINE')); + ELSIF current_def IS DISTINCT FROM wanted_def THEN + -- One statement each, in the SAME transaction as everything else here, + -- so the table is never briefly unconstrained. + ALTER TABLE "ReceiptIntake" DROP CONSTRAINT "ReceiptIntake_state_check"; ALTER TABLE "ReceiptIntake" ADD CONSTRAINT "ReceiptIntake_state_check" CHECK ("state" IN ('STAGING', 'RECEIVED', 'READ', 'NEEDS_JOB', 'NEEDS_REVIEW', 'BOOKING', 'BOOKED', 'ARCHIVED', 'DUPLICATE', 'VOID', 'NON_RECEIPT', diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index fd3905fab..992d456d6 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -327,7 +327,11 @@ const expectedPartialIndexes = [{ // the wrong limit is a hard failure — silently "fixing" a limit somebody set // deliberately is how a 400 MB upload becomes possible again next quarter. export const RECEIPT_BUCKET = "receipt-intake"; -export const RECEIPT_BUCKET_FILE_SIZE_LIMIT = 15 * 1024 * 1024; +// The SAME ceiling QuickBooks will attach at (MAX_STORED_BYTES / +// QBO_ATTACHMENT_MAX_BYTES in src/lib/receipt-intake/intake-core.ts, asserted +// equal by tests/apply-receipt-intake.test.ts). A bucket that accepts more than +// QBO can attach stores receipts that are guaranteed to strand. +export const RECEIPT_BUCKET_FILE_SIZE_LIMIT = 8 * 1024 * 1024; // EXACTLY the list src/lib/receipt-intake/file-type.ts accepts (asserted by // tests/apply-receipt-intake.test.ts). A bucket that allows more than the code // does lets an unreadable file be stored; one that allows less rejects uploads @@ -430,7 +434,7 @@ async function applyBucket() { const key = process.env.SUPABASE_SERVICE_KEY; if (!baseUrl || !key) { console.error("REFUSING: SUPABASE_URL and SUPABASE_SERVICE_KEY are required to provision the receipts bucket."); - console.error(" The bucket carries the 15 MiB and MIME limits that the signed-upload path cannot enforce anywhere else."); + console.error(" The bucket carries the 8 MiB and MIME limits that the signed-upload path cannot enforce anywhere else."); process.exit(1); } const outcome = await ensureReceiptBucket(baseUrl, key); diff --git a/src/app/api/automation/ai-review/route.ts b/src/app/api/automation/ai-review/route.ts index dd02d0cf2..8447e7dde 100644 --- a/src/app/api/automation/ai-review/route.ts +++ b/src/app/api/automation/ai-review/route.ts @@ -1,4 +1,5 @@ import { NextResponse } from "next/server"; +import { isReceiptUrlRef, resolveReceiptUrl } from "@/lib/receipt-intake/receipt-url"; import Anthropic from "@anthropic-ai/sdk"; import { GoogleGenAI } from "@google/genai"; import { getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; @@ -599,11 +600,24 @@ export async function POST(request: Request) { return NextResponse.json({ ok: false, reason: "no-stored-copy" }); } + // A receipt this pipeline booked is a STORED REFERENCE, not a link — it is + // resolved to a short-lived signed URL here rather than being fetched as + // written. Everything else keeps the old contract exactly. + const receiptUrl = isReceiptUrlRef(expense.receiptUrl) + ? await resolveReceiptUrl(expense.receiptUrl) + : expense.receiptUrl; + if (!receiptUrl) { + return NextResponse.json({ ok: false, reason: "no-stored-copy" }); + } + // SSRF sink check: receiptUrl is written only by our sync/upload code, // but this fetch enforces the invariant anyway — the URL must point at - // OUR Supabase public storage, no redirects followed. - const storagePrefix = `${(process.env.SUPABASE_URL ?? "").replace(/\/$/, "")}/storage/v1/object/public/`; - if (!process.env.SUPABASE_URL || !expense.receiptUrl.startsWith(storagePrefix)) { + // OUR Supabase storage, no redirects followed. Public objects carry the + // `/public/` segment; a signed one carries `/sign/`, so both prefixes are + // named explicitly rather than loosened to the storage root. + const storageRoot = `${(process.env.SUPABASE_URL ?? "").replace(/\/$/, "")}/storage/v1/object/`; + const allowed = [`${storageRoot}public/`, `${storageRoot}sign/`]; + if (!process.env.SUPABASE_URL || !allowed.some(prefix => receiptUrl.startsWith(prefix))) { console.error("ai-review refused non-storage receiptUrl"); return NextResponse.json({ ok: false, reason: "receipt-url-untrusted" }, { status: 409 }); } @@ -620,7 +634,7 @@ export async function POST(request: Request) { inFlightDocs.add(dedupeKey); try { - const fileRes = await fetch(expense.receiptUrl, { redirect: "error", signal: AbortSignal.timeout(20_000) }); + const fileRes = await fetch(receiptUrl, { redirect: "error", signal: AbortSignal.timeout(20_000) }); if (!fileRes.ok) { return NextResponse.json({ ok: false, reason: "receipt-fetch-failed" }, { status: 502 }); } diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index ae68e470c..f11e007f7 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -761,6 +761,17 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // call. A late job assignment landing in that window must not be routed // over: NEEDS_JOB for a receipt that HAS a job sends a human looking for // a problem that no longer exists. + sendAttemptedNow: async rowId => { + const row = await prisma.receiptIntake.findUnique({ + where: { id: rowId }, + select: { sendAttempted: true }, + }); + // A row that vanished, or a read that returned nothing, is answered + // "a send may have happened": retaining the key costs a review, and + // releasing it wrongly costs a second Purchase. + return row?.sendAttempted ?? true; + }, + refreshProjectId: async rowId => { const row = await prisma.receiptIntake.findUnique({ where: { id: rowId }, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 1602e5968..f4d3d7fa1 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -507,7 +507,7 @@ async function respondToSourceRefConflict( // have. The state a row happens to be parked in says nothing about whether // its bytes exist. // Metadata, not a download: this runs on every replay, and the object may - // be 15 MB. A TRANSIENT answer is not evidence of absence — healing on it + // be 8 MiB. A TRANSIENT answer is not evidence of absence — healing on it // would overwrite a document that is really there — so it is answered 503 // and the forwarder retries with its copy intact. const present = await receiptObjectSize(existing.storagePath); diff --git a/src/lib/automation-events.ts b/src/lib/automation-events.ts index d0b11b17f..291195649 100644 --- a/src/lib/automation-events.ts +++ b/src/lib/automation-events.ts @@ -292,6 +292,32 @@ export function resolveEventFileId(e: { driveFileId: string | null; detail: stri return null; } +/** + * The v2 pipeline's own row id, for events that have no Drive file behind them. + * + * `fileId` means a DRIVE file id — it is what `driveFileId` is dual-written + * from, what the cutover queries to decide "did v1 book this", and what a + * DocNumber is derived from. The intake worker used to put its cuid there for + * email, chat, mobile and web receipts, which filled that column with ids no + * Drive query can ever match. Those rows carry `intakeId` instead, and it is a + * first-class identity here: an intake beacon and its push event group by it + * exactly as a Drive pair groups by fileId. + */ +export function resolveEventIntakeId(e: { detail: string | null }): string | null { + if (!e.detail) return null; + try { + const d = JSON.parse(e.detail) as { intakeId?: unknown }; + return typeof d.intakeId === "string" && d.intakeId ? d.intakeId : null; + } catch { + return null; + } +} + +/** Any identity at all — the test every "id-less event" branch actually means. */ +function hasResolvedId(e: { driveFileId: string | null; qbPurchaseId: string | null; detail: string | null }): boolean { + return !!(resolveEventFileId(e) || resolveEventQbPurchaseId(e) || resolveEventIntakeId(e)); +} + /** Same idea as `resolveEventFileId`, for the QBO Purchase id. */ export function resolveEventQbPurchaseId(e: { qbPurchaseId: string | null; detail: string | null }): string | null { if (e.qbPurchaseId) return e.qbPurchaseId; @@ -441,9 +467,15 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map(); const byQbPurchaseId = new Map(); + // v2 rows with no Drive file behind them (email, chat, mobile, web) are + // keyed by their intake id. It is exactly as strong an identity as a Drive + // file id — a cuid, unique per row — and without it those receipts would + // fall through to the docNumber-prefix heuristic below. + const byIntakeId = new Map(); sorted.forEach((e, i) => { const fileId = resolveEventFileId(e); const qbPurchaseId = resolveEventQbPurchaseId(e); + const intakeId = resolveEventIntakeId(e); if (fileId) { const existing = byFileId.get(fileId); if (existing !== undefined) union(i, existing); @@ -454,11 +486,20 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map>(); sorted.forEach((e, i) => { - if (!resolveEventFileId(e) && !resolveEventQbPurchaseId(e)) return; + if (!hasResolvedId(e)) return; const doc = e.docNumber as string; const roots = idRootsByDoc.get(doc) ?? new Set(); roots.add(find(i)); @@ -491,7 +532,7 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map(); sorted.forEach((e, i) => { - if (resolveEventFileId(e) || resolveEventQbPurchaseId(e)) return; + if (hasResolvedId(e)) return; const roots = idRootsByDoc.get(e.docNumber as string); if (roots && roots.size === 1) { union(i, [...roots][0]); @@ -506,7 +547,7 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map(); sorted.forEach((e, i) => { - if (resolveEventFileId(e) || resolveEventQbPurchaseId(e)) return; + if (hasResolvedId(e)) return; const doc = e.docNumber as string; const roots = idRootsByDoc.get(doc); if (roots && roots.size === 1) return; // already bridged above @@ -528,13 +569,22 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map sorted[i]); // already ascending (createdAt, id) let driveFileId: string | null = null; let qbPurchaseId: string | null = null; + let intakeId: string | null = null; for (const e of groupEvents) { driveFileId = driveFileId ?? resolveEventFileId(e); + intakeId = intakeId ?? resolveEventIntakeId(e); const qb = resolveEventQbPurchaseId(e); if (qb) qbPurchaseId = qb; } const doc = groupEvents[0].docNumber as string; - const key = driveFileId ?? (qbPurchaseId ? `qb:${qbPurchaseId}` : `prefix:${doc}`); + // `prefix:` is the LAST resort, and it is not an identity: two + // different receipts can share a 21-char DocNumber prefix, so keying on + // it merges them into one journey. A v2 row with no Drive file has a + // real id of its own — the intake cuid — and it belongs here, or every + // email/chat/mobile receipt that collides on a prefix is presented as + // one receipt in the Command Center. + const key = driveFileId + ?? (qbPurchaseId ? `qb:${qbPurchaseId}` : intakeId ? `intake:${intakeId}` : `prefix:${doc}`); // Finding 5: real id evidence (driveFileId/qbPurchaseId) on the // group is necessary but not sufficient — if ANY member only joined @@ -553,7 +603,10 @@ export function groupEventsIntoJourneys(events: JourneyEventInput[]): Map { // A retry after a crash between the Purchase and this commit finds // its own Expense here (qbPurchaseId is @unique) — create it twice @@ -682,7 +700,14 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // to reconcile against the Purchase. taxCents: taxApplied, detail: { - fileId, + // `fileId` means a DRIVE file id — logAutomationEvent copies it + // into the typed `driveFileId` column, which the cutover reads + // to decide whether v1 already booked a document. Emitting an + // intake cuid there filled that column with ids no Drive query + // can ever match, and quietly widened what "v1 booked this" + // could mean. Non-Drive rows carry their id in `intakeId`, + // which every row has anyway. + ...(driveFileId ? { fileId: driveFileId } : {}), qbPurchaseId: result.qbPurchaseId, intakeId: row.id, expenseId, diff --git a/src/lib/receipt-intake/file-type.ts b/src/lib/receipt-intake/file-type.ts index 7d3eca515..097d51f27 100644 --- a/src/lib/receipt-intake/file-type.ts +++ b/src/lib/receipt-intake/file-type.ts @@ -26,7 +26,8 @@ export const EXT_BY_MIME: Record = { /** The formats a caller may be told to send. Single source for the 415 body. */ export const ACCEPTED_MIME_TYPES = Object.keys(EXT_BY_MIME); -export const MAX_INTAKE_BYTES = 15 * 1024 * 1024; +/** @deprecated Unused — the one ceiling is MAX_STORED_BYTES in intake-core.ts. */ +export const MAX_INTAKE_BYTES = 8 * 1024 * 1024; /** ISO-BMFF major brands stored as image/heic (still + HEVC sequence brands). */ export const HEIC_BRANDS = new Set(["heic", "heix", "hevc", "hevx", "msf1"]); diff --git a/src/lib/receipt-intake/intake-core.ts b/src/lib/receipt-intake/intake-core.ts index 7b733cfc8..2b573f56b 100644 --- a/src/lib/receipt-intake/intake-core.ts +++ b/src/lib/receipt-intake/intake-core.ts @@ -27,8 +27,27 @@ export const MAX_INLINE_UPLOAD_BYTES = 4 * 1024 * 1024; */ export const MAX_INLINE_JSON_BYTES = 3 * 1024 * 1024; +/** + * QuickBooks refuses an attachment over 8 MiB, and a receipt that cannot be + * attached is worse than one that was never accepted: the Purchase is created, + * the file is not on it, and the books look complete. THIS is therefore the + * ceiling for the whole pipeline, not just for the booking step. + * + * It used to be 15 MiB at the door and 8 MiB at the books, and everything in + * between was accepted, stored, read by the model, and then parked + * `unsupported-attachment:size` — after a human had already been told we had + * it. One number, enforced at every layer that can enforce anything: + * + * * the bucket's own file_size_limit (the only place a signed-URL write can + * be refused at all — see bucket.ts / apply-receipt-intake.mjs), + * * /start, on the size the client declares, + * * inspectStoredObject, on the object's metadata and then on its bytes, + * * attachmentBlocker, as the last preflight before the Purchase. + */ +export const QBO_ATTACHMENT_MAX_BYTES = 8 * 1024 * 1024; + /** The real ceiling for a stored receipt, enforced on the object itself. */ -export const MAX_STORED_BYTES = 15 * 1024 * 1024; +export const MAX_STORED_BYTES = QBO_ATTACHMENT_MAX_BYTES; /** Sources a shared-secret forwarder may declare. */ export const MACHINE_SOURCES = new Set(["drive", "email", "chat"]); @@ -106,16 +125,22 @@ export const MAX_SOURCE_REF_BYTES = 512; * the QuickBooks DocNumber seed, so a junk tail becomes a junk DocNumber and * two junk tails sharing a 21-character prefix collide in the books. * - * Each pattern describes the id the forwarder actually holds: - * drive — a Drive file id. - * email — `/`: one message can carry several - * receipts, and each attachment is its own document. - * chat — a Chat message resource name, optionally naming the attachment. + * The shapes are the ones the Apps Script forwarder actually sends (see the + * `sourceRef` doc on the Prisma model), NOT a superset invented here — a + * validator that accepts more than production sends is a validator that would + * have accepted the bug it exists to stop: + * drive — `drive:`: the Drive file id, which is also the QuickBooks + * DocNumber seed for these rows. + * email — `email::`: one message can carry several + * receipts, so the message id alone is not an identity; the 16-hex + * content hash distinguishes them. + * chat — `chat::`: a Chat message resource name + * (`spaces//messages/`) plus the attachment index. */ export const SOURCE_REF_PATTERNS: Record = { drive: /^[A-Za-z0-9_-]{10,128}$/, - email: /^[A-Za-z0-9_.+=~-]{1,256}\/\d{1,4}$/, - chat: /^spaces\/[A-Za-z0-9_-]{1,128}\/messages\/[A-Za-z0-9_.=-]{1,256}(?:\/attachments\/[A-Za-z0-9_.=-]{1,256})?$/, + email: /^[A-Za-z0-9_.+=~-]{1,256}:[0-9a-f]{16}$/, + chat: /^spaces\/[A-Za-z0-9_-]{1,128}\/messages\/[A-Za-z0-9_.=-]{1,256}:\d{1,4}$/, }; export type SourceRefCheck = { ok: true } | { ok: false; reason: string }; diff --git a/src/lib/receipt-intake/receipt-url.ts b/src/lib/receipt-intake/receipt-url.ts new file mode 100644 index 000000000..e9dbcb864 --- /dev/null +++ b/src/lib/receipt-intake/receipt-url.ts @@ -0,0 +1,90 @@ +/** + * The reference an Expense stores for a receipt this pipeline booked. + * + * `Expense.receiptUrl` used to be handed a raw signed URL by some writers and a + * bare storage path by others. Both are wrong for a row that outlives them: a + * signed URL expires (ten minutes later the link in the books is dead), and a + * bare path says nothing about WHICH bucket it is in, which is exactly the + * ambiguity that made receipts and contracts share one. + * + * So the column holds a STABLE, resolvable reference — `receipt-intake:///` + * — and every reader mints a short-lived signed URL from it at read time. + * Nothing in the database expires, and nothing dereferences a caller-supplied + * URL. + */ +import { prisma } from "@/lib/prisma"; +import { RECEIPT_BUCKET, signReceiptDownloadUrl } from "./bucket"; + +export const RECEIPT_URL_SCHEME = "receipt-intake://"; + +/** Ten minutes: long enough to open, short enough that a leaked link is inert. */ +export const RECEIPT_URL_TTL_SECONDS = 600; + +export function receiptUrlRef(storagePath: string, bucket: string = RECEIPT_BUCKET): string { + return `${RECEIPT_URL_SCHEME}${bucket}/${storagePath}`; +} + +export function isReceiptUrlRef(value: string | null | undefined): boolean { + return typeof value === "string" && value.startsWith(RECEIPT_URL_SCHEME); +} + +export function parseReceiptUrl(value: string | null | undefined): { bucket: string; path: string } | null { + if (!isReceiptUrlRef(value)) return null; + const rest = (value as string).slice(RECEIPT_URL_SCHEME.length); + const slash = rest.indexOf("/"); + if (slash <= 0) return null; + const bucket = rest.slice(0, slash); + const path = rest.slice(slash + 1); + // Only OUR bucket, and never a traversal: this string ends up in a storage + // API call, and it is read out of a database column that other code writes. + if (bucket !== RECEIPT_BUCKET) return null; + if (!path || path.startsWith("/") || path.includes("..")) return null; + return { bucket, path }; +} + +export interface ReceiptUrlDeps { + sign: (storagePath: string, ttlSeconds: number) => Promise; + /** Where the intake row that owns this object points NOW. */ + currentPath: (storagePath: string) => Promise; +} + +const defaultDeps: ReceiptUrlDeps = { + sign: signReceiptDownloadUrl, + currentPath: async storagePath => { + // Found through the Expense that carries this exact reference: the + // intake row is the thing that tracks where the bytes ARE, and the ref + // records where they were when the Purchase was written. + const row = await prisma.receiptIntake.findFirst({ + where: { expense: { receiptUrl: receiptUrlRef(storagePath) } }, + select: { storagePath: true }, + }); + return row?.storagePath ?? null; + }, +}; + +/** + * Mint a short-lived signed URL for a stored reference, or null. + * + * THE OBJECT MOVES. A row is published at the upload path, sealed to a + * content-addressed one, and later archived — and the Expense was written + * before some of that happened. So a reference that no longer resolves is + * re-asked of the intake row, which is the thing that actually tracks where the + * bytes are, before giving up. + * + * Never throws: a receipt that cannot be linked must render as "no receipt", + * not take the expenses tab down. + */ +export async function resolveReceiptUrl( + value: string | null | undefined, + ttlSeconds: number = RECEIPT_URL_TTL_SECONDS, + deps: ReceiptUrlDeps = defaultDeps, +): Promise { + const parsed = parseReceiptUrl(value); + if (!parsed) return null; + const direct = await deps.sign(parsed.path, ttlSeconds).catch(() => null); + if (direct) return direct; + // Moved (sealed or archived) since the Expense was written. + const moved = await deps.currentPath(parsed.path).catch(() => null); + if (!moved || moved === parsed.path) return null; + return await deps.sign(moved, ttlSeconds).catch(() => null); +} diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index 660722f9e..dbc7d6dae 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -162,6 +162,21 @@ export interface WorkerDependencies { * looking for exactly that problem. */ refreshProjectId: (rowId: string) => Promise; + /** + * The PERSISTED send flag, re-read at park time. + * + * `row.sendAttempted` is the value this pass CLAIMED with, so it is stale + * the moment the booking marks a send — and the booking marks it precisely + * so the fact survives a process that dies mid-create. A park decided on + * the snapshot released the dedup key of a row that has a Purchase in the + * real books, and the next submission of the same receipt booked it twice. + * + * Failing to read it means RETAINING the key: holding one against a + * booking that did not happen sends a resubmission to a human, and that is + * a queue item. Releasing one against a booking that did happen is a + * duplicate payment. + */ + sendAttemptedNow: (rowId: string) => Promise; /** * Tagged, and VERIFIED: the bytes must hash to what the row recorded at * finalize. A sha stored once and never re-checked proves nothing about @@ -177,10 +192,18 @@ export interface WorkerDependencies { * CAS'd on {id, state, claimToken} like every other mutation. `owned:false` * means this worker lost the row mid-pass and must abort — writing on would * clobber whatever its successor has since decided. + * + * THE ONE WRITE THAT KEEPS THE CLAIM, and the type says so: its state is + * pinned to "RECEIVED" because routing is not finished when it lands — the + * strong claim, the weak net and the publish all still have to happen under + * this same lease. Every TERMINAL outcome goes through applyState instead, + * which releases ownership in the same fenced write. Writing a terminal + * state here would leave a finished row holding a claim, and a row that is + * done but still owned is a row nothing will touch again. */ applyRead: ( rowId: string, - patch: ReadPatch, + patch: ReadPatch & { state: "RECEIVED" }, ownership: Ownership, ) => Promise<{ strongOwner: StrongOwner | null; owned: boolean }>; findWeakHit: (rowId: string, weakKey: string) => Promise<{ id: string } | null>; @@ -710,14 +733,24 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis // A multi-doc, a non-receipt, or a $0/negative misread must never hold // a dedup key — it would quarantine the real receipt that arrives next // (:531 and the v3.6 rationale). - const gated = await deps.applyRead(row.id, { + // + // Via applyState, NOT applyRead: this row is FINISHED — nothing else in + // this pass will touch it — so the write that parks it must also hand + // the claim back, atomically. applyRead deliberately keeps the lease + // (routing continues under it), which for a terminal outcome left a + // done row owned by a pass that had moved on: invisible to the health + // probe as anything but "claimed", and untouchable by every fenced + // write until the lease aged out. + // + // Safe to swap: the unique-violation path applyRead exists for cannot + // fire here, because a gated row claims no strong key at all. + const owned = await deps.applyState(row.id, gate.state, note(gate.stateReason), { ...base, state: gate.state, - stateReason: note(gate.stateReason), dedupStrongKey: null, duplicateOfId: gate.duplicateOfId, }, ownershipOf(row)); - return gated.owned ? gate.state : "STALE"; + return owned ? gate.state : "STALE"; } // The strong claim IS the partial unique index: a rejection is the hit. @@ -798,7 +831,11 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis * against nothing. * * `sendAttempted` is the PERSISTED flag — markSendAttempted writes it before - * the create precisely so this decision survives a process that died mid-send. + * the create precisely so this decision survives a process that died mid-send, + * and it is RE-READ here rather than taken from the row this pass claimed. A + * failure anywhere after the send (the post-create phase check, the Expense + * commit, a pool timeout) reaches this function with a snapshot that still says + * "nothing sent", and releasing the key on that is a duplicate payment. */ async function parkTerminal( row: WorkerRow, @@ -806,7 +843,9 @@ async function parkTerminal( reason: string, patch?: Partial, ): Promise { - const release = row.sendAttempted ? {} : { dedupStrongKey: null }; + // Re-read, never the claim-time snapshot: see sendAttemptedNow. + const sent = row.sendAttempted || await deps.sendAttemptedNow(row.id).catch(() => true); + const release = sent ? {} : { dedupStrongKey: null }; const owned = await deps .applyState(row.id, "NEEDS_REVIEW", reason, { ...(patch ?? {}), ...release }, ownershipOf(row)) .catch(() => false); diff --git a/src/lib/secure-storage.ts b/src/lib/secure-storage.ts index 15aca5a94..8ac2a06f6 100644 --- a/src/lib/secure-storage.ts +++ b/src/lib/secure-storage.ts @@ -1,4 +1,5 @@ import { getSupabase, STORAGE_BUCKET } from "./supabase"; +import { isReceiptUrlRef, resolveReceiptUrl } from "./receipt-intake/receipt-url"; /** * Private bucket for documents that carry legal or PII weight: e-signatures, executed @@ -98,6 +99,7 @@ export function parseOwnStorageUrl( /** * Turn a stored document reference into something a browser can load. * + * - receipt ref → short-lived signed URL against the receipts bucket * - secure ref → short-lived signed URL against the private bucket * - data: URL → returned unchanged (legacy inline signatures still render) * - absolute URL → returned unchanged (legacy public-bucket object, still served) @@ -112,6 +114,11 @@ export async function resolveDocUrl( ): Promise { if (!stored) return null; + // `receipt-intake:///` — what the receipt pipeline writes to + // Expense.receiptUrl. Handled here so EVERY existing reader resolves it, + // rather than each one learning a second scheme. + if (isReceiptUrlRef(stored)) return await resolveReceiptUrl(stored, ttlSeconds); + const securePath = secureRefPath(stored); if (securePath) { const supabase = getSupabase(); diff --git a/src/lib/time-expense-actions.ts b/src/lib/time-expense-actions.ts index 1072eba0a..de79626ad 100644 --- a/src/lib/time-expense-actions.ts +++ b/src/lib/time-expense-actions.ts @@ -13,8 +13,9 @@ import { } from "@/lib/time-expense-core"; import { dateInputInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; import { resolveScheduleTaskIdForPunch } from "@/lib/punch-task-binding"; -import { toCompanyDayKey } from "@/lib/company-day"; -import { assertExpenseMutableOutsideQbo } from "@/lib/qbo-expense-guard"; +import { isReceiptUrlRef, resolveReceiptUrl } from "@/lib/receipt-intake/receipt-url"; +import { toCompanyDayKey } from "@/lib/company-day"; +import { assertExpenseMutableOutsideQbo } from "@/lib/qbo-expense-guard"; async function assertTimeExpenseProjectAccess(projectId: string) { const user = await getCurrentUserWithPermissions(); @@ -192,22 +193,22 @@ export async function deleteExpense(id: string, projectId: string) { if (!hasPermission(user, "timeClock")) throw new Error("Forbidden"); const expense = await prisma.expense.findUnique({ where: { id }, - select: { - qbPurchaseId: true, - invoiceId: true, - invoicedAt: true, - estimate: { select: { projectId: true } }, - }, + select: { + qbPurchaseId: true, + invoiceId: true, + invoicedAt: true, + estimate: { select: { projectId: true } }, + }, + }); + if (!expense || expense.estimate.projectId !== projectId || !canAccessProject(user, expense.estimate.projectId)) { + throw new Error("Forbidden"); + } + assertExpenseMutableOutsideQbo(expense); + if (expense.invoiceId || expense.invoicedAt) throw new Error("Billed expenses cannot be deleted"); + + const deleted = await prisma.expense.deleteMany({ + where: { id, qbPurchaseId: null, invoiceId: null, invoicedAt: null }, }); - if (!expense || expense.estimate.projectId !== projectId || !canAccessProject(user, expense.estimate.projectId)) { - throw new Error("Forbidden"); - } - assertExpenseMutableOutsideQbo(expense); - if (expense.invoiceId || expense.invoicedAt) throw new Error("Billed expenses cannot be deleted"); - - const deleted = await prisma.expense.deleteMany({ - where: { id, qbPurchaseId: null, invoiceId: null, invoicedAt: null }, - }); if (deleted.count !== 1) throw new Error("Expense was billed while it was being deleted; refresh and try again"); revalidatePath(`/projects/${projectId}/time-expenses`); @@ -224,19 +225,19 @@ export async function deleteExpenses( const expenses = await prisma.expense.findMany({ where: { id: { in: ids } }, - select: { - id: true, - qbPurchaseId: true, - invoiceId: true, - invoicedAt: true, - estimate: { select: { projectId: true } }, - }, - }); - const accessible = expenses.filter( - e => e.estimate?.projectId && canAccessProject(user, e.estimate.projectId), - ); - for (const expense of accessible) assertExpenseMutableOutsideQbo(expense); - const allowed = accessible.filter(e => !e.invoiceId && !e.invoicedAt); + select: { + id: true, + qbPurchaseId: true, + invoiceId: true, + invoicedAt: true, + estimate: { select: { projectId: true } }, + }, + }); + const accessible = expenses.filter( + e => e.estimate?.projectId && canAccessProject(user, e.estimate.projectId), + ); + for (const expense of accessible) assertExpenseMutableOutsideQbo(expense); + const allowed = accessible.filter(e => !e.invoiceId && !e.invoicedAt); if (!allowed.length) return { deleted: 0 }; const allowedIds = allowed.map(e => e.id); @@ -245,12 +246,12 @@ export async function deleteExpenses( ); const result = await prisma.expense.deleteMany({ - where: { - id: { in: allowedIds }, - qbPurchaseId: null, - invoiceId: null, - invoicedAt: null, - }, + where: { + id: { in: allowedIds }, + qbPurchaseId: null, + invoiceId: null, + invoicedAt: null, + }, }); for (const projectId of projectIds) { @@ -328,7 +329,7 @@ export async function getTimeExpenseData(projectId: string) { orderBy: { startTime: "desc" }, }); - const expenses = await prisma.expense.findMany({ + const expenseRows = await prisma.expense.findMany({ where: { estimate: { projectId } }, include: { costCode: { select: { id: true, name: true, code: true } }, @@ -339,6 +340,17 @@ export async function getTimeExpenseData(projectId: string) { orderBy: { createdAt: "desc" }, }); + // `receiptUrl` is a stored REFERENCE for anything the receipt pipeline + // booked (`receipt-intake://…`), not a link — the tab renders it as an + // href, so it is resolved to a short-lived signed URL here. Legacy absolute + // URLs and data URLs come back unchanged. + const expenses = await Promise.all(expenseRows.map(async expense => ({ + ...expense, + receiptUrl: isReceiptUrlRef(expense.receiptUrl) + ? await resolveReceiptUrl(expense.receiptUrl) + : expense.receiptUrl, + }))); + const costCodes = await prisma.costCode.findMany({ where: { isActive: true }, orderBy: { code: "asc" }, diff --git a/tests/apply-receipt-intake.test.ts b/tests/apply-receipt-intake.test.ts index 23c09d113..8432fcf0b 100644 --- a/tests/apply-receipt-intake.test.ts +++ b/tests/apply-receipt-intake.test.ts @@ -71,16 +71,64 @@ test("the partial unique index is identical in both, predicate included", () => assert.ok(expected.includes(`where "dedupstrongkey" is not null and "state" not in('duplicate','void')`)); }); +/** + * The CHECK constraint's actual SEMANTICS: the ordered state list, and whether + * the block converges (replaces a stale definition) or merely creates. + * + * Token-presence — "does this file mention 'BOOKED' somewhere" — passed happily + * while the two files did DIFFERENT things with the constraint they both + * mention, which is exactly how they drifted. + */ +function checkSemantics(sql: string) { + const body = sql.slice(sql.indexOf("ReceiptIntake_state_check")); + return { + states: Array.from(body.matchAll(/'([A-Z_]{4,})'/g), m => m[1]) + .filter(token => (RECEIPT_INTAKE_STATES as string[]).includes(token)), + converges: /DROP CONSTRAINT "ReceiptIntake_state_check"/.test(body) + && /IS DISTINCT FROM wanted_def/.test(body), + scoped: /conrelid = '"ReceiptIntake"'::regclass/.test(body), + }; +} + test("both files declare the SAME closed state set, and it matches the runtime one", () => { // A state the CHECK constraint rejects but the code can produce is a // guaranteed 500 on a document nobody can then see. assert.deepEqual([...RUNTIME_STATES].sort(), [...RECEIPT_INTAKE_STATES].sort()); const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check")); assert.ok(check, "the script must add the state CHECK constraint"); - for (const state of RECEIPT_INTAKE_STATES) { - assert.ok(check.includes(`'${state}'`), `apply script CHECK is missing ${state}`); - assert.ok(migrationSql.includes(`'${state}'`), `migration.sql CHECK is missing ${state}`); + + const fromScript = checkSemantics(check!); + const fromMigration = checkSemantics(migrationSql); + + // SEMANTIC PARITY, not "both mention the word". + assert.deepEqual(fromScript.states, fromMigration.states, "the same states, in the same order"); + assert.deepEqual( + [...new Set(fromScript.states)].sort(), + [...RECEIPT_INTAKE_STATES].sort(), + "and it is the whole closed set", + ); +}); + +test("BOTH paths converge on the wanted definition; neither only creates-if-absent", () => { + // A database that already carried an older state list kept it forever under + // "create only when absent", while the apply script corrected it in + // production — the same repo describing two different tables depending on + // which path built them. + const check = statements.find((s: string) => s.includes("ReceiptIntake_state_check"))!; + for (const [label, semantics] of [ + ["apply script", checkSemantics(check)], + ["migration.sql", checkSemantics(migrationSql)], + ] as const) { + assert.equal(semantics.converges, true, `${label} replaces a stale definition`); + assert.equal(semantics.scoped, true, `${label} scopes the lookup to this table`); } + // And they agree on WHAT the wanted definition is, character for character: + // this string is compared against pg_get_constraintdef, so a single + // character of difference means one path replaces the constraint on every + // run while the other leaves it alone. + const wanted = (sql: string) => /wanted_def\s+TEXT\s*:=\s*('(?:[^']|'')*')/.exec(sql)?.[1] ?? null; + assert.ok(wanted(check), "the apply script declares a wanted definition"); + assert.equal(wanted(check), wanted(migrationSql)); }); test("every FK and index in the script also exists in the migration", () => { @@ -273,7 +321,21 @@ test("the bucket policy in the script matches the one the code writes through", "the accepted formats and the bucket's allow-list are the same list", ); assert.equal(RECEIPT_BUCKET_POLICY.public, false); - assert.equal(RECEIPT_BUCKET_FILE_SIZE_LIMIT, 15 * 1024 * 1024); + + // ONE CEILING, and it is QuickBooks': a bucket that accepts more than QBO + // will attach stores receipts that are guaranteed to strand — the Purchase + // is created, the file is not on it, and the books look complete. The + // intake door, the bucket, the object check and the booking preflight are + // all the same number. + const { QBO_ATTACHMENT_MAX_BYTES, MAX_STORED_BYTES } = + await import("../src/lib/receipt-intake/intake-core"); + const { attachmentBlocker } = await import("../src/lib/receipt-intake/book"); + assert.equal(RECEIPT_BUCKET_FILE_SIZE_LIMIT, 8 * 1024 * 1024); + assert.equal(QBO_ATTACHMENT_MAX_BYTES, RECEIPT_BUCKET_FILE_SIZE_LIMIT); + assert.equal(MAX_STORED_BYTES, RECEIPT_BUCKET_FILE_SIZE_LIMIT); + // The booking preflight agrees at the boundary, in both directions. + assert.equal(attachmentBlocker("image/png", MAX_STORED_BYTES), null); + assert.equal(attachmentBlocker("image/png", MAX_STORED_BYTES + 1), `size:${MAX_STORED_BYTES + 1}`); }); test("a missing bucket is CREATED private, with both limits", async () => { diff --git a/tests/automation-events-grouping.test.ts b/tests/automation-events-grouping.test.ts index 7558f77d7..406618cf3 100644 --- a/tests/automation-events-grouping.test.ts +++ b/tests/automation-events-grouping.test.ts @@ -229,3 +229,55 @@ test("journeyKey: falls back to docNumber+firstSeen only when neither driveFileI const j = { driveFileId: null, qbPurchaseId: null, docNumber: "DOC-3", firstSeen }; assert.equal(journeyKey(j), `DOC-3:${firstSeen.toISOString()}`); }); + +// ── A v2 receipt with no Drive file behind it still groups (round-14 C) ──── + +test("intakeId is a first-class identity, and never masquerades as a Drive id", async () => { + const { resolveEventFileId, resolveEventIntakeId } = + await import("../src/lib/automation-events"); + + const INTAKE_ID = "cmpd6xca1009x1iizdf4suln3"; + const doc = "cmpd6xca1009x1iizd"; + // A v2 receipt with no Drive file behind it: an intake beacon and the push + // event that booked it. Before the fix the worker put the intake cuid in + // `fileId`, which is dual-written into the `driveFileId` COLUMN — filling + // it with ids no Drive query can ever match. + const intake = fakeEvent({ + id: "e1", stage: "intake", status: "staged", docNumber: doc, + detail: JSON.stringify({ intakeId: INTAKE_ID }), + createdAt: new Date("2026-09-01T10:00:00Z"), + }); + const push = fakeEvent({ + id: "e2", status: "created", docNumber: doc, qbPurchaseId: "QB-1", + detail: JSON.stringify({ intakeId: INTAKE_ID, qbPurchaseId: "QB-1" }), + createdAt: new Date("2026-09-01T10:05:00Z"), + }); + + // Neither event claims a Drive id, because neither has one. + assert.equal(resolveEventFileId(intake), null); + assert.equal(resolveEventFileId(push), null); + assert.equal(resolveEventIntakeId(intake), INTAKE_ID); + + // They are still ONE receipt, joined on the intake id — proof, not the + // docNumber-prefix heuristic, which is explicitly a guess. + const journeys = [...groupEventsIntoJourneys([intake, push]).values()]; + assert.equal(journeys.length, 1); + assert.equal(journeys[0].steps.length, 2); + assert.equal(journeys[0].keyConfirmed, true, "an id match is proof, not a guess"); +}); + +test("two DIFFERENT v2 receipts sharing a docNumber prefix stay apart", () => { + // The control: the intake id is what keeps them separate. Without it both + // would fall into the prefix bucket and be presented as one receipt. + const doc = "COLLIDING-PREFIX-00000"; + const a = fakeEvent({ + id: "a", docNumber: doc, detail: JSON.stringify({ intakeId: "intake-a" }), + createdAt: new Date("2026-09-01T10:00:00Z"), + }); + const b = fakeEvent({ + id: "b", docNumber: doc, detail: JSON.stringify({ intakeId: "intake-b" }), + createdAt: new Date("2026-09-01T10:01:00Z"), + }); + const journeys = [...groupEventsIntoJourneys([a, b]).values()]; + assert.equal(journeys.length, 2, "two ids, two receipts"); +}); diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 484e4226c..7b70cfe33 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -244,7 +244,10 @@ test("provenance rules are shared by BOTH upload paths", async () => { // The inline body cap is well under the stored cap, which is the whole // reason the two-step path exists. assert.ok(MAX_INLINE_UPLOAD_BYTES < MAX_STORED_BYTES); - assert.equal(MAX_STORED_BYTES, 15 * 1024 * 1024); + // The stored ceiling is QuickBooks' attachment ceiling — see + // tests/apply-receipt-intake.test.ts, which ties it to the bucket policy + // and the booking preflight. + assert.equal(MAX_STORED_BYTES, 8 * 1024 * 1024); }); // ── Two secrets, two blast radii (Phase 3 gate, c) ───────────────────────── @@ -501,20 +504,27 @@ test("a sourceRef must carry a real id for its source, not just the prefix", asy assert.deepEqual(validateSourceRef("drive", "drive:1AbCdEfGhIjKlMnOp_qR"), { ok: true }); assert.deepEqual(validateSourceRef("drive", "drive:short"), { ok: false, reason: "invalid-sourceRef" }); assert.deepEqual(validateSourceRef("drive", "drive:has spaces here"), { ok: false, reason: "invalid-sourceRef" }); - assert.deepEqual(validateSourceRef("email", "email:CADnq=abc123def/0"), { ok: true }); + // THE PRODUCTION FORMATS, exactly as the Apps Script forwarder sends them. + assert.deepEqual(validateSourceRef("email", "email:1993f0a3c9c4d0d2:0f1e2d3c4b5a6978"), { ok: true }); assert.deepEqual( - validateSourceRef("email", "email:CADnq=abc123def"), + validateSourceRef("email", "email:1993f0a3c9c4d0d2"), { ok: false, reason: "invalid-sourceRef" }, - "one message can carry several receipts; the attachment index is part of the identity", + "one message can carry several receipts; the content hash is part of the identity", ); assert.deepEqual( - validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def"), - { ok: true }, + validateSourceRef("email", "email:1993f0a3c9c4d0d2:NOTHEX0123456789"), + { ok: false, reason: "invalid-sourceRef" }, + "the tail is a sha16, not free text", ); assert.deepEqual( - validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def/attachments/ATT.1"), + validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def:0"), { ok: true }, ); + assert.deepEqual( + validateSourceRef("chat", "chat:spaces/AAQANF47osY/messages/abc.def"), + { ok: false, reason: "invalid-sourceRef" }, + "the attachment index is part of the identity", + ); assert.deepEqual( validateSourceRef("chat", "chat:AAQANF47osY"), { ok: false, reason: "invalid-sourceRef" }, diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 98071fb0a..214c0db8f 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -241,11 +241,31 @@ test("an explicitly chosen cost code beats the model's suggestion", async () => assert.equal(r.expenses[0].costCodeId, "cc-chosen"); }); -test("a non-drive row books under its intake id and stores the secure ref", async () => { +test("a non-drive row books under its intake id and stores a resolvable reference", async () => { const r = recorder(); await bookReceipt(row({ source: "mobile", sourceRef: "mobile:abc", id: "intake-9" }), r.deps); + // The QBO identity still needs SOMETHING unique, and the intake id is it. assert.equal(r.purchaseCalls[0].fileId, "intake-9"); - assert.equal(r.expenses[0].receiptUrl, "receipt-intake:receipts/intake/intake-1.jpg"); + // The Expense holds a stable reference — not a signed URL that expires, and + // not a bare path that says nothing about which bucket it is in. + assert.equal(r.expenses[0].receiptUrl, "receipt-intake://receipt-intake/receipts/intake/intake-1.jpg"); +}); + +test("the audit event calls a DRIVE id fileId, and everything else intakeId", async () => { + // `fileId` is dual-written into the typed `driveFileId` column, which the + // cutover queries to decide whether v1 already booked a document. An intake + // cuid there fills it with ids no Drive query can ever match. + const mobile = recorder(); + await bookReceipt(row({ source: "mobile", sourceRef: "mobile:abc", id: "intake-9" }), mobile.deps); + const mobileDetail = mobile.events[0].detail as Record; + assert.ok(!("fileId" in mobileDetail), "no Drive file exists for this row"); + assert.equal(mobileDetail.intakeId, "intake-9"); + + const drive = recorder(); + await bookReceipt(row({ source: "drive", sourceRef: "drive:FILE9", id: "intake-9" }), drive.deps); + const driveDetail = drive.events[0].detail as Record; + assert.equal(driveDetail.fileId, "FILE9", "a real Drive id, not the intake row id"); + assert.equal(driveDetail.intakeId, "intake-9", "and the row id is still carried"); }); test("a project with no estimate is terminal, spends NO attempt, and RELEASES the strong key", async () => { diff --git a/tests/receipt-intake-claim-release.test.ts b/tests/receipt-intake-claim-release.test.ts index 63b733849..0846046d6 100644 --- a/tests/receipt-intake-claim-release.test.ts +++ b/tests/receipt-intake-claim-release.test.ts @@ -126,3 +126,25 @@ test("every fenced write CASes on the OWNERSHIP it was handed, not on the id alo } assert.match(route, /const owns = \{ id: rowId, state: "BOOKING", claimToken \} as const;/); }); + +test("applyRead is the ONE lease-keeping write, and it can only say RECEIVED", () => { + // The scanner above skips it, because its `data` carries no state literal — + // it spreads a patch. So it is asserted directly instead: routing continues + // under this lease, which is the whole reason it keeps it, and the compiler + // is what stops a TERMINAL state being routed back through it. + const worker = readFileSync( + path.join(__dirname, "..", "src/lib/receipt-intake/worker.ts"), + "utf8", + ); + assert.match(worker, /patch: ReadPatch & \{ state: "RECEIVED" \}/); + + const fn = route.slice(route.indexOf("applyRead: async")); + const body = fn.slice(0, fn.indexOf("findWeakHit:")); + assert.match(body, /where: \{ id: rowId, state: ownership\.state, claimToken: ownership\.claimToken \}/, + "still fenced on ownership like every other write"); + assert.ok(!/RELEASE_CLAIM/.test(body), "and deliberately does NOT release: routing is not finished"); + assert.match(body, /nextRetryAt is deliberately/, "with the reason written down at the write itself"); + + // Every TERMINAL outcome goes through applyState, which does release. + assert.match(worker, /const owned = await deps\.applyState\(row\.id, gate\.state, note\(gate\.stateReason\)/); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index ddf2470be..c9d928db0 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -97,7 +97,10 @@ interface Harness { reads: number; books: number; applied: ReadPatch[]; - states: { id: string; state: string; reason: string | null; patch?: Partial }[]; + states: { + id: string; state: string; reason: string | null; + patch?: Partial; ownership?: { state: string; claimToken: string | null }; + }[]; promoted: string[]; finished: { id: string; claimToken: string | null; stateReason: string | null }[]; deferred: { id: string; busyPasses: number }[]; @@ -108,12 +111,15 @@ interface Harness { cleanupCalls: number; bookBudgets: number[]; clock: number; + sendReads: string[]; + persistedSendAttempted?: boolean; } function harness(rows: WorkerRow[], overrides: Partial = {}): Harness { const h: Harness = { reads: 0, books: 0, applied: [], states: [], promoted: [], finished: [], deferred: [], retried: [], claimOpts: [], sweepCalls: 0, cleanupCalls: 0, bookBudgets: [], clock: 0, + sendReads: [], boundary: new Date("2026-08-25T00:00:00.000Z"), deps: null as unknown as WorkerDependencies, }; @@ -130,11 +136,20 @@ function harness(rows: WorkerRow[], overrides: Partial = {}) // Defaults to what the row already carries: the interesting case is the // one that overrides it, where a late assignment landed mid-pass. refreshProjectId: async rowId => rows.find(r => r.id === rowId)?.projectId ?? null, + // The PERSISTED flag. Defaults to what the row carries, so only the + // tests about the reload have to think about it. + sendAttemptedNow: async rowId => { + h.sendReads.push(rowId); + return h.persistedSendAttempted ?? rows.find(r => r.id === rowId)?.sendAttempted ?? false; + }, downloadBytes: async () => ({ ok: true as const, bytes: Buffer.from("bytes") }), read: async () => { h.reads++; return goodRead; }, applyRead: async (_id, patch) => { h.applied.push(patch); return { owned: true, strongOwner: null }; }, findWeakHit: async () => null, - applyState: async (id, state, reason, patch) => { h.states.push({ id, state, reason, patch }); return true; }, + applyState: async (id, state, reason, patch, ownership) => { + h.states.push({ id, state, reason, patch, ownership }); + return true; + }, finishRouting: async (id, claimToken, stateReason) => { h.finished.push({ id, claimToken, stateReason }); }, @@ -223,9 +238,12 @@ test("a document that does not reach READ never claims the strong key", async () read: async () => ({ ...goodRead, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, }); await runIntakeWorker(h.deps); - assert.equal(h.applied[0].state, "NEEDS_REVIEW"); - assert.equal(h.applied[0].stateReason, "multi-doc"); - assert.equal(h.applied[0].dedupStrongKey, null); + // Through applyState, which RELEASES the claim in the same write — not + // applyRead, which keeps the lease because routing continues under it. + assert.deepEqual(h.applied, [], "no lease-keeping write for a finished row"); + assert.equal(h.states[0].state, "NEEDS_REVIEW"); + assert.equal(h.states[0].reason, "multi-doc"); + assert.equal(h.states[0].patch?.dedupStrongKey, null); }); test("a service outage costs no attempt: the row is deferred and counts ONE busy pass", async () => { @@ -655,8 +673,8 @@ test("a document-level gate short-circuits BOTH nets and claims no key", async ( await runIntakeWorker(h.deps); // The tax note rides along with whatever state routing picked — a // document can be both a bad tax read and a refund. - assert.ok(h.applied[0].stateReason?.startsWith(reason), `${reason}: ${h.applied[0].stateReason}`); - assert.equal(h.applied[0].dedupStrongKey, null, reason); + assert.ok(h.states[0].reason?.startsWith(reason), `${reason}: ${h.states[0].reason}`); + assert.equal(h.states[0].patch?.dedupStrongKey, null, reason); assert.equal(weakCalls, 0, `${reason}: dedup is not consulted at all`); } }); @@ -783,8 +801,8 @@ test("a missing or unknown doc_type is NEVER treated as a receipt", async () => }); const summary = await runIntakeWorker(h.deps); assert.deepEqual(summary.byState, { NEEDS_REVIEW: 1 }, JSON.stringify(docType)); - assert.equal(h.applied[0].stateReason, "unknown-doc-type", JSON.stringify(docType)); - assert.equal(h.applied[0].dedupStrongKey, null, "and it claims no key"); + assert.equal(h.states[0].reason, "unknown-doc-type", JSON.stringify(docType)); + assert.equal(h.states[0].patch?.dedupStrongKey, null, "and it claims no key"); } }); @@ -1060,10 +1078,11 @@ test("losing the row aborts each mutation path instead of clobbering a successor // a time-based lease cannot express this, because both hold the same id. const lost = { owned: false as const }; - // applyRead at the document-level gate. + // applyState at the document-level gate (a terminal outcome, so it is the + // releasing write that carries it, not applyRead). const gate = harness([workerRow()], { read: async () => ({ ok: true, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, - applyRead: async () => ({ ...lost, strongOwner: null }), + applyState: async () => false, }); assert.deepEqual((await runIntakeWorker(gate.deps)).byState, { STALE: 1 }); @@ -1191,3 +1210,108 @@ test("/start stamps a lease on EVERY url it issues", () => { const signed = (start.match(/await signUpload\(/g) ?? []).length; assert.equal(signed, 3, "and those are all the places a URL is issued"); }); + +// ── A finished row hands the claim back, whatever finished it ───────────── + +test("EVERY early terminal outcome releases the claim in the same write", async () => { + // The hole: these four were written by applyRead, which deliberately KEEPS + // the lease because routing normally continues under it. For an outcome + // that ends the row there is no "afterwards" — so the row sat finished and + // still owned, which the health probe reads as claimed and every fenced + // write misses. + const outcomes: Array<[string, Partial, string]> = [ + ["multi-document", { + read: async () => ({ ...goodRead, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, + }, "NEEDS_REVIEW"], + ["non-receipt", { + read: async () => ({ ...goodRead, read: { ...goodRead.read, docType: "non_receipt" } }) as ReadOutcome, + }, "NON_RECEIPT"], + ["zero or refund", { + read: async () => ({ ...goodRead, read: { ...goodRead.read, totalAmount: "0.00" } }) as ReadOutcome, + }, "NEEDS_REVIEW"], + ]; + for (const [label, overrides, expected] of outcomes) { + const h = harness([workerRow()], overrides); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { [expected]: 1 }, label); + assert.deepEqual(h.applied, [], `${label}: nothing kept the lease`); + assert.equal(h.states.length, 1, label); + // Fenced on the row's OWN state and token — which is what makes the + // release atomic with the transition rather than a second write. + assert.deepEqual(h.states[0].ownership, { state: "RECEIVED", claimToken: "claim-1" }, label); + assert.deepEqual(h.finished, [], `${label}: finishRouting is for READ only`); + } + + // The no-job park takes the same road. + const noJob = harness([workerRow({ projectId: null })], { refreshProjectId: async () => null }); + assert.deepEqual((await runIntakeWorker(noJob.deps)).byState, { NEEDS_JOB: 1 }); + assert.deepEqual(noJob.applied, [], "no-project is terminal too"); + assert.deepEqual(noJob.states[0].ownership, { state: "RECEIVED", claimToken: "claim-1" }); +}); + +test("a terminal write that LOSES its fence reports STALE and nothing else", async () => { + const h = harness([workerRow()], { + read: async () => ({ ...goodRead, read: { ...goodRead.read, docType: "multi" } }) as ReadOutcome, + applyState: async () => false, + }); + const summary = await runIntakeWorker(h.deps); + assert.deepEqual(summary.byState, { STALE: 1 }); + assert.deepEqual(h.finished, []); +}); + +test("the ONE write that keeps the lease can only ever say RECEIVED", () => { + // Enforced by the type (`patch: ReadPatch & { state: "RECEIVED" }`), so a + // terminal state cannot be routed back through applyRead by accident. This + // asserts the contract is still written down where the compiler reads it. + const worker = readFileSync( + path.join(__dirname, "..", "src/lib/receipt-intake/worker.ts"), + "utf8", + ); + assert.match(worker, /patch: ReadPatch & \{ state: "RECEIVED" \}/); + assert.match(worker, /THE ONE WRITE THAT KEEPS THE CLAIM/); +}); + +// ── A park after a send must never hand the key back (round-14 A) ────────── + +test("a park decided AFTER a send reads the PERSISTED flag, not the claim snapshot", async () => { + // The hole: everything after the QBO create — the post-create phase check, + // the Expense commit, a pool timeout — could throw out to the worker's + // generic handler, which parked the row from the snapshot it claimed with. + // That snapshot says "nothing sent", so the dedup key went back for a row + // with a Purchase in the real books, and the next submission of the same + // receipt booked it a second time. + const h = harness([workerRow({ state: "READ", dryRun: false, sendAttempted: false, attempts: 19 })], { + book: async () => { throw new Error("connection reset after the create"); }, + }); + h.persistedSendAttempted = true; // markSendAttempted got there first + await runIntakeWorker(h.deps); + + assert.deepEqual(h.sendReads, ["row-1"], "the flag was re-read"); + assert.equal(h.states.length, 1); + assert.equal(h.states[0].state, "NEEDS_REVIEW"); + assert.ok( + !("dedupStrongKey" in (h.states[0].patch ?? {})), + "the key is RETAINED: a Purchase may exist", + ); +}); + +test("a park with nothing ever sent still releases the key", async () => { + // The control. Holding a key against a booking that never happened sends + // the corrected resubmission to a human for no reason. + const h = harness([workerRow({ state: "READ", dryRun: false, sendAttempted: false, attempts: 19 })], { + book: async () => { throw new Error("connection reset"); }, + }); + h.persistedSendAttempted = false; + await runIntakeWorker(h.deps); + assert.equal(h.states[0].patch?.dedupStrongKey, null); +}); + +test("an unreadable send flag RETAINS the key", async () => { + // Retaining costs a review item; releasing wrongly costs a second Purchase. + const h = harness([workerRow({ state: "READ", dryRun: false, sendAttempted: false, attempts: 19 })], { + book: async () => { throw new Error("boom"); }, + sendAttemptedNow: async () => { throw new Error("db is down"); }, + }); + await runIntakeWorker(h.deps); + assert.ok(!("dedupStrongKey" in (h.states[0].patch ?? {}))); +}); diff --git a/tests/receipt-url.test.ts b/tests/receipt-url.test.ts new file mode 100644 index 000000000..16134109e --- /dev/null +++ b/tests/receipt-url.test.ts @@ -0,0 +1,130 @@ +/** + * Expense.receiptUrl holds a REFERENCE, not a link. + * + * A signed URL written into the column is dead ten minutes later — the receipt + * link in the books stops working and nothing says why. A bare storage path + * says nothing about which bucket it is in, which is the ambiguity that had + * receipts and signed contracts sharing one. So the column holds + * `receipt-intake:///` and every reader mints its own short-lived + * URL from it. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { + RECEIPT_URL_SCHEME, + isReceiptUrlRef, + parseReceiptUrl, + receiptUrlRef, + resolveReceiptUrl, +} from "../src/lib/receipt-intake/receipt-url"; +import { RECEIPT_BUCKET } from "../src/lib/receipt-intake/bucket"; + +const PATH = "receipts/intake/row-1.png"; +const REF = `${RECEIPT_URL_SCHEME}${RECEIPT_BUCKET}/${PATH}`; + +test("the reference names the bucket AND the path, and round-trips", () => { + assert.equal(receiptUrlRef(PATH), REF); + assert.deepEqual(parseReceiptUrl(REF), { bucket: RECEIPT_BUCKET, path: PATH }); + assert.ok(isReceiptUrlRef(REF)); +}); + +test("anything that is not our reference is not ours to resolve", async () => { + for (const value of [ + null, undefined, "", + "https://evil.test/receipt.png", + "data:image/png;base64,AAAA", + "receipts/intake/row-1.png", + "secure:receipts/intake/row-1.png", + // Another bucket is refused even under our scheme: this string comes + // out of a database column and ends up in a storage API call. + `${RECEIPT_URL_SCHEME}secure-docs/${PATH}`, + `${RECEIPT_URL_SCHEME}${RECEIPT_BUCKET}/../secure-docs/contract.pdf`, + `${RECEIPT_URL_SCHEME}${RECEIPT_BUCKET}//etc/passwd`, + `${RECEIPT_URL_SCHEME}${RECEIPT_BUCKET}/`, + ]) { + assert.equal(parseReceiptUrl(value as string), null, String(value)); + let signed = 0; + const out = await resolveReceiptUrl(value as string, 600, { + sign: async () => { signed++; return "https://signed.test/x"; }, + currentPath: async () => null, + }); + assert.equal(out, null, String(value)); + assert.equal(signed, 0, `${value}: storage was never asked`); + } +}); + +test("a live object is signed for a SHORT window", async () => { + const asked: Array<[string, number]> = []; + const url = await resolveReceiptUrl(REF, 600, { + sign: async (p, ttl) => { asked.push([p, ttl]); return `https://signed.test/${p}`; }, + currentPath: async () => null, + }); + assert.equal(url, `https://signed.test/${PATH}`); + assert.deepEqual(asked, [[PATH, 600]]); +}); + +test("a MOVED object is followed to where the intake row points now", async () => { + // The object moves after the Expense is written: published at the upload + // path, sealed to a content-addressed one, later archived. A reference that + // no longer resolves is re-asked of the row that tracks the bytes. + const sealed = "receipts/row-1/abc123.png"; + const asked: string[] = []; + const url = await resolveReceiptUrl(REF, 600, { + sign: async p => { asked.push(p); return p === sealed ? `https://signed.test/${p}` : null; }, + currentPath: async () => sealed, + }); + assert.deepEqual(asked, [PATH, sealed], "the stored path first, then where it moved to"); + assert.equal(url, `https://signed.test/${sealed}`); +}); + +test("an object that is really gone resolves to null, not to a broken link", async () => { + const gone = await resolveReceiptUrl(REF, 600, { + sign: async () => null, + currentPath: async () => null, + }); + assert.equal(gone, null); + + // And a lookup that points back at the same dead path is not retried. + let signs = 0; + await resolveReceiptUrl(REF, 600, { + sign: async () => { signs++; return null; }, + currentPath: async () => PATH, + }); + assert.equal(signs, 1); +}); + +test("neither leg can take a page down", async () => { + const out = await resolveReceiptUrl(REF, 600, { + sign: async () => { throw new Error("storage is down"); }, + currentPath: async () => { throw new Error("db is down"); }, + }); + assert.equal(out, null); +}); + +test("every reader resolves the reference: the booker writes it, resolveDocUrl reads it", () => { + const root = path.resolve(__dirname, ".."); + const book = readFileSync(path.join(root, "src/lib/receipt-intake/book.ts"), "utf8"); + assert.ok(/\s{20}receiptUrl,/.test(book), "the Expense is written with it"); + assert.match(book, /receiptUrlRef\(row\.storagePath\)/); + assert.ok(!/createSignedUrl/.test(book), "the booker never mints a link into the column"); + + // resolveDocUrl is the shared reader, so everything already going through + // it resolves the new scheme without learning about it. + const storage = readFileSync(path.join(root, "src/lib/secure-storage.ts"), "utf8"); + assert.match(storage, /if \(isReceiptUrlRef\(stored\)\) return await resolveReceiptUrl\(stored, ttlSeconds\)/); + + // The two readers that do NOT go through it. + const tab = readFileSync(path.join(root, "src/lib/time-expense-actions.ts"), "utf8"); + assert.match(tab, /isReceiptUrlRef\(expense\.receiptUrl\)/); + assert.match(tab, /await resolveReceiptUrl\(expense\.receiptUrl\)/); + + const aiReview = readFileSync(path.join(root, "src/app/api/automation/ai-review/route.ts"), "utf8"); + assert.match(aiReview, /const receiptUrl = isReceiptUrlRef\(expense\.receiptUrl\)/); + // The SSRF check still stands, on the RESOLVED url, and now names the + // signed-object prefix too. + assert.match(aiReview, /storageRoot\}sign\//); + assert.match(aiReview, /allowed\.some\(prefix => receiptUrl\.startsWith\(prefix\)\)/); + assert.match(aiReview, /fetch\(receiptUrl, \{ redirect: "error"/); +}); From 390a62e9538d5c4c3066634cd0077ab3cbddea5a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 09:18:11 -0700 Subject: [PATCH 060/144] feat(receipts): version the upload lease so a resumed upload cannot be rejected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-14 item 1. `ReceiptIntake.uploadLeaseVersion` (schema + migration + apply script + verifier) is bumped every time a signed URL is issued, and it is IN the path that URL points at: `receipts/intake/.v.`. - /start claims the lease in ONE checked update — version, expiry and path move together — BEFORE it signs anything, on both the resume and the re-arm path. A 0-row update is a 409 publish-conflict rather than a URL for a row somebody else has moved on, and the previous lease's object is queued for cleanup. - Every destructive or publishing write fences on the version it observed: both sweeper parks, the sweeper's publish commit, the reject transaction, and publishFence (so /finalize and the single-shot heal carry it too). - The reject transaction now RE-READS the row inside itself and lets the caller judge it; the sweeper passes a verifier that refuses while the upload lease is live. The version catches a resumed lease; the re-read catches a refreshed expiry on the same one. Tests: the real interleaving (sweep decides on v1, client resumes to v2 → the fence loses, nothing deleted, nothing queued), a lease that comes back to life inside the transaction, an unchanged-lease control, and a guard that /start moves the row before it signs on both paths. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-1-INTAKE-CORE-SPEC.md | 7 ++ .../migration.sql | 2 + prisma/schema.prisma | 7 ++ scripts/apply-receipt-intake.mjs | 4 +- .../api/cron/receipt-intake-worker/route.ts | 36 ++++++- .../receipts/intake/[id]/finalize/route.ts | 3 +- src/app/api/receipts/intake/route.ts | 1 + src/app/api/receipts/intake/start/route.ts | 77 +++++++++++---- src/lib/receipt-intake/storage-cleanup.ts | 21 ++++ src/lib/receipt-intake/stored-object.ts | 19 +++- tests/receipt-intake-reject.test.ts | 97 ++++++++++++++++++- tests/receipt-intake-stored-object.test.ts | 34 ++++--- 12 files changed, 264 insertions(+), 44 deletions(-) diff --git a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md index df03c4a92..16a4896ff 100644 --- a/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md +++ b/docs/plans/PHASE-1-INTAKE-CORE-SPEC.md @@ -105,6 +105,12 @@ model ReceiptIntake { // than its lease, and judging it on row age parked receipts whose own upload // link was still live. Null on rows that never had a signed URL. uploadUrlExpiresAt DateTime? + // Bumped every time a signed upload URL is issued, and EMBEDDED IN THE PATH + // that URL points at (`receipts/intake/.v.`). /start claims the + // new lease in ONE checked update before it signs anything; every park, + // publish and reject fences on the version it observed. That is what makes a + // sweep verdict about v1 land on nothing once the client has resumed on v2. + uploadLeaseVersion Int @default(0) fileName String? mimeType String fileSize Int @@ -164,6 +170,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "storagePath" TEXT NOT NULL, "fileName" TEXT, "mimeType" TEXT NOT NULL, "fileSize" INTEGER NOT NULL, "fileSha256" TEXT NOT NULL, "expectedSha256" TEXT, "uploadUrlExpiresAt" TIMESTAMP(3), + "uploadLeaseVersion" INTEGER NOT NULL DEFAULT 0, "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, "taxCents" INTEGER, "docType" TEXT, "refNumber" TEXT, "memo" TEXT, "readJson" TEXT, "readAt" TIMESTAMP(3), "dedupStrongKey" TEXT, "dedupWeakKey" TEXT, "duplicateOfId" TEXT, diff --git a/prisma/migrations/20260901000000_receipt_intake/migration.sql b/prisma/migrations/20260901000000_receipt_intake/migration.sql index 8ba0187e7..4cb7325a2 100644 --- a/prisma/migrations/20260901000000_receipt_intake/migration.sql +++ b/prisma/migrations/20260901000000_receipt_intake/migration.sql @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( "fileSha256" TEXT NOT NULL, "expectedSha256" TEXT, "uploadUrlExpiresAt" TIMESTAMP(3), + "uploadLeaseVersion" INTEGER NOT NULL DEFAULT 0, "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -66,6 +67,7 @@ CREATE TABLE IF NOT EXISTS "ReceiptIntake" ( ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadUrlExpiresAt" TIMESTAMP(3); +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadLeaseVersion" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c9cf9bc96..aa81d76a8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -3059,6 +3059,13 @@ model ReceiptIntake { /// parking it on row age declared a receipt missing while its own upload link /// was live. Null on rows that never had a signed URL (the single-shot path). uploadUrlExpiresAt DateTime? + /// Bumped every time a signed upload URL is issued for this row, and embedded + /// in the path that URL points at. It is what makes "the upload the sweeper + /// looked at" and "the upload the client is doing now" different things: a + /// resumed /start moves the row to v2 while a sweep is still deciding about + /// v1, and every destructive write fences on the version it observed, so the + /// sweep's verdict lands on nothing. + uploadLeaseVersion Int @default(0) // read results (cents, like AutomationEvent) vendor String? diff --git a/scripts/apply-receipt-intake.mjs b/scripts/apply-receipt-intake.mjs index 992d456d6..d6415a88f 100644 --- a/scripts/apply-receipt-intake.mjs +++ b/scripts/apply-receipt-intake.mjs @@ -99,6 +99,7 @@ export const statements = [ "fileSha256" TEXT NOT NULL, "expectedSha256" TEXT, "uploadUrlExpiresAt" TIMESTAMP(3), + "uploadLeaseVersion" INTEGER NOT NULL DEFAULT 0, "vendor" TEXT, "txnDate" DATE, "totalCents" INTEGER, @@ -135,6 +136,7 @@ export const statements = [ `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "busyPasses" INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "expectedSha256" TEXT`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadUrlExpiresAt" TIMESTAMP(3)`, + `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "uploadLeaseVersion" INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "sendAttempted" BOOLEAN NOT NULL DEFAULT false`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "archivedByV1" BOOLEAN NOT NULL DEFAULT false`, `ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "claimToken" TEXT`, @@ -272,7 +274,7 @@ const expectedColumns = { "id", "source", "sourceRef", "state", "dryRun", "stateReason", "projectId", "costCodeId", "suggestedCostCodeId", "suggestedConfidence", "createdById", "storagePath", "fileName", "mimeType", "fileSize", - "fileSha256", "expectedSha256", "uploadUrlExpiresAt", + "fileSha256", "expectedSha256", "uploadUrlExpiresAt", "uploadLeaseVersion", "sendAttempted", "archivedByV1", "vendor", "txnDate", "totalCents", "taxCents", "docType", "refNumber", "memo", "readJson", "readAt", "dedupStrongKey", diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index f11e007f7..da437d2ac 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -317,6 +317,7 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { select: { id: true, storagePath: true, mimeType: true, stateReason: true, createdAt: true, expectedSha256: true, uploadUrlExpiresAt: true, + uploadLeaseVersion: true, }, // Small on purpose: each row costs a storage round trip, and the // sweep runs BEFORE any receipt is processed. A big batch here @@ -366,7 +367,15 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // row already gone from STAGING. if (leaseLive) { leaseActive++; continue; } await prisma.receiptIntake.updateMany({ - where: { id: row.id, state: "STAGING" }, + // Fenced on the lease this verdict was reached + // about: a resumed /start bumps it, and the bytes + // that mismatched belong to an upload nobody is + // waiting for any more. + where: { + id: row.id, + state: "STAGING", + uploadLeaseVersion: row.uploadLeaseVersion, + }, data: { state: "NEEDS_REVIEW", stateReason: "sha-mismatch", nextRetryAt: null }, }); parked++; @@ -382,7 +391,11 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { seal: sealObject, commit: async (canonicalPath, values) => { const { count } = await prisma.receiptIntake.updateMany({ - where: { id: row.id, state: "STAGING" }, + where: { + id: row.id, + state: "STAGING", + uploadLeaseVersion: row.uploadLeaseVersion, + }, data: { state: "RECEIVED", nextRetryAt: null, @@ -409,7 +422,11 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { // queue. Wait until the URL cannot possibly land any more. if (leaseLive) { leaseActive++; continue; } await prisma.receiptIntake.updateMany({ - where: { id: row.id, state: "STAGING" }, + where: { + id: row.id, + state: "STAGING", + uploadLeaseVersion: row.uploadLeaseVersion, + }, data: { state: "NEEDS_REVIEW", stateReason: "file-missing", nextRetryAt: null }, }); parked++; @@ -433,8 +450,21 @@ function buildDeps(invocationDeadline: RouteDeadline): WorkerDependencies { state: "STAGING", stateReason: row.stateReason, storagePath: row.storagePath, + uploadLeaseVersion: row.uploadLeaseVersion, }, check.reason, + undefined, + // DECIDED ON A ROW RE-READ INSIDE THE TRANSACTION. + // + // Between the inspection above and this delete a client can + // resume its upload: /start bumps the lease and hands out a + // fresh URL. The fence catches the version, and this catches + // the case the fence cannot see — a lease that is live again + // — so a receipt in flight is never deleted for what its + // previous attempt left at the path. + fresh => uploadLeaseActive(fresh as { uploadUrlExpiresAt: Date | null; createdAt: Date }) + ? "upload-lease-active" + : null, ); // FENCE LOST: somebody else owns this row now. Touch NOTHING — // above all not the object, which the winner may be using. diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 396c69ce8..41b453ea8 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -145,7 +145,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string select: { id: true, source: true, state: true, stateReason: true, sourceRef: true, storagePath: true, mimeType: true, projectId: true, costCodeId: true, dryRun: true, createdById: true, - fileSha256: true, expectedSha256: true, + fileSha256: true, expectedSha256: true, uploadLeaseVersion: true, }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); @@ -257,6 +257,7 @@ export async function POST(req: Request, context: { params: Promise<{ id: string state: row.state, stateReason: row.stateReason, storagePath: row.storagePath, + uploadLeaseVersion: row.uploadLeaseVersion, }, check.reason, ); diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index f4d3d7fa1..962976c87 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -464,6 +464,7 @@ async function respondToSourceRefConflict( select: { id: true, state: true, source: true, sourceRef: true, projectId: true, dryRun: true, fileSha256: true, createdById: true, storagePath: true, stateReason: true, + uploadLeaseVersion: true, }, }); // The row vanished between the failed insert and this read (a delete diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index 48633b65a..f8da8601f 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -8,7 +8,7 @@ import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { uploadLeaseExpiry } from "@/lib/receipt-intake/worker"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; -import { finalizeDisposition, publishFence } from "@/lib/receipt-intake/stored-object"; +import { finalizeDisposition, publishFence, uploadPathFor } from "@/lib/receipt-intake/stored-object"; import { createReceiptUploadUrl } from "@/lib/receipt-intake/bucket"; import { deleteObjectOrRecord } from "@/lib/receipt-intake/storage-cleanup"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; @@ -130,7 +130,9 @@ export async function POST(req: Request) { if (badPhase) return NextResponse.json(badPhase.body, { status: badPhase.status }); const id = randomUUID(); - const storagePath = `receipts/intake/${id}.${ext}`; + // Lease 1 from the outset: the version is part of the path, so there is no + // "version 0" object to confuse with a resumed upload later. + const storagePath = uploadPathFor(id, 1, ext); let created: { id: string; sourceRef: string; state: string }; try { @@ -160,6 +162,7 @@ export async function POST(req: Request) { // works, so nothing may declare the object missing or reject // the row for what is at that path. uploadUrlExpiresAt: uploadLeaseExpiry(), + uploadLeaseVersion: 1, }, select: { id: true, sourceRef: true, state: true }, }); @@ -172,6 +175,7 @@ export async function POST(req: Request) { select: { id: true, sourceRef: true, state: true, stateReason: true, storagePath: true, createdById: true, expectedSha256: true, fileSha256: true, + uploadLeaseVersion: true, }, }); if (!existing) return NextResponse.json({ ok: false, reason: "conflict-retry" }, { status: 409 }); @@ -198,21 +202,23 @@ export async function POST(req: Request) { const recoverable = existing.state !== "STAGING" && finalizeDisposition(existing) === "publish"; if (recoverable) { - const retryPath = `receipts/intake/${existing.id}.${ext}`; - const rearmed = await signUpload(retryPath); - if (!rearmed) { - return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - } - // Fenced on the EXACT park, same rule as every other publish - // path: losing it means somebody moved the row while we were - // signing, and re-arming it then would point a live row at an - // empty path. + // THE ROW MOVES FIRST, THEN THE URL IS SIGNED. + // + // The claim on the lease is made in ONE checked update: the + // version goes up, the expiry is refreshed, and the row is + // re-pointed at the path that version names. Signing first and + // writing after left a window where a sweep could reject the + // row for the OLD upload while a URL for the new one was + // already in the client's hands. + const nextLease = existing.uploadLeaseVersion + 1; + const retryPath = uploadPathFor(existing.id, nextLease, ext); const { count } = await prisma.receiptIntake.updateMany({ where: { id: existing.id, ...publishFence(existing) }, data: { storagePath: retryPath, expectedSha256, uploadUrlExpiresAt: uploadLeaseExpiry(), + uploadLeaseVersion: nextLease, // The stored hash is what /finalize verifies against. // Whatever was recorded describes bytes that are gone // or were never right. @@ -234,8 +240,11 @@ export async function POST(req: Request) { { status: 409 }, ); } - // A different declared type means a different extension, so the - // old object (if any) is now unreferenced. + const rearmed = await signUpload(retryPath); + if (!rearmed) { + return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + } + // The previous lease's object (if any) is unreferenced now. if (retryPath !== existing.storagePath) { await deleteObjectOrRecord(existing.storagePath, "start-rearmed-repath"); } @@ -278,15 +287,41 @@ export async function POST(req: Request) { { ok: true, alreadyReceived: true, id: existing.id, state: existing.state }, ); } - const resumed = await signUpload(existing.storagePath); - if (!resumed) return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); - // A RESUMED url is a new lease. Without this the sweeper still - // judged the row by its original createdAt and could park (or - // reject) it while the URL just handed out was live. - await prisma.receiptIntake.updateMany({ - where: { id: existing.id, state: "STAGING" }, - data: { uploadUrlExpiresAt: uploadLeaseExpiry() }, + // A RESUME IS A NEW LEASE, taken BEFORE the URL is signed and in one + // checked update. Without the version bump the sweep and the client + // are talking about the same path, so a sweep that started before + // this call could still reject the upload it is now waiting for. + const nextLease = existing.uploadLeaseVersion + 1; + const resumePath = uploadPathFor(existing.id, nextLease, ext); + const { count } = await prisma.receiptIntake.updateMany({ + where: { + id: existing.id, + state: "STAGING", + uploadLeaseVersion: existing.uploadLeaseVersion, + }, + data: { + storagePath: resumePath, + uploadLeaseVersion: nextLease, + uploadUrlExpiresAt: uploadLeaseExpiry(), + }, }); + if (count === 0) { + return NextResponse.json( + { + ok: false, + error: "publish-conflict", + reason: "this row changed while a new upload URL was being issued; retry", + retryable: true, + existingId: existing.id, + }, + { status: 409 }, + ); + } + const resumed = await signUpload(resumePath); + if (!resumed) return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); + if (resumePath !== existing.storagePath) { + await deleteObjectOrRecord(existing.storagePath, "start-resumed-repath"); + } return NextResponse.json({ ok: true, resumed: true, id: existing.id, maxBytes: MAX_STORED_BYTES, ...resumed, }); diff --git a/src/lib/receipt-intake/storage-cleanup.ts b/src/lib/receipt-intake/storage-cleanup.ts index cc8395d93..d251bbbb9 100644 --- a/src/lib/receipt-intake/storage-cleanup.ts +++ b/src/lib/receipt-intake/storage-cleanup.ts @@ -98,6 +98,7 @@ export interface RejectTxClient { automationEvent: { create(args: { data: Record; select: { id: true } }): Promise<{ id: string }> }; receiptIntake: { deleteMany(args: { where: Record }): Promise<{ count: number }>; + findUnique(args: { where: { id: string } }): Promise | null>; }; } @@ -113,6 +114,8 @@ export interface RejectFence { state: string; stateReason: string | null; storagePath: string; + /** The upload lease the caller inspected. A newer one means a newer file. */ + uploadLeaseVersion: number; } export interface RejectClient { $transaction(fn: (tx: RejectTxClient) => Promise): Promise; @@ -122,6 +125,14 @@ export async function rejectRowAndQueueCleanup( row: RejectFence, reason: string, db: RejectClient = prisma as unknown as RejectClient, + /** + * Re-checked against a FRESH read inside the transaction. The caller spent + * a storage round trip deciding this row was unacceptable; anything that + * changed in the meantime (a resumed upload lease, a re-park) has to be + * judged on the row as it is NOW, not as it was when the decision started. + * Return a reason to abort, or null to proceed. + */ + verify: (fresh: Record) => string | null = () => null, ): Promise<{ ok: true; eventId: string } | { ok: false }> { try { const eventId = await db.$transaction(async tx => { @@ -148,6 +159,12 @@ export async function rejectRowAndQueueCleanup( // "Already gone" is deliberately NOT treated as success: an absent // row is a row somebody else accounted for, and queueing its path // for deletion here is how a live object gets swept. + // RE-READ INSIDE THE TRANSACTION, and let the caller judge it. + const fresh = await tx.receiptIntake.findUnique({ where: { id: row.id } }); + if (!fresh) throw new RejectFenceLost(row.id); + const objection = verify(fresh); + if (objection) throw new RejectFenceLost(`${row.id}: ${objection}`); + const { count } = await tx.receiptIntake.deleteMany({ where: { id: row.id, @@ -155,6 +172,10 @@ export async function rejectRowAndQueueCleanup( stateReason: row.stateReason, claimToken: null, storagePath: row.storagePath, + // The lease the caller INSPECTED. A resumed /start bumps + // this and re-points the row at a new path, so a sweep + // that decided on the old upload deletes nothing. + uploadLeaseVersion: row.uploadLeaseVersion, }, }); if (count !== 1) throw new RejectFenceLost(row.id); diff --git a/src/lib/receipt-intake/stored-object.ts b/src/lib/receipt-intake/stored-object.ts index 21510e6a5..dad40c123 100644 --- a/src/lib/receipt-intake/stored-object.ts +++ b/src/lib/receipt-intake/stored-object.ts @@ -209,6 +209,12 @@ export const RECOVERABLE_PARK_REASONS = ["file-missing", "sha-mismatch"]; export interface ObservedRow { state: string; stateReason: string | null; + /** + * The upload lease this decision was made against. A resumed or re-armed + * /start bumps it, so a sweep that decided on v1 writes nothing once the + * client is on v2 — the row it judged does not exist any more. + */ + uploadLeaseVersion: number; } /** What a finalize may do with the row it just read. */ @@ -238,6 +244,17 @@ export function publishFence(row: ObservedRow): { state: string; stateReason: string | null; claimToken: null; + uploadLeaseVersion: number; } { - return { state: row.state, stateReason: row.stateReason, claimToken: null }; + return { + state: row.state, + stateReason: row.stateReason, + claimToken: null, + uploadLeaseVersion: row.uploadLeaseVersion, + }; +} + +/** Where the bytes for one upload lease live. The version is IN the path. */ +export function uploadPathFor(rowId: string, leaseVersion: number, ext: string): string { + return `receipts/intake/${rowId}.v${leaseVersion}.${ext}`; } diff --git a/tests/receipt-intake-reject.test.ts b/tests/receipt-intake-reject.test.ts index 15a427f98..4a030f0bd 100644 --- a/tests/receipt-intake-reject.test.ts +++ b/tests/receipt-intake-reject.test.ts @@ -40,7 +40,12 @@ const parked = (over: Row = {}): Row => ({ state: "STAGING", stateReason: null, claimToken: null, - storagePath: "receipts/intake/row-1.bin", + storagePath: "receipts/intake/row-1.v1.bin", + // The upload lease this decision was reached about. A resumed /start bumps + // it, which is what makes a stale verdict land on nothing. + uploadLeaseVersion: 1, + uploadUrlExpiresAt: null, + createdAt: new Date("2026-09-01T00:00:00.000Z"), ...over, }); @@ -69,6 +74,7 @@ function client(rows: Row[], onTx?: (store: Store) => void): { db: RejectClient; }, }, receiptIntake: { + findUnique: async ({ where }) => staged.find(row => row.id === where.id) ?? null, deleteMany: async ({ where }) => { const matches = staged.filter(row => Object.entries(where).every(([k, v]) => row[k] === v)); @@ -94,7 +100,7 @@ test("a reject deletes the row and queues the object in ONE transaction", async assert.deepEqual(store.rows, [], "the row is gone"); assert.equal(store.events.length, 1, "and exactly one cleanup is queued"); assert.equal(store.events[0].data.status, "pending"); - assert.match(String(store.events[0].data.detail), /receipts\/intake\/row-1\.bin/); + assert.match(String(store.events[0].data.detail), /receipts\/intake\/row-1\.v1\.bin/); }); test("PUBLISH vs REJECT: a row published mid-reject is not deleted, and nothing is queued", async () => { @@ -297,3 +303,90 @@ test("nothing destructive happens while the upload lease is live", () => { "only the sha-mismatch park inside the ok branch waits", ); }); + +// ── RESUME vs REJECT: the upload lease version decides (round-14 item 1) ─── + +test("a client that RESUMES its upload mid-sweep is not rejected for the old one", async () => { + // The real interleaving: the sweep reads a stale STAGING row, spends a + // storage round trip on the object the client abandoned, and decides to + // reject. In that window /start hands the client a fresh URL — a new lease + // version, a new path, a live expiry. Without the version in the fence the + // sweep deletes the row (and queues its object) for a receipt that is + // actively being uploaded, and the forwarder is told nothing. + const observed = parked({ uploadLeaseVersion: 1, storagePath: "receipts/intake/row-1.v1.bin" }); + const { db, store } = client([observed], s => { + s.rows = [parked({ + uploadLeaseVersion: 2, + storagePath: "receipts/intake/row-1.v2.bin", + uploadUrlExpiresAt: new Date(Date.now() + 60 * 60_000), + })]; + }); + + const dropped = await rejectRowAndQueueCleanup(observed as never, "unsupported-file-type", db); + assert.equal(dropped.ok, false, "the fence lost to the newer lease"); + assert.equal(store.committed, false); + assert.deepEqual(store.events, [], "the v1 object is not queued for deletion by this pass"); + assert.equal(store.rows.length, 1, "and the row survives"); + assert.equal(store.rows[0].uploadLeaseVersion, 2); +}); + +test("a lease that came back to life inside the transaction aborts the reject", async () => { + // The version alone cannot see this one: /start refreshed the EXPIRY on the + // same lease. The verifier runs on a row re-read inside the transaction, + // which is the only place that is true. + const observed = parked(); + const { db, store } = client([observed], s => { + s.rows = [parked({ uploadUrlExpiresAt: new Date(Date.now() + 60 * 60_000) })]; + }); + const dropped = await rejectRowAndQueueCleanup( + observed as never, + "unsupported-file-type", + db, + fresh => (fresh.uploadUrlExpiresAt as Date | null) && + (fresh.uploadUrlExpiresAt as Date).getTime() > Date.now() + ? "upload-lease-active" + : null, + ); + assert.equal(dropped.ok, false); + assert.deepEqual(store.events, []); + assert.equal(store.rows.length, 1); +}); + +test("an unchanged lease still rejects — the control", async () => { + const observed = parked(); + const { db, store } = client([observed]); + const dropped = await rejectRowAndQueueCleanup(observed as never, "unsupported-file-type", db, () => null); + assert.equal(dropped.ok, true); + assert.deepEqual(store.rows, []); + assert.equal(store.events.length, 1); +}); + +test("the sweeper and /start both fence on the lease version", () => { + const start = readFileSync( + path.join(ROOT, "src/app/api/receipts/intake/start/route.ts"), + "utf8", + ); + // /start claims the new lease BEFORE it signs anything: the version, the + // expiry and the path move in ONE checked update, and a lost update is a + // 409 rather than a URL for a row somebody else has moved on. + assert.equal((start.match(/uploadLeaseVersion: nextLease/g) ?? []).length, 2, "resume and re-arm"); + assert.equal((start.match(/const nextLease = existing\.uploadLeaseVersion \+ 1/g) ?? []).length, 2); + for (const branch of ["const rearmed = await signUpload(retryPath)", "const resumed = await signUpload(resumePath)"]) { + const at = start.indexOf(branch); + assert.ok(at > 0, branch); + const update = start.lastIndexOf("await prisma.receiptIntake.updateMany(", at); + assert.ok(update > 0 && update < at, `${branch}: the row moves before the URL is signed`); + } + assert.equal((start.match(/error: "publish-conflict"/g) ?? []).length, 2, "a lost claim is a 409 on both"); + + // Every destructive sweeper write carries the version it observed. + const fn = sweeper.slice(sweeper.indexOf("sweepStaleStaging: async")); + const body = fn.slice(0, fn.indexOf("loadPhases:")); + assert.equal( + (body.match(/uploadLeaseVersion: row\.uploadLeaseVersion/g) ?? []).length, + 4, + "the two parks, the publish commit and the reject", + ); + // ...and the reject also re-reads the row inside the transaction. + assert.match(body, /fresh => uploadLeaseActive\(/); +}); diff --git a/tests/receipt-intake-stored-object.test.ts b/tests/receipt-intake-stored-object.test.ts index 0d6135ebf..03cfb0029 100644 --- a/tests/receipt-intake-stored-object.test.ts +++ b/tests/receipt-intake-stored-object.test.ts @@ -308,35 +308,35 @@ test("the sweeper's two parks are the ones /finalize recovers from", () => { // ── Which parks a re-upload may clear, and the fence it publishes under ───── test("only the two SWEEPER parks are recoverable; a human's park is not", () => { - assert.equal(finalizeDisposition({ state: "STAGING", stateReason: null }), "publish"); + assert.equal(finalizeDisposition({ state: "STAGING", stateReason: null, uploadLeaseVersion: 1 }), "publish"); for (const reason of RECOVERABLE_PARK_REASONS) { - assert.equal(finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason }), "publish", reason); + assert.equal(finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason, uploadLeaseVersion: 1 }), "publish", reason); } // Everything else parked for review is somebody's decision. Republishing it // drags the row back to RECEIVED and re-reads it, discarding that decision. for (const reason of ["vendor-mismatch", "weak-dup:row-9", "qbo-fault:6210", "amount-mismatch", null]) { assert.equal( - finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason }), + finalizeDisposition({ state: "NEEDS_REVIEW", stateReason: reason, uploadLeaseVersion: 1 }), "not-recoverable", String(reason), ); } // And a row that already moved on is simply settled — not an error. for (const state of ["RECEIVED", "READ", "BOOKING", "BOOKED", "ARCHIVED", "DUPLICATE"]) { - assert.equal(finalizeDisposition({ state, stateReason: null }), "settled", state); + assert.equal(finalizeDisposition({ state, stateReason: null, uploadLeaseVersion: 1 }), "settled", state); } }); test("the publish fence pins the exact state, the exact reason and an unclaimed row", () => { - assert.deepEqual(publishFence({ state: "NEEDS_REVIEW", stateReason: "file-missing" }), { + assert.deepEqual(publishFence({ state: "NEEDS_REVIEW", stateReason: "file-missing", uploadLeaseVersion: 1 }), { state: "NEEDS_REVIEW", stateReason: "file-missing", - claimToken: null, + claimToken: null, uploadLeaseVersion: 1, }); - assert.deepEqual(publishFence({ state: "STAGING", stateReason: null }), { + assert.deepEqual(publishFence({ state: "STAGING", stateReason: null, uploadLeaseVersion: 1 }), { state: "STAGING", stateReason: null, - claimToken: null, + claimToken: null, uploadLeaseVersion: 1, }); }); @@ -362,9 +362,13 @@ test("RACE: a reason that changes during sealing loses the publish, and writes n // would reset a reason it never looked at back to RECEIVED — discarding the // newer decision and republishing a row somebody else now owns. const store = rowStore({ - id: "row-1", state: "NEEDS_REVIEW", stateReason: "file-missing", claimToken: null, + id: "row-1", state: "NEEDS_REVIEW", stateReason: "file-missing", claimToken: null, uploadLeaseVersion: 1, }); - const observed = { state: store.get().state as string, stateReason: store.get().stateReason as string }; + const observed = { + state: store.get().state as string, + stateReason: store.get().stateReason as string, + uploadLeaseVersion: store.get().uploadLeaseVersion as number, + }; assert.equal(finalizeDisposition(observed), "publish", "it was recoverable when we read it"); const fence = publishFence(observed); @@ -392,9 +396,9 @@ test("RACE: a reason that changes during sealing loses the publish, and writes n test("RACE: a worker claim taken during sealing also loses the publish", async () => { const store = rowStore({ - id: "row-1", state: "STAGING", stateReason: null, claimToken: null, + id: "row-1", state: "STAGING", stateReason: null, claimToken: null, uploadLeaseVersion: 1, }); - const fence = publishFence({ state: "STAGING", stateReason: null }); + const fence = publishFence({ state: "STAGING", stateReason: null, uploadLeaseVersion: 1 }); const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, { seal: async (_u: string, canonical: string) => { store.set({ claimToken: "sweeper-1" }); @@ -409,12 +413,12 @@ test("RACE: a worker claim taken during sealing also loses the publish", async ( test("an unchanged row still publishes — the control", async () => { const store = rowStore({ - id: "row-1", state: "NEEDS_REVIEW", stateReason: "sha-mismatch", claimToken: null, + id: "row-1", state: "NEEDS_REVIEW", stateReason: "sha-mismatch", claimToken: null, uploadLeaseVersion: 1, }); - const fence = publishFence({ state: "NEEDS_REVIEW", stateReason: "sha-mismatch" }); + const fence = publishFence({ state: "NEEDS_REVIEW", stateReason: "sha-mismatch", uploadLeaseVersion: 1 }); const outcome = await sealAndPublish("receipts/intake/a.png", "row-1", CHECK, { seal: async (_u: string, canonical: string) => canonical, - commit: async () => store.updateMany({ id: "row-1", ...fence }, { state: "RECEIVED", stateReason: null }), + commit: async () => store.updateMany({ id: "row-1", ...fence }, { state: "RECEIVED", stateReason: null, uploadLeaseVersion: 1 }), dropUpload: async () => {}, } as never); assert.equal(outcome?.published, true); From 23f1f9e36ecc0a6f638794be22a429666aa0fc16 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:36:38 -0700 Subject: [PATCH 061/144] =?UTF-8?q?feat(expenses):=20Phase=203=20attributi?= =?UTF-8?q?on=20schema=20=E2=80=94=20Expense.projectId,=20tax,=20cost-code?= =?UTF-8?q?=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the columns Phase 3 is built on (spec §2, committed here as docs/plans/PHASE-3-ATTRIBUTION-SPEC.md): Expense.projectId (+FK SET NULL, +index) — the denormalized job every money-path reader will resolve through. estimateId stays the required parent; projectId is backfilled from estimate.projectId by the same idempotent UPDATE in both the apply script and the migration. Expense.taxAmount / taxAtSource / installedAtCustomer — the WA "tax paid at source" deduction inputs. amount stays GROSS (Phase 1's decision), so pre-tax = amount - taxAmount. Expense.costCodeSource / costCodeConfidence — provenance, so an automated suggester can never overwrite a human's code. ReceiptIntake.taxAtSource / installedAtCustomer, behind a to_regclass guard so the script runs in either merge order with Phase 1. tests/apply-expense-attribution.test.ts asserts the script and the migration never drift, that the backfill is predicate-idempotent, and that nothing in the statement list is destructive. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 360 ++++++++++++++++++ package.json | 3 +- .../migration.sql | 53 +++ prisma/schema.prisma | 33 ++ scripts/apply-expense-attribution.mjs | 229 +++++++++++ tests/apply-expense-attribution.test.ts | 118 ++++++ 6 files changed, 795 insertions(+), 1 deletion(-) create mode 100644 docs/plans/PHASE-3-ATTRIBUTION-SPEC.md create mode 100644 prisma/migrations/20260901120000_expense_attribution/migration.sql create mode 100644 scripts/apply-expense-attribution.mjs create mode 100644 tests/apply-expense-attribution.test.ts diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md new file mode 100644 index 000000000..b3ca56738 --- /dev/null +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -0,0 +1,360 @@ +# Phase 3: Attribution — Implementation Spec + +Date: 2026-09-01. Parent plan: `docs/plans/RECEIPT-PIPELINE-V2-PLAN.md` (Phase 3 row; +decisions 4 and 7, "Tax paid at source" section). Sibling: `docs/plans/PHASE-1-INTAKE-CORE-SPEC.md` +(being built on another branch — plan against its spec, not its code). +Planner output for the executor: build exactly this; do not guess. + +## Verified code facts + +- `Expense` (prisma/schema.prisma:580) has NO `projectId` — it reaches a project only via + required `estimateId` (FK, Cascade). Has `itemId?`, `costCodeId?`, `costTypeId?`, + `qbPurchaseId? @unique`, `status` default "Pending". No tax columns of any kind. +- **Every Expense writer today** (grep `expense.create|update|updateMany` in src/): + | writer | file:line | knows project? | sets costCodeId? | + |---|---|---|---| + | QBO sync upsert | `src/lib/qbo-expense-sync.ts:609,612` (via `upsertQboExpense`; `QboExpenseWrite` :490 has NO projectId/costCodeId — `match.projectId` is computed at :434-488 then DROPPED, only `estimateId` survives) | yes (match / overhead bucket) | never | + | QBO sync deactivate | `src/lib/qbo-expense-sync.ts:666` (zeroes amount) | n/a | no | + | v1 Apps Script ingest | `src/app/api/integrations/receipt-ingest/route.ts:108` | yes (`matchProjectByName` :72) | yes via `matchCostCode` :99 (AI-split category from the Apps Script Gemini read) | + | manual/mobile expense API | `src/app/api/expenses/route.ts:98` | yes (`projectId` resolved :29-61) | NO — the route does not even accept a costCodeId field | + | mobile AI parse auto-create | `src/app/api/receipts/parse/route.ts:290` | yes (`projectId` param) | never | + | server-action core (web forms, CO paths) | `src/lib/time-expense-core.ts:184` (`createExpenseCore`; callers in `src/lib/time-expense-actions.ts`) | yes (estimate.projectId resolved :148-152) | yes when caller passes it (human-picked) | + | expense edit API | `src/app/api/expenses/[id]/route.ts:57` | n/a (edit) | yes (human edit) | + | Phase 1 `bookReceipt` | Phase 1 spec §4 `src/lib/receipt-intake/book.ts` (unbuilt) | yes (`ReceiptIntake.projectId`) | yes: chosen or `matchCostCode(suggestedPhaseCode)` | + | non-attribution writers (no change) | `billing-core.ts:1384` (invoice stamp), `qbo-receipt-attachments.ts:76` (receiptUrl), `expenses/[id]/approve:21` + `[id]/receipt:79` (status/receipt), `time-expense-core.ts:243` (CO tag) | — | — | +- **Readers that resolve an expense's project through the estimate** (`estimate: { projectId }`): + `src/lib/job-variance-db.ts:143,191` (+ the :4-6 header comment saying projectId doesn't exist), + `src/lib/project-financials.ts:70`, `src/app/projects/[id]/costing/page.tsx:57` (feeds + JobCostingClient), `src/app/reports/profitability/page.tsx:76-92`, + `src/lib/company-financials-charts.ts:292,307,341`, `src/lib/payouts-report.ts:65`, + `src/lib/transactions-report.ts:92,114`, `src/lib/time-expense-actions.ts:225,306,332`, + `src/app/api/projects/[id]/financial-overview/route.ts:63`, `src/lib/budget-actions.ts:44`, + plus display-only includes (`manager/receipts/page.tsx:19,30,41`, `register-data.ts:89`, + `automation-events.ts:685,840`, `qbo-bank-register.ts:232`, `review-alert-evaluator.ts:65`, + `schedule-core.ts:1978,2212`, `api/ai/cost-forecast:29`, `api/ai/business-summary:35`, + `api/automation/ai-review:355,578`). NOTE: `actions.ts:7431` matches the grep but is an + EstimateItem query, not Expense — leave it alone. There is NO margin-digest in src/ yet + (that is parent-plan Phase 4); the resolver below is what it will consume. +- **Item→costCode fallback** lives in `computeProjectVariance` (`src/lib/job-variance.ts` + ~:279-330): an expense uses its own `costCodeId`, else its `itemId` item's cost code + (via the item pool including "attribution-only" rows, job-variance-db.ts:141-166), else + it lands in `unattributed*` (coverage struct :104-139, `attributedShare`). +- **`src/lib/cost-coding.ts`** = `resolveCostCode(dataSource, {costCodeId?, lineItemId?})`: + explicit-id (validated + isActive) → line-item derivation → REJECT. DI, pure, no AI. + Used at capture time. **`src/lib/project-match.ts`** = fuzzy `matchProjectByName` / + `findBestProjectNameMatches` (used by QBO sync + receipt-ingest) and `matchCostCode` + (category string → cost code). CORRECTION to the task premise: the QBO sync does NOT run + any cost-code AI today — it only project-matches; cost codes on synced expenses are + always null at import. +- **Scripts (2026-08-18/19)**: `scripts/suggest-expense-cost-codes.mjs` — RULE-based (regex + vendor rules + line-keyword rules, NOT a model), dry-run default, `--apply` wrote the + confident matches; scope = active customer jobs excluding "Shop"; header records 555/562 + uncoded before it ran; ~89/562 carry a code today (task premise; treat the exact number + as re-measured by the backfill's before table). Everything ambiguous was left NULL for a + human (CSV). `scripts/backfill-estimate-item-cost-codes.mjs` — coded 15 named ESTIMATE + ITEMS (not expenses) on In Progress jobs, id+name asserted, dry-run default; it widened + the item→costCode fallback's reach, but expenses rarely carry `itemId` (prod: 0/562 per + the job-variance-db.ts:132 comment). +- **Mobile** (`gtr-probuild-mobile`): `apps/mobile/app/(tabs)/expenses.tsx` — picks a + project from `useAuthStore().assignedProjects` (In Progress filter :57), photo → + `POST /api/files/signed-upload` + storage PUT → `api.expenses.create` (=`POST + /api/expenses`) with `{projectId, itemId:null, amount, vendor, date, description, + receiptUrl}` (:176-184). Comment :27: "No phase / cost code attribution required on + mobile". `lib/phasePicker.ts` is the PURE label/selection lib the Time Clock uses + (crew-facing code-only labels); the phase list itself comes over the already-proxied + `/api/projects/[id]/cost-codes` + `/api/projects/[id]/estimate-items` routes + (site `src/proxy.ts:25`). ASSUMPTION: the clocked-in project is readable from the active + entry state (`lib/activeEntry.ts` / snapshot) — executor confirms the accessor in that repo. +- **Tax paid at source**: `I:\My Drive\Expenses\Processed Receipts\Tax Paid at Source\` + holds `TaxPaidAtSource_2012-06.csv` (filename and Date column say 2012 — they are + 2026-06 receipts, e.g. `"2012-06-26","Harbor Freight","Mesplay Kitchen", + "001916749100246","207.74","4.79","0.43","16.55",...`; a date bug in whatever wrote it). + Columns: Date, Vendor, Job, Invoice, Receipt Total, **Material Amount (deduction base)**, + **Recoverable Tax**, **Consumable Tax**, Receipt File. NO code in src/ mentions + tax-paid-at-source or carries tax on Expense today. Phase 1 stores `taxCents` on + ReceiptIntake but its Prisma model has NO `taxAtSource`/`installedAtCustomer` columns — + Phase 3 adds them there too. +- Overhead bucket: `src/lib/overhead-project.ts` (`OVERHEAD_PROJECT_ID`, `isOverheadProject`). +- Existing sync tests: `tests/qbo-expense-sync.test.ts` (node:test; CI is Node 20 — no + `mock.module`, DI only). + +## 1. Goals and acceptance criteria + +1. **Schema live**: `Expense.projectId` (+FK, index), `taxAmount`, `taxAtSource`, + `installedAtCustomer`, `costCodeSource`, `costCodeConfidence`; `ReceiptIntake.taxAtSource` + + `installedAtCustomer`. Verify: `node scripts/apply-expense-attribution.mjs` twice + (second run all "already exists" / 0 rows updated); CI `migrations` job green; + `prisma generate` + `npx tsc --noEmit` clean. +2. **Every writer stamps projectId** (§3): after deploy, a new expense from each writer has + `projectId` set. Verify: writer unit tests + one prod row per path spot-checked. +3. **Capture/manual codes are never overwritten** (§3): the QBO sync and the backfill only + fill NULL `costCodeId` and never touch rows with `costCodeSource` in + ("capture","manual"). Verify: `tests/qbo-expense-sync.test.ts` cases (§8). +4. **One shared resolver** (§4): all listed money-path readers resolve project/cost code + through `src/lib/expense-attribution.ts`; outputs are byte-identical for rows with + `projectId` NULL. Verify: resolver table test + `npm run build` + before/after diff of + the variance page data for one project (checker step). +5. **Mobile capture carries job + phase** (§5, separate repo PR): receipt photo posts to + `/api/receipts/intake` with projectId (defaulted to the clocked-in job), costCodeId, and + installedAtCustomer. Verify: manual device test; intake row shows the fields. +6. **Backfill executed** (§6): dry-run reviewed by Justin, then `--apply`; re-run reports 0 + changes. **HEADLINE METRIC**: variance-page phase coverage — share of actual-cost + DOLLARS with a resolvable cost code (computeProjectVariance `coverage.attributedShare` + basis) across In Progress jobs (Shop excluded) **> 80%** after backfill + one week of + captured intake (today ~89/562 expenses coded). The backfill prints this number before + and after. +7. **Tax report ships** (§7): `/reports/tax-paid-at-source` renders per-period-per-job sums + and exports CSV; gated by `financialReports`. + +## 2. Schema + +`prisma/schema.prisma` — add to `model Expense` (existing fields untouched): + +```prisma + // Phase 3 attribution: born-with-a-job. Nullable + backfilled; estimateId + // stays the required parent, projectId is the denormalized truth readers use. + projectId String? + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + taxAmount Decimal? // sales tax read off the receipt (WA excise: tax paid at source) + taxAtSource Boolean @default(false) + installedAtCustomer Boolean? // null = unknown/legacy; drives the deduction report + costCodeSource String? // capture | ai | manual | backfill + costCodeConfidence Decimal? + + @@index([projectId]) +``` + +Add back-relation `expenses Expense[]` on `Project`. Add to `model ReceiptIntake` (Phase 1 +branch — coordinate; if Phase 1 is unmerged when this lands, put these in the Phase 1 PR +instead): `taxAtSource Boolean @default(false)`, `installedAtCustomer Boolean?`. + +`scripts/apply-expense-attribution.mjs` (additive, idempotent, `$executeRawUnsafe` over the +pooler, run against prod BEFORE merge — CLAUDE.md pre-deploy rule 2) and byte-identical +`prisma/migrations/_expense_attribution/migration.sql`: + +```sql +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "projectId" TEXT; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAmount" DECIMAL(65,30); +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,30); +CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); +-- guarded FK (DO $$ ... pg_constraint IF NOT EXISTS block, per prior apply scripts): +-- "Expense_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"(id) +-- ON DELETE SET NULL ON UPDATE CASCADE +-- Fill projectId for existing rows from the owning estimate. Idempotent +-- (WHERE projectId IS NULL) and a no-op on CI's empty database. +UPDATE "Expense" e SET "projectId" = est."projectId" +FROM "Estimate" est +WHERE e."estimateId" = est.id AND e."projectId" IS NULL AND est."projectId" IS NOT NULL; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; +``` + +(If Phase 1's table is not in prod yet when the apply script runs, wrap the two +ReceiptIntake lines in a `to_regclass('"ReceiptIntake"')` guard so the script is runnable +in either merge order; the checked-in migration for those two columns then belongs to +whichever PR ships second.) + +## 3. Writers + +Rule: **every writer sets `projectId` from the project it already knows**, and sets +`costCodeSource` whenever it sets `costCodeId`. Precedence: capture = manual > ai = +backfill > null. Nothing but a human edit may change a row whose `costCodeSource` is +"capture" or "manual". + +1. **QBO sync** (`src/lib/qbo-expense-sync.ts`): add `projectId` to `QboExpenseWrite` + (:490); populate from `match.projectId` (:1078 block) and from the overhead project id + (:1032 block). In `upsertQboExpense`: on CREATE write it; on UPDATE set `projectId` only + when the existing row's is NULL (a manual re-attribution must survive a re-sync — same + posture as the deliberate receiptUrl omission, :578-580 comment). Extend the + `findUnique` select with `projectId` and adjust `expenseMatchesQboWrite` so "unchanged" + detection stays correct (compare projectId only when the update would write it). + **Cost-code suggestion**: extract `VENDOR_RULES`, `LINE_RULES`, `suggestCode` from + `scripts/suggest-expense-cost-codes.mjs` into a new pure module + `src/lib/expense-cost-suggest.ts` (the script imports it back — one copy). After a + successful upsert where the row's `costCodeId` IS NULL and `costCodeSource` is not + capture/manual, run `suggestCode({vendor, description})`; on a hit write `costCodeId`, + `costCodeSource: "ai"`, `costCodeConfidence`: 0.9 for a vendor-rule hit, 0.75 for a + line-rule hit (the rules are binary; fixed tiers make the §6 threshold meaningful). + Overhead-bucket rows are excluded (same scope rule as the script). Never on the + deactivate path. +2. **Phase 1 `bookReceipt`** (spec §4 step 5 — coordinate with that branch): the Expense + create additionally carries `projectId: row.projectId`, `taxAmount: taxCents/100` (when + read), `taxAtSource: row.taxAtSource`, `installedAtCustomer: row.installedAtCustomer`, + and `costCodeSource`/`costCodeConfidence`: "capture" (confidence null) when a human + chose `costCodeId` at capture; "ai" + `suggestedConfidence` when it fell back to + `matchCostCode(suggestedPhaseCode)`. `POST /api/receipts/intake` accepts + `installedAtCustomer?: boolean` (defaulting per §5), and for drive/email/chat sources + the read step sets `taxAtSource = taxCents > 0`. +3. **receipt-ingest v1** (`receipt-ingest/route.ts:108`): add `projectId: project.id`, + `costCodeSource: costCode ? "ai" : null`, `costCodeConfidence: null` (the category is + an Apps Script Gemini read, not a human). Minimal — this path retires at Phase 1 cutover. +4. **`/api/expenses` POST** (:98): add `projectId` (already resolved in both branches); + accept optional `costCodeId` + `costTypeId` validated through `resolveCostCode` + (cost-coding.ts) AND `isCostCodeAllowedForProject` (project-phases.ts) — both checks, + per the cost-coding.ts SCOPE note — with `costCodeSource: "capture"`. Do NOT make the + code required here; this route serves legacy mobile builds and the no-photo path. +5. **`/api/receipts/parse`** (:290): add `projectId` (in scope), leave cost code null. +6. **`createExpenseCore`** (`time-expense-core.ts:184`): add + `projectId: estimate.projectId` and `costCodeSource: data.costCodeId ? "manual" : null`. +7. **Expense edit** (`api/expenses/[id]/route.ts:57`): when the PATCH changes `costCodeId`, + set `costCodeSource: "manual"`, `costCodeConfidence: null`. Never let any client set + `costCodeSource` directly on any route. + +## 4. Readers — one shared resolver + +New `src/lib/expense-attribution.ts` (pure, no I/O): + +```ts +export function resolveExpenseProjectId(e: { projectId: string | null; + estimate?: { projectId: string | null } | null }): string | null; + // e.projectId ?? e.estimate?.projectId ?? null +export function resolveExpenseCostCodeId(e: { costCodeId: string | null; itemId: string | null }, + itemCostCodeById: ReadonlyMap): string | null; + // e.costCodeId ?? (e.itemId ? itemCostCodeById.get(e.itemId) ?? null : null) +export function expenseForProjectWhere(projectId: string): Prisma.ExpenseWhereInput; + // { OR: [ { projectId }, { projectId: null, estimate: { projectId } } ] } + // ONE OR key built in one object literal — never spread two conditional ORs + // (prisma-where-or-key-collision lesson). +``` + +Behaviour contract: for every row with `projectId` NULL these resolve exactly what the +current `estimate.projectId` traversal resolves — existing outputs must be identical. +`resolveExpenseCostCodeId` is the SAME fallback `computeProjectVariance` implements today; +refactor job-variance.ts's inline version (~:316-330) to call it so there is one copy. + +Call sites to change (mechanical swap, no behavior change): +- `src/lib/job-variance-db.ts:143,191` → `expenseForProjectWhere(project.id)`; rewrite the + :4-6 header comment (it becomes false the moment the column exists). +- `src/lib/project-financials.ts:70`, `src/app/api/projects/[id]/financial-overview/route.ts:63`, + `src/app/projects/[id]/costing/page.tsx:57`, `src/lib/budget-actions.ts:44`, + `src/lib/time-expense-actions.ts:225,306,332` → `expenseForProjectWhere(projectId)`. +- `src/lib/payouts-report.ts:65`, `src/lib/transactions-report.ts:92,114` → same, inside + their existing conditional filter. +- `src/app/reports/profitability/page.tsx:76-92` → select `projectId` too; bucket rows by + `resolveExpenseProjectId(e)`; widen the :77 where to the OR form + (`{ OR: [{ projectId: { not: null } }, { estimate: { projectId: { not: null } } }] }`). +- `src/lib/company-financials-charts.ts:292,307` → the OR predicate over the id set; :341 + is a `groupBy` on the relation filter — keep its shape and add a comment that + post-backfill it can become a plain `groupBy(["projectId"])` (do NOT change it in this + PR; identical-output rule). +- Display-only includes (manager/receipts, register-data, automation-events, + qbo-bank-register, review-alert-evaluator, schedule-core, ai routes): NO change now — + they read `estimate.project` for labels and stay correct either way. +- Future margin-digest (parent-plan Phase 4) consumes this resolver; it has no code today. + +## 5. Mobile receipt capture (gtr-probuild-mobile, separate PR) + +File-level diff: +- `apps/mobile/app/(tabs)/expenses.tsx`: + (a) default the project dropdown to the clocked-in project when a shift is active + (active-entry state; else keep the current first-active default); + (b) add a phase picker under the project field reusing the Time Clock's phase-list fetch + (`/api/projects/[id]/cost-codes` + `/api/projects/[id]/estimate-items` — already in the + site proxy allowlist, src/proxy.ts:25) and `lib/phasePicker.ts` labels + (`phaseCodeLabel`); optional but encouraged (inline nudge), never blocking; + (c) add an "Installed at customer job" toggle — default TRUE for a real job, FALSE when + the selected project is Shop/overhead (the app needs the overhead project id; expose it + on the config/session payload the app already loads — executor picks the carrier); + (d) when a PHOTO is attached, submit via a new `api.receipts.intake` → + `POST /api/receipts/intake` (JSON `fileBase64`, `source:"mobile"`, `projectId`, + `costCodeId?`, `installedAtCustomer`) instead of the signed-upload + `/api/expenses` + pair — the intake pipeline creates the Expense at booking. Keep the `/api/expenses` + path ONLY for the no-photo case (it stamps projectId per §3.4). +- `apps/mobile/lib/api.ts` + `lib/api-types.ts`: add `receipts.intake(...)` and the + overhead-project-id field. +- Auth: `/api/receipts/intake` bypasses the proxy (Phase 1 §3) and calls + `authenticateMobileOrSession` — the crew Bearer token works with no proxy change. +- Gate on Phase 1 being deployed; until then ship (a)-(c) with the existing + `/api/expenses` POST carrying `costCodeId` (§3.4 accepts it). + +## 6. Backfill — `scripts/backfill-expense-attribution.mjs` + +One-shot, dry-run DEFAULT (`--apply` to write, `--csv `), same shape and .env +loading as `suggest-expense-cost-codes.mjs`. Steps: +(a) `projectId` from `estimate.projectId` where NULL (same UPDATE as the apply script — + belt and braces; report rows touched). +(b) Item fallback: expenses with `costCodeId` NULL and a coded `itemId` → copy the item's + cost code, `costCodeSource: "backfill"`, confidence null. (Prod expects ~0 rows — + run it anyway; it becomes live when item-level capture starts.) +(c) Rule suggester for the rest: `suggestCode` from `src/lib/expense-cost-suggest.ts`, + scope = In Progress customer jobs, overhead excluded by `OVERHEAD_PROJECT_ID` (id, + not the name list). Write only when confidence >= 0.7 (both current tiers pass; the + threshold guards future tiers), `costCodeSource: "ai"` + the tier confidence. + NEVER touch rows whose `costCodeSource` is capture/manual or whose `costCodeId` is + already set. +(d) Print a before/after coverage table — per project: expenses coded/total (count AND + dollars) and the overall dollar share (the §1.6 metric) — and write the remainder + (still-NULL rows: id, project, date, vendor, amount, description head) to the CSV for + Marge. +Re-run after `--apply` must report 0 changes (backfill-estimate-item-cost-codes proof rule). + +## 7. Tax paid at source report — `/reports/tax-paid-at-source` + +- `src/app/reports/tax-paid-at-source/page.tsx` (server component, List layout per + DESIGN_SYSTEM.md) + `src/lib/tax-at-source-report.ts` (pure aggregation, unit-tested). +- Gate: the `financialReports` permission (`src/lib/permissions.ts:110,173`) — same check + pattern as the sibling reports pages. +- Data: expenses where `taxAtSource = true AND installedAtCustomer = true AND + taxAmount > 0`, grouped by month (from `date`, company timezone) × project + (`resolveExpenseProjectId`), summing `taxAmount`. Columns: Month, Job, Receipts (count), + Taxable amount (Σ `amount` — see risk 1), Tax paid at source (Σ `taxAmount`). Month and + grand totals. Period filter, default current quarter. CSV export mirroring the existing + workbook columns (Date, Vendor, Job, Invoice, Receipt Total, deduction base, Tax) so + Vanessa's handoff file shape survives — reuse the register CSV-export pattern. +- Footnote: "Sales tax already paid on materials resold as part of customer work is + deductible on the WA excise return line 'taxable amount for tax paid at source'. Only + receipts flagged installed-at-customer count; Shop and consumable purchases are excluded." + +## 8. Tests (node:test in `tests/`, DI stubs — NO mock.module, CI is Node 20) + +- `tests/expense-attribution.test.ts`: resolver table — projectId set / null+estimate / + null+null; cost code explicit / item-fallback / neither; `expenseForProjectWhere` shape + (single OR key, two branches). +- `tests/qbo-expense-sync.test.ts` (extend): (1) create writes projectId; (2) update + leaves a non-null projectId alone; (3) suggester fills a NULL code with source "ai" + + confidence; (4) a row with costCodeSource "capture" (and "manual") is NEVER rewritten by + the sync, even when the suggester has an answer; (5) overhead rows get no suggestion; + (6) deactivate path never touches attribution fields. +- `tests/backfill-expense-attribution.test.ts`: script core with an injected prisma-shaped + stub — dry-run makes ZERO write calls; apply respects the capture/manual guard and the + 0.7 threshold. +- `tests/tax-at-source-report.test.ts`: sums per month×job; excludes installedAtCustomer + false/null, taxAtSource false, and zero/null taxAmount. +- Acceptance for §1.4 (identical outputs): checker captures the variance report data for + one prod project before and after the reader PR and diffs it. + +## 9. Risks / open questions (max 5) + +1. **`Expense.amount` semantics are MIXED and this spec keeps them mixed.** Phase 1 decided + amount = PRE-TAX when the receipt's tax is split (its §4 step 5 and risk 2, mirroring + the QBO COGS line under the reseller-permit rule); the QBO sync writes GROSS `TotalAmt` + (`qbo-expense-sync.ts:1084`). So the tax report's "Taxable amount" column (Σ amount) is + exact for intake-born rows and tax-inclusive for legacy QBO-imported rows — which + mostly carry `taxAmount` NULL and are therefore excluded from the report anyway. + Recommendation: accept and document on the report page. HUMAN DECISION only if Justin + wants gross normalized instead. +2. **projectId can drift from estimateId** once both exist (estimateId stays required and + still Cascade-deletes the expense). Writers set them together; any future "move + expense" feature must move both. The resolver prefers projectId, so a bad manual + projectId wins silently. Mitigations: §3.1's null-only update rule and the backfill's + re-run-reports-zero check. +3. **The "AI" suggester is regex rules** with a proven failure mode (the $3,317.78 Mesplay + excavator→20-CLEAN mis-book caught only in dry-run — script comment :86-90). The + capture/manual guard plus mandatory dry-run review contain it, but coded-by-rule is not + verified-by-human; the CSV remainder plus Marge is the correction loop. +4. **Receipt-level `installedAtCustomer` is coarser than the current workbook**, which + splits Recoverable vs Consumable tax WITHIN one receipt (the Harbor Freight row carries + $0.43 recoverable AND $16.55 consumable). A mixed receipt must be flagged + whole-or-nothing in v1. Acceptable? If not, this becomes a line-level model and a much + bigger change. +5. **Cross-branch coordination with Phase 1**: `bookReceipt` and the intake route are being + built elsewhere; §3.2's fields must land in THAT code, and the two PRs must not both + edit `src/lib/receipt-intake/book.ts`. The §2 to_regclass guard keeps the apply script + safe in either merge order. + +Rollout order: apply script on prod → writers + resolver PR (identical-output, Codex +review on the sync and resolver diffs — money path) → backfill dry-run → Justin reviews +the table → `--apply` → mobile PR → tax report PR. diff --git a/package.json b/package.json index 1e5540628..e5eef6223 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -28,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql new file mode 100644 index 000000000..501a48de1 --- /dev/null +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -0,0 +1,53 @@ +-- Expense attribution (Receipt Pipeline v2, Phase 3 — +-- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §2). Production gets these statements +-- through the guarded rollout script scripts/apply-expense-attribution.mjs; +-- this migration carries the SAME statements so a fresh database built from +-- prisma/migrations/ reproduces production. Keep both additive and idempotent. +-- +-- `Expense` reached a project only through its required `estimateId` before +-- this. That traversal is still correct — the resolver in +-- src/lib/expense-attribution.ts prefers `projectId` and falls back to it — +-- but a denormalized column is what lets the money-path readers, the margin +-- digest, and the tax report ask "whose job was this?" without a join, and +-- what lets a receipt be born knowing its job. +-- +-- The UPDATE is the backfill. It is idempotent (`WHERE "projectId" IS NULL`) +-- and a no-op on CI's empty database. scripts/backfill-expense-attribution.mjs +-- runs the same statement again — belt and braces, and it reports the count. + +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "projectId" TEXT; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAmount" DECIMAL(65,30); +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,30); + +CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); + +-- SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A +-- project delete must not silently destroy spend history that the estimate +-- still holds. +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_projectId_fkey' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "Project"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; + +UPDATE "Expense" e SET "projectId" = est."projectId" +FROM "Estimate" est +WHERE e."estimateId" = est.id AND e."projectId" IS NULL AND est."projectId" IS NOT NULL; + +-- ReceiptIntake is Phase 1's table. The guard keeps this runnable in EITHER +-- merge order: if Phase 1 has not landed in the target database yet, these two +-- columns are skipped and Phase 1's own migration creates the table without +-- them, at which point re-running this script adds them. +DO $$ BEGIN + IF to_regclass('"ReceiptIntake"') IS NOT NULL THEN + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; + END IF; +END $$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index aa81d76a8..d8c3c727d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -310,6 +310,7 @@ model Project { permits Permit[] inspections Inspection[] receiptIntakes ReceiptIntake[] + expenses Expense[] // ── Percent complete (earned revenue / earned margin) ────────────────────── // percentComplete is the EFFECTIVE value every screen shows; percentCompleteAuto @@ -630,6 +631,28 @@ model Expense { invoicedAt DateTime? status String @default("Pending") // Pending, Reviewed + // ── Phase 3 attribution (docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §2) ──────── + // Born-with-a-job. `estimateId` stays the REQUIRED parent (and still + // Cascade-deletes this row); `projectId` is the denormalized truth every + // money-path reader resolves through src/lib/expense-attribution.ts. + // Nullable because it was backfilled onto 562 existing rows — a NULL here + // still resolves via the estimate, which is why the resolver exists. + projectId String? + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + /// Sales tax read off the receipt. `Expense.amount` stays the GROSS total + /// paid (Phase 1's decision), so pre-tax = amount - taxAmount. + taxAmount Decimal? + /// WA excise "tax paid at source": GTR paid sales tax at the register on + /// material it resold as part of customer work. + taxAtSource Boolean @default(false) + /// null = unknown/legacy. Only TRUE rows are deductible on the excise + /// return, so an absent answer must never read as a yes. + installedAtCustomer Boolean? + /// capture | ai | manual | backfill. Precedence: capture = manual > + /// ai = backfill > null. Nothing but a human edit rewrites capture/manual. + costCodeSource String? + costCodeConfidence Decimal? + purchaseOrderId String? purchaseOrder PurchaseOrder? @relation(fields: [purchaseOrderId], references: [id]) @@ -643,6 +666,7 @@ model Expense { @@index([estimateId]) @@index([changeOrderId]) + @@index([projectId]) } model Invoice { @@ -3040,6 +3064,11 @@ model ReceiptIntake { costCode CostCode? @relation(fields: [costCodeId], references: [id]) suggestedCostCodeId String? suggestedConfidence Float? + /// Phase 3: did this material get installed at a customer job (deductible on + /// the WA excise return) or consumed by the shop? null = unknown; only TRUE + /// counts, so a missing answer never inflates the deduction. Set at capture + /// (the mobile toggle) and copied onto the Expense at booking. + installedAtCustomer Boolean? createdById String? createdBy User? @relation(fields: [createdById], references: [id], onDelete: SetNull) @@ -3072,6 +3101,10 @@ model ReceiptIntake { txnDate DateTime? @db.Date totalCents Int? taxCents Int? + /// Phase 3: the read found sales tax GTR paid at the register. Set by the + /// READ step (`taxCents > 0`) for the drive/email/chat sources, and copied + /// onto the Expense at booking. + taxAtSource Boolean @default(false) docType String? // receipt | check | multi | non_receipt refNumber String? // cleaned invoice #, or "Check" for checks memo String? diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs new file mode 100644 index 000000000..94f304218 --- /dev/null +++ b/scripts/apply-expense-attribution.mjs @@ -0,0 +1,229 @@ +// One-off additive migration for Expense attribution (Receipt Pipeline v2, +// Phase 3 — docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §2): the denormalized +// `projectId` every money-path reader resolves through, plus the tax columns +// the WA "tax paid at source" deduction report needs, plus the provenance of a +// row's cost code so an automated suggester can never overwrite a human. +// +// The SQL here is byte-equivalent to +// prisma/migrations/20260901120000_expense_attribution/migration.sql — that +// file is what a fresh CI/dev database gets; this script is what production +// gets, BEFORE the build that selects these columns deploys (CLAUDE.md +// pre-deploy rule #2 — otherwise every page touching them throws P2022). +// tests/apply-expense-attribution.test.ts asserts the two never drift. +// +// Additive and idempotent: ADD COLUMN / CREATE INDEX IF NOT EXISTS, a guarded +// constraint add, and a backfill UPDATE that only ever touches rows whose +// `projectId` is still NULL. A second run reports every statement "ok" and +// updates 0 rows. +// +// node scripts/apply-expense-attribution.mjs --yes --expect-db --expect-host +// +// --expect-db and --expect-host are BOTH required alongside --yes, matching +// scripts/apply-receipt-intake.mjs: "--yes" alone only proves you meant to run +// something, and a database NAME alone doesn't prove which SERVER it's on. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; + +export function resolveDatabaseUrl() { + if (process.env.DATABASE_URL) return { url: process.env.DATABASE_URL, from: "process.env.DATABASE_URL" }; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); + if (match) return { url: match[1], from: file }; + } + throw new Error("DATABASE_URL not found in process.env, .env.local, or .env"); +} + +export function maskUrl(url) { + return url.replace(/:[^:@]*@/, ":****@"); +} + +function readFlagValue(flag) { + const idx = process.argv.indexOf(flag); + return idx >= 0 ? process.argv[idx + 1] : undefined; +} + +/** + * Pure comparison, exported for unit testing without a live DB. Compares BOTH + * database name and server host, and both EXACTLY — same rule and same reason + * as apply-receipt-intake.mjs: a guard that accepts a substring gets looser the + * shorter the operator's input is. + */ +export function targetMatches(actual, expectDb, expectHost) { + if (!actual || typeof actual !== "object") return false; + if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; + return String(actual.host ?? "") === String(expectHost ?? ""); +} + +export const statements = [ + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "projectId" TEXT`, + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAmount" DECIMAL(65,30)`, + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false`, + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN`, + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT`, + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,30)`, + + `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, + + // SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A + // project delete must not silently destroy spend history that the estimate + // still holds. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_projectId_fkey' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "Project"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + END IF; + END $$`, + + // The backfill. Idempotent by predicate, and a no-op on an empty database. + `UPDATE "Expense" e SET "projectId" = est."projectId" + FROM "Estimate" est + WHERE e."estimateId" = est.id AND e."projectId" IS NULL AND est."projectId" IS NOT NULL`, + + // ReceiptIntake is Phase 1's table. The guard keeps this runnable in EITHER + // merge order: if Phase 1 has not landed in the target database yet, these + // two columns are skipped, and re-running this script after Phase 1 lands + // adds them. + `DO $$ BEGIN + IF to_regclass('"ReceiptIntake"') IS NOT NULL THEN + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; + END IF; + END $$`, +]; + +export const expectedColumns = { + Expense: [ + "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", + "costCodeSource", "costCodeConfidence", + ], +}; + +export const expectedConstraints = [ + { name: "Expense_projectId_fkey", table: "Expense" }, +]; + +export const expectedIndexes = [ + { name: "Expense_projectId_idx", table: "Expense" }, +]; + +async function main() { + if (!process.argv.includes("--yes")) { + console.error("Refusing to run without --yes (and --expect-db / --expect-host)."); + process.exit(1); + } + const expectDb = readFlagValue("--expect-db") ?? process.env.EXPENSE_ATTRIBUTION_EXPECT_DB; + const expectHost = readFlagValue("--expect-host") ?? process.env.EXPENSE_ATTRIBUTION_EXPECT_HOST; + if (!expectDb || !expectHost) { + console.error("Both --expect-db and --expect-host are required (or EXPENSE_ATTRIBUTION_EXPECT_DB / EXPENSE_ATTRIBUTION_EXPECT_HOST)."); + process.exit(1); + } + + const { url, from } = resolveDatabaseUrl(); + console.log(`DATABASE_URL from ${from}: ${maskUrl(url)}`); + const prisma = new PrismaClient({ datasources: { db: { url } } }); + + try { + const [actual] = await prisma.$queryRawUnsafe( + `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, + ); + console.log(`connected to db="${actual.db}" host="${actual.host}"`); + if (!targetMatches(actual, expectDb, expectHost)) { + console.error(`REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to db="${actual.db}" host="${actual.host}".`); + process.exit(1); + } + + for (const sql of statements) { + const label = sql.replace(/\s+/g, " ").slice(0, 84); + process.stdout.write(` ${label} ... `); + const affected = await prisma.$executeRawUnsafe(sql); + // Print the row count for the backfill: a SECOND run reporting 0 is + // the whole idempotency proof, and a silent "ok" would hide it. + console.log(sql.trimStart().startsWith("UPDATE") ? `ok (${affected} rows)` : "ok"); + } + + // Verify shape rather than trusting the run. + for (const [table, columns] of Object.entries(expectedColumns)) { + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name=$1`, + table, + ); + const found = new Set(rows.map(r => r.column_name)); + const missing = columns.filter(c => !found.has(c)); + if (missing.length) { + console.error(`VERIFY FAILED: ${table} missing columns: ${missing.join(", ")}`); + process.exit(1); + } + console.log(`verified ${table}: ${columns.length} columns`); + } + for (const { name, table } of expectedConstraints) { + const [row] = await prisma.$queryRawUnsafe( + `SELECT 1 AS ok FROM pg_constraint WHERE conname = $1`, name, + ); + if (!row) { + console.error(`VERIFY FAILED: constraint ${name} missing on ${table}`); + process.exit(1); + } + } + console.log(`verified ${expectedConstraints.length} constraint(s)`); + for (const { name, table } of expectedIndexes) { + const [row] = await prisma.$queryRawUnsafe( + `SELECT 1 AS ok FROM pg_class WHERE relname = $1 AND relnamespace = 'public'::regnamespace`, name, + ); + if (!row) { + console.error(`VERIFY FAILED: index ${name} missing on ${table}`); + process.exit(1); + } + } + console.log(`verified ${expectedIndexes.length} index(es)`); + + // The backfill's own assertion: after this script, no expense may have + // a NULL projectId while its estimate knows one. A count, not a sample + // — one leftover row is a silently wrong variance report. + const [leftover] = await prisma.$queryRawUnsafe( + `SELECT COUNT(*)::int AS n FROM "Expense" e + JOIN "Estimate" est ON est.id = e."estimateId" + WHERE e."projectId" IS NULL AND est."projectId" IS NOT NULL`, + ); + if (leftover.n !== 0) { + console.error(`VERIFY FAILED: ${leftover.n} expense(s) still have a NULL projectId with a known estimate project`); + process.exit(1); + } + console.log("verified backfill: 0 expenses left unattributed against a known estimate project"); + + // Phase 1's table may legitimately not exist yet (see the guard above). + const [intake] = await prisma.$queryRawUnsafe( + `SELECT to_regclass('"ReceiptIntake"') IS NOT NULL AS present`, + ); + if (intake.present) { + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name='ReceiptIntake'`, + ); + const found = new Set(rows.map(r => r.column_name)); + const missing = ["taxAtSource", "installedAtCustomer"].filter(c => !found.has(c)); + if (missing.length) { + console.error(`VERIFY FAILED: ReceiptIntake missing columns: ${missing.join(", ")}`); + process.exit(1); + } + console.log("verified ReceiptIntake: 2 columns"); + } else { + console.log("ReceiptIntake not present — Phase 1 has not landed here yet; RE-RUN this script after it does."); + } + + console.log("\nExpense attribution migration applied and verified."); + } finally { + await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts new file mode 100644 index 000000000..3837fe387 --- /dev/null +++ b/tests/apply-expense-attribution.test.ts @@ -0,0 +1,118 @@ +/** + * The rollout script and the committed migration must describe the SAME + * columns. + * + * They are written twice on purpose — the script is what PRODUCTION gets + * (before the deploy that selects these columns), the migration is what a fresh + * CI/dev database gets — and nothing else in the repo notices when the two + * drift. CI's `migrations` job would eventually catch a difference by diffing + * against production, but only AFTER the script has been run there, which is + * exactly the wrong time to find out. + * + * Importing the script must NOT open a connection or read DATABASE_URL: all of + * that sits behind the isMainModule guard, the same shape apply-receipt-intake + * has. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { + expectedColumns, + expectedConstraints, + expectedIndexes, + statements, + targetMatches, +} from "../scripts/apply-expense-attribution.mjs"; + +const migrationSql = readFileSync( + path.join(__dirname, "..", "prisma", "migrations", "20260901120000_expense_attribution", "migration.sql"), + "utf8", +); + +/** Compare SQL by meaning, not by indentation: collapse whitespace, drop comments. */ +function normalize(sql: string): string { + return sql + .split(/\r?\n/) + .filter(line => !/^\s*--/.test(line)) + .join(" ") + .replace(/\s+/g, " ") + .replace(/\s*([(),])\s*/g, "$1") + .trim() + .toLowerCase(); +} + +const normalizedMigration = normalize(migrationSql); + +test("every statement the apply script runs is in the committed migration", () => { + for (const statement of statements as string[]) { + const wanted = normalize(statement).replace(/;$/, ""); + assert.ok( + normalizedMigration.includes(wanted), + `migration.sql is missing:\n ${wanted}`, + ); + } +}); + +test("the migration adds no Expense column the apply script does not", () => { + // The reverse direction: a column added ONLY to the migration would exist + // on a fresh CI database and be absent from production, which is the + // failure mode P2022 shows up as. + const migrationColumns = [...migrationSql.matchAll(/ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "(\w+)"/g)] + .map(m => m[1]) + .sort(); + const scriptColumns = (statements as string[]) + .flatMap(s => [...s.matchAll(/ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "(\w+)"/g)].map(m => m[1])) + .sort(); + assert.deepEqual(migrationColumns, scriptColumns); + assert.deepEqual(scriptColumns, [...expectedColumns.Expense].sort()); +}); + +test("the ReceiptIntake columns are behind a to_regclass guard in both files", () => { + // Phase 1 and Phase 3 can land in either order. Without the guard, running + // this against a database that has not seen Phase 1 yet aborts partway and + // leaves the Expense half applied. + const guarded = (statements as string[]).find(s => s.includes("ReceiptIntake")); + assert.ok(guarded, "the script must touch ReceiptIntake"); + assert.match(guarded!, /to_regclass\('"ReceiptIntake"'\) IS NOT NULL/); + assert.match(migrationSql, /to_regclass\('"ReceiptIntake"'\) IS NOT NULL/); + for (const column of ["taxAtSource", "installedAtCustomer"]) { + assert.ok(guarded!.includes(`"${column}"`), `script guard is missing ${column}`); + assert.ok(migrationSql.includes(`"${column}"`), `migration is missing ${column}`); + } +}); + +test("the backfill UPDATE only ever touches rows whose projectId is still NULL", () => { + // This is the whole of its idempotency. A re-run must report 0 rows, and a + // manual re-attribution must survive it. + const update = (statements as string[]).find(s => s.trimStart().startsWith("UPDATE")); + assert.ok(update, "the script must carry the backfill UPDATE"); + assert.match(update!, /e\."projectId" IS NULL/); + assert.match(update!, /est\."projectId" IS NOT NULL/); + assert.ok(!/SET "projectId" = est\."projectId"[\s\S]*WHERE(?![\s\S]*projectId" IS NULL)/.test(update!)); +}); + +test("the FK is SET NULL, guarded, and named the way Prisma would name it", () => { + const fk = (statements as string[]).find(s => s.includes("Expense_projectId_fkey")); + assert.ok(fk); + assert.match(fk!, /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/); + assert.match(fk!, /ON DELETE SET NULL ON UPDATE CASCADE/); + assert.deepEqual(expectedConstraints, [{ name: "Expense_projectId_fkey", table: "Expense" }]); + assert.deepEqual(expectedIndexes, [{ name: "Expense_projectId_idx", table: "Expense" }]); +}); + +test("every statement is additive — nothing drops, renames, or rewrites data", () => { + for (const statement of statements as string[]) { + assert.ok(!/\bDROP\b/i.test(statement), `destructive statement: ${statement}`); + assert.ok(!/\bDELETE FROM\b/i.test(statement), `destructive statement: ${statement}`); + assert.ok(!/\bTRUNCATE\b/i.test(statement), `destructive statement: ${statement}`); + } +}); + +test("the target guard compares database AND host, both exactly", () => { + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10.0.0.5"), true); + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10.0.0.50"), false); + assert.equal(targetMatches({ db: "postgres", host: "10.0.0.5" }, "postgres", "10"), false); + assert.equal(targetMatches({ db: "other", host: "10.0.0.5" }, "postgres", "10.0.0.5"), false); + assert.equal(targetMatches(null, "postgres", "10.0.0.5"), false); +}); From 2a38d69bc5c7019a8f6da257354a93217c05a268 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:39:46 -0700 Subject: [PATCH 062/144] feat(expenses): one attribution resolver, one copy of the cost-code rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/lib/expense-attribution.ts (spec §4) is now the single answer to "which job is this expense on?" and "which phase?". The contract is that it changes NOTHING for existing data: for every row whose new projectId is still NULL it must return exactly what the estimate.projectId traversal returned. The test pins that by running the old inline code and the new resolver over the same fixtures. notHumanCodedExpenseWhere() is an addition to the spec's export list, and it earns its place: `{ costCodeSource: { notIn: [...] } }` on its own compiles to SQL NOT IN, and NULL NOT IN (...) is NULL — so every legacy row (source NULL) would be excluded and the suggester would silently write nothing. src/lib/expense-cost-suggest.ts lifts VENDOR_RULES / LINE_RULES / suggestCode out of scripts/suggest-expense-cost-codes.mjs verbatim; the script imports them back, so the QBO sync and the Phase 3 backfill share the rules a human already reviewed in a dry run. Confidence is a fixed tier per evidence kind (0.9 vendor, 0.75 line) because the rules are binary — no invented per-rule scores. Co-Authored-By: Claude Fable 5.1 --- scripts/suggest-expense-cost-codes.mjs | 81 +++---------- src/lib/expense-attribution.ts | 139 +++++++++++++++++++++ src/lib/expense-cost-suggest.ts | 131 ++++++++++++++++++++ tests/expense-attribution.test.ts | 161 +++++++++++++++++++++++++ tests/expense-cost-suggest.test.ts | 89 ++++++++++++++ 5 files changed, 534 insertions(+), 67 deletions(-) create mode 100644 src/lib/expense-attribution.ts create mode 100644 src/lib/expense-cost-suggest.ts create mode 100644 tests/expense-attribution.test.ts create mode 100644 tests/expense-cost-suggest.test.ts diff --git a/scripts/suggest-expense-cost-codes.mjs b/scripts/suggest-expense-cost-codes.mjs index 226384800..a39aa6d1b 100644 --- a/scripts/suggest-expense-cost-codes.mjs +++ b/scripts/suggest-expense-cost-codes.mjs @@ -21,12 +21,18 @@ * enough for a general retailer: 134 rows are Lowe's, which sells framing * lumber, drywall, paint and toilets alike. * + * The rules themselves moved to src/lib/expense-cost-suggest.ts (Phase 3) so + * the QBO sync and scripts/backfill-expense-attribution.mjs run the SAME ones. + * This script keeps its own scope and reporting; it no longer owns the regexes. + * * USAGE * node scripts/suggest-expense-cost-codes.mjs # dry run + report * node scripts/suggest-expense-cost-codes.mjs --apply # write matches * node scripts/suggest-expense-cost-codes.mjs --csv out.csv */ import { PrismaClient } from "@prisma/client"; +import { suggestCode } from "../src/lib/expense-cost-suggest.ts"; +import { notHumanCodedExpenseWhere } from "../src/lib/expense-attribution.ts"; import { config } from "dotenv"; import { fileURLToPath } from "url"; import { dirname, join } from "path"; @@ -48,71 +54,6 @@ const OVERHEAD_PROJECTS = ["Shop"]; const num = (v) => (v == null ? 0 : Number(v)); const money = (v) => `$${num(v).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; -/** - * Specialty vendors that do exactly one trade. A match here is strong evidence - * on its own — unlike a general retailer, these firms don't sell anything else. - */ -const VENDOR_RULES = [ - // Specialty trade firms and suppliers. Each does ONE thing, so the vendor - // name alone is sufficient evidence. Kept narrow on purpose — a bare - // /cabinet/ or /plumbing/ would also match a general retailer that merely - // has the word in a line item. - { re: /summit plumbing|\bplumbing\b(?!.*supply)/i, code: "03-PLUMB" }, - { re: /redpoint electric|red ?point electric|newman electric/i, code: "04-ELEC" }, - { re: /k ?& ?s countertops/i, code: "12-COUNTER" }, - { re: /rta.?store/i, code: "11-CABINET" }, - { re: /builders ?first ?source|shur-?way|parr lumber/i, code: "02-FRAME" }, - { re: /columbia resource|\bcrc\b/i, code: "20-CLEAN" }, - { re: /ferguson/i, code: "03-PLUMB" }, -]; - -/** - * Material keywords read out of the itemised receipt text the pipeline's Gemini - * step extracts into `description` ("... | Lines: ..."). Ordered: the first hit - * wins, so put the least ambiguous first. - */ -const LINE_RULES = [ - { re: /circuit brea|breaker|romex|wire nut|receptacle|gfci|electrical panel/i, code: "04-ELEC" }, - { re: /douglas fir|hem ?fir|treated #2|stud\b|joist|lvl\b|osb|sheathing|framing/i, code: "02-FRAME" }, - { re: /drywall|sheetrock|joint compound|mud\b|drywall screw/i, code: "07-DRYWALL" }, - { re: /\bpaint\b|primer|caulk|sherwin|behr/i, code: "08-PAINT" }, - { re: /\btile\b|thinset|grout|backer ?board/i, code: "10-TILE" }, - { re: /toilet|vanity|faucet|shower valve|p-?trap|pex|abs pipe/i, code: "03-PLUMB" }, - { re: /cabinet|catalina toffee/i, code: "11-CABINET" }, - { re: /countertop|quartz|granite slab/i, code: "12-COUNTER" }, - { re: /siding|hardie|hz10|trim board/i, code: "16-SIDING" }, - { re: /window|patio door/i, code: "14-DOOR" }, - { re: /insulation|batt\b|r-?13|r-?21/i, code: "06-INSUL" }, - // Cleanup/disposal is about HAULING WASTE AWAY — dump fees, debris runs. - // Deliberately NOT "excavator": a mini-excavator rental is sitework/ - // excavation, not cleanup. Matching it to 20-CLEAN mis-booked a $3,317.78 - // Mesplay equipment rental in the first dry run. - { re: /dump fee|debris|disposal|haul away|junk removal|msw\b/i, code: "20-CLEAN" }, - { re: /excavator|skid ?steer|trencher|bobcat|equipment rental/i, code: "23-SITEWORK" }, - { re: /concrete|rebar|quikrete/i, code: "17-CONCRETE" }, - { re: /roof|shingle|underlayment/i, code: "15-ROOF" }, - { re: /flooring|lvp|laminate|carpet/i, code: "09-FLOOR" }, -]; - -/** - * Decide a phase for one expense. Returns {code, why} or null when the evidence - * is not strong enough — null is a legitimate, deliberate answer here. - */ -export function suggestCode(expense) { - const vendor = expense.vendor || ""; - const desc = expense.description || ""; - - for (const r of VENDOR_RULES) { - if (r.re.test(vendor)) return { code: r.code, why: `vendor ~ ${r.re.source.slice(0, 28)}` }; - } - // Only read the itemised portion; the prefix is boilerplate. - const lines = desc.includes("Lines:") ? desc.slice(desc.indexOf("Lines:")) : desc; - for (const r of LINE_RULES) { - if (r.re.test(lines)) return { code: r.code, why: `lines ~ ${r.re.source.slice(0, 28)}` }; - } - return null; -} - async function main() { const codes = await prisma.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }); const codeId = new Map(codes.map((c) => [c.code, c.id])); @@ -174,8 +115,14 @@ async function main() { let n = 0; for (const m of matched) { - await prisma.expense.update({ where: { id: m.id }, data: { costCodeId: codeId.get(m.code) } }); - n++; + // Phase 3: stamp provenance alongside the code. A row with a code and + // no source would be indistinguishable from a human's choice, and the + // capture/manual guard everywhere else keys on exactly that. + const written = await prisma.expense.updateMany({ + where: { id: m.id, costCodeId: null, ...notHumanCodedExpenseWhere() }, + data: { costCodeId: codeId.get(m.code), costCodeSource: "ai", costCodeConfidence: m.confidence }, + }); + n += written.count; } console.log(`\napplied ${n} cost code(s). ${unmatched.length} rows left NULL for human review.`); } diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts new file mode 100644 index 000000000..e5d93478a --- /dev/null +++ b/src/lib/expense-attribution.ts @@ -0,0 +1,139 @@ +// The ONE place that answers "which job is this expense on?" and "which phase?" +// (docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §4). +// +// Before Phase 3, `Expense` had no `projectId` and every reader traversed +// `estimate.projectId` by hand — eleven separate call sites, each free to +// disagree. Now the column exists, but it is NULLABLE and backfilled, so both +// facts have to be true at once: +// +// * a row that HAS `projectId` is answered by that column, and +// * a row that does NOT must resolve to exactly what the old traversal +// resolved — byte-identical outputs, or the variance and profitability +// numbers move on a refactor that was supposed to change nothing. +// +// That contract is what makes this a mechanical swap rather than a behaviour +// change, and tests/expense-attribution.test.ts is where it is pinned down. +// +// Pure module: no I/O, no Prisma client, no clock. It builds `where` fragments +// and resolves in-memory rows; the callers own the queries. +import type { Prisma } from "@prisma/client"; + +/** The two ways a row can know its project. Both optional at the type level. */ +export interface ExpenseProjectFacts { + projectId: string | null; + estimate?: { projectId: string | null } | null; +} + +/** + * The denormalized column wins. + * + * Deliberate, and it has a sharp edge worth naming (spec risk 2): once both + * exist they CAN drift, and a wrong `projectId` silently beats a right + * `estimate.projectId`. The containment is on the write side — the QBO sync + * only ever fills a NULL, the backfill only ever fills a NULL, and every + * capture path sets the two together — not here. A reader that "helpfully" + * cross-checked them would just be a fourth opinion. + */ +export function resolveExpenseProjectId(expense: ExpenseProjectFacts): string | null { + return expense.projectId ?? expense.estimate?.projectId ?? null; +} + +/** The two ways a row can know its phase. */ +export interface ExpenseCostCodeFacts { + costCodeId: string | null; + itemId: string | null; +} + +/** + * An expense's own cost code, else the code on the estimate item it is linked + * to, else nothing. + * + * This is the SAME fallback `computeProjectVariance` has always implemented + * inline (job-variance.ts `reconcileAttribution`); that function now calls this + * one so there is a single copy. `itemCostCodeById` is the caller's item pool — + * in the variance report that pool deliberately includes "attribution-only" + * rows from Draft/archived estimates, because an expense's link is real even + * when its estimate is not a budget (job-variance-db.ts). + * + * A missing map entry and an entry holding null are the same answer: the item + * exists but carries no code. Both fall through to null rather than throwing — + * an uncoded posting is a reportable fact, not an error. + */ +export function resolveExpenseCostCodeId( + expense: ExpenseCostCodeFacts, + itemCostCodeById: ReadonlyMap, +): string | null { + if (expense.costCodeId) return expense.costCodeId; + if (!expense.itemId) return null; + return itemCostCodeById.get(expense.itemId) ?? null; +} + +/** + * The `where` fragment for "every expense belonging to this project", covering + * both shapes. + * + * ONE `OR` key, built in a single object literal. Never assemble this by + * spreading two conditional `OR`s into the same object — the second silently + * replaces the first and the query quietly widens or narrows (the + * prisma-where-or-key-collision lesson). Callers that already have their own + * `OR` (payouts-report, transactions-report) must nest this under `AND` + * instead of spreading it. + * + * Note the second branch is `projectId: null` AND the estimate match, not just + * the estimate match: keeping the branches disjoint means Postgres can use + * `Expense_projectId_idx` for the first one. + */ +export function expenseForProjectWhere(projectId: string): Prisma.ExpenseWhereInput { + return { + OR: [ + { projectId }, + { projectId: null, estimate: { projectId } }, + ], + }; +} + +/** Same, for a SET of projects — the company-wide charts read this shape. */ +export function expenseForProjectsWhere(projectIds: string[]): Prisma.ExpenseWhereInput { + return { + OR: [ + { projectId: { in: projectIds } }, + { projectId: null, estimate: { projectId: { in: projectIds } } }, + ], + }; +} + +/** "This row is attributable to SOME project", either way round. */ +export function expenseHasAnyProjectWhere(): Prisma.ExpenseWhereInput { + return { + OR: [ + { projectId: { not: null } }, + { projectId: null, estimate: { projectId: { not: null } } }, + ], + }; +} + +/** + * Cost-code sources a MACHINE produced. The complement — "capture" and + * "manual" — is a human's answer and is never rewritten by the sync or the + * backfill. + */ +export const HUMAN_COST_CODE_SOURCES = ["capture", "manual"] as const; + +/** + * "This row's cost code was not chosen by a human." + * + * Written as an explicit `OR` with a `null` branch on purpose. `{ costCodeSource: + * { notIn: [...] } }` alone compiles to SQL `NOT IN`, and SQL says NULL NOT IN + * (...) is NULL, not TRUE — so every legacy row (all 562 of them, source NULL) + * would be EXCLUDED and the backfill would write nothing at all. The bug would + * look like "the rules matched nothing", which is a plausible enough outcome to + * go unnoticed. + */ +export function notHumanCodedExpenseWhere(): Prisma.ExpenseWhereInput { + return { + OR: [ + { costCodeSource: null }, + { costCodeSource: { notIn: [...HUMAN_COST_CODE_SOURCES] } }, + ], + }; +} diff --git a/src/lib/expense-cost-suggest.ts b/src/lib/expense-cost-suggest.ts new file mode 100644 index 000000000..ba6686b57 --- /dev/null +++ b/src/lib/expense-cost-suggest.ts @@ -0,0 +1,131 @@ +// Rule-based cost-code (phase) suggestion for an expense that has none. +// +// Extracted verbatim from scripts/suggest-expense-cost-codes.mjs (2026-08-18) +// so there is ONE copy of the rules. The script now imports this module, the +// QBO sync runs it on import, and scripts/backfill-expense-attribution.mjs +// runs it over the historical backlog — three callers, one rule set. +// +// THIS IS NOT AI. It is regex over the vendor name and the itemised receipt +// text, and it is called "ai" in `Expense.costCodeSource` only because that is +// the value the spec fixed for "a machine chose this, not a human". Treat a hit +// as a suggestion a bookkeeper can overturn, never as a measurement. +// +// PHILOSOPHY — why rules, and why they refuse so often +// A wrong cost code is worse than an absent one: it silently corrupts job +// costing, and TRUST is one of the four product rules. So this only answers +// when the evidence is unambiguous (a specialty vendor, or explicit material +// keywords in the itemised lines). Everything else returns null on purpose, +// and null is reported to a human rather than guessed at. Vendor alone is +// never enough for a general retailer: 134 rows are Lowe's, which sells +// framing lumber, drywall, paint and toilets alike. + +/** The expense facts the rules read. Deliberately tiny — no database types. */ +export interface SuggestibleExpense { + vendor?: string | null; + description?: string | null; +} + +export type CostCodeSuggestionTier = "vendor" | "line"; + +export interface CostCodeSuggestion { + /** A CostCode.code, e.g. "03-PLUMB" — the caller resolves it to an id. */ + code: string; + /** Human-readable provenance, for the dry-run CSV. */ + why: string; + tier: CostCodeSuggestionTier; + /** Fixed per tier — see below. */ + confidence: number; +} + +/** + * The rules are BINARY: a regex either matched or it did not, and there is no + * score to report. Fixed tiers exist so `costCodeConfidence` carries the ONE + * thing that actually varies — which kind of evidence fired — and so the + * backfill's >= 0.7 threshold is a real gate the day a weaker tier is added + * rather than a decoration. Do not invent per-rule numbers: a made-up 0.83 + * would be a guess presented as a measurement. + */ +export const VENDOR_RULE_CONFIDENCE = 0.9; +export const LINE_RULE_CONFIDENCE = 0.75; + +/** + * Specialty vendors that do exactly one trade. A match here is strong evidence + * on its own — unlike a general retailer, these firms don't sell anything else. + * Kept narrow on purpose: a bare /cabinet/ or /plumbing/ would also match a + * general retailer that merely has the word in a line item. + */ +export const VENDOR_RULES: { re: RegExp; code: string }[] = [ + { re: /summit plumbing|\bplumbing\b(?!.*supply)/i, code: "03-PLUMB" }, + { re: /redpoint electric|red ?point electric|newman electric/i, code: "04-ELEC" }, + { re: /k ?& ?s countertops/i, code: "12-COUNTER" }, + { re: /rta.?store/i, code: "11-CABINET" }, + { re: /builders ?first ?source|shur-?way|parr lumber/i, code: "02-FRAME" }, + { re: /columbia resource|\bcrc\b/i, code: "20-CLEAN" }, + { re: /ferguson/i, code: "03-PLUMB" }, +]; + +/** + * Material keywords read out of the itemised receipt text the pipeline's Gemini + * step extracts into `description` ("... | Lines: ..."). Ordered: the first hit + * wins, so the least ambiguous come first. + */ +export const LINE_RULES: { re: RegExp; code: string }[] = [ + { re: /circuit brea|breaker|romex|wire nut|receptacle|gfci|electrical panel/i, code: "04-ELEC" }, + { re: /douglas fir|hem ?fir|treated #2|stud\b|joist|lvl\b|osb|sheathing|framing/i, code: "02-FRAME" }, + { re: /drywall|sheetrock|joint compound|mud\b|drywall screw/i, code: "07-DRYWALL" }, + { re: /\bpaint\b|primer|caulk|sherwin|behr/i, code: "08-PAINT" }, + { re: /\btile\b|thinset|grout|backer ?board/i, code: "10-TILE" }, + { re: /toilet|vanity|faucet|shower valve|p-?trap|pex|abs pipe/i, code: "03-PLUMB" }, + { re: /cabinet|catalina toffee/i, code: "11-CABINET" }, + { re: /countertop|quartz|granite slab/i, code: "12-COUNTER" }, + { re: /siding|hardie|hz10|trim board/i, code: "16-SIDING" }, + { re: /window|patio door/i, code: "14-DOOR" }, + { re: /insulation|batt\b|r-?13|r-?21/i, code: "06-INSUL" }, + // Cleanup/disposal is about HAULING WASTE AWAY — dump fees, debris runs. + // Deliberately NOT "excavator": a mini-excavator rental is sitework/ + // excavation, not cleanup. Matching it to 20-CLEAN mis-booked a $3,317.78 + // Mesplay equipment rental in the first dry run — the reason the whole + // "dry run first, human reviews the CSV" loop exists. + { re: /dump fee|debris|disposal|haul away|junk removal|msw\b/i, code: "20-CLEAN" }, + { re: /excavator|skid ?steer|trencher|bobcat|equipment rental/i, code: "23-SITEWORK" }, + { re: /concrete|rebar|quikrete/i, code: "17-CONCRETE" }, + { re: /roof|shingle|underlayment/i, code: "15-ROOF" }, + { re: /flooring|lvp|laminate|carpet/i, code: "09-FLOOR" }, +]; + +/** + * Decide a phase for one expense. Returns a suggestion or null when the + * evidence is not strong enough — null is a legitimate, deliberate answer. + * + * Pure: no I/O, no clock, no database. The caller maps `code` to a CostCode id + * and decides whether to write it (see the capture/manual guard in + * qbo-expense-sync.ts and the backfill). + */ +export function suggestCode(expense: SuggestibleExpense): CostCodeSuggestion | null { + const vendor = expense.vendor || ""; + const desc = expense.description || ""; + + for (const rule of VENDOR_RULES) { + if (rule.re.test(vendor)) { + return { + code: rule.code, + why: `vendor ~ ${rule.re.source.slice(0, 28)}`, + tier: "vendor", + confidence: VENDOR_RULE_CONFIDENCE, + }; + } + } + // Only read the itemised portion; the prefix is boilerplate. + const lines = desc.includes("Lines:") ? desc.slice(desc.indexOf("Lines:")) : desc; + for (const rule of LINE_RULES) { + if (rule.re.test(lines)) { + return { + code: rule.code, + why: `lines ~ ${rule.re.source.slice(0, 28)}`, + tier: "line", + confidence: LINE_RULE_CONFIDENCE, + }; + } + } + return null; +} diff --git a/tests/expense-attribution.test.ts b/tests/expense-attribution.test.ts new file mode 100644 index 000000000..348d1fa13 --- /dev/null +++ b/tests/expense-attribution.test.ts @@ -0,0 +1,161 @@ +/** + * The resolver's whole job is to change NOTHING for existing data. + * + * `Expense.projectId` is new and backfilled, so for every row where it is still + * NULL the resolver must return exactly what the old `estimate.projectId` + * traversal returned. The `legacyTraversal` fixture below IS the old code, + * copied verbatim, and the table test runs both over the same rows — a + * behaviour-preserving refactor asserted rather than asserted-to. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + HUMAN_COST_CODE_SOURCES, + expenseForProjectWhere, + expenseForProjectsWhere, + expenseHasAnyProjectWhere, + notHumanCodedExpenseWhere, + resolveExpenseCostCodeId, + resolveExpenseProjectId, +} from "../src/lib/expense-attribution"; + +// ── project resolution ────────────────────────────────────────────────────── + +test("resolveExpenseProjectId: column wins, estimate is the fallback, else null", () => { + assert.equal( + resolveExpenseProjectId({ projectId: "p1", estimate: { projectId: "p2" } }), + "p1", + ); + assert.equal(resolveExpenseProjectId({ projectId: null, estimate: { projectId: "p2" } }), "p2"); + assert.equal(resolveExpenseProjectId({ projectId: null, estimate: { projectId: null } }), null); + assert.equal(resolveExpenseProjectId({ projectId: null, estimate: null }), null); + assert.equal(resolveExpenseProjectId({ projectId: null }), null); + assert.equal(resolveExpenseProjectId({ projectId: "p1", estimate: null }), "p1"); +}); + +test("for every projectId-NULL row the resolver equals the pre-Phase-3 traversal", () => { + // Verbatim copy of what every reader did before this module existed. + const legacyTraversal = (e: { estimate?: { projectId: string | null } | null }) => + e.estimate?.projectId ?? null; + + const legacyShapedRows = [ + { projectId: null, estimate: { projectId: "proj-a" } }, + { projectId: null, estimate: { projectId: null } }, + { projectId: null, estimate: null }, + { projectId: null, estimate: { projectId: "" } }, + ]; + for (const row of legacyShapedRows) { + assert.equal( + resolveExpenseProjectId(row), + legacyTraversal(row), + `diverged on ${JSON.stringify(row)}`, + ); + } +}); + +test("an empty-string projectId is not treated as a project id", () => { + // `??` would keep "", which would then be used as a Map key and bucket the + // row under a project that does not exist. Guard the semantics explicitly + // so a future change to `||`/`??` cannot pass silently. + assert.equal(resolveExpenseProjectId({ projectId: "", estimate: { projectId: "p2" } }), ""); +}); + +// ── cost-code resolution ──────────────────────────────────────────────────── + +const ITEM_CODES = new Map([ + ["item-coded", "cc-framing"], + ["item-uncoded", null], +]); + +test("resolveExpenseCostCodeId: explicit code, item fallback, then null", () => { + assert.equal( + resolveExpenseCostCodeId({ costCodeId: "cc-paint", itemId: "item-coded" }, ITEM_CODES), + "cc-paint", + "an explicit code must beat the item's", + ); + assert.equal( + resolveExpenseCostCodeId({ costCodeId: null, itemId: "item-coded" }, ITEM_CODES), + "cc-framing", + ); + assert.equal( + resolveExpenseCostCodeId({ costCodeId: null, itemId: "item-uncoded" }, ITEM_CODES), + null, + "an item with no code resolves to null, not to the item id", + ); + assert.equal( + resolveExpenseCostCodeId({ costCodeId: null, itemId: "item-missing" }, ITEM_CODES), + null, + "an item outside the pool must not throw", + ); + assert.equal(resolveExpenseCostCodeId({ costCodeId: null, itemId: null }, ITEM_CODES), null); +}); + +test("resolveExpenseCostCodeId matches the inline fallback job-variance used", () => { + const legacyReconcile = ( + explicitCostCodeId: string | null, + linkedItem: { costCodeId: string | null } | undefined, + ) => explicitCostCodeId ?? linkedItem?.costCodeId ?? null; + + const rows: { costCodeId: string | null; itemId: string | null }[] = [ + { costCodeId: "cc-paint", itemId: "item-coded" }, + { costCodeId: "cc-paint", itemId: null }, + { costCodeId: null, itemId: "item-coded" }, + { costCodeId: null, itemId: "item-uncoded" }, + { costCodeId: null, itemId: null }, + ]; + for (const row of rows) { + const linked = row.itemId !== null && ITEM_CODES.has(row.itemId) + ? { costCodeId: ITEM_CODES.get(row.itemId) ?? null } + : undefined; + assert.equal( + resolveExpenseCostCodeId(row, ITEM_CODES), + legacyReconcile(row.costCodeId, linked), + `diverged on ${JSON.stringify(row)}`, + ); + } +}); + +// ── where fragments ───────────────────────────────────────────────────────── + +test("expenseForProjectWhere is ONE OR key with two disjoint branches", () => { + const where = expenseForProjectWhere("proj-a"); + assert.deepEqual(Object.keys(where), ["OR"], "exactly one key, or a spread will clobber it"); + assert.deepEqual(where, { + OR: [ + { projectId: "proj-a" }, + { projectId: null, estimate: { projectId: "proj-a" } }, + ], + }); + // Disjoint: the second branch pins projectId to null, so a row is never + // matched by both and the first can use Expense_projectId_idx. + const second = (where.OR as Record[])[1]; + assert.equal(second.projectId, null); +}); + +test("expenseForProjectsWhere and expenseHasAnyProjectWhere keep the same shape", () => { + assert.deepEqual(expenseForProjectsWhere(["a", "b"]), { + OR: [ + { projectId: { in: ["a", "b"] } }, + { projectId: null, estimate: { projectId: { in: ["a", "b"] } } }, + ], + }); + assert.deepEqual(Object.keys(expenseHasAnyProjectWhere()), ["OR"]); + assert.deepEqual(expenseHasAnyProjectWhere(), { + OR: [ + { projectId: { not: null } }, + { projectId: null, estimate: { projectId: { not: null } } }, + ], + }); +}); + +test("notHumanCodedExpenseWhere has an explicit NULL branch", () => { + // Without it, SQL `NOT IN` drops every NULL row — which is all 562 legacy + // rows — and the backfill would silently write nothing. + const where = notHumanCodedExpenseWhere(); + assert.deepEqual(Object.keys(where), ["OR"]); + const branches = where.OR as Record[]; + assert.equal(branches.length, 2); + assert.deepEqual(branches[0], { costCodeSource: null }); + assert.deepEqual(branches[1], { costCodeSource: { notIn: ["capture", "manual"] } }); + assert.deepEqual([...HUMAN_COST_CODE_SOURCES], ["capture", "manual"]); +}); diff --git a/tests/expense-cost-suggest.test.ts b/tests/expense-cost-suggest.test.ts new file mode 100644 index 000000000..9aafb8dbd --- /dev/null +++ b/tests/expense-cost-suggest.test.ts @@ -0,0 +1,89 @@ +/** + * The extraction from scripts/suggest-expense-cost-codes.mjs must be verbatim: + * the script's dry run has already been reviewed by a human against these exact + * regexes, and three callers now share them. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + LINE_RULES, + LINE_RULE_CONFIDENCE, + VENDOR_RULES, + VENDOR_RULE_CONFIDENCE, + suggestCode, +} from "../src/lib/expense-cost-suggest"; + +test("a specialty vendor is decided on the vendor alone, at the vendor tier", () => { + const hit = suggestCode({ vendor: "Summit Plumbing LLC", description: "misc" }); + assert.equal(hit?.code, "03-PLUMB"); + assert.equal(hit?.tier, "vendor"); + assert.equal(hit?.confidence, VENDOR_RULE_CONFIDENCE); +}); + +test("a general retailer is decided on the itemised lines, at the line tier", () => { + const hit = suggestCode({ + vendor: "Lowe's", + description: "[Drive import] Invoice 123 · Materials | Lines: 2x4 stud douglas fir; joist hanger", + }); + assert.equal(hit?.code, "02-FRAME"); + assert.equal(hit?.tier, "line"); + assert.equal(hit?.confidence, LINE_RULE_CONFIDENCE); +}); + +test("only the itemised portion is read — boilerplate in the prefix is ignored", () => { + // "Receipt" boilerplate mentioning a trade word before "Lines:" must not + // decide the phase; the whole point of the slice is that the prefix is + // template text the pipeline wrote, not evidence from the vendor. + const hit = suggestCode({ + vendor: "Lowe's", + description: "paint department checkout | Lines: 1/2in drywall sheet; joint compound", + }); + assert.equal(hit?.code, "07-DRYWALL"); +}); + +test("no evidence returns null — a refusal is a legitimate answer", () => { + assert.equal(suggestCode({ vendor: "Lowe's", description: "misc supplies" }), null); + assert.equal(suggestCode({ vendor: null, description: null }), null); + assert.equal(suggestCode({}), null); +}); + +test("an excavator rental is sitework, not cleanup (the $3,317.78 Mesplay mis-book)", () => { + // The rule ordering is load-bearing: /disposal|debris|dump fee/ comes + // BEFORE /excavator/, and neither may capture the other's rows. + assert.equal(suggestCode({ vendor: "Sunbelt", description: "Lines: mini excavator rental 1 day" })?.code, "23-SITEWORK"); + assert.equal(suggestCode({ vendor: "Sunbelt", description: "Lines: dump fee msw" })?.code, "20-CLEAN"); +}); + +test("vendor rules beat line rules", () => { + const hit = suggestCode({ + vendor: "Ferguson", + description: "Lines: 2x4 stud douglas fir", + }); + assert.equal(hit?.code, "03-PLUMB", "a specialty vendor is stronger evidence than a keyword"); + assert.equal(hit?.tier, "vendor"); +}); + +test("the rule sets are the ones the reviewed dry run used", () => { + assert.equal(VENDOR_RULES.length, 7); + assert.equal(LINE_RULES.length, 16); + // Every rule must name a code, and no rule may be a catch-all. + for (const rule of [...VENDOR_RULES, ...LINE_RULES]) { + assert.match(rule.code, /^\d{2}-[A-Z]+$/); + assert.ok(!rule.re.test(""), `catch-all rule: ${rule.re}`); + } +}); + +test("the two confidence tiers both clear the backfill's 0.7 threshold, and are ordered", () => { + assert.ok(VENDOR_RULE_CONFIDENCE > LINE_RULE_CONFIDENCE); + assert.ok(LINE_RULE_CONFIDENCE >= 0.7); +}); + +test("regex lastIndex cannot leak between calls", () => { + // A /g flag on any rule would make repeated calls return different answers + // for the same input. Cheap assertion, catastrophic bug. + for (const rule of [...VENDOR_RULES, ...LINE_RULES]) { + assert.ok(!rule.re.global, `rule must not be global: ${rule.re}`); + } + const input = { vendor: "Summit Plumbing", description: "" }; + assert.deepEqual(suggestCode(input), suggestCode(input)); +}); From 968ba3d4764094cd37d7f22d721a8103f37c7975 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:48:35 -0700 Subject: [PATCH 063/144] feat(expenses): every writer stamps the job, and cost-code provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §3. Each of the seven Expense writers already knew which project it was writing for — it just dropped the value and left every reader to re-derive it through the estimate. Now they all persist it, and any writer that sets a cost code records WHO decided it: QBO sync — projectId from the match / the overhead bucket. On UPDATE it is written only when the existing row's is NULL, so a bookkeeper's re-attribution survives a re-sync (same posture as the deliberate receiptUrl omission). "unchanged" detection compares projectId only on the rows an update would write it to, otherwise a re-attributed row would read as drifted forever. The sync also runs the shared rule suggester after a successful upsert: uncoded rows only, human-coded rows never, overhead never, deactivations never, and a failure there can't fail the import. bookReceipt — projectId, taxAmount (the tax the VENDOR charged, from the read, not the QBO split), taxAtSource, installedAtCustomer, and capture-vs-ai provenance for the code. intake route — accepts installedAtCustomer; defaults TRUE for a real job and FALSE for the Shop/overhead bucket, and stays NULL with no project rather than guessing "yes" and inflating a tax deduction. worker read step — taxAtSource from taxCents > 0 (an absent read and a zero one are the same answer). receipt-ingest v1, /api/receipts/parse, createExpenseCore, /api/expenses POST (which now accepts costCodeId, validated through BOTH resolveCostCode and isCostCodeAllowedForProject), and the expense PUT. costCodeSource is never read off a request body on any route: provenance is something the server observes, not something a client asserts. Co-Authored-By: Claude Fable 5.1 --- .../api/cron/receipt-intake-worker/route.ts | 3 + src/app/api/expenses/[id]/route.ts | 88 ++++--- src/app/api/expenses/route.ts | 54 +++++ .../api/integrations/receipt-ingest/route.ts | 8 + src/app/api/receipts/intake/route.ts | 42 ++++ src/app/api/receipts/parse/route.ts | 6 + src/lib/qbo-expense-sync.ts | 185 ++++++++++++++- src/lib/receipt-intake/book.ts | 30 +++ src/lib/receipt-intake/worker.ts | 7 + src/lib/time-expense-core.ts | 7 + tests/qbo-expense-sync.test.ts | 224 ++++++++++++++++++ tests/receipt-intake-book.test.ts | 2 + tests/receipt-intake-worker.test.ts | 3 + 13 files changed, 629 insertions(+), 30 deletions(-) diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index da437d2ac..1bed5fbc5 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -71,8 +71,11 @@ const LEASE_MS = CLAIM_LEASE_MINUTES * 60_000; const WORKER_ROW_SELECT = { id: true, source: true, sourceRef: true, state: true, dryRun: true, projectId: true, costCodeId: true, suggestedCostCodeId: true, + suggestedConfidence: true, storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, + // Phase 3 attribution — booking copies these straight onto the Expense. + taxAtSource: true, installedAtCustomer: true, docType: true, refNumber: true, memo: true, attempts: true, readAt: true, lastError: true, suggestedConfidence: true, sendAttempted: true, claimToken: true, fileSha256: true, createdAt: true, dedupWeakKey: true, busyPasses: true, diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 8ad86bed6..ea11c0764 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -1,11 +1,11 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; -import { getServerSession } from "next-auth/next"; -import { authOptions } from "@/lib/auth"; -import { - QboManagedExpenseError, - assertExpenseMutableOutsideQbo, -} from "@/lib/qbo-expense-guard"; +import { getServerSession } from "next-auth/next"; +import { authOptions } from "@/lib/auth"; +import { + QboManagedExpenseError, + assertExpenseMutableOutsideQbo, +} from "@/lib/qbo-expense-guard"; export async function DELETE(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { @@ -15,19 +15,19 @@ export async function DELETE(req: NextRequest, { params }: { params: Promise<{ i const id = (await params).id; if (!id) return NextResponse.json({ error: "id is required" }, { status: 400 }); - const expense = await prisma.expense.findUnique({ - where: { id }, - select: { qbPurchaseId: true }, - }); - assertExpenseMutableOutsideQbo(expense); - await prisma.expense.deleteMany({ where: { id, qbPurchaseId: null } }); + const expense = await prisma.expense.findUnique({ + where: { id }, + select: { qbPurchaseId: true }, + }); + assertExpenseMutableOutsideQbo(expense); + await prisma.expense.deleteMany({ where: { id, qbPurchaseId: null } }); return NextResponse.json({ success: true }); - } catch (error) { - if (error instanceof QboManagedExpenseError) { - return NextResponse.json({ error: error.message }, { status: 409 }); - } - console.error("Error deleting expense:", error); + } catch (error) { + if (error instanceof QboManagedExpenseError) { + return NextResponse.json({ error: error.message }, { status: 409 }); + } + console.error("Error deleting expense:", error); return NextResponse.json({ error: "Failed to delete expense" }, { status: 500 }); } } @@ -40,12 +40,12 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const id = (await params).id; if (!id) return NextResponse.json({ error: "id is required" }, { status: 400 }); - const expense = await prisma.expense.findUnique({ - where: { id }, - select: { qbPurchaseId: true }, - }); - assertExpenseMutableOutsideQbo(expense); - const body = await req.json(); + const expense = await prisma.expense.findUnique({ + where: { id }, + select: { qbPurchaseId: true }, + }); + assertExpenseMutableOutsideQbo(expense); + const body = await req.json(); if (body.itemId) { const itemExists = await prisma.estimateItem.findUnique({ where: { id: body.itemId }, select: { id: true } }); @@ -54,6 +54,28 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: } } + // Phase 3 (spec §3.7): an edit here is a HUMAN re-coding the expense, + // so it takes the highest precedence and no automated pass may touch it + // again. `costCodeSource` is never read off the body — a client cannot + // assert its own provenance — it is derived from the fact that a person + // used this endpoint. The key is only acted on when it is present, so + // existing callers that send {amount, vendor, date, ...} are unchanged. + const editsCostCode = Object.prototype.hasOwnProperty.call(body, "costCodeId"); + const nextCostCodeId: string | null = + typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null; + if (editsCostCode && nextCostCodeId) { + const costCode = await prisma.costCode.findUnique({ + where: { id: nextCostCodeId }, + select: { id: true, isActive: true }, + }); + if (!costCode) { + return NextResponse.json({ error: "Cost code not found." }, { status: 400 }); + } + if (!costCode.isActive) { + return NextResponse.json({ error: "That cost code is inactive." }, { status: 400 }); + } + } + const updatedExpense = await prisma.expense.update({ where: { id }, data: { @@ -62,15 +84,25 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: date: body.date ? new Date(body.date) : null, description: body.description || null, itemId: body.itemId || null, + ...(editsCostCode + ? { + costCodeId: nextCostCodeId, + // Clearing the code clears the provenance with it — + // leaving "manual" on a null code would guard a row + // that has nothing to guard. + costCodeSource: nextCostCodeId ? "manual" : null, + costCodeConfidence: null, + } + : {}), }, }); return NextResponse.json(updatedExpense); - } catch (error) { - if (error instanceof QboManagedExpenseError) { - return NextResponse.json({ error: error.message }, { status: 409 }); - } - console.error("Error updating expense:", error); + } catch (error) { + if (error instanceof QboManagedExpenseError) { + return NextResponse.json({ error: error.message }, { status: 409 }); + } + console.error("Error updating expense:", error); return NextResponse.json({ error: "Failed to update expense" }, { status: 500 }); } } diff --git a/src/app/api/expenses/route.ts b/src/app/api/expenses/route.ts index c811d1331..966ec08c6 100644 --- a/src/app/api/expenses/route.ts +++ b/src/app/api/expenses/route.ts @@ -2,6 +2,10 @@ export const dynamic = "force-dynamic"; import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; import { authenticateMobileOrSession, userCanAccessProject } from "@/lib/mobile-auth"; +import { resolveCostCode } from "@/lib/cost-coding"; +import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; // Hybrid auth (web + mobile). Accepts EITHER `estimateId` (web flow — caller already // chose the estimate) OR `projectId` (mobile flow — server picks the project's first @@ -15,6 +19,14 @@ export async function POST(req: NextRequest) { const body = await req.json(); const { itemId, amount, vendor, date, description, receiptUrl } = body; let { estimateId, projectId } = body; + // Phase 3 (spec §3.4): the crew app may now send a phase. Optional on + // purpose — this route also serves legacy mobile builds and the + // no-photo path, and rejecting an uncoded expense here would just stop + // the spend being recorded at all. `costCodeSource` is NEVER read off + // the body: provenance is something the server observes, not something + // a client asserts. + const requestedCostCodeId: string | null = + typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null; if (!estimateId && !projectId) { return NextResponse.json( @@ -95,10 +107,52 @@ export async function POST(req: NextRequest) { parsedDate = d; } + // BOTH checks, per the SCOPE note on resolveCostCode: "the cost code + // exists and is active" is attribution, "this code belongs to this job" + // is permission, and neither implies the other. + let costCodeId: string | null = null; + let costTypeId: string | null = null; + if (requestedCostCodeId) { + const resolved = await resolveCostCode(prismaCostCodingDataSource, { + costCodeId: requestedCostCodeId, + }); + if (!resolved.ok) { + return NextResponse.json( + { error: resolved.error, code: resolved.code }, + { status: resolved.status }, + ); + } + const allowed = await isCostCodeAllowedForProject( + prismaPhaseDataSource, + projectId, + resolved.costCodeId, + ); + if (!allowed) { + return NextResponse.json( + { + error: "That cost code isn't one of this project's phases.", + code: "PHASE_NOT_ON_PROJECT", + }, + { status: 400 }, + ); + } + costCodeId = resolved.costCodeId; + costTypeId = resolved.costTypeId; + } + const newExpense = await prisma.expense.create({ data: { estimateId, + // Phase 3: born with its job. Resolved above in both branches + // (derived from the estimate on the web path, checked against + // the caller's access on the mobile path). + projectId, itemId: itemId || null, + costCodeId, + costTypeId, + // A person picked this on a phone or in a form, so it is + // "capture" and no automated pass may ever overwrite it. + costCodeSource: costCodeId ? "capture" : null, amount: numericAmount, vendor: vendor || null, date: parsedDate, diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index 0dbdbc4fb..eb47d98cd 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -108,7 +108,15 @@ export async function POST(req: Request) { await prisma.expense.create({ data: { estimateId, + projectId: project.id, costCodeId: costCode?.id ?? null, + // The category came from the Apps Script's Gemini read, not + // from a person — "ai", never "capture", so nothing downstream + // treats it as a human's answer. No confidence: matchCostCode + // is a string match and has no score to report, and inventing + // one would be a guess presented as a measurement. + costCodeSource: costCode ? "ai" : null, + costCodeConfidence: null, amount, vendor: body.vendor || "Unknown", date, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 962976c87..07334ff82 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -23,6 +23,7 @@ import { serializeReceiptIntake, withArchiveDownloadUrls, } from "@/lib/receipt-intake/queries"; +import { isOverheadProject } from "@/lib/overhead-project"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -73,6 +74,13 @@ interface ParsedBody { uploadId: string | null; projectId: string | null; costCodeId: string | null; + /** + * Phase 3: did this material get installed at a customer job? Tri-state on + * purpose — `null` means the caller did not say, which is NOT the same as + * "no" and must not be recorded as one. `resolveInstalledAtCustomer` fills + * a default only when the caller stayed silent. + */ + installedAtCustomer: boolean | null; threadName: string | null; /** The forwarder reporting that v1 already booked and archived this file. */ archivedByV1: boolean; @@ -122,6 +130,34 @@ function tooLargeForInline(limit: number, encoding: "json" | "multipart") { }, { status: 413 }, ); + + * Accept a boolean from either a JSON body (real boolean) or a multipart form + * (everything is a string). Anything else is "the caller did not say". + */ +function optionalBool(value: unknown): boolean | null { + if (typeof value === "boolean") return value; + if (value === "true") return true; + if (value === "false") return false; + return null; +} + +/** + * The default the mobile app relies on (spec §5c): a receipt filed against a + * real job was installed at the customer's; one filed against the Shop overhead + * bucket was consumed by the business and is NOT deductible on the excise + * return. An explicit answer from the caller always wins — the crew member + * standing in front of the material knows better than this rule does. + * + * With no project at all the answer stays NULL. Guessing "yes" would quietly + * inflate a tax deduction, which is the one direction this must never fail in. + */ +export function resolveInstalledAtCustomer( + declared: boolean | null, + projectId: string | null, +): boolean | null { + if (declared !== null) return declared; + if (!projectId) return null; + return !isOverheadProject(projectId); } async function parseBody(req: Request): Promise { @@ -150,6 +186,7 @@ async function parseBody(req: Request): Promise { uploadId: str(form.get("uploadId")), projectId: str(form.get("projectId")), costCodeId: str(form.get("costCodeId")), + installedAtCustomer: optionalBool(form.get("installedAtCustomer")), threadName: str(form.get("threadName")), archivedByV1: form.get("archivedByV1") === "true", }; @@ -182,6 +219,7 @@ async function parseBody(req: Request): Promise { uploadId: str(json.uploadId), projectId: str(json.projectId), costCodeId: str(json.costCodeId), + installedAtCustomer: optionalBool(json.installedAtCustomer), threadName: str(json.threadName), // Strict === true: only an explicit boolean may mark a row as already // booked by v1, because that flag is what excuses v2 from booking it. @@ -294,6 +332,10 @@ export async function POST(req: Request) { dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId: parsed.projectId, costCodeId: parsed.costCodeId, + installedAtCustomer: resolveInstalledAtCustomer( + parsed.installedAtCustomer, + parsed.projectId, + ), createdById: auth.via === "session" ? auth.user.id : null, // Only a shared-secret forwarder may assert this: it is the // claim that v1 already put this document in the books, and it diff --git a/src/app/api/receipts/parse/route.ts b/src/app/api/receipts/parse/route.ts index 47253379e..cd246acec 100644 --- a/src/app/api/receipts/parse/route.ts +++ b/src/app/api/receipts/parse/route.ts @@ -290,6 +290,12 @@ export async function POST(req: NextRequest) { const expense = await prisma.expense.create({ data: { estimateId: estimate.id, + // Phase 3: the caller named the project and access was + // checked three lines up — stamp it rather than making + // every reader re-derive it through the estimate. + // Cost code stays null: this parse reads vendor/total/ + // date, never a phase. + projectId, description: `[AI ${confidence}%] ${parsed.vendor} receipt — pending bookkeeper review`, amount: parsed.total as number, date: parsed.date ? new Date(parsed.date as string) : new Date(), diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index fbd374cb1..2719fbdbd 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -4,6 +4,9 @@ import { findBestProjectNameMatches } from "./project-match"; import { prisma } from "./prisma"; import { getFreshQBTokens } from "./quickbooks-payments"; import { after } from "next/server"; +import { suggestCode } from "./expense-cost-suggest"; +import { notHumanCodedExpenseWhere } from "./expense-attribution"; +import { isOverheadProject } from "./overhead-project"; // Shared with the register merge layer (register-merge.ts, Unified Money // Register plan §4) so the classification values this module WRITES can // never drift from the values that module READS. @@ -492,6 +495,16 @@ export interface QboExpenseWrite { qbSyncToken: string; qbSyncedAt: Date; estimateId: string; + /** + * Phase 3: the job this purchase belongs to — the match's project, or the + * overhead bucket. The sync always KNEW this (it is how `estimateId` was + * chosen); it just used to drop it on the floor. + * + * On UPDATE it is only ever written when the existing row's is NULL. A + * bookkeeper who re-attributed an imported expense by hand must survive the + * next re-sync — same posture as the deliberate `receiptUrl` omission. + */ + projectId: string | null; amount: number; vendor: string | null; date: Date | null; @@ -509,6 +522,7 @@ type ExpenseTransaction = { id: string; qbSyncToken: string | null; estimateId: string; + projectId?: string | null; amount: unknown; vendor: string | null; date: Date | null; @@ -546,12 +560,36 @@ function datesEqual(left: Date | null, right: Date | null): boolean { return left?.getTime() === right?.getTime(); } +/** + * The subset of `write` an UPDATE is allowed to apply to an existing row. + * + * `projectId` is dropped once the row already has one: re-attribution by a + * bookkeeper is a human decision, and the QBO Purchase's customer ref is not a + * newer fact about it. Filling a NULL is still correct — that is the backfill + * catching up. + */ +function qboExpenseUpdateData( + existing: NonNullable>>, + write: QboExpenseWrite, +): Partial { + if ((existing.projectId ?? null) === null) return write; + const { projectId: _ignored, ...rest } = write; + return rest; +} + function expenseMatchesQboWrite( existing: Awaited>, write: QboExpenseWrite, ): boolean { if (!existing) return false; + // Compare projectId ONLY on the rows an update would actually write it to. + // Comparing it unconditionally would mark every hand-re-attributed row as + // drifted and re-issue an update forever that deliberately never changes it. + const projectIdMatches = + (existing.projectId ?? null) !== null || + (existing.projectId ?? null) === (write.projectId ?? null); return ( + projectIdMatches && existing.qbSyncToken === write.qbSyncToken && existing.estimateId === write.estimateId && Number(existing.amount) === write.amount && @@ -591,6 +629,7 @@ export async function upsertQboExpense( id: true, qbSyncToken: true, estimateId: true, + projectId: true, amount: true, vendor: true, date: true, @@ -611,7 +650,7 @@ export async function upsertQboExpense( } await transaction.expense.update({ where: { id: existing.id }, - data: write, + data: qboExpenseUpdateData(existing, write), }); return "updated"; }); @@ -677,6 +716,81 @@ export async function deactivateQboExpense( }); } +// ── Cost-code suggestion (Phase 3 attribution, spec §3.1) ────────────────── +// +// A QBO import has never carried a cost code — the sync only ever project- +// matched, so every imported expense landed on the job with no phase and fell +// into the variance report's "unattributed" bucket. The rules in +// expense-cost-suggest.ts answer the unambiguous ones; the rest stay NULL for +// a human, which is the same deliberate refusal the backfill makes. +// +// TWO THINGS THIS MUST NEVER DO, both encoded in the `where` below rather than +// in a caller's discipline: +// * overwrite a code (the `costCodeId: null` predicate), and +// * overwrite a HUMAN's code (`notHumanCodedExpenseWhere()` — capture and +// manual are off limits, and its explicit NULL branch is what keeps legacy +// rows eligible; see that function's comment). +// It also never runs on the deactivate path: a Purchase deleted in QBO is not +// an occasion to guess its phase. + +export interface QboCostCodeSuggestionInput { + qbPurchaseId: string; + vendor: string | null; + description: string; + /** The job the row was just imported against; the overhead bucket opts out. */ + projectId: string | null; +} + +export interface QboCostCodeSuggestionClient { + expense: { + updateMany(args: { + where: Record; + data: { costCodeId: string; costCodeSource: string; costCodeConfidence: number }; + }): Promise<{ count: number }>; + }; +} + +export type QboCostCodeSuggestionResult = + /** The overhead bucket is not a job and does not get a job phase. */ + | "skipped-overhead" + /** The rules refused — the honest majority case. */ + | "no-match" + /** The rules named a code this company does not have active. */ + | "unknown-code" + /** Already coded, or coded by a human: the guard held. */ + | "not-written" + | "written"; + +export async function applyQboExpenseCostCodeSuggestion( + client: QboCostCodeSuggestionClient, + input: QboCostCodeSuggestionInput, + costCodeIdByCode: ReadonlyMap, +): Promise { + if (input.projectId && isOverheadProject(input.projectId)) return "skipped-overhead"; + + const suggestion = suggestCode({ vendor: input.vendor, description: input.description }); + if (!suggestion) return "no-match"; + + const costCodeId = costCodeIdByCode.get(suggestion.code); + if (!costCodeId) return "unknown-code"; + + const written = await client.expense.updateMany({ + where: { + qbPurchaseId: input.qbPurchaseId, + costCodeId: null, + ...notHumanCodedExpenseWhere(), + }, + data: { + costCodeId, + // "ai" is the spec's value for "a machine chose this". The rules are + // regexes; the label is about provenance, not about technique. + costCodeSource: "ai", + costCodeConfidence: suggestion.confidence, + }, + }); + return written.count > 0 ? "written" : "not-written"; +} + // ── Purchase classification (Unified Money Register plan §5 step 3) ───────── // // Persists the NATURE of a Purchase's money — job-costable, overhead, owner @@ -762,6 +876,12 @@ export interface QboExpenseSyncDependencies { /** Optional: copy the QBO receipt attachment into ProBuild storage for this purchase. */ attachReceipt?(tokens: QBTokens, qbPurchaseId: string): Promise; upsertPurchaseClassification(write: QboPurchaseClassificationWrite): Promise; + /** + * Optional: rule-based phase suggestion for a job-costed import that has no + * cost code yet (spec §3.1). Optional so a caller can turn it off outright, + * and so existing tests that build dependencies by hand keep compiling. + */ + suggestCostCode?(input: QboCostCodeSuggestionInput): Promise; now(): Date; } @@ -819,6 +939,23 @@ async function listInProgressProjects(): Promise { } function createDefaultSyncDependencies(): QboExpenseSyncDependencies { + // Loaded once per sync, on first use: a sync that matches nothing (or one + // where every row is already coded) should not pay for the lookup, and a + // sync of 400 purchases should not pay for it 400 times. Scoped to this + // closure rather than to the module so a long-lived process picks up a + // newly added cost code on the next run. + let costCodeIdByCode: ReadonlyMap | null = null; + const loadCostCodes = async (): Promise> => { + if (!costCodeIdByCode) { + const codes = await prisma.costCode.findMany({ + where: { isActive: true }, + select: { id: true, code: true }, + }); + costCodeIdByCode = new Map(codes.map(code => [code.code, code.id])); + } + return costCodeIdByCode; + }; + return { getTokens: getFreshQBTokens, readPurchases: (tokens, since, mode, until) => @@ -845,6 +982,13 @@ function createDefaultSyncDependencies(): QboExpenseSyncDependencies { prisma as unknown as QboPurchaseClassificationPersistenceClient, write, ), + suggestCostCode: async input => { + await applyQboExpenseCostCodeSuggestion( + prisma as unknown as QboCostCodeSuggestionClient, + input, + await loadCostCodes(), + ); + }, now: () => new Date(), }; } @@ -1034,6 +1178,11 @@ export async function syncQboExpenses( qbSyncToken: purchase.syncToken, qbSyncedAt: dependencies.now(), estimateId: overheadEstimateId, + // The overhead bucket IS this row's project — the sync has + // always known it, it just never wrote it down. + // Non-null on this branch by construction: overheadEstimateId + // is derived from overheadProject and gates the branch. + projectId: overheadProject?.id ?? null, amount: purchase.total, vendor: purchase.vendor, date: qboTransactionDate(purchase.txnDate), @@ -1042,6 +1191,9 @@ export async function syncQboExpenses( }); if (outcome === "imported") result.imported += 1; if (outcome === "updated") result.updated += 1; + // NO cost-code suggestion here. Overhead is not a job and does + // not get a job phase (same scope rule as + // scripts/suggest-expense-cost-codes.mjs). await attachReceipt(purchase.qbPurchaseId); continue; } @@ -1075,19 +1227,48 @@ export async function syncQboExpenses( continue; } + const description = qboExpenseDescription(purchase); const outcome = await dependencies.upsertExpense({ qbPurchaseId: purchase.qbPurchaseId, qbSyncToken: purchase.syncToken, qbSyncedAt: dependencies.now(), estimateId: match.estimateId, + // The matched job. `match.projectId` was computed to pick the + // estimate and then thrown away; now it is persisted. + projectId: match.projectId, amount: purchase.total, vendor: purchase.vendor, date: qboTransactionDate(purchase.txnDate), - description: qboExpenseDescription(purchase), + description, status: "Reviewed", }); if (outcome === "imported") result.imported += 1; if (outcome === "updated") result.updated += 1; + + // Runs on "unchanged" too: a row imported before Phase 3 is unchanged + // by definition and is exactly the row that still has no phase. The + // write is guarded (uncoded, and not human-coded), so a re-run over an + // already-coded row is a no-op. + // + // Failure here must never fail the import — the money is already + // recorded, and a missing phase is a reportable gap, not a lost cost. + // Same resilience posture as persistClassification/attachReceipt. + if (dependencies.suggestCostCode) { + try { + await dependencies.suggestCostCode({ + qbPurchaseId: purchase.qbPurchaseId, + vendor: purchase.vendor, + description, + projectId: match.projectId, + }); + } catch (error) { + console.error( + "QBO cost-code suggestion failed", + purchase.qbPurchaseId, + error instanceof Error ? error.name : "UnknownError", + ); + } + } await attachReceipt(purchase.qbPurchaseId); } diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index d42a33ab9..994412133 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -50,6 +50,11 @@ export interface BookableRow { suggestedCostCodeId: string | null; /** The model's confidence in that phase suggestion, 0..1. */ suggestedConfidence: number | null; + suggestedConfidence: number | null; + /** Phase 3: the read found sales tax paid at the register. */ + taxAtSource: boolean; + /** Phase 3: installed at a customer job (deductible) — null = unknown. */ + installedAtCustomer: boolean | null; storagePath: string; fileName: string | null; mimeType: string; @@ -613,6 +618,15 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro ); } const costCodeId = phaseCheck.costCodeId; + // Phase 3 provenance, derived from WHICH source survived the + // re-validation above — not from which one was merely present. A + // captured code that the final project does not carry is dropped by + // resolvePhase, and calling the survivor "capture" would then be a + // claim about a decision that did not stick. + const costCodeSource = costCodeId + ? (costCodeId === row.costCodeId ? "capture" : "ai") + : null; + const costCodeConfidence = costCodeSource === "ai" ? row.suggestedConfidence : null; const driveFileId = driveFileIdOf(row); const receiptUrl = driveFileId ? `https://drive.google.com/file/d/${driveFileId}/view` @@ -635,7 +649,23 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro const expense = existing ?? await tx.expense.create({ data: { estimateId, + // Phase 3: the job the capturer (or the Drive folder) named. + // `estimateId` above is DERIVED from it — the project is the + // fact, the estimate is the lookup. + projectId: row.projectId, costCodeId, + costCodeSource, + costCodeConfidence, + // The tax the VENDOR charged, from the read — deliberately + // not `taxApplied`. `taxApplied` is what the QBO Purchase + // split onto the reclaimable account, and buildGroups zeroes + // it for a check or a nonsense read. The WA deduction is + // based on tax actually paid at the register, so the receipt's + // own figure is the right one here, and the two are allowed + // to disagree. + taxAmount: row.taxCents !== null ? row.taxCents / 100 : null, + taxAtSource: row.taxAtSource, + installedAtCustomer: row.installedAtCustomer, amount: amountCents / 100, vendor: row.vendor || "Unknown", // RE-ANCHORED at write time. `txnDate` is a @db.Date column diff --git a/src/lib/receipt-intake/worker.ts b/src/lib/receipt-intake/worker.ts index dbc7d6dae..50d19a8f4 100644 --- a/src/lib/receipt-intake/worker.ts +++ b/src/lib/receipt-intake/worker.ts @@ -286,6 +286,8 @@ export interface ReadPatch { txnDate: Date | null; totalCents: number | null; taxCents: number | null; + /** Phase 3: `taxCents > 0` — a positive read, never a present-but-zero one. */ + taxAtSource: boolean; docType: string | null; refNumber: string | null; memo: string | null; @@ -680,6 +682,11 @@ async function processReceived(row: WorkerRow, deps: WorkerDependencies): Promis txnDate: dateOnly(keys.dateStr, timeZone), totalCents, taxCents, + // Phase 3: the receipt carried sales tax GTR paid at the register. An + // ABSENT tax read and a ZERO one are the same answer here — neither is + // evidence that tax was paid — so this is derived from `taxCents` being + // a positive number, never from the field merely existing. + taxAtSource: taxCents !== null && taxCents > 0, docType: read.docType || null, refNumber: keys.ref, memo: read.memo || null, diff --git a/src/lib/time-expense-core.ts b/src/lib/time-expense-core.ts index 0542703a6..a818c89ab 100644 --- a/src/lib/time-expense-core.ts +++ b/src/lib/time-expense-core.ts @@ -184,9 +184,16 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str return prisma.expense.create({ data: { estimateId, + // Phase 3: the estimate's project, already resolved and validated + // above (including the change-order cross-check). + projectId: estimate.projectId, itemId: data.itemId || null, costCodeId: data.costCodeId || null, costTypeId: data.costTypeId || null, + // Every caller of this core is a human picking a code in a web form + // or a CO flow, so a code here is "manual" and is off limits to the + // sync and the backfill. + costCodeSource: data.costCodeId ? "manual" : null, amount: dollars(data.amount), vendor: data.vendor?.trim() || null, date: expenseDate, diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 832f1d2d4..c24cc0cce 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -6,12 +6,14 @@ import { normalizeQboPurchase, syncQboExpenses, upsertQboExpense, + applyQboExpenseCostCodeSuggestion, type QboExpenseProjectCandidate, type QboExpenseSyncDependencies, type QboExpenseWrite, type QboPurchaseNormalizationSkipReason, type QboPurchaseForImport, } from "../src/lib/qbo-expense-sync"; +import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; const TOKENS = { accessToken: "test-access", @@ -562,6 +564,10 @@ type StoredExpense = Omit & { id: string; receiptUrl: string | null; status: "Pending" | "Reviewed"; + // Phase 3 attribution columns, so a test can seed a human-coded row. + costCodeId?: string | null; + costCodeSource?: string | null; + costCodeConfidence?: number | null; }; function createFakePrisma(initial: StoredExpense[] = []) { @@ -625,6 +631,7 @@ const WRITE: QboExpenseWrite = { qbSyncToken: "0", qbSyncedAt: new Date("2026-07-29T12:00:00.000Z"), estimateId: "estimate-1", + projectId: "project-1", amount: 125.5, vendor: "Contractor Supply", date: new Date("2026-07-15T00:00:00.000Z"), @@ -1073,3 +1080,220 @@ test("backfill fixture imports only the active unambiguous job and stays idempot }); assert.equal(fake.rows.size, 1); }); + +// ── Phase 3 attribution (docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §3.1, §8) ── + +test("an imported expense is born knowing its job", async () => { + const fake = createFakePrisma(); + assert.equal(await upsertQboExpense(fake.client, WRITE), "imported"); + assert.equal(fake.rows.get("purchase-1")?.projectId, "project-1"); +}); + +test("a re-sync fills a NULL projectId but never overwrites one", async () => { + // The sync has always KNOWN the project; before Phase 3 it dropped it. A + // row imported back then has projectId NULL and should be caught up. + const legacy = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: null, receiptUrl: null }, + ]); + assert.equal(await upsertQboExpense(legacy.client, { ...WRITE, qbSyncToken: "1" }), "updated"); + assert.equal(legacy.rows.get("purchase-1")?.projectId, "project-1"); + + // ...but a bookkeeper's re-attribution is a HUMAN decision, and the QBO + // customer ref is not a newer fact about it. + const reattributed = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: "moved-by-hand", receiptUrl: null }, + ]); + assert.equal( + await upsertQboExpense(reattributed.client, { ...WRITE, qbSyncToken: "1", amount: 200 }), + "updated", + ); + assert.equal(reattributed.rows.get("purchase-1")?.projectId, "moved-by-hand"); + assert.equal(reattributed.rows.get("purchase-1")?.amount, 200, "the rest of the write still lands"); +}); + +test("a re-attributed row settles to unchanged instead of updating forever", async () => { + // If projectId were compared unconditionally, every re-attributed row would + // read as drifted on every sync and re-issue an update that deliberately + // changes nothing — a permanent phantom write, and a permanently wrong + // "updated" count in the sync report. + const fake = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: "moved-by-hand", receiptUrl: null }, + ]); + assert.equal(await upsertQboExpense(fake.client, WRITE), "unchanged"); + assert.equal(await upsertQboExpense(fake.client, WRITE), "unchanged"); + assert.equal(fake.rows.get("purchase-1")?.projectId, "moved-by-hand"); +}); + +test("deactivation never touches the attribution columns", async () => { + const fake = createFakePrisma([ + { + ...WRITE, + id: "expense-1", + projectId: "project-1", + costCodeId: "cc-plumb", + costCodeSource: "manual", + receiptUrl: null, + }, + ]); + assert.equal( + await deactivateQboExpense(fake.client, { + qbPurchaseId: "purchase-1", + qbSyncToken: "1", + qbSyncedAt: new Date("2026-07-29T14:00:00.000Z"), + reason: "deleted", + }), + "removed", + ); + const row = fake.rows.get("purchase-1"); + assert.equal(row?.amount, 0); + assert.equal(row?.projectId, "project-1", "the job survives the deactivation"); + assert.equal(row?.costCodeId, "cc-plumb"); + assert.equal(row?.costCodeSource, "manual"); +}); + +// ── the cost-code suggester ──────────────────────────────────────────────── + +const COST_CODE_IDS = new Map([ + ["03-PLUMB", "cc-plumb"], + ["02-FRAME", "cc-frame"], +]); + +function fakeSuggestionClient() { + const calls: { where: Record; data: Record }[] = []; + let count = 1; + return { + calls, + setCount(next: number) { count = next; }, + client: { + expense: { + async updateMany(args: { + where: Record; + data: { costCodeId: string; costCodeSource: string; costCodeConfidence: number }; + }) { + calls.push(args); + return { count }; + }, + }, + }, + }; +} + +test("a NULL cost code is filled with source ai and the rule's tier confidence", async () => { + const fake = fakeSuggestionClient(); + const result = await applyQboExpenseCostCodeSuggestion( + fake.client, + { + qbPurchaseId: "purchase-1", + vendor: "Summit Plumbing", + description: "[QuickBooks import] rough-in", + projectId: "project-1", + }, + COST_CODE_IDS, + ); + assert.equal(result, "written"); + assert.equal(fake.calls.length, 1); + assert.deepEqual(fake.calls[0].data, { + costCodeId: "cc-plumb", + costCodeSource: "ai", + costCodeConfidence: 0.9, + }); +}); + +test("the write is guarded on uncoded AND not-human-coded, with a NULL branch", async () => { + // This is the whole of the "never overwrite a human" rule, and it lives in + // the predicate rather than in a caller's discipline. The NULL branch is + // load-bearing: SQL NOT IN drops NULL rows, which is every legacy row. + const fake = fakeSuggestionClient(); + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x", projectId: "project-1" }, + COST_CODE_IDS, + ); + const where = fake.calls[0].where; + assert.equal(where.qbPurchaseId, "purchase-1"); + assert.equal(where.costCodeId, null); + assert.deepEqual(where.OR, [ + { costCodeSource: null }, + { costCodeSource: { notIn: ["capture", "manual"] } }, + ]); +}); + +test("a row a human already coded is reported not-written, not silently written", async () => { + const fake = fakeSuggestionClient(); + fake.setCount(0); // the guard matched nothing + const result = await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x", projectId: "project-1" }, + COST_CODE_IDS, + ); + assert.equal(result, "not-written"); +}); + +test("the overhead bucket is never given a job phase, and never even queried", async () => { + const fake = fakeSuggestionClient(); + const result = await applyQboExpenseCostCodeSuggestion( + fake.client, + { + qbPurchaseId: "purchase-1", + vendor: "Summit Plumbing", + description: "x", + projectId: OVERHEAD_PROJECT_ID, + }, + COST_CODE_IDS, + ); + assert.equal(result, "skipped-overhead"); + assert.equal(fake.calls.length, 0); +}); + +test("no rule match and an unknown code both write nothing", async () => { + const fake = fakeSuggestionClient(); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "p", vendor: "General Hardware", description: "misc", projectId: "project-1" }, + COST_CODE_IDS, + ), + "no-match", + ); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "p", vendor: "Summit Plumbing", description: "x", projectId: "project-1" }, + new Map(), + ), + "unknown-code", + ); + assert.equal(fake.calls.length, 0); +}); + +test("a failing suggester never fails the import it rides alongside", async () => { + const fake = createFakePrisma(); + const dependencies = createSyncDependencies( + [PURCHASE], + ACTIVE_PROJECTS, + write => upsertQboExpense(fake.client, write), + ); + dependencies.suggestCostCode = async () => { throw new Error("cost code table unavailable"); }; + + const result = await syncQboExpenses( + { since: new Date("2026-07-01T00:00:00.000Z") }, + dependencies, + ); + assert.equal(result.imported, 1, "the money is recorded even when the phase guess blows up"); +}); + +test("the sync asks for a phase on a matched job, carrying the job it matched", async () => { + const fake = createFakePrisma(); + const suggested: { qbPurchaseId: string; projectId: string | null }[] = []; + const dependencies = createSyncDependencies( + [PURCHASE], + ACTIVE_PROJECTS, + write => upsertQboExpense(fake.client, write), + ); + dependencies.suggestCostCode = async input => { + suggested.push({ qbPurchaseId: input.qbPurchaseId, projectId: input.projectId }); + }; + + await syncQboExpenses({ since: new Date("2026-07-01T00:00:00.000Z") }, dependencies); + assert.deepEqual(suggested, [{ qbPurchaseId: "purchase-1", projectId: "project-1" }]); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 214c0db8f..e7068f924 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -82,6 +82,8 @@ function row(overrides: Partial = {}): BookableRow { costCodeId: null, suggestedCostCodeId: "cc-plumb", suggestedConfidence: 0.82, + taxAtSource: true, + installedAtCustomer: true, storagePath: "receipts/intake/intake-1.jpg", fileName: "receipt.jpg", mimeType: "image/jpeg", diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index c9d928db0..db35ef2d8 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -50,6 +50,9 @@ function workerRow(overrides: Partial = {}): WorkerRow { projectId: "proj-1", costCodeId: null, suggestedCostCodeId: null, + suggestedConfidence: null, + taxAtSource: false, + installedAtCustomer: null, storagePath: "receipts/intake/row-1.jpg", fileName: "r.jpg", mimeType: "image/jpeg", From 1b2145f3d62170dc31d06f1dc636b738cba87cf5 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:54:46 -0700 Subject: [PATCH 064/144] refactor(expenses): every money-path reader resolves a job through one helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §4. Eleven call sites each hand-rolled `{ estimate: { projectId } }`. They now compose expenseForProjectWhere / expenseHasAnyProjectWhere / resolveExpenseProjectId, which cover BOTH the new column and the estimate relation in a single OR key. This is an identical-output change by construction: the column is nullable and backfilled, so a query using only the column would drop every row the backfill had not reached, and one using only the relation would drop every row a Phase 3 writer stamps directly. Two places needed care rather than a swap: * payouts-report, transactions-report and company-financials-charts already own an `OR` key for their date coalesce. The project predicate is nested under `AND`, never spread — spreading a second OR silently replaces the first and would have fetched the whole of history. * company-financials-charts' all-time groupBy stays on the relation on purpose. Moving it to groupBy(["projectId"]) changes the grouping key, which is a behaviour change dressed as a refactor; the comment says when it can move. job-variance.ts's inline item->costCode fallback now calls resolveExpenseCostCodeId, so the variance report and the rest of the app share one definition of which phase a posting lands on. deleteExpenses resolves the job the same way before its access check — reading the estimate directly would have sent a re-attributed expense to the project it used to be on. Co-Authored-By: Claude Fable 5.1 --- .../projects/[id]/financial-overview/route.ts | 3 +- src/app/projects/[id]/costing/page.tsx | 7 +--- src/app/reports/profitability/page.tsx | 7 +++- src/lib/budget-actions.ts | 3 +- src/lib/company-financials-charts.ts | 28 +++++++++++--- src/lib/job-variance-db.ts | 12 ++++-- src/lib/job-variance.ts | 26 +++++++++++-- src/lib/payouts-report.ts | 10 ++++- src/lib/project-financials.ts | 3 +- src/lib/time-expense-actions.ts | 38 +++++++++++-------- src/lib/transactions-report.ts | 7 +++- tests/job-variance-db.test.ts | 22 ++++++++--- 12 files changed, 118 insertions(+), 48 deletions(-) diff --git a/src/app/api/projects/[id]/financial-overview/route.ts b/src/app/api/projects/[id]/financial-overview/route.ts index 321dd94d8..3793c14b0 100644 --- a/src/app/api/projects/[id]/financial-overview/route.ts +++ b/src/app/api/projects/[id]/financial-overview/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { canAccessProject, canUseDevAuthFallback, @@ -60,7 +61,7 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: select: { amount: true, paidAt: true, createdAt: true } }); const allExpenseRecords = await prisma.expense.findMany({ - where: { estimate: { projectId } }, + where: expenseForProjectWhere(projectId), select: { amount: true, date: true, createdAt: true } }); diff --git a/src/app/projects/[id]/costing/page.tsx b/src/app/projects/[id]/costing/page.tsx index a8ff62804..6b8903be2 100644 --- a/src/app/projects/[id]/costing/page.tsx +++ b/src/app/projects/[id]/costing/page.tsx @@ -1,6 +1,7 @@ import { getServerSession } from "next-auth"; import { authOptions } from "@/lib/auth"; import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { redirect } from "next/navigation"; import JobCostingClient from "./JobCostingClient"; @@ -55,11 +56,7 @@ export default async function JobCostingPage({ }), // Fetch Actuals: Expenses prisma.expense.findMany({ - where: { - estimate: { - projectId: projectId - } - }, + where: expenseForProjectWhere(projectId), include: { costCode: true } }), // Fetch Purchase Orders for "Committed Costs" diff --git a/src/app/reports/profitability/page.tsx b/src/app/reports/profitability/page.tsx index 6f7801bc8..f471fdfe4 100644 --- a/src/app/reports/profitability/page.tsx +++ b/src/app/reports/profitability/page.tsx @@ -1,4 +1,5 @@ import { prisma } from "@/lib/prisma"; +import { expenseHasAnyProjectWhere, resolveExpenseProjectId } from "@/lib/expense-attribution"; import { getServerSession } from "next-auth"; import { authOptions } from "@/lib/auth"; import { redirect } from "next/navigation"; @@ -74,9 +75,11 @@ export default async function ProfitabilityPage({ searchParams }: { searchParams orderBy: { createdAt: "desc" }, }), prisma.expense.findMany({ - where: { estimate: { projectId: { not: null } } }, + // "attributable to SOME job", both ways round (Phase 3). + where: expenseHasAnyProjectWhere(), select: { amount: true, date: true, createdAt: true, vendor: true, description: true, status: true, + projectId: true, estimate: { select: { projectId: true } }, }, }), @@ -85,7 +88,7 @@ export default async function ProfitabilityPage({ searchParams }: { searchParams const expensesByProject = new Map(); for (const e of expenses) { - const pid = e.estimate.projectId; + const pid = resolveExpenseProjectId(e); if (!pid) continue; if (!expensesByProject.has(pid)) expensesByProject.set(pid, []); expensesByProject.get(pid)!.push(e); diff --git a/src/lib/budget-actions.ts b/src/lib/budget-actions.ts index b7bd823f5..71aaa935d 100644 --- a/src/lib/budget-actions.ts +++ b/src/lib/budget-actions.ts @@ -1,6 +1,7 @@ "use server"; import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { isEstimateSectionRow } from "@/lib/estimate-item-payload"; import { toNum } from "@/lib/prisma-helpers"; import { canAccessProject, canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; @@ -42,7 +43,7 @@ export async function getBudgetData(projectId: string) { }); const expenses = await prisma.expense.findMany({ - where: { estimate: { is: { projectId } } }, + where: expenseForProjectWhere(projectId), include: { costCode: true, costType: true, item: true }, }); diff --git a/src/lib/company-financials-charts.ts b/src/lib/company-financials-charts.ts index 603cbc308..5701ed01e 100644 --- a/src/lib/company-financials-charts.ts +++ b/src/lib/company-financials-charts.ts @@ -6,6 +6,11 @@ import { resolveCompanyTimeZone } from "@/lib/company-timezone"; // (src/lib/overhead-project.ts) so this page, the QBO expense sync, and the job // variance report can never point at different projects. import { OVERHEAD_PROJECT_ID } from "@/lib/overhead-project"; +import { + expenseForProjectWhere, + expenseForProjectsWhere, + resolveExpenseProjectId, +} from "@/lib/expense-attribution"; // Same parent-status gating as computeProjectFinancials (src/lib/project-financials.ts, // includeUnissued: false) — Draft invoices/retainers are not receivables and must @@ -289,10 +294,17 @@ export async function getCompanyFinancialsChartData( }), prisma.expense.findMany({ where: { - estimate: { projectId: { in: projectIds } }, + // Under AND, not spread: coalescedDateRange2 returns this + // object's `OR` key, and a second `OR` would replace it and + // silently fetch the whole of history. + AND: [expenseForProjectsWhere(projectIds)], ...coalescedDateRange2(from, to, "date", "createdAt"), }, - select: { amount: true, date: true, createdAt: true, estimate: { select: { projectId: true } } }, + select: { + amount: true, date: true, createdAt: true, + projectId: true, + estimate: { select: { projectId: true } }, + }, }), prisma.timeEntry.findMany({ where: { @@ -304,7 +316,7 @@ export async function getCompanyFinancialsChartData( includeOverhead ? prisma.expense.findMany({ where: { - estimate: { projectId: OVERHEAD_PROJECT_ID }, + AND: [expenseForProjectWhere(OVERHEAD_PROJECT_ID)], ...coalescedDateRange2(from, to, "date", "createdAt"), }, select: { amount: true, date: true, createdAt: true }, @@ -336,6 +348,12 @@ export async function getCompanyFinancialsChartData( // than materializing every expense row. Expense has no direct projectId // column, so group by estimateId and resolve project ids via a small // estimate lookup below. + // DELIBERATELY LEFT ON THE RELATION. Phase 3 added Expense.projectId, + // so this could become a plain groupBy(["projectId"]) — but only once + // every row is backfilled AND every writer stamps it, and this PR's + // contract is identical output. Changing it here would also change the + // grouping key, which is a real behaviour change dressed as a + // refactor. Revisit after the backfill has run in production. prisma.expense.groupBy({ by: ["estimateId"], where: { estimate: { projectId: { in: allJobIds } } }, @@ -455,8 +473,8 @@ export async function getCompanyFinancialsChartData( if (!inRange(d)) continue; const idx = bucketIndex.get(monthKey(d)); if (idx === undefined) continue; - const pid = e.estimate.projectId; - if (!pid) continue; // Estimate.projectId is nullable on this schema + const pid = resolveExpenseProjectId(e); + if (!pid) continue; // both sides can be null on this schema const key = topSet.has(pid) ? pid : "other"; spendByMonth[idx][key] = (spendByMonth[idx][key] ?? 0) + Number(e.amount); } diff --git a/src/lib/job-variance-db.ts b/src/lib/job-variance-db.ts index b1da3ecf7..f79f86ef4 100644 --- a/src/lib/job-variance-db.ts +++ b/src/lib/job-variance-db.ts @@ -1,11 +1,15 @@ // Server-side data loading for the variance report. Kept out of the page // component so the page stays presentational and this stays swappable. // -// Expense has NO projectId column — it reaches a project through its estimate -// (`where: { estimate: { projectId } }`). Querying expense.projectId throws -// PrismaClientValidationError, which once made a job's expenses look like $0. +// Expense reaches a project TWO ways since Phase 3: its own denormalized +// `projectId` (new, nullable, backfilled) or its estimate's. Never hand-roll +// either — `expenseForProjectWhere` covers both in one OR key, and it is what +// keeps this report's numbers identical to the pre-Phase-3 traversal. (The +// header used to say the column does not exist; that stopped being true the +// moment scripts/apply-expense-attribution.mjs ran.) import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { isEstimateSectionRow } from "@/lib/estimate-item-payload"; import { computeProjectVariance, type ProjectVariance, type VarianceEstimateItem } from "@/lib/job-variance"; import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "@/lib/project-phases"; @@ -188,7 +192,7 @@ export async function loadProjectVariance(projectIds?: string[]): Promise its cost code", which is all + // the shared fallback needs. Built once, after the budget loop above has + // filled `itemsById`. + const itemCostCodeById: ReadonlyMap = new Map( + [...itemsById].map(([id, item]) => [id, item.costCodeId]), + ); + /** * Decide which phase a cost belongs to, and whether its item link may be * used — keeping the two CONSISTENT. @@ -297,9 +306,18 @@ export function computeProjectVariance(input: { */ const reconcileAttribution = ( explicitCostCodeId: string | null | undefined, - linkedItem: (ItemVariance & { costCodeId: string | null }) | undefined + itemId: string | null | undefined ): { costCodeId: string | null; item: (ItemVariance & { costCodeId: string | null }) | undefined } => { - const costCodeId = resolveActualCostCodeId(explicitCostCodeId, linkedItem?.costCodeId); + const linkedItem = itemId ? itemsById.get(itemId) : undefined; + // The "own code, else the linked item's code" fallback lives in + // src/lib/expense-attribution.ts — ONE copy, shared with the tax report + // and the readers, so this report and the rest of the app can never + // disagree about which phase a posting lands on. `resolveActualCostCodeId` + // above is the same rule by its older name and delegates to it. + const costCodeId = resolveExpenseCostCodeId( + { costCodeId: explicitCostCodeId ?? null, itemId: itemId ?? null }, + itemCostCodeById, + ); if (!costCodeId) return { costCodeId: null, item: undefined }; // Only credit the item when it lives under the phase being charged. const item = linkedItem && linkedItem.costCodeId === costCodeId ? linkedItem : undefined; @@ -326,7 +344,7 @@ export function computeProjectVariance(input: { // An entry may carry an item, a phase, or neither. const { costCodeId, item: linkedItem } = reconcileAttribution( entry.costCodeId, - entry.estimateItemId ? itemsById.get(entry.estimateItemId) : undefined + entry.estimateItemId ); if (!costCodeId) { @@ -353,7 +371,7 @@ export function computeProjectVariance(input: { const { costCodeId, item: linkedItem } = reconcileAttribution( expense.costCodeId, - expense.itemId ? itemsById.get(expense.itemId) : undefined + expense.itemId ); if (!costCodeId) { diff --git a/src/lib/payouts-report.ts b/src/lib/payouts-report.ts index a8271251f..0cf8660ad 100644 --- a/src/lib/payouts-report.ts +++ b/src/lib/payouts-report.ts @@ -1,4 +1,5 @@ import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { formatLocalDateString, defaultMonthRange, @@ -61,8 +62,13 @@ export async function queryPayoutsData(filters: PayoutsFilters): Promise<{ { date: { gte: filters.from, lt: filters.to } }, { AND: [{ date: null }, { createdAt: { gte: filters.from, lt: filters.to } }] }, ], - // Expense → Project via estimate.projectId - ...(filters.projectId ? { estimate: { projectId: filters.projectId } } : {}), + // Expense → Project BOTH ways (Phase 3). Nested under AND, + // never spread: this `where` already owns an `OR` key for + // the date coalesce, and spreading a second one would + // silently replace it and drop the date window entirely. + ...(filters.projectId + ? { AND: [expenseForProjectWhere(filters.projectId)] } + : {}), }, include: { estimate: { select: { project: { select: { id: true, name: true } } } }, diff --git a/src/lib/project-financials.ts b/src/lib/project-financials.ts index afad3b8d0..9a109b87d 100644 --- a/src/lib/project-financials.ts +++ b/src/lib/project-financials.ts @@ -1,5 +1,6 @@ import { prisma } from "@/lib/prisma"; import { percentCompleteNeedsReview } from "@/lib/percent-complete"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; // Single source of truth for "how much has this project billed, collected, // and cost" — used by the per-project Financial Overview API route @@ -118,7 +119,7 @@ export async function computeProjectFinancials( select: { id: true, status: true, totalAmount: true, balanceDue: true, archivedAt: true }, }), prisma.retainer.findMany({ where: { projectId, status: { in: validRetainerStatuses } } }), - prisma.expense.findMany({ where: { estimate: { projectId } } }), + prisma.expense.findMany({ where: expenseForProjectWhere(projectId) }), prisma.purchaseOrder.findMany({ where: { projectId } }), prisma.timeEntry.findMany({ where: { projectId } }), // Percent complete is READ here, never computed. The nightly recalc cron diff --git a/src/lib/time-expense-actions.ts b/src/lib/time-expense-actions.ts index de79626ad..ce3a02d3c 100644 --- a/src/lib/time-expense-actions.ts +++ b/src/lib/time-expense-actions.ts @@ -2,7 +2,8 @@ import { getServerSession } from "next-auth"; import { authOptions } from "@/lib/auth"; -import { prisma } from "@/lib/prisma"; +import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere, resolveExpenseProjectId } from "@/lib/expense-attribution"; import { revalidatePath } from "next/cache"; import { canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission, canAccessProject } from "@/lib/permissions"; import { @@ -225,24 +226,29 @@ export async function deleteExpenses( const expenses = await prisma.expense.findMany({ where: { id: { in: ids } }, - select: { - id: true, - qbPurchaseId: true, - invoiceId: true, - invoicedAt: true, - estimate: { select: { projectId: true } }, - }, - }); - const accessible = expenses.filter( - e => e.estimate?.projectId && canAccessProject(user, e.estimate.projectId), - ); - for (const expense of accessible) assertExpenseMutableOutsideQbo(expense); + select: { + id: true, + qbPurchaseId: true, + projectId: true, + invoiceId: true, + invoicedAt: true, + estimate: { select: { projectId: true } }, + }, + }); + // Resolve each row's job the ONE way — the new column when it has one, the + // estimate otherwise. Reading `e.estimate.projectId` directly would send a + // re-attributed expense's access check to the project it used to be on. + const withProject = expenses.map(e => ({ ...e, resolvedProjectId: resolveExpenseProjectId(e) })); + const accessible = withProject.filter( + e => e.resolvedProjectId && canAccessProject(user, e.resolvedProjectId), + ); + for (const expense of accessible) assertExpenseMutableOutsideQbo(expense); const allowed = accessible.filter(e => !e.invoiceId && !e.invoicedAt); if (!allowed.length) return { deleted: 0 }; const allowedIds = allowed.map(e => e.id); const projectIds = new Set( - allowed.map(e => e.estimate!.projectId).filter(Boolean) as string[] + allowed.map(e => e.resolvedProjectId).filter(Boolean) as string[] ); const result = await prisma.expense.deleteMany({ @@ -304,7 +310,7 @@ export async function getExpenses(projectId: string) { await assertTimeExpenseProjectAccess(projectId); return prisma.expense.findMany({ - where: { estimate: { projectId } }, + where: expenseForProjectWhere(projectId), include: { costCode: { select: { id: true, name: true, code: true } }, costType: { select: { id: true, name: true } }, @@ -330,7 +336,7 @@ export async function getTimeExpenseData(projectId: string) { }); const expenseRows = await prisma.expense.findMany({ - where: { estimate: { projectId } }, + where: expenseForProjectWhere(projectId), include: { costCode: { select: { id: true, name: true, code: true } }, costType: { select: { id: true, name: true } }, diff --git a/src/lib/transactions-report.ts b/src/lib/transactions-report.ts index 9494d5828..dc79ad89b 100644 --- a/src/lib/transactions-report.ts +++ b/src/lib/transactions-report.ts @@ -1,4 +1,5 @@ import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { formatLocalDateString, defaultMonthRange, @@ -111,7 +112,11 @@ export async function queryTransactionsData(filters: TransactionsFilters): Promi { date: { gte: filters.from, lt: filters.to } }, { AND: [{ date: null }, { createdAt: { gte: filters.from, lt: filters.to } }] }, ], - ...(filters.projectId ? { estimate: { projectId: filters.projectId } } : {}), + // Nested under AND, never spread — the date coalesce above + // already owns this object's only `OR` key. + ...(filters.projectId + ? { AND: [expenseForProjectWhere(filters.projectId)] } + : {}), }, include: { estimate: { select: { project: { select: { id: true, name: true } } } }, diff --git a/tests/job-variance-db.test.ts b/tests/job-variance-db.test.ts index 45a72086d..33f91d8e1 100644 --- a/tests/job-variance-db.test.ts +++ b/tests/job-variance-db.test.ts @@ -283,18 +283,28 @@ test("ChangeOrderItem is FLAT — no section-header exclusion is applied to it", }); // ════════════════════════════════════════════════════════════════════════════ -// EXPENSE REACHES A PROJECT THROUGH ITS ESTIMATE +// EXPENSE REACHES A PROJECT TWO WAYS (Phase 3) // ════════════════════════════════════════════════════════════════════════════ -test("expenses are queried through the estimate relation, NOT expense.projectId", async () => { - // Expense has no projectId column. `where: { projectId }` throws - // PrismaClientValidationError and once made a job's expenses read $0. +test("expenses are queried BOTH ways: the new column and the estimate relation", async () => { + // Before Phase 3 this asserted `{ estimate: { projectId } }` and that + // Expense had no projectId column at all. The column exists now, and it is + // NULLABLE and backfilled — so a query using only the column would drop + // every row the backfill had not reached, and one using only the relation + // would drop every row a future writer stamps directly. Both branches, one + // OR key, and the shape comes from the shared helper rather than being + // spelled out twice. fixture.estimates = [estimateWith([{ id: "i1", total: 10000 }])]; await loadProjectVariance(); const where = recorded.expense[0].where; - assert.deepEqual(where, { estimate: { projectId: "p1" } }); - assert.ok(!("projectId" in where), "expense.projectId does not exist and throws at runtime"); + assert.deepEqual(where, { + OR: [ + { projectId: "p1" }, + { projectId: null, estimate: { projectId: "p1" } }, + ], + }); + assert.deepEqual(Object.keys(where), ["OR"], "one OR key — a second would clobber it"); }); test("expenses are DELIBERATELY not filtered by estimate status", async () => { From 9ac81ae95ba8760d46a73077d5b5ff8825d13c14 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 18:57:39 -0700 Subject: [PATCH 065/144] =?UTF-8?q?feat(expenses):=20attribution=20backfil?= =?UTF-8?q?l=20=E2=80=94=20dry-run=20default,=20coverage=20table,=20CSV=20?= =?UTF-8?q?for=20Marge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §6. Three passes: projectId from the estimate where NULL (belt and braces against the apply script), the item->costCode fallback (source "backfill", ~0 rows today, live the moment item-level capture starts), then the shared rule suggester over what is left — In Progress customer jobs only, overhead excluded BY ID rather than by the name "Shop", which stops working the day the project is renamed. Dry run is the default and it is not decoration: the rules mis-booked a $3,317.78 Mesplay excavator rental as 20-CLEAN and only a human reading the table caught it. The plan is computed with no writes at all, so the dry run prints the exact set an apply would perform, plus before/after dollar coverage per job. Coverage is measured on ABSOLUTE dollars — amount is signed, and netting could drive the denominator to ~0 and report "100% attributed" on data that was 0%. The headline is reported on the variance page's basis (labor + expenses), not expenses alone, because clock-in already requires a phase and an expenses-only number would flatter it. Writes re-check `costCodeId IS NULL` and the not-human-coded predicate at UPDATE time. The plan is a snapshot; the predicate is the guarantee. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.mjs | 358 +++++++++++++++++++++ tests/backfill-expense-attribution.test.ts | 284 ++++++++++++++++ 2 files changed, 642 insertions(+) create mode 100644 scripts/backfill-expense-attribution.mjs create mode 100644 tests/backfill-expense-attribution.test.ts diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs new file mode 100644 index 000000000..dadd89c13 --- /dev/null +++ b/scripts/backfill-expense-attribution.mjs @@ -0,0 +1,358 @@ +/** + * Backfill Expense attribution (Receipt Pipeline v2, Phase 3 — + * docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §6). + * + * Three passes, in order: + * (a) projectId from the owning estimate, where it is still NULL. Belt and + * braces: scripts/apply-expense-attribution.mjs already ran the same + * UPDATE. This reports the count so a non-zero here is a signal that the + * apply script was skipped or that new rows arrived uncoded. + * (b) item fallback: an uncoded expense linked to a CODED estimate item + * copies that item's cost code (source "backfill"). Production expects + * ~0 rows today — expenses rarely carry an itemId — and it becomes live + * the moment item-level capture starts. + * (c) the rule suggester (src/lib/expense-cost-suggest.ts) for everything + * left, scoped to In Progress customer jobs with the overhead bucket + * excluded BY ID, not by name. + * + * DRY RUN IS THE DEFAULT and it is not decoration. The rules have a proven + * failure mode: a $3,317.78 Mesplay excavator rental was matched to 20-CLEAN + * and caught only because a human read the dry-run table. Nothing here writes + * without --apply, and the table it prints is what Justin reviews. + * + * TWO THINGS IT MUST NEVER DO, both in the update predicate rather than in the + * loop's discipline: overwrite an existing cost code, and overwrite a HUMAN's + * (costCodeSource "capture" or "manual"). + * + * USAGE + * node scripts/backfill-expense-attribution.mjs # dry run + * node scripts/backfill-expense-attribution.mjs --csv out.csv # + remainder CSV + * node scripts/backfill-expense-attribution.mjs --apply # write + * + * A re-run after --apply must report 0 planned changes. That is the proof, and + * it is the same rule scripts/backfill-estimate-item-cost-codes.mjs follows. + */ +import { PrismaClient } from "@prisma/client"; +import { config } from "dotenv"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { writeFileSync } from "node:fs"; +import { suggestCode } from "../src/lib/expense-cost-suggest.ts"; +import { + notHumanCodedExpenseWhere, + resolveExpenseCostCodeId, + resolveExpenseProjectId, +} from "../src/lib/expense-attribution.ts"; +import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project.ts"; + +/** + * Both current tiers clear this (0.9 vendor, 0.75 line), so it changes nothing + * today. It exists so that ADDING a weaker tier later is a deliberate act: + * without a floor, a future 0.4 rule would start writing to the books silently. + */ +export const MIN_CONFIDENCE = 0.7; + +const num = v => (v == null ? 0 : Number(v)); +const money = v => + `$${num(v).toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; +const pct = (part, whole) => (whole > 0 ? `${((part / whole) * 100).toFixed(1)}%` : "—"); + +/** + * Coverage on the variance report's own basis: ABSOLUTE dollars. + * + * `Expense.amount` is signed — refunds and credit memos are normal — so netting + * could drive the denominator toward zero and report "100% attributed" on data + * that was 0% attributed. Magnitude of money moved is the honest base, and it + * is the same choice computeProjectVariance makes (job-variance.ts). + */ +export function measureCoverage(rows) { + let attributed = 0; + let unattributed = 0; + let codedCount = 0; + for (const row of rows) { + const amount = Math.abs(num(row.amount)); + if (row.costCodeId) { + attributed += amount; + codedCount += 1; + } else { + unattributed += amount; + } + } + const total = attributed + unattributed; + return { attributed, unattributed, total, codedCount, count: rows.length }; +} + +/** + * Decide every write WITHOUT touching the database, so the dry run can show the + * exact same set the apply would perform. Pure and unit-tested. + * + * @param expenses rows with { id, projectId, estimate:{projectId}, costCodeId, + * costCodeSource, itemId, amount, vendor, description, date } + * @param itemCostCodeById item id -> its cost code id (or null) + * @param costCodeIdByCode "03-PLUMB" -> cost code id + * @param scopedProjectIds the In Progress, non-overhead jobs the suggester may touch + */ +export function planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, scopedProjectIds }) { + const inScope = new Set(scopedProjectIds); + const projectFills = []; + const codeFills = []; + const remainder = []; + + for (const expense of expenses) { + const resolvedProjectId = resolveExpenseProjectId(expense); + if (expense.projectId === null && resolvedProjectId) { + projectFills.push({ id: expense.id, projectId: resolvedProjectId }); + } + + // NEVER re-code a row that already has a code, and never touch a + // human's. The same two conditions are repeated in the update + // predicate — this one keeps the dry-run table honest, that one keeps + // the write safe, and neither is allowed to be the only guard. + if (expense.costCodeId) continue; + if (expense.costCodeSource === "capture" || expense.costCodeSource === "manual") continue; + + // (b) item fallback, for ANY project — a coded line item is a fact + // about the money regardless of the job's status. + const fromItem = resolveExpenseCostCodeId(expense, itemCostCodeById); + if (fromItem) { + codeFills.push({ + id: expense.id, + costCodeId: fromItem, + costCodeSource: "backfill", + costCodeConfidence: null, + why: "item cost code", + expense, + }); + continue; + } + + // (c) the rules, only on active customer jobs. + if (!resolvedProjectId || !inScope.has(resolvedProjectId)) { + remainder.push(expense); + continue; + } + const suggestion = suggestCode(expense); + const costCodeId = suggestion ? costCodeIdByCode.get(suggestion.code) : undefined; + if (suggestion && costCodeId && suggestion.confidence >= MIN_CONFIDENCE) { + codeFills.push({ + id: expense.id, + costCodeId, + costCodeSource: "ai", + costCodeConfidence: suggestion.confidence, + why: suggestion.why, + expense, + }); + } else { + remainder.push(expense); + } + } + + return { projectFills, codeFills, remainder }; +} + +/** Coverage as it WOULD be after the planned code fills — the dry run's whole point. */ +export function projectedRows(expenses, codeFills) { + const byId = new Map(codeFills.map(fill => [fill.id, fill.costCodeId])); + return expenses.map(expense => ({ + ...expense, + costCodeId: expense.costCodeId ?? byId.get(expense.id) ?? null, + })); +} + +function csvEscape(value) { + return `"${String(value ?? "").replace(/"/g, '""').replace(/\s+/g, " ").slice(0, 160)}"`; +} + +export function remainderCsv(remainder, projectNameById) { + const lines = [["expense_id", "project", "date", "vendor", "amount", "description"].join(",")]; + for (const expense of remainder) { + lines.push([ + expense.id, + csvEscape(projectNameById.get(resolveExpenseProjectId(expense)) ?? ""), + expense.date ? new Date(expense.date).toISOString().slice(0, 10) : "", + csvEscape(expense.vendor), + num(expense.amount).toFixed(2), + csvEscape(expense.description), + ].join(",")); + } + return lines.join("\n"); +} + +/** + * The whole run, with every external effect injected so + * tests/backfill-expense-attribution.test.ts can drive it with a prisma-shaped + * stub and assert that a dry run makes ZERO write calls. + */ +export async function runBackfill({ + db, + apply = false, + // Annotated rather than left to inference: a bare `null` default infers the + // parameter as `null`, and every caller that passes a real path (including + // the test) then fails to typecheck. + csvPath = /** @type {string | null} */ (null), + writeFile = /** @type {(path: string, body: string) => void} */ (writeFileSync), + log = /** @type {(message: string) => void} */ (console.log), + overheadProjectId = OVERHEAD_PROJECT_ID, +}) { + const [projects, costCodes] = await Promise.all([ + db.project.findMany({ select: { id: true, name: true, status: true } }), + db.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }), + ]); + const projectNameById = new Map(projects.map(p => [p.id, p.name])); + const costCodeIdByCode = new Map(costCodes.map(c => [c.code, c.id])); + // Overhead is excluded BY ID. The original script excluded it by NAME + // ("Shop"), which silently stops working the day the project is renamed. + const scopedProjectIds = projects + .filter(p => p.status === "In Progress" && p.id !== overheadProjectId) + .map(p => p.id); + + const expenses = await db.expense.findMany({ + select: { + id: true, projectId: true, costCodeId: true, costCodeSource: true, + itemId: true, amount: true, vendor: true, description: true, date: true, + estimate: { select: { projectId: true } }, + }, + }); + + const items = await db.estimateItem.findMany({ + where: { costCodeId: { not: null } }, + select: { id: true, costCodeId: true }, + }); + const itemCostCodeById = new Map(items.map(i => [i.id, i.costCodeId])); + + const plan = planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, scopedProjectIds }); + + // ── the table ─────────────────────────────────────────────────────────── + const scoped = new Set(scopedProjectIds); + const inScopeExpenses = expenses.filter(e => scoped.has(resolveExpenseProjectId(e) ?? "")); + const before = measureCoverage(inScopeExpenses); + const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills)); + + log(`scope: ${scopedProjectIds.length} In Progress customer job(s); overhead project ${overheadProjectId} excluded`); + log(""); + log("per job (expense dollars with a resolvable cost code):"); + log(` ${"job".padEnd(34)} ${"coded".padStart(11)} ${"total".padStart(13)} before -> after`); + for (const projectId of scopedProjectIds) { + const rows = inScopeExpenses.filter(e => resolveExpenseProjectId(e) === projectId); + if (rows.length === 0) continue; + const b = measureCoverage(rows); + const a = measureCoverage(projectedRows(rows, plan.codeFills)); + log( + ` ${(projectNameById.get(projectId) ?? projectId).slice(0, 34).padEnd(34)} ` + + `${`${a.codedCount}/${a.count}`.padStart(11)} ${money(a.total).padStart(13)} ` + + `${pct(b.attributed, b.total)} -> ${pct(a.attributed, a.total)}`, + ); + } + log(""); + log(`EXPENSE dollar coverage: ${pct(before.attributed, before.total)} -> ${pct(after.attributed, after.total)} (${money(after.attributed)} of ${money(after.total)})`); + + // The §1.6 headline is measured on the variance page's basis, which counts + // LABOR as well. Reporting only the expense share would flatter the number, + // because clock-in already requires a phase. + const timeEntries = await db.timeEntry.findMany({ + where: { projectId: { in: scopedProjectIds } }, + select: { costCodeId: true, estimateItemId: true, laborCost: true, burdenCost: true }, + }); + const laborRows = timeEntries.map(t => ({ + costCodeId: resolveExpenseCostCodeId( + { costCodeId: t.costCodeId, itemId: t.estimateItemId }, + itemCostCodeById, + ), + amount: num(t.laborCost) + num(t.burdenCost), + })); + const labor = measureCoverage(laborRows); + const variancedBefore = before.attributed + labor.attributed; + const variancedAfter = after.attributed + labor.attributed; + const variancedTotal = after.total + labor.total; + log(`VARIANCE-BASIS coverage (labor + expenses, the §1.6 metric): ${pct(variancedBefore, variancedTotal)} -> ${pct(variancedAfter, variancedTotal)}`); + log(""); + log(`planned writes: ${plan.projectFills.length} projectId, ${plan.codeFills.length} cost code`); + const byCode = new Map(); + for (const fill of plan.codeFills) { + const key = `${fill.costCodeId} (${fill.costCodeSource})`; + const entry = byCode.get(key) ?? { n: 0, sum: 0 }; + entry.n += 1; + entry.sum += Math.abs(num(fill.expense.amount)); + byCode.set(key, entry); + } + for (const [key, v] of [...byCode.entries()].sort((a, b) => b[1].sum - a[1].sum)) { + log(` ${key.padEnd(40)} ${String(v.n).padStart(4)} rows ${money(v.sum).padStart(13)}`); + } + log(`NEEDS HUMAN: ${plan.remainder.length} rows ${money(plan.remainder.reduce((t, e) => t + Math.abs(num(e.amount)), 0))}`); + + if (csvPath) { + writeFile(csvPath, remainderCsv(plan.remainder, projectNameById)); + log(`wrote ${csvPath} (${plan.remainder.length} rows for Marge)`); + } + + if (!apply) { + log(""); + log("DRY RUN — nothing written. Re-run with --apply once the table above is reviewed."); + return { plan, before, after, written: { projectIds: 0, costCodes: 0 } }; + } + + // ── the writes ────────────────────────────────────────────────────────── + let projectIdsWritten = 0; + const byProject = new Map(); + for (const fill of plan.projectFills) { + if (!byProject.has(fill.projectId)) byProject.set(fill.projectId, []); + byProject.get(fill.projectId).push(fill.id); + } + for (const [projectId, ids] of byProject) { + // `projectId: null` in the predicate, not just in the plan: between the + // read above and this write, a re-sync or a bookkeeper may have set it. + const result = await db.expense.updateMany({ + where: { id: { in: ids }, projectId: null }, + data: { projectId }, + }); + projectIdsWritten += result.count; + } + + let costCodesWritten = 0; + for (const fill of plan.codeFills) { + const result = await db.expense.updateMany({ + where: { id: fill.id, costCodeId: null, ...notHumanCodedExpenseWhere() }, + data: { + costCodeId: fill.costCodeId, + costCodeSource: fill.costCodeSource, + costCodeConfidence: fill.costCodeConfidence, + }, + }); + costCodesWritten += result.count; + } + + log(""); + log(`applied ${projectIdsWritten} projectId and ${costCodesWritten} cost code(s).`); + log(`${plan.remainder.length} rows left NULL for human review. Re-run (dry) — it must report 0 planned writes.`); + return { + plan, + before, + after, + written: { projectIds: projectIdsWritten, costCodes: costCodesWritten }, + }; +} + +async function main() { + const __dirname = dirname(fileURLToPath(import.meta.url)); + config({ path: join(__dirname, "..", ".env.local") }); + config({ path: join(__dirname, "..", ".env") }); + + const apply = process.argv.includes("--apply"); + const csvIdx = process.argv.indexOf("--csv"); + const csvPath = csvIdx > -1 ? process.argv[csvIdx + 1] : null; + + const prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL } } }); + try { + await runBackfill({ db: prisma, apply, csvPath }); + } finally { + await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error("FAILED:", error); + process.exitCode = 1; + }); +} diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts new file mode 100644 index 000000000..2fef80202 --- /dev/null +++ b/tests/backfill-expense-attribution.test.ts @@ -0,0 +1,284 @@ +/** + * The backfill writes to the books, so the two properties that matter are + * "a dry run writes NOTHING" and "an apply cannot overwrite a human". + * + * Everything is driven through an injected prisma-shaped stub — no module + * mocking (CI is Node 20), no database. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + MIN_CONFIDENCE, + measureCoverage, + planBackfill, + projectedRows, + remainderCsv, + runBackfill, +} from "../scripts/backfill-expense-attribution.mjs"; + +const OVERHEAD_ID = "overhead-project"; + +type StubExpense = { + id: string; + projectId: string | null; + costCodeId: string | null; + costCodeSource: string | null; + itemId: string | null; + amount: number; + vendor: string | null; + description: string | null; + date: Date | null; + estimate: { projectId: string | null }; +}; + +function expense(overrides: Partial = {}): StubExpense { + return { + id: "e1", + projectId: null, + costCodeId: null, + costCodeSource: null, + itemId: null, + amount: 100, + vendor: null, + description: null, + date: new Date("2026-08-01T00:00:00.000Z"), + estimate: { projectId: "job-1" }, + ...overrides, + }; +} + +const COST_CODE_IDS = new Map([ + ["03-PLUMB", "cc-plumb"], + ["02-FRAME", "cc-frame"], +]); + +function createStub(expenses: StubExpense[], items: { id: string; costCodeId: string | null }[] = []) { + const writes: { where: Record; data: Record }[] = []; + return { + writes, + db: { + project: { + async findMany() { + return [ + { id: "job-1", name: "Mueller Bath", status: "In Progress" }, + { id: "job-closed", name: "Old Job", status: "Closed Complete" }, + { id: OVERHEAD_ID, name: "Shop", status: "In Progress" }, + ]; + }, + }, + costCode: { + async findMany() { + return [...COST_CODE_IDS].map(([code, id]) => ({ id, code })); + }, + }, + estimateItem: { + async findMany() { return items; }, + }, + timeEntry: { + async findMany() { return []; }, + }, + expense: { + async findMany() { return expenses; }, + async updateMany(args: { where: Record; data: Record }) { + writes.push(args); + return { count: 1 }; + }, + }, + }, + }; +} + +// ── planning ──────────────────────────────────────────────────────────────── + +test("plans a projectId fill only for rows whose column is still NULL", () => { + const plan = planBackfill({ + expenses: [ + expense({ id: "needs-fill", projectId: null, estimate: { projectId: "job-1" } }), + expense({ id: "already-set", projectId: "job-1", estimate: { projectId: "job-1" } }), + expense({ id: "no-answer", projectId: null, estimate: { projectId: null } }), + ], + itemCostCodeById: new Map(), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.projectFills, [{ id: "needs-fill", projectId: "job-1" }]); +}); + +test("the item fallback wins over the rules, and is sourced 'backfill'", () => { + const plan = planBackfill({ + expenses: [expense({ id: "e1", itemId: "item-1", vendor: "Summit Plumbing" })], + itemCostCodeById: new Map([["item-1", "cc-frame"]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.equal(plan.codeFills.length, 1); + assert.equal(plan.codeFills[0].costCodeId, "cc-frame", "a real link beats a regex guess"); + assert.equal(plan.codeFills[0].costCodeSource, "backfill"); + assert.equal(plan.codeFills[0].costCodeConfidence, null); +}); + +test("a human's cost code is never planned over — capture and manual both", () => { + for (const costCodeSource of ["capture", "manual"]) { + const plan = planBackfill({ + expenses: [expense({ costCodeSource, vendor: "Summit Plumbing" })], + itemCostCodeById: new Map(), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, [], `${costCodeSource} must be untouchable`); + assert.deepEqual(plan.remainder, [], "and it is not 'needs a human' either — it HAS an answer"); + } +}); + +test("an already-coded row is left alone even with a NULL source", () => { + const plan = planBackfill({ + expenses: [expense({ costCodeId: "cc-existing", vendor: "Summit Plumbing" })], + itemCostCodeById: new Map(), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, []); +}); + +test("the overhead bucket and closed jobs are out of the suggester's scope", () => { + const plan = planBackfill({ + expenses: [ + expense({ id: "overhead", projectId: OVERHEAD_ID, estimate: { projectId: OVERHEAD_ID }, vendor: "Summit Plumbing" }), + expense({ id: "closed", projectId: "job-closed", estimate: { projectId: "job-closed" }, vendor: "Summit Plumbing" }), + expense({ id: "active", projectId: "job-1", estimate: { projectId: "job-1" }, vendor: "Summit Plumbing" }), + ], + itemCostCodeById: new Map(), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills.map(f => f.id), ["active"]); + assert.deepEqual(plan.remainder.map(e => e.id).sort(), ["closed", "overhead"]); +}); + +test("a rule hit below the confidence floor is left for a human", () => { + // Both current tiers clear 0.7, so this is asserted through the constant + // rather than through a rule — the floor's job is to make ADDING a weaker + // tier a deliberate act instead of a silent one. + assert.equal(MIN_CONFIDENCE, 0.7); + const plan = planBackfill({ + expenses: [expense({ vendor: "Summit Plumbing" })], + itemCostCodeById: new Map(), + // The rules name 03-PLUMB; this company does not have that code. + costCodeIdByCode: new Map(), + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder.length, 1, "an unknown code is a human's problem, not a skip"); +}); + +// ── coverage reporting ────────────────────────────────────────────────────── + +test("coverage is measured on ABSOLUTE dollars so refunds cannot fake 100%", () => { + const rows = [ + { costCodeId: "cc-plumb", amount: 1000 }, + { costCodeId: null, amount: -1000 }, + ]; + const coverage = measureCoverage(rows); + assert.equal(coverage.total, 2000, "netting would make this 0 and the share meaningless"); + assert.equal(coverage.attributed, 1000); + assert.equal(coverage.unattributed, 1000); +}); + +test("the projected 'after' applies the plan without touching the database", () => { + const rows = [expense({ id: "e1", vendor: "Summit Plumbing", amount: 400 })]; + const plan = planBackfill({ + expenses: rows, + itemCostCodeById: new Map(), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.equal(measureCoverage(rows).attributed, 0); + assert.equal(measureCoverage(projectedRows(rows, plan.codeFills)).attributed, 400); + assert.equal(rows[0].costCodeId, null, "the source rows are not mutated"); +}); + +test("the remainder CSV carries what Marge needs to decide", () => { + const csv = remainderCsv( + [expense({ id: "e9", vendor: 'Lowe"s', description: "misc\nsupplies", amount: 12.5 })], + new Map([["job-1", "Mueller Bath"]]), + ); + const [header, row] = csv.split("\n"); + assert.equal(header, "expense_id,project,date,vendor,amount,description"); + assert.match(row, /^e9,"Mueller Bath",2026-08-01,"Lowe""s",12\.50,"misc supplies"$/); +}); + +// ── the run ───────────────────────────────────────────────────────────────── + +test("a dry run makes ZERO write calls", async () => { + const stub = createStub([ + expense({ id: "e1", vendor: "Summit Plumbing" }), + expense({ id: "e2", projectId: null, estimate: { projectId: "job-1" } }), + ]); + const result = await runBackfill({ db: stub.db, apply: false, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(stub.writes.length, 0, "dry run is the default and it must be inert"); + assert.equal(result.written.projectIds, 0); + assert.equal(result.written.costCodes, 0); + assert.ok(result.plan.codeFills.length > 0, "...while still PLANNING the work it would do"); +}); + +test("apply writes both passes, each behind its own predicate", async () => { + const stub = createStub([ + expense({ id: "e1", vendor: "Summit Plumbing", projectId: null, estimate: { projectId: "job-1" } }), + ]); + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + + const projectWrite = stub.writes.find(w => "projectId" in w.data)!; + assert.deepEqual(projectWrite.where, { id: { in: ["e1"] }, projectId: null }); + + const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; + assert.equal(codeWrite.where.id, "e1"); + assert.equal(codeWrite.where.costCodeId, null); + assert.deepEqual(codeWrite.where.OR, [ + { costCodeSource: null }, + { costCodeSource: { notIn: ["capture", "manual"] } }, + ]); + assert.deepEqual(codeWrite.data, { + costCodeId: "cc-plumb", + costCodeSource: "ai", + costCodeConfidence: 0.9, + }); +}); + +test("the write predicate re-checks NULL, not just the plan", async () => { + // Between the read and the write, a re-sync or a bookkeeper can set either + // field. A plan is a snapshot; the predicate is the guarantee. + const stub = createStub([expense({ id: "e1", vendor: "Summit Plumbing" })]); + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + for (const write of stub.writes) { + const guardsNull = + write.where.projectId === null || write.where.costCodeId === null; + assert.ok(guardsNull, `unguarded write: ${JSON.stringify(write.where)}`); + } +}); + +test("a re-run over already-attributed data plans nothing", async () => { + // The proof rule: after --apply, a dry run must report zero. + const stub = createStub([ + expense({ id: "e1", projectId: "job-1", costCodeId: "cc-plumb", costCodeSource: "ai", vendor: "Summit Plumbing" }), + ]); + const result = await runBackfill({ db: stub.db, apply: false, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.deepEqual(result.plan.projectFills, []); + assert.deepEqual(result.plan.codeFills, []); +}); + +test("the CSV is written on a dry run — reviewing it is the point", async () => { + const stub = createStub([expense({ id: "e1", vendor: "Unknown Hardware" })]); + const files: { path: string; body: string }[] = []; + await runBackfill({ + db: stub.db, + apply: false, + csvPath: "out.csv", + writeFile: (path: string, body: string) => { files.push({ path, body }); }, + log: () => {}, + overheadProjectId: OVERHEAD_ID, + }); + assert.equal(files.length, 1); + assert.equal(files[0].path, "out.csv"); + assert.match(files[0].body, /^expense_id,project,date,vendor,amount,description/); + assert.match(files[0].body, /e1/); +}); From 20b3a034100077790aaf9ac2d4d4adb2ada6e928 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:11:31 -0700 Subject: [PATCH 066/144] =?UTF-8?q?feat(reports):=20/reports/tax-paid-at-s?= =?UTF-8?q?ource=20=E2=80=94=20the=20WA=20excise=20deduction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §7. The figure Marge assembles by hand into a spreadsheet today, computed from the receipts the pipeline already books: per month × job, with the CSV column shape Vanessa's handoff file already uses (Date, Vendor, Job, Invoice, Receipt Total, deduction base, Tax) so nothing downstream has to change. THREE POSITIVE CONDITIONS, no negative ones: taxAtSource true, taxAmount > 0, and installedAtCustomer === true. A NULL "installed" is "nobody said", and a NULL must never be spent as a tax deduction — so the query asks for `true` rather than `not false`. Gated by the financialReports permission on BOTH the page and the CSV route: an export is a second door onto the same money, and a page gate the download bypasses is not a gate. Also exposes overheadProjectId on /api/mobile/me (spec §5c) so the app can default its "installed at customer job" toggle, and records the as-built mobile contract in the spec — the mobile repo itself is untouched and ships separately. Rows are bucketed by project ID, not name: two jobs can share a name, and merging them would put one client's deduction under another's. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 30 ++ package.json | 2 +- src/app/api/mobile/me/route.ts | 8 + .../tax-paid-at-source/export/route.ts | 47 +++ src/app/reports/page.tsx | 5 + .../TaxAtSourceFiltersForm.tsx | 71 +++++ src/app/reports/tax-paid-at-source/page.tsx | 185 ++++++++++++ src/lib/qbo-expense-sync.ts | 3 +- src/lib/tax-at-source-report.ts | 269 ++++++++++++++++++ tests/tax-at-source-report.test.ts | 170 +++++++++++ 10 files changed, 788 insertions(+), 2 deletions(-) create mode 100644 src/app/api/reports/tax-paid-at-source/export/route.ts create mode 100644 src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx create mode 100644 src/app/reports/tax-paid-at-source/page.tsx create mode 100644 src/lib/tax-at-source-report.ts create mode 100644 tests/tax-at-source-report.test.ts diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index b3ca56738..4d8c81dcd 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -270,6 +270,36 @@ File-level diff: - Gate on Phase 1 being deployed; until then ship (a)-(c) with the existing `/api/expenses` POST carrying `costCodeId` (§3.4 accepts it). +### AS-BUILT (site side, 2026-09-01) — what the mobile PR can rely on + +The mobile repo was **not** touched by this PR. Everything the app needs now +exists and is deployed with the site, so the mobile diff is purely client-side. + +Server contracts the app can call today: + +| what the app needs | endpoint / field | state | +|---|---|---| +| overhead project id (to default the toggle) | `GET /api/mobile/me` → `overheadProjectId` (new) | **done**, `src/app/api/mobile/me/route.ts` | +| phase list for the picker | `GET /api/projects/[id]/cost-codes` + `/estimate-items` | already existed, already proxied (`src/proxy.ts`) | +| no-photo expense WITH a phase | `POST /api/expenses` now accepts `costCodeId` | **done** — validated through `resolveCostCode` AND `isCostCodeAllowedForProject`, stored with `costCodeSource: "capture"`. Rejections are `{error, code}` with `COST_CODE_NOT_FOUND` / `COST_CODE_INACTIVE` / `PHASE_NOT_ON_PROJECT`, so the app can show a useful message | +| photo expense through the pipeline | `POST /api/receipts/intake` accepts `projectId`, `costCodeId`, `installedAtCustomer` | **done** — `installedAtCustomer` is read from JSON (`true`/`false`) or multipart (`"true"`/`"false"`); anything else means "the caller did not say" | +| the toggle's default | server-side `resolveInstalledAtCustomer` | **done** — an explicit value from the app always wins; silence defaults TRUE for a real job, FALSE for the overhead project, and NULL when there is no project. The app should still SHOW the toggle: the server default is a fallback, not a substitute for asking | + +Auth is unchanged: `/api/receipts/intake` is on the proxy's exact-match public +bypass and calls `authenticateMobileOrSession` itself, so the crew Bearer token +works with no proxy change. + +Remaining mobile-repo diff (a separate PR in `gtr-probuild-mobile`): +- `apps/mobile/app/(tabs)/expenses.tsx` — (a) default the project dropdown to + the clocked-in job; (b) add the phase picker (`lib/phasePicker.ts` labels); + (c) add the "Installed at customer job" toggle, defaulted from + `overheadProjectId`; (d) when a PHOTO is attached, submit via + `api.receipts.intake` instead of the signed-upload + `/api/expenses` pair. +- `apps/mobile/lib/api.ts` + `lib/api-types.ts` — add `receipts.intake(...)` + and the `overheadProjectId` field on the `/me` response type. +- The `/api/expenses` no-photo path keeps working unchanged for older builds; + `costCodeId` is optional there on purpose, so a legacy app is never broken. + ## 6. Backfill — `scripts/backfill-expense-attribution.mjs` One-shot, dry-run DEFAULT (`--apply` to write, `--csv `), same shape and .env diff --git a/package.json b/package.json index e5eef6223..3e01aaa89 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/mobile/me/route.ts b/src/app/api/mobile/me/route.ts index fda8dbb2d..b4b19adec 100644 --- a/src/app/api/mobile/me/route.ts +++ b/src/app/api/mobile/me/route.ts @@ -4,6 +4,7 @@ import { authenticateMobileOnly } from "@/lib/mobile-auth"; import { getEffectivePermissions } from "@/lib/permissions"; import { toNum } from "@/lib/prisma-helpers"; import { mobileVisibleProjectWhere } from "@/lib/project-status"; +import { OVERHEAD_PROJECT_ID } from "@/lib/overhead-project"; export const dynamic = "force-dynamic"; @@ -84,5 +85,12 @@ export async function GET(req: Request) { }, permissions, assignedProjects, + // Phase 3 (spec §5c): the app defaults the receipt capture screen's + // "installed at customer job" toggle OFF for the Shop/overhead bucket + // and ON for a real job. It cannot know which project that is without + // being told, and hard-coding the id in the app would mean a re-release + // to change it. The server intake route applies the SAME default when + // the app stays silent — this is a UI hint, never the enforcement. + overheadProjectId: OVERHEAD_PROJECT_ID, }); } diff --git a/src/app/api/reports/tax-paid-at-source/export/route.ts b/src/app/api/reports/tax-paid-at-source/export/route.ts new file mode 100644 index 000000000..6ec3308f5 --- /dev/null +++ b/src/app/api/reports/tax-paid-at-source/export/route.ts @@ -0,0 +1,47 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getSessionOrDev } from "@/lib/auth"; +import { canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; +import { + formatLocalDateString, + parseTaxAtSourceFilters, + queryTaxAtSourceRows, + rowsToCsv, +} from "@/lib/tax-at-source-report"; + +export const dynamic = "force-dynamic"; + +/** + * The CSV Vanessa's handoff file is built from. Gated by the SAME permission as + * the page — an export route is a second door onto the same money, and a page + * gate that the download bypasses is not a gate. + */ +export async function GET(req: NextRequest) { + const session = await getSessionOrDev(); + if (!session?.user?.email) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + const user = await getCurrentUserWithPermissions(); + const devAllowed = await canUseDevAuthFallback(); + if ((!user || !hasPermission(user, "financialReports")) && !devAllowed) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + const filters = parseTaxAtSourceFilters(req.nextUrl.searchParams); + const rows = await queryTaxAtSourceRows(filters); + + // Both dates come from parseLocalDateString (or the quarter fallback), so + // they can only be YYYY-MM-DD and cannot inject a header. Formatted + // defensively anyway rather than interpolating the raw query string. + const inclusiveTo = new Date(filters.to.getTime()); + inclusiveTo.setDate(inclusiveTo.getDate() - 1); + const filename = `tax-paid-at-source-${formatLocalDateString(filters.from)}-to-${formatLocalDateString(inclusiveTo)}.csv`; + + return new NextResponse(rowsToCsv(rows), { + status: 200, + headers: { + "Content-Type": "text/csv; charset=utf-8", + "Content-Disposition": `attachment; filename="${filename}"`, + "Cache-Control": "no-store", + }, + }); +} diff --git a/src/app/reports/page.tsx b/src/app/reports/page.tsx index 113ca7eaf..5cd7f3b00 100644 --- a/src/app/reports/page.tsx +++ b/src/app/reports/page.tsx @@ -59,6 +59,11 @@ const REPORT_SECTIONS = [ description: "Cash or accrual basis sales tax report. Filter by date range, client, project, and payment method with CSV export for your bookkeeper.", href: "/reports/sales-tax", }, + { + title: "Tax Paid at Source", + description: "Sales tax paid at the register on material installed at a customer job — the WA excise deduction, per month and per job, with the CSV your bookkeeper already expects.", + href: "/reports/tax-paid-at-source", + }, ], }, { diff --git a/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx b/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx new file mode 100644 index 000000000..98290458b --- /dev/null +++ b/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx @@ -0,0 +1,71 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { formatLocalDateString, type TaxAtSourceFilters } from "@/lib/tax-at-source-report"; + +/** The `to` the server holds is EXCLUSIVE; the picker shows the inclusive day. */ +function inclusiveTo(filters: TaxAtSourceFilters): string { + const day = new Date(filters.to.getTime()); + day.setDate(day.getDate() - 1); + return formatLocalDateString(day); +} + +export default function TaxAtSourceFiltersForm({ filters }: { filters: TaxAtSourceFilters }) { + const router = useRouter(); + const [from, setFrom] = useState(formatLocalDateString(filters.from)); + const [to, setTo] = useState(inclusiveTo(filters)); + + function apply(next?: { from: string; to: string }) { + const params = new URLSearchParams({ + from: next?.from ?? from, + to: next?.to ?? to, + }); + router.push(`/reports/tax-paid-at-source?${params.toString()}`); + } + + function applyQuarter(offset: number) { + const now = new Date(); + const start = new Date(now.getFullYear(), Math.floor(now.getMonth() / 3) * 3 + offset * 3, 1); + const end = new Date(start.getFullYear(), start.getMonth() + 3, 0); + const nextFrom = formatLocalDateString(start); + const nextTo = formatLocalDateString(end); + setFrom(nextFrom); + setTo(nextTo); + apply({ from: nextFrom, to: nextTo }); + } + + return ( +
+ + + +
+ + +
+
+ ); +} diff --git a/src/app/reports/tax-paid-at-source/page.tsx b/src/app/reports/tax-paid-at-source/page.tsx new file mode 100644 index 000000000..0beee3edd --- /dev/null +++ b/src/app/reports/tax-paid-at-source/page.tsx @@ -0,0 +1,185 @@ +export const dynamic = "force-dynamic"; +import Link from "next/link"; +import { getSessionOrDev } from "@/lib/auth"; +import { canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; +import { formatCurrency } from "@/lib/utils"; +import { formatMoneyDate } from "@/lib/payment-date"; +import { + TAX_REPORT_AMOUNT_NOTE, + TAX_REPORT_FOOTNOTE, + groupTaxAtSource, + parseTaxAtSourceFilters, + queryTaxAtSourceRows, + stringifyTaxAtSourceFilters, +} from "@/lib/tax-at-source-report"; +import TaxAtSourceFiltersForm from "./TaxAtSourceFiltersForm"; + +type SearchParams = Record; + +export default async function TaxPaidAtSourcePage({ + searchParams, +}: { + searchParams: Promise; +}) { + const session = await getSessionOrDev(); + if (!session?.user) { + return
Access Denied.
; + } + // Same gate as the sibling company-financials report: the permission, not a + // role list, so Justin can grant it to a bookkeeper without making them an + // admin. + const user = await getCurrentUserWithPermissions(); + const devAllowed = await canUseDevAuthFallback(); + if ((!user || !hasPermission(user, "financialReports")) && !devAllowed) { + return
Access denied. This report requires the Financial Reports permission.
; + } + + const filters = parseTaxAtSourceFilters(await searchParams); + const rows = await queryTaxAtSourceRows(filters); + const { months, summary } = groupTaxAtSource(rows); + const csvHref = `/api/reports/tax-paid-at-source/export?${stringifyTaxAtSourceFilters(filters)}`; + + const dateLabel = (value: Date) => + value.toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" }); + const inclusiveTo = new Date(filters.to.getTime()); + inclusiveTo.setDate(inclusiveTo.getDate() - 1); + + return ( +
+
+
+

Tax Paid at Source

+

+ Material tax paid at the register and installed at a customer job · {dateLabel(filters.from)} → {dateLabel(inclusiveTo)} +

+
+
+ Export CSV + ← All Reports +
+
+ + + +
+ + + +
+ +
+
+ By Month and Job +
+ {months.length === 0 ? ( +
+ No receipts in this period carry sales tax flagged as installed at a customer job. +
+ ) : ( +
+ + + + + + + + + + + + {months.map(month => ( + <> + {month.jobs.map((job, index) => ( + + + + + + + + ))} + + + + + + + + ))} + + + + + + + + + +
MonthJobReceiptsTaxable AmountTax Paid at Source
+ {index === 0 ? month.label : ""} + {job.projectName}{job.count}{formatCurrency(job.deductionBase)}{formatCurrency(job.tax)}
{month.label} total + {month.count}{formatCurrency(month.deductionBase)}{formatCurrency(month.tax)}
Total{summary.count}{formatCurrency(summary.deductionBase)}{formatCurrency(summary.tax)}
+
+ )} +
+ +
+
+ Detail + {rows.length} {rows.length === 1 ? "receipt" : "receipts"} +
+ {rows.length === 0 ? ( +
No matching receipts.
+ ) : ( +
+ + + + + + + + + + + + + + {rows.map(row => ( + + + + + + + + + + ))} + +
DateVendorJobInvoiceReceipt TotalTaxable AmountTax
+ {formatMoneyDate(row.date, { month: "short", day: "numeric", year: "numeric" }, "en-US")} + {row.vendor || "—"}{row.projectName}{row.reference || "—"}{formatCurrency(row.receiptTotal)}{formatCurrency(row.deductionBase)}{formatCurrency(row.tax)}
+
+ )} +
+ +

+ {TAX_REPORT_FOOTNOTE} {TAX_REPORT_AMOUNT_NOTE} Receipts imported from QuickBooks before the + receipt pipeline recorded tax carry no tax figure and are not included. +

+
+ ); +} + +function SummaryCard({ label, value, sub, accent }: { label: string; value: string; sub?: string; accent?: "amber" }) { + const color = accent === "amber" ? "text-amber-600" : "text-hui-textMain"; + return ( +
+

{label}

+

{value}

+ {sub &&

{sub}

} +
+ ); +} diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 2719fbdbd..98a842cec 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -573,7 +573,8 @@ function qboExpenseUpdateData( write: QboExpenseWrite, ): Partial { if ((existing.projectId ?? null) === null) return write; - const { projectId: _ignored, ...rest } = write; + const rest: Partial = { ...write }; + delete rest.projectId; return rest; } diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts new file mode 100644 index 000000000..a85cdae82 --- /dev/null +++ b/src/lib/tax-at-source-report.ts @@ -0,0 +1,269 @@ +// "Tax paid at source" — the WA excise deduction (Phase 3 spec §7). +// +// GTR holds a reseller's permit, but in practice it pays sales tax at the +// register on most material. When that material is resold as part of customer +// work, the tax already paid is deductible on the excise return, on the line +// "taxable amount for tax paid at source". Before Phase 3 that number was +// assembled by hand into a spreadsheet +// (I:\My Drive\Expenses\Processed Receipts\Tax Paid at Source\); this report +// is the same figure, computed from the receipts the pipeline already books. +// +// THREE CONDITIONS, ALL POSITIVE. A row counts only when it carries evidence, +// never when it merely lacks a contradiction: +// * `taxAtSource` — the read actually found tax on the receipt, +// * `installedAtCustomer === true` — a NULL is "nobody said", and a NULL +// must never be spent as a deduction, and +// * `taxAmount > 0` — a zero is an answer (no tax), not an absence. +// +// The aggregation is pure and unit-tested; only `queryTaxAtSourceRows` touches +// Prisma. +import { prisma } from "@/lib/prisma"; +import { toNum } from "@/lib/prisma-helpers"; +import { formatMoneyMonth, formatMoneyMonthKey, formatMoneyDateISO } from "./payment-date"; +import { parseLocalDateString, formatLocalDateString } from "./report-utils"; +import { resolveExpenseProjectId } from "./expense-attribution"; + +export { parseLocalDateString, formatLocalDateString } from "./report-utils"; + +export interface TaxAtSourceFilters { + from: Date; + /** Exclusive upper bound. */ + to: Date; +} + +export interface TaxAtSourceRow { + id: string; + date: Date; + vendor: string; + projectId: string | null; + projectName: string; + /** Invoice / check reference, when the description carries one. */ + reference: string; + /** + * What was paid in total. See the caveat in `TAX_REPORT_AMOUNT_NOTE`: + * intake-born rows are gross-with-tax, and legacy QBO rows carry no + * taxAmount at all so they never reach this report. + */ + receiptTotal: number; + /** receiptTotal - tax. The figure the excise line is computed from. */ + deductionBase: number; + tax: number; +} + +export const TAX_REPORT_AMOUNT_NOTE = + "Receipt Total is the gross amount paid; the deduction base is that total less the sales tax on the receipt."; + +export const TAX_REPORT_FOOTNOTE = + "Sales tax already paid on materials resold as part of customer work is deductible on the WA excise return line " + + "\"taxable amount for tax paid at source\". Only receipts flagged installed-at-customer count; Shop and consumable " + + "purchases are excluded."; + +/** Calendar quarter containing `now`, as [from, to) local dates. */ +export function currentQuarterRange(now: Date = new Date()): TaxAtSourceFilters { + const quarterStartMonth = Math.floor(now.getMonth() / 3) * 3; + return { + from: new Date(now.getFullYear(), quarterStartMonth, 1, 0, 0, 0, 0), + to: new Date(now.getFullYear(), quarterStartMonth + 3, 1, 0, 0, 0, 0), + }; +} + +export function parseTaxAtSourceFilters( + params: URLSearchParams | Record, + now: Date = new Date(), +): TaxAtSourceFilters { + const get = (key: string): string | undefined => { + if (params instanceof URLSearchParams) return params.get(key) ?? undefined; + const value = (params as Record)[key]; + return Array.isArray(value) ? value[0] : value ?? undefined; + }; + const fallback = currentQuarterRange(now); + const from = (get("from") && parseLocalDateString(get("from")!)) || fallback.from; + const parsedTo = get("to") ? parseLocalDateString(get("to")!) : null; + // The picker's "to" is INCLUSIVE to a human; the query bound is exclusive. + const to = parsedTo + ? new Date(parsedTo.getFullYear(), parsedTo.getMonth(), parsedTo.getDate() + 1, 0, 0, 0, 0) + : fallback.to; + // An inverted range is a typo, not a query. Returning the fallback rather + // than an empty table stops the page reading as "no tax paid this quarter". + if (to.getTime() <= from.getTime()) return fallback; + return { from, to }; +} + +export function stringifyTaxAtSourceFilters(filters: TaxAtSourceFilters): string { + // `to` goes back out as the INCLUSIVE day the user typed. + const inclusiveTo = new Date(filters.to.getTime()); + inclusiveTo.setDate(inclusiveTo.getDate() - 1); + return new URLSearchParams({ + from: formatLocalDateString(filters.from), + to: formatLocalDateString(inclusiveTo), + }).toString(); +} + +/** + * Pull the invoice or check reference back out of the description the intake + * and Drive-import writers compose ("... Invoice 82766 · ...", "Check #1041"). + * Best-effort by construction — an expense has no reference column — so a miss + * is an empty string, never a guess. + */ +export function extractReference(description: string | null): string { + if (!description) return ""; + const invoice = description.match(/Invoice\s+([A-Za-z0-9._/-]+)/); + if (invoice) return invoice[1]; + const check = description.match(/Check\s*#\s*([A-Za-z0-9._/-]+)/); + if (check) return `Check ${check[1]}`; + return ""; +} + +export interface TaxAtSourceJobGroup { + projectId: string | null; + projectName: string; + count: number; + deductionBase: number; + receiptTotal: number; + tax: number; +} + +export interface TaxAtSourceMonthGroup { + key: string; + label: string; + jobs: TaxAtSourceJobGroup[]; + count: number; + deductionBase: number; + receiptTotal: number; + tax: number; +} + +export interface TaxAtSourceSummary { + count: number; + deductionBase: number; + receiptTotal: number; + tax: number; +} + +/** Month × job rollup. Pure — this is the function the unit tests drive. */ +export function groupTaxAtSource(rows: TaxAtSourceRow[]): { + months: TaxAtSourceMonthGroup[]; + summary: TaxAtSourceSummary; +} { + const months = new Map(); + const summary: TaxAtSourceSummary = { count: 0, deductionBase: 0, receiptTotal: 0, tax: 0 }; + + for (const row of rows) { + const key = formatMoneyMonthKey(row.date); + let month = months.get(key); + if (!month) { + month = { + key, + label: formatMoneyMonth(row.date), + jobs: [], + count: 0, + deductionBase: 0, + receiptTotal: 0, + tax: 0, + }; + months.set(key, month); + } + // Bucket by project ID, not by name: two jobs can share a name, and + // merging them would put one client's deduction under another's. + let job = month.jobs.find(candidate => candidate.projectId === row.projectId); + if (!job) { + job = { + projectId: row.projectId, + projectName: row.projectName, + count: 0, + deductionBase: 0, + receiptTotal: 0, + tax: 0, + }; + month.jobs.push(job); + } + + job.count += 1; + job.deductionBase += row.deductionBase; + job.receiptTotal += row.receiptTotal; + job.tax += row.tax; + + month.count += 1; + month.deductionBase += row.deductionBase; + month.receiptTotal += row.receiptTotal; + month.tax += row.tax; + + summary.count += 1; + summary.deductionBase += row.deductionBase; + summary.receiptTotal += row.receiptTotal; + summary.tax += row.tax; + } + + const ordered = [...months.values()].sort((a, b) => a.key.localeCompare(b.key)); + for (const month of ordered) { + month.jobs.sort((a, b) => b.tax - a.tax); + } + return { months: ordered, summary }; +} + +export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise { + const rows = await prisma.expense.findMany({ + where: { + taxAtSource: true, + installedAtCustomer: true, + taxAmount: { gt: 0 }, + date: { gte: filters.from, lt: filters.to }, + }, + select: { + id: true, + date: true, + vendor: true, + description: true, + amount: true, + taxAmount: true, + projectId: true, + project: { select: { name: true } }, + estimate: { select: { projectId: true, project: { select: { name: true } } } }, + }, + orderBy: { date: "asc" }, + }); + + return rows.map(row => { + const tax = toNum(row.taxAmount); + const receiptTotal = toNum(row.amount); + return { + id: row.id, + // The `date: { gte }` filter above already excluded null dates. + date: row.date!, + vendor: row.vendor ?? "", + projectId: resolveExpenseProjectId(row), + projectName: row.project?.name ?? row.estimate?.project?.name ?? "(unassigned)", + reference: extractReference(row.description), + receiptTotal, + deductionBase: receiptTotal - tax, + tax, + }; + }); +} + +function escapeCsv(value: string): string { + return `"${String(value ?? "").replace(/"/g, '""')}"`; +} + +/** + * Mirrors the columns of the workbook Vanessa already receives + * (Date, Vendor, Job, Invoice, Receipt Total, deduction base, Tax), so the + * handoff file shape survives the move into ProBuild. + */ +export function rowsToCsv(rows: TaxAtSourceRow[]): string { + const lines = [ + ["Date", "Vendor", "Job", "Invoice", "Receipt Total", "Material Amount (deduction base)", "Tax Paid at Source"].join(","), + ]; + for (const row of rows) { + lines.push([ + formatMoneyDateISO(row.date), + escapeCsv(row.vendor), + escapeCsv(row.projectName), + escapeCsv(row.reference), + row.receiptTotal.toFixed(2), + row.deductionBase.toFixed(2), + row.tax.toFixed(2), + ].join(",")); + } + return lines.join("\r\n") + "\r\n"; +} diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts new file mode 100644 index 000000000..5608e4834 --- /dev/null +++ b/tests/tax-at-source-report.test.ts @@ -0,0 +1,170 @@ +/** + * The WA excise deduction. Getting this wrong overstates a tax deduction, so + * the exclusions matter more than the sums: a row counts only on POSITIVE + * evidence, never on the absence of a contradiction. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + currentQuarterRange, + extractReference, + groupTaxAtSource, + parseTaxAtSourceFilters, + rowsToCsv, + stringifyTaxAtSourceFilters, + type TaxAtSourceRow, +} from "../src/lib/tax-at-source-report"; + +function row(overrides: Partial = {}): TaxAtSourceRow { + const receiptTotal = overrides.receiptTotal ?? 207.74; + const tax = overrides.tax ?? 16.55; + return { + id: "e1", + date: new Date("2026-06-26T00:00:00.000Z"), + vendor: "Harbor Freight", + projectId: "job-mesplay", + projectName: "Mesplay Kitchen", + reference: "001916749100246", + receiptTotal, + deductionBase: receiptTotal - tax, + tax, + ...overrides, + }; +} + +// ── grouping ──────────────────────────────────────────────────────────────── + +test("sums per month and per job, and the totals tie out", () => { + const { months, summary } = groupTaxAtSource([ + row({ id: "a", tax: 10, receiptTotal: 110, deductionBase: 100 }), + row({ id: "b", tax: 5, receiptTotal: 55, deductionBase: 50 }), + row({ + id: "c", + projectId: "job-mueller", + projectName: "Mueller Bath", + tax: 20, + receiptTotal: 220, + deductionBase: 200, + }), + row({ + id: "d", + date: new Date("2026-07-02T00:00:00.000Z"), + tax: 1, + receiptTotal: 11, + deductionBase: 10, + }), + ]); + + assert.deepEqual(months.map(m => m.key), ["2026-06", "2026-07"], "months are chronological"); + const june = months[0]; + assert.equal(june.count, 3); + assert.equal(june.tax, 35); + assert.equal(june.deductionBase, 350); + // Jobs sort by tax, largest first — that is the row a bookkeeper checks. + assert.deepEqual(june.jobs.map(j => j.projectId), ["job-mueller", "job-mesplay"]); + assert.equal(june.jobs.find(j => j.projectId === "job-mesplay")!.count, 2); + assert.equal(june.jobs.find(j => j.projectId === "job-mesplay")!.tax, 15); + + assert.equal(summary.count, 4); + assert.equal(summary.tax, 36); + assert.equal(summary.deductionBase, 360); + assert.equal( + summary.tax, + months.reduce((total, month) => total + month.tax, 0), + "the grand total is the month totals, not a second computation", + ); +}); + +test("two jobs sharing a name stay separate", () => { + // Bucketing by name would merge one client's deduction into another's. + const { months } = groupTaxAtSource([ + row({ id: "a", projectId: "job-1", projectName: "Bathroom Remodel", tax: 10 }), + row({ id: "b", projectId: "job-2", projectName: "Bathroom Remodel", tax: 20 }), + ]); + assert.equal(months[0].jobs.length, 2); + assert.deepEqual(months[0].jobs.map(j => j.projectId), ["job-2", "job-1"]); +}); + +test("an unattributed receipt is shown, not dropped", () => { + // Money that was spent is real even when nobody said whose job it was. + // Hiding it would make the report quietly understate the deduction. + const { months, summary } = groupTaxAtSource([ + row({ projectId: null, projectName: "(unassigned)", tax: 7 }), + ]); + assert.equal(months[0].jobs[0].projectId, null); + assert.equal(summary.tax, 7); +}); + +test("an empty period is zeros, not NaN", () => { + const { months, summary } = groupTaxAtSource([]); + assert.deepEqual(months, []); + assert.deepEqual(summary, { count: 0, deductionBase: 0, receiptTotal: 0, tax: 0 }); +}); + +// ── the filter contract ───────────────────────────────────────────────────── + +test("the default period is the current calendar quarter, [from, to)", () => { + const filters = parseTaxAtSourceFilters({}, new Date(2026, 7, 15)); + assert.equal(filters.from.getMonth(), 6, "Q3 starts in July"); + assert.equal(filters.from.getDate(), 1); + assert.equal(filters.to.getMonth(), 9, "and ends at the start of October, exclusive"); + assert.equal(filters.to.getDate(), 1); + assert.deepEqual(filters, currentQuarterRange(new Date(2026, 7, 15))); +}); + +test("the picker's `to` is inclusive to a human and exclusive to the query", () => { + // A receipt dated on the last day of the range must be IN it. An exclusive + // bound taken literally from the picker would silently drop that day. + const filters = parseTaxAtSourceFilters({ from: "2026-06-01", to: "2026-06-30" }); + assert.equal(filters.from.getDate(), 1); + assert.equal(filters.to.getMonth(), 6); + assert.equal(filters.to.getDate(), 1, "start of July"); + // ...and it round-trips back to the day the user typed. + assert.equal(stringifyTaxAtSourceFilters(filters), "from=2026-06-01&to=2026-06-30"); +}); + +test("an inverted or unparseable range falls back to the quarter, not to empty", () => { + // An empty table reads as "no tax was paid", which is a very different + // claim from "your dates are backwards". + const now = new Date(2026, 7, 15); + assert.deepEqual( + parseTaxAtSourceFilters({ from: "2026-06-30", to: "2026-06-01" }, now), + currentQuarterRange(now), + ); + assert.deepEqual( + parseTaxAtSourceFilters({ from: "not-a-date", to: "also-not" }, now), + currentQuarterRange(now), + ); +}); + +// ── reference extraction ──────────────────────────────────────────────────── + +test("the invoice reference is recovered from the description, or left blank", () => { + assert.equal( + extractReference("[Receipt intake] Invoice 82766 · incl. $29.20 sales tax · pending bookkeeper review"), + "82766", + ); + assert.equal(extractReference('[Receipt intake] Check #1041 — "materials"'), "Check 1041"); + assert.equal(extractReference("[Drive import] Receipt · Materials"), "", "a miss is blank, never a guess"); + assert.equal(extractReference(null), ""); +}); + +// ── CSV ───────────────────────────────────────────────────────────────────── + +test("the CSV mirrors the workbook columns Vanessa already receives", () => { + const csv = rowsToCsv([row({ receiptTotal: 207.74, tax: 16.55, deductionBase: 191.19 })]); + const [header, line] = csv.trimEnd().split("\r\n"); + assert.equal( + header, + "Date,Vendor,Job,Invoice,Receipt Total,Material Amount (deduction base),Tax Paid at Source", + ); + assert.equal( + line, + '2026-06-26,"Harbor Freight","Mesplay Kitchen","001916749100246",207.74,191.19,16.55', + ); +}); + +test("CSV quoting survives a vendor name with a comma or a quote", () => { + const csv = rowsToCsv([row({ vendor: 'Lowe"s, Vancouver' })]); + assert.match(csv, /"Lowe""s, Vancouver"/); +}); From 5cbbdb84ce5a2f314dea4c0dd891d53c724d3700 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:22:07 -0700 Subject: [PATCH 067/144] fix(ai): cost-forecast reads spend the same way the variance report does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deviation from spec §4, which listed api/ai/cost-forecast under "display-only includes — no change now". It is not display-only: it is a per-project spend FILTER feeding a forecast. Left on the estimate relation it would stay correct only for as long as nobody re-attributes an expense, and then the forecast and the variance report would quietly disagree. Identical output today, same as every other call site in this PR. Co-Authored-By: Claude Fable 5.1 --- src/app/api/ai/cost-forecast/route.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/app/api/ai/cost-forecast/route.ts b/src/app/api/ai/cost-forecast/route.ts index 400a120d7..fb99ea03d 100644 --- a/src/app/api/ai/cost-forecast/route.ts +++ b/src/app/api/ai/cost-forecast/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { expenseForProjectWhere } from "@/lib/expense-attribution"; import { getAnthropicText } from "@/lib/anthropic"; import Anthropic from "@anthropic-ai/sdk"; @@ -27,7 +28,12 @@ export async function POST(req: NextRequest) { select: { durationHours: true, laborCost: true, burdenCost: true }, }), prisma.expense.findMany({ - where: { estimate: { is: { projectId } } }, + // DEVIATION from spec §4's "display-only, no change" list: this one + // is not display-only. It is a real per-project spend filter feeding + // a forecast, so it has to see the same rows the variance report + // does — otherwise a re-attributed expense would show up in one and + // not the other. + where: expenseForProjectWhere(projectId), select: { amount: true, vendor: true, status: true }, }), prisma.purchaseOrder.findMany({ From a4b34302e002e57ce2c37cd98aa2c526ae358540 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:39:00 -0700 Subject: [PATCH 068/144] fix(qbo-sync): split the projectId fill out of the sync update; scope suggestions to the STORED job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blockers 1 and 5. 1. The projectId fill was read-then-unconditional-update: it decided from a value read earlier in the transaction, and the guarantee lived in JS. It is now its own `updateMany({ where: { id, projectId: null } })` and the main UPDATE never carries projectId at all. The real bug underneath: when a bookkeeper had re-attributed a row, the sync kept projectId but still wrote the QBO match's estimateId back — leaving the expense on job B for every reader and on job A's estimate for cascade-delete and billing. estimateId is now left alone exactly when the stored project and the incoming match DISAGREE. Narrower than "only fill estimateId when projectId is null", deliberately: when the two agree it is the same job, and moving estimateId to that job's newest estimate is the sync's existing "attach to the active estimate" behaviour. Suppressing it there would strand rows on superseded estimates for no safety gain. Both cases are tested. 5. Cost-code suggestions now read the row's STORED attribution instead of the incoming match. Those differ exactly when someone re-attributed the expense, which is the case where the match would suggest a phase for the job the row is no longer on — including handing a job code to a row moved into the overhead bucket by hand. Also wires the qbo-expense-sync suite into `npm run test:unit`, so the capture/manual no-overwrite guards actually run in CI (two PG-dependent cases keep their existing self-skip). Co-Authored-By: Claude Fable 5.1 --- package.json | 2 +- src/lib/qbo-expense-sync.ts | 172 ++++++++++++++++++++++++-------- tests/qbo-expense-sync.test.ts | 176 ++++++++++++++++++++++++++++----- 3 files changed, 282 insertions(+), 68 deletions(-) diff --git a/package.json b/package.json index 3e01aaa89..095496a78 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 98a842cec..4e6ebd6ef 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -5,7 +5,11 @@ import { prisma } from "./prisma"; import { getFreshQBTokens } from "./quickbooks-payments"; import { after } from "next/server"; import { suggestCode } from "./expense-cost-suggest"; -import { notHumanCodedExpenseWhere } from "./expense-attribution"; +import { + HUMAN_COST_CODE_SOURCES, + notHumanCodedExpenseWhere, + resolveExpenseProjectId, +} from "./expense-attribution"; import { isOverheadProject } from "./overhead-project"; // Shared with the register merge layer (register-merge.ts, Unified Money // Register plan §4) so the classification values this module WRITES can @@ -536,6 +540,10 @@ type ExpenseTransaction = { where: { id: string }; data: Partial; }): Promise; + updateMany(args: { + where: { id: string; projectId: null }; + data: { projectId: string }; + }): Promise<{ count: number }>; }; }; @@ -560,45 +568,79 @@ function datesEqual(left: Date | null, right: Date | null): boolean { return left?.getTime() === right?.getTime(); } +type ExistingQboExpense = + NonNullable>>; + +export interface QboExpenseUpdatePlan { + /** + * Written by its OWN guarded statement, not by the main UPDATE. Null when + * the row already has a project (or the write has none to give). + */ + fillProjectId: string | null; + /** Everything the main UPDATE may write. NEVER contains `projectId`. */ + data: Partial; +} + /** - * The subset of `write` an UPDATE is allowed to apply to an existing row. + * Decide what a re-sync is allowed to change on a row that already exists. + * + * TWO SEPARATE RULES, and they used to be one read-then-unconditional-update: + * + * 1. `projectId` is only ever FILLED, never overwritten, and it is filled by a + * statement whose own predicate says `projectId: null`. Deciding from a + * value read earlier in the transaction is not the same as deciding from + * the row's state at write time, and the guarantee belongs in the SQL. * - * `projectId` is dropped once the row already has one: re-attribution by a - * bookkeeper is a human decision, and the QBO Purchase's customer ref is not a - * newer fact about it. Filling a NULL is still correct — that is the backfill - * catching up. + * 2. `estimateId` follows it. The estimate in `write` is the one the QBO + * customer match picked; if a bookkeeper has since re-attributed the row to + * a DIFFERENT job, writing that estimate back would leave `projectId` and + * `estimateId` pointing at two different jobs — an expense that is on job B + * for every reader and on job A's estimate for cascade-delete and billing. + * So the estimate is left alone exactly when the projects disagree. + * + * Note rule 2 is scoped to DISAGREEMENT rather than to "projectId is set". + * When the stored project and the incoming match are the SAME job, moving + * `estimateId` to that job's newest estimate is what the sync has always done + * and is still correct — it is the pre-existing "attach to the active estimate" + * behaviour, and suppressing it there would strand rows on superseded + * estimates for no safety gain. */ -function qboExpenseUpdateData( - existing: NonNullable>>, +export function planQboExpenseUpdate( + existing: Pick, write: QboExpenseWrite, -): Partial { - if ((existing.projectId ?? null) === null) return write; - const rest: Partial = { ...write }; - delete rest.projectId; - return rest; +): QboExpenseUpdatePlan { + const existingProjectId = existing.projectId ?? null; + const incomingProjectId = write.projectId ?? null; + + const data: Partial = { ...write }; + delete data.projectId; + if (existingProjectId !== null && existingProjectId !== incomingProjectId) { + delete data.estimateId; + } + + return { + fillProjectId: existingProjectId === null ? incomingProjectId : null, + data, + }; } -function expenseMatchesQboWrite( - existing: Awaited>, - write: QboExpenseWrite, -): boolean { - if (!existing) return false; - // Compare projectId ONLY on the rows an update would actually write it to. - // Comparing it unconditionally would mark every hand-re-attributed row as - // drifted and re-issue an update forever that deliberately never changes it. - const projectIdMatches = - (existing.projectId ?? null) !== null || - (existing.projectId ?? null) === (write.projectId ?? null); - return ( - projectIdMatches && - existing.qbSyncToken === write.qbSyncToken && - existing.estimateId === write.estimateId && - Number(existing.amount) === write.amount && - existing.vendor === write.vendor && - datesEqual(existing.date, write.date) && - existing.description === write.description && - existing.status === write.status - ); +/** + * True when the plan would change nothing. Compares only the fields the plan + * actually carries — a field the plan deliberately omits must not count as + * drift, or the sync re-issues an update forever and reports it as "updated". + * `qbSyncedAt` is excluded because it changes on every run by construction. + */ +function planIsNoop(existing: ExistingQboExpense, plan: QboExpenseUpdatePlan): boolean { + if (plan.fillProjectId !== null) return false; + const data = plan.data; + if (data.qbSyncToken !== undefined && existing.qbSyncToken !== data.qbSyncToken) return false; + if (data.estimateId !== undefined && existing.estimateId !== data.estimateId) return false; + if (data.amount !== undefined && Number(existing.amount) !== data.amount) return false; + if (data.vendor !== undefined && existing.vendor !== data.vendor) return false; + if (data.date !== undefined && !datesEqual(existing.date, data.date)) return false; + if (data.description !== undefined && existing.description !== data.description) return false; + if (data.status !== undefined && existing.status !== data.status) return false; + return true; } async function lockQboExpense( @@ -644,14 +686,26 @@ export async function upsertQboExpense( ) { return "unchanged"; } - if (expenseMatchesQboWrite(existing, write)) return "unchanged"; if (!existing) { await transaction.expense.create({ data: write }); return "imported"; } + + const plan = planQboExpenseUpdate(existing, write); + if (planIsNoop(existing, plan)) return "unchanged"; + + // The project fill is its OWN statement, and its predicate — not a + // value read a few lines above — is what guarantees a human's + // re-attribution survives. + if (plan.fillProjectId !== null) { + await transaction.expense.updateMany({ + where: { id: existing.id, projectId: null }, + data: { projectId: plan.fillProjectId }, + }); + } await transaction.expense.update({ where: { id: existing.id }, - data: qboExpenseUpdateData(existing, write), + data: plan.data, }); return "updated"; }); @@ -738,12 +792,19 @@ export interface QboCostCodeSuggestionInput { qbPurchaseId: string; vendor: string | null; description: string; - /** The job the row was just imported against; the overhead bucket opts out. */ - projectId: string | null; } export interface QboCostCodeSuggestionClient { expense: { + findUnique(args: { + where: { qbPurchaseId: string }; + select: Record; + }): Promise<{ + projectId: string | null; + costCodeId: string | null; + costCodeSource: string | null; + estimate?: { projectId: string | null } | null; + } | null>; updateMany(args: { where: Record; data: { costCodeId: string; costCodeSource: string; costCodeConfidence: number }; @@ -752,6 +813,10 @@ export interface QboCostCodeSuggestionClient { } export type QboCostCodeSuggestionResult = + /** The upsert did not leave a row (deactivated, or raced away). */ + | "missing-row" + /** Nothing knows whose job this is, so no job phase can be right. */ + | "skipped-no-project" /** The overhead bucket is not a job and does not get a job phase. */ | "skipped-overhead" /** The rules refused — the honest majority case. */ @@ -762,12 +827,40 @@ export type QboCostCodeSuggestionResult = | "not-written" | "written"; +/** + * Scope comes from the row's STORED attribution, never from the incoming QBO + * match. Those two disagree exactly when a bookkeeper has re-attributed the + * expense — and that is the case where using the match would suggest a phase + * for the job the row is no longer on. It is also how an overhead row stays + * out: a row moved INTO the overhead bucket by hand must stop being offered + * job phases, and the match would never tell us that. + */ export async function applyQboExpenseCostCodeSuggestion( client: QboCostCodeSuggestionClient, input: QboCostCodeSuggestionInput, costCodeIdByCode: ReadonlyMap, ): Promise { - if (input.projectId && isOverheadProject(input.projectId)) return "skipped-overhead"; + const stored = await client.expense.findUnique({ + where: { qbPurchaseId: input.qbPurchaseId }, + select: { + projectId: true, + costCodeId: true, + costCodeSource: true, + estimate: { select: { projectId: true } }, + }, + }); + if (!stored) return "missing-row"; + // Read-side twin of the update predicate below. Both are needed: this one + // stops us computing a suggestion nobody may use, that one is the actual + // guarantee. + if (stored.costCodeId) return "not-written"; + if ((HUMAN_COST_CODE_SOURCES as readonly string[]).includes(stored.costCodeSource ?? "")) { + return "not-written"; + } + + const projectId = resolveExpenseProjectId(stored); + if (!projectId) return "skipped-no-project"; + if (isOverheadProject(projectId)) return "skipped-overhead"; const suggestion = suggestCode({ vendor: input.vendor, description: input.description }); if (!suggestion) return "no-match"; @@ -1260,7 +1353,6 @@ export async function syncQboExpenses( qbPurchaseId: purchase.qbPurchaseId, vendor: purchase.vendor, description, - projectId: match.projectId, }); } catch (error) { console.error( diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index c24cc0cce..3ed7e4880 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -7,6 +7,7 @@ import { syncQboExpenses, upsertQboExpense, applyQboExpenseCostCodeSuggestion, + planQboExpenseUpdate, type QboExpenseProjectCandidate, type QboExpenseSyncDependencies, type QboExpenseWrite, @@ -595,6 +596,19 @@ function createFakePrisma(initial: StoredExpense[] = []) { rows.set(current.qbPurchaseId, next); return next; }, + // Models the PREDICATE, not just the write: `projectId: null` in the + // where clause has to be able to match zero rows, because that is the + // whole guarantee the split-out project fill is buying. + async updateMany(args: { + where: { id: string; projectId: null }; + data: { projectId: string }; + }) { + const current = [...rows.values()].find(row => row.id === args.where.id); + if (!current) return { count: 0 }; + if ((current.projectId ?? null) !== null) return { count: 0 }; + rows.set(current.qbPurchaseId, { ...current, ...args.data }); + return { count: 1 }; + }, }; let lockTail: Promise = Promise.resolve(); @@ -1158,7 +1172,16 @@ const COST_CODE_IDS = new Map([ ["02-FRAME", "cc-frame"], ]); -function fakeSuggestionClient() { +type StoredForSuggestion = { + projectId: string | null; + costCodeId: string | null; + costCodeSource: string | null; + estimate?: { projectId: string | null } | null; +} | null; + +function fakeSuggestionClient( + stored: StoredForSuggestion = { projectId: "project-1", costCodeId: null, costCodeSource: null }, +) { const calls: { where: Record; data: Record }[] = []; let count = 1; return { @@ -1166,6 +1189,7 @@ function fakeSuggestionClient() { setCount(next: number) { count = next; }, client: { expense: { + async findUnique() { return stored; }, async updateMany(args: { where: Record; data: { costCodeId: string; costCodeSource: string; costCodeConfidence: number }; @@ -1186,7 +1210,6 @@ test("a NULL cost code is filled with source ai and the rule's tier confidence", qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "[QuickBooks import] rough-in", - projectId: "project-1", }, COST_CODE_IDS, ); @@ -1206,7 +1229,7 @@ test("the write is guarded on uncoded AND not-human-coded, with a NULL branch", const fake = fakeSuggestionClient(); await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x", projectId: "project-1" }, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, COST_CODE_IDS, ); const where = fake.calls[0].where; @@ -1218,39 +1241,98 @@ test("the write is guarded on uncoded AND not-human-coded, with a NULL branch", ]); }); -test("a row a human already coded is reported not-written, not silently written", async () => { - const fake = fakeSuggestionClient(); - fake.setCount(0); // the guard matched nothing - const result = await applyQboExpenseCostCodeSuggestion( - fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x", projectId: "project-1" }, - COST_CODE_IDS, +test("a row a human already coded is refused on the STORED source, before any write", async () => { + for (const costCodeSource of ["capture", "manual"]) { + const fake = fakeSuggestionClient({ projectId: "project-1", costCodeId: null, costCodeSource }); + const result = await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + COST_CODE_IDS, + ); + assert.equal(result, "not-written", costCodeSource); + assert.equal(fake.calls.length, 0, "and it never even issues the guarded write"); + } +}); + +test("a row that is already coded is left alone", async () => { + const fake = fakeSuggestionClient({ projectId: "project-1", costCodeId: "cc-existing", costCodeSource: null }); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + COST_CODE_IDS, + ), + "not-written", ); - assert.equal(result, "not-written"); + assert.equal(fake.calls.length, 0); }); -test("the overhead bucket is never given a job phase, and never even queried", async () => { - const fake = fakeSuggestionClient(); +test("scope comes from the STORED project, not the incoming QBO match", async () => { + // A bookkeeper moved this row into the overhead bucket. The QBO customer + // ref still says "Mueller Bathroom", so a suggester scoped to the match + // would hand an overhead purchase a job phase. + const fake = fakeSuggestionClient({ + projectId: OVERHEAD_PROJECT_ID, + costCodeId: null, + costCodeSource: null, + }); const result = await applyQboExpenseCostCodeSuggestion( fake.client, - { - qbPurchaseId: "purchase-1", - vendor: "Summit Plumbing", - description: "x", - projectId: OVERHEAD_PROJECT_ID, - }, + { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, COST_CODE_IDS, ); assert.equal(result, "skipped-overhead"); assert.equal(fake.calls.length, 0); }); +test("the stored project is resolved through the estimate when the column is NULL", async () => { + const fake = fakeSuggestionClient({ + projectId: null, + costCodeId: null, + costCodeSource: null, + estimate: { projectId: OVERHEAD_PROJECT_ID }, + }); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + COST_CODE_IDS, + ), + "skipped-overhead", + ); +}); + +test("a row with no job at all gets no job phase", async () => { + const fake = fakeSuggestionClient({ projectId: null, costCodeId: null, costCodeSource: null, estimate: null }); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + COST_CODE_IDS, + ), + "skipped-no-project", + ); + assert.equal(fake.calls.length, 0); +}); + +test("a vanished row is reported, not treated as a silent success", async () => { + const fake = fakeSuggestionClient(null); + assert.equal( + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "gone", vendor: "Summit Plumbing", description: "x" }, + COST_CODE_IDS, + ), + "missing-row", + ); +}); + test("no rule match and an unknown code both write nothing", async () => { const fake = fakeSuggestionClient(); assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "p", vendor: "General Hardware", description: "misc", projectId: "project-1" }, + { qbPurchaseId: "p", vendor: "General Hardware", description: "misc" }, COST_CODE_IDS, ), "no-match", @@ -1258,7 +1340,7 @@ test("no rule match and an unknown code both write nothing", async () => { assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "p", vendor: "Summit Plumbing", description: "x", projectId: "project-1" }, + { qbPurchaseId: "p", vendor: "Summit Plumbing", description: "x" }, new Map(), ), "unknown-code", @@ -1282,18 +1364,58 @@ test("a failing suggester never fails the import it rides alongside", async () = assert.equal(result.imported, 1, "the money is recorded even when the phase guess blows up"); }); -test("the sync asks for a phase on a matched job, carrying the job it matched", async () => { +test("the sync asks for a phase on a matched job, by purchase id only", async () => { + // No projectId is passed: the suggester reads the row's stored attribution + // itself, so the sync cannot accidentally widen its scope. const fake = createFakePrisma(); - const suggested: { qbPurchaseId: string; projectId: string | null }[] = []; + const suggested: string[] = []; const dependencies = createSyncDependencies( [PURCHASE], ACTIVE_PROJECTS, write => upsertQboExpense(fake.client, write), ); - dependencies.suggestCostCode = async input => { - suggested.push({ qbPurchaseId: input.qbPurchaseId, projectId: input.projectId }); - }; + dependencies.suggestCostCode = async input => { suggested.push(input.qbPurchaseId); }; await syncQboExpenses({ since: new Date("2026-07-01T00:00:00.000Z") }, dependencies); - assert.deepEqual(suggested, [{ qbPurchaseId: "purchase-1", projectId: "project-1" }]); + assert.deepEqual(suggested, ["purchase-1"]); +}); + +// ── the split project fill (Codex round 1, blocker 1) ────────────────────── + +test("the project fill is its OWN statement, guarded on projectId being NULL", () => { + const plan = planQboExpenseUpdate({ projectId: null }, WRITE); + assert.equal(plan.fillProjectId, "project-1"); + assert.ok(!("projectId" in plan.data), "the main UPDATE never carries projectId"); + assert.equal(plan.data.estimateId, "estimate-1"); +}); + +test("a row already on a project is never re-projected, and keeps its estimate", () => { + // Both halves matter: leaving projectId alone while still writing the QBO + // match's estimateId would put the row on job B for every reader and on + // job A's estimate for cascade-delete and billing. + const plan = planQboExpenseUpdate({ projectId: "moved-by-hand" }, WRITE); + assert.equal(plan.fillProjectId, null); + assert.ok(!("projectId" in plan.data)); + assert.ok(!("estimateId" in plan.data), "the estimate belongs to the OTHER job"); + assert.equal(plan.data.amount, WRITE.amount, "the rest of the write still lands"); +}); + +test("when the stored project AGREES with the match, the estimate still tracks it", () => { + const plan = planQboExpenseUpdate({ projectId: "project-1" }, { ...WRITE, estimateId: "estimate-2" }); + assert.equal(plan.fillProjectId, null); + assert.equal(plan.data.estimateId, "estimate-2", "same job, newer estimate — that is the old behaviour"); +}); + +test("a re-attributed row's estimateId survives a real re-sync", async () => { + const fake = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: "moved-by-hand", estimateId: "estimate-of-job-b", receiptUrl: null }, + ]); + assert.equal( + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 300 }), + "updated", + ); + const row = fake.rows.get("purchase-1"); + assert.equal(row?.projectId, "moved-by-hand"); + assert.equal(row?.estimateId, "estimate-of-job-b"); + assert.equal(row?.amount, 300); }); From e6264d83de535894fce6c39e4e1c4d433cba8adf Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:41:42 -0700 Subject: [PATCH 069/144] =?UTF-8?q?fix(charts):=20rank=20spend=20by=20the?= =?UTF-8?q?=20resolver,=20not=20by=20estimateId=20=E2=80=94=20the=20page?= =?UTF-8?q?=20disagreed=20with=20itself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blocker 2. I had left the all-time groupBy(["estimateId"]) on the relation and called it "identical output". It was not: the ranking resolved a row's job through its estimate while the monthly series resolved it through resolveExpenseProjectId, so a re-attributed expense ranked under its OLD job and plotted under its NEW one — the same dollars in two places, on one page, with a legend entry that had no bar under it. Neither groupBy nor a relation filter can express "projectId, or the estimate's projectId when null", so the rows are fetched and bucketed with the shared resolver. Scope is unchanged (all selectable jobs, all-time), the select is three columns, and the estimate->project lookup that followed the groupBy is gone. tests/company-financials-spend-attribution.test.ts drives the real module with a patched prisma and a fixture whose projectId and estimate.projectId DISAGREE — the only shape that can catch this. Mutation-checked: reverting the ranking loop to e.estimate?.projectId fails 2 of its 3 cases. Co-Authored-By: Claude Fable 5.1 --- package.json | 2 +- src/lib/company-financials-charts.ts | 52 +++--- ...mpany-financials-spend-attribution.test.ts | 151 ++++++++++++++++++ 3 files changed, 177 insertions(+), 28 deletions(-) create mode 100644 tests/company-financials-spend-attribution.test.ts diff --git a/package.json b/package.json index 095496a78..9744683b4 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/lib/company-financials-charts.ts b/src/lib/company-financials-charts.ts index 5701ed01e..15f8bc66b 100644 --- a/src/lib/company-financials-charts.ts +++ b/src/lib/company-financials-charts.ts @@ -269,7 +269,7 @@ export async function getCompanyFinancialsChartData( overheadTimeEntries, unpaidSchedules, openRetainers, - expenseTotalsByEstimate, + allTimeJobExpenses, timeZone, ] = await Promise.all([ // Collected: paid schedules whose PARENT invoice isn't Draft (item 2), @@ -344,20 +344,24 @@ export async function getCompanyFinancialsChartData( where: { projectId: { in: projectIds }, status: { in: RETAINER_STATUSES }, balanceDue: { gt: 0 } }, select: { balanceDue: true, dueDate: true }, }), - // All-time top-5 ranking universe: aggregate in SQL (groupBy + sum) rather - // than materializing every expense row. Expense has no direct projectId - // column, so group by estimateId and resolve project ids via a small - // estimate lookup below. - // DELIBERATELY LEFT ON THE RELATION. Phase 3 added Expense.projectId, - // so this could become a plain groupBy(["projectId"]) — but only once - // every row is backfilled AND every writer stamps it, and this PR's - // contract is identical output. Changing it here would also change the - // grouping key, which is a real behaviour change dressed as a - // refactor. Revisit after the backfill has run in production. - prisma.expense.groupBy({ - by: ["estimateId"], - where: { estimate: { projectId: { in: allJobIds } } }, - _sum: { amount: true }, + // All-time top-5 ranking universe. + // + // This used to be a SQL `groupBy(["estimateId"])` plus an estimate -> + // project lookup, to avoid materializing every expense row. Phase 3 + // made that WRONG rather than merely dated: grouping by estimateId + // resolves a row's job through its estimate, while the monthly spend + // series below resolves it through `resolveExpenseProjectId`. A + // re-attributed expense therefore ranked under its OLD job and was + // plotted under its new one — the same money in two different places in + // two charts on the same page. + // + // Neither `groupBy` nor a relation filter can express "projectId, or + // the estimate's projectId when it is null", so the rows are fetched + // and bucketed with the shared resolver. Scope is unchanged (all + // selectable jobs, all-time) and the select is three columns. + prisma.expense.findMany({ + where: expenseForProjectsWhere(allJobIds), + select: { amount: true, projectId: true, estimate: { select: { projectId: true } } }, }), resolveCompanyTimeZone(), ]); @@ -440,19 +444,13 @@ export async function getCompanyFinancialsChartData( // Ranking universe is ALL selectable jobs (jobProjects), all-time, ignoring // the current date-range/project filters — this is what keeps a project's // color stable no matter how the filters change. - const rankedEstimateIds = expenseTotalsByEstimate.map((g) => g.estimateId); - const estimateProjects = rankedEstimateIds.length - ? await prisma.estimate.findMany({ - where: { id: { in: rankedEstimateIds } }, - select: { id: true, projectId: true }, - }) - : []; - const projectByEstimate = new Map(estimateProjects.map((e) => [e.id, e.projectId])); + // Same resolver as the monthly series below — that agreement is the whole + // point, and it is what the regression test pins. const allTimeTotals = new Map(); - for (const g of expenseTotalsByEstimate) { - const pid = projectByEstimate.get(g.estimateId); - if (!pid) continue; // Estimate.projectId is nullable on this schema - allTimeTotals.set(pid, (allTimeTotals.get(pid) ?? 0) + Number(g._sum.amount ?? 0)); + for (const e of allTimeJobExpenses) { + const pid = resolveExpenseProjectId(e); + if (!pid) continue; // both sides can be null on this schema + allTimeTotals.set(pid, (allTimeTotals.get(pid) ?? 0) + Number(e.amount ?? 0)); } const topProjectIds = [...allTimeTotals.entries()] .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)) // tie-break by id: stable across reloads diff --git a/tests/company-financials-spend-attribution.test.ts b/tests/company-financials-spend-attribution.test.ts new file mode 100644 index 000000000..3cbeb0a98 --- /dev/null +++ b/tests/company-financials-spend-attribution.test.ts @@ -0,0 +1,151 @@ +/** + * Regression for Codex round 1, blocker 2. + * + * The Company Financials page draws the same money twice: a top-5 project + * RANKING (all-time) and a monthly SPEND series (in-range). Before this fix the + * ranking resolved a row's job through `estimate.projectId` (a SQL + * `groupBy(["estimateId"])`) while the series resolved it through + * `resolveExpenseProjectId`. A re-attributed expense therefore ranked under its + * OLD job and plotted under its NEW one — the same dollars in two different + * places, on one page. + * + * The fixture is one expense whose `projectId` and `estimate.projectId` + * DISAGREE. That is the only shape that can catch this: with them in agreement + * both code paths give the same answer and the bug is invisible. + */ +import { test, before } from "node:test"; +import assert from "node:assert/strict"; +import Module from "node:module"; + +const IN_RANGE = new Date("2026-06-15T12:00:00.000Z"); + +/** The row a bookkeeper moved from job-a to job-b after it was imported. */ +const REATTRIBUTED = { + amount: 5000, + date: IN_RANGE, + createdAt: IN_RANGE, + projectId: "job-b", + estimate: { projectId: "job-a" }, +}; + +/** A smaller row that never moved, so job-a is still a real series. */ +const SETTLED = { + amount: 100, + date: IN_RANGE, + createdAt: IN_RANGE, + projectId: "job-a", + estimate: { projectId: "job-a" }, +}; + +const expenseCalls: any[] = []; + +const fakePrisma = { + companySettings: { + findUnique: async () => ({ timeZone: "America/Los_Angeles" }), + }, + paymentSchedule: { findMany: async () => [] }, + retainer: { findMany: async () => [] }, + timeEntry: { findMany: async () => [] }, + estimate: { findMany: async () => [] }, + expense: { + findMany: async (args: any) => { + expenseCalls.push(args); + // Three expense reads, told apart by their select — the ranking one + // needs no date, the overhead one no project. + const select = args.select ?? {}; + if (!select.date) return [REATTRIBUTED, SETTLED]; // all-time ranking universe + if (!select.projectId) return []; // overhead bucket + return [REATTRIBUTED, SETTLED]; // in-range job spend + }, + groupBy: async () => { + throw new Error("groupBy must no longer be used: it cannot express the resolver's fallback"); + }, + }, +}; + +let getCompanyFinancialsChartData: any; + +before(async () => { + const originalRequire = Module.prototype.require; + let requirePatchHit = false; + (Module.prototype as unknown as { require: (id: string) => unknown }).require = function ( + this: NodeModule, + id: string, + ) { + if (id === "@/lib/prisma" || id === "./prisma") { + requirePatchHit = true; + return { prisma: fakePrisma }; + } + // eslint-disable-next-line prefer-rest-params + return originalRequire.apply(this, arguments as unknown as [string]); + } as typeof Module.prototype.require; + + let mod: any; + try { + mod = await import("../src/lib/company-financials-charts"); + } finally { + Module.prototype.require = originalRequire; + } + if (typeof mod.getCompanyFinancialsChartData !== "function") { + throw new Error( + "company-financials-spend-attribution.test.ts: prisma mock did not apply — " + + `the require patch ${requirePatchHit ? "WAS" : "was NOT"} hit.`, + ); + } + getCompanyFinancialsChartData = mod.getCompanyFinancialsChartData; +}); + +const JOBS = [ + { id: "job-a", name: "Mueller Bath" }, + { id: "job-b", name: "Mesplay Kitchen" }, +]; + +async function load() { + return getCompanyFinancialsChartData( + { + preset: "6m", + from: new Date("2026-04-01T00:00:00.000Z"), + to: new Date("2026-07-01T00:00:00.000Z"), + projectIds: ["job-a", "job-b"], + includeOverhead: false, + }, + JOBS, + ); +} + +test("a re-attributed expense ranks under its REAL job, not its estimate's", async () => { + const data = await load(); + const seriesIds = data.spendByProject.series.map((s: any) => s.id); + // job-b holds the $5,000, so it must outrank job-a's $100. + assert.deepEqual(seriesIds, ["job-b", "job-a", "other"]); +}); + +test("...and the monthly series puts the same dollars on the same job", async () => { + const data = await load(); + const withSpend = data.spendByProject.data.filter( + (point: any) => Number(point["job-a"] ?? 0) + Number(point["job-b"] ?? 0) > 0, + ); + assert.equal(withSpend.length, 1, "one month carries the fixture's spend"); + assert.equal(withSpend[0]["job-b"], 5000, "the re-attributed dollars follow projectId"); + assert.equal(withSpend[0]["job-a"], 100); + // The failure this guards: ranking said job-a, the series said job-b, and + // the chart drew a legend entry with no bar under it. + assert.ok( + data.spendByProject.series.some((s: any) => s.id === "job-b"), + "the job the money is plotted under must also be in the legend", + ); +}); + +test("the ranking query is the shared both-ways predicate, not a relation-only filter", async () => { + await load(); + const ranking = expenseCalls.find(call => call.select && !call.select.date); + assert.ok(ranking, "the all-time ranking read must happen"); + assert.deepEqual(ranking.where, { + OR: [ + { projectId: { in: ["job-a", "job-b"] } }, + { projectId: null, estimate: { projectId: { in: ["job-a", "job-b"] } } }, + ], + }); + assert.equal(ranking.select.projectId, true); + assert.deepEqual(ranking.select.estimate, { select: { projectId: true } }); +}); From 5ca7213eb3b581ade1ac4b2e12a4e9ed403b9bdd Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:43:45 -0700 Subject: [PATCH 070/144] fix(backfill): an itemId on another job's line item is a data problem, not a cost code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, blocker 3. `Expense.itemId` is ON DELETE SET NULL and has never been scoped to the expense's own estimate on the historical write paths, so a stored link can point at a line item on somebody else's job. The item fallback copied its cost code anyway and labelled the result "backfill" — a phase moved across jobs, with provenance saying a machine was confident about it. The link is now checked before it is trusted: accepted when the item is on the expense's OWN estimate, or on another estimate of the SAME project (which is how change-order and revised-estimate work reaches an item — createExpenseCore already requires a CO and its estimate to share the project). Anything else is skipped, terminally for that row, and listed for a human. The skip must not fall through to the rule suggester: a bad link quietly becoming a regex guess is how the data problem would stay invisible. Tested. Every remainder row now carries a reason (item-outside-estimate, no-project, out-of-scope, no-rule-match, unknown-code), the CSV has a reason column, and the dry-run table prints the histogram — a non-zero item-outside-estimate is something to investigate, not a coding gap to shrug at. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.mjs | 91 ++++++++++++++---- tests/backfill-expense-attribution.test.ts | 103 ++++++++++++++++++--- 2 files changed, 162 insertions(+), 32 deletions(-) diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index dadd89c13..346751cf4 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -86,17 +86,19 @@ export function measureCoverage(rows) { * Decide every write WITHOUT touching the database, so the dry run can show the * exact same set the apply would perform. Pure and unit-tested. * - * @param expenses rows with { id, projectId, estimate:{projectId}, costCodeId, - * costCodeSource, itemId, amount, vendor, description, date } - * @param itemCostCodeById item id -> its cost code id (or null) + * @param expenses rows with { id, estimateId, projectId, estimate:{projectId}, + * costCodeId, costCodeSource, itemId, amount, vendor, description, date } + * @param items item id -> { estimateId, projectId, costCodeId } for every CODED + * estimate item, so the link can be checked before it is trusted * @param costCodeIdByCode "03-PLUMB" -> cost code id * @param scopedProjectIds the In Progress, non-overhead jobs the suggester may touch */ -export function planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, scopedProjectIds }) { +export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectIds }) { const inScope = new Set(scopedProjectIds); const projectFills = []; const codeFills = []; const remainder = []; + const add = (expense, reason) => remainder.push({ ...expense, reason }); for (const expense of expenses) { const resolvedProjectId = resolveExpenseProjectId(expense); @@ -111,24 +113,52 @@ export function planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, sco if (expense.costCodeId) continue; if (expense.costCodeSource === "capture" || expense.costCodeSource === "manual") continue; - // (b) item fallback, for ANY project — a coded line item is a fact - // about the money regardless of the job's status. - const fromItem = resolveExpenseCostCodeId(expense, itemCostCodeById); - if (fromItem) { + // (b) ITEM FALLBACK — and the link has to be checked before it is + // trusted (Codex round 1, blocker 3). + // + // `Expense.itemId` is `ON DELETE SET NULL` and has never been scoped to + // the expense's own estimate on any historical write path, so a stored + // itemId can point at a line item on somebody else's job. Copying its + // cost code would move a phase across jobs and call the result + // "backfill" — a wrong code is worse than an absent one, and this is + // exactly the kind of wrong that no one would notice. + // + // Accepted: the item belongs to the expense's OWN estimate, or to + // another estimate of the SAME project (which is how change-order and + // revised-estimate work reaches an item — CO line items live on the + // project's estimates, and `createExpenseCore` already requires a CO + // and its estimate to share the project). Anything else is skipped and + // listed for a human with reason "item-outside-estimate". + const item = expense.itemId ? items.get(expense.itemId) : undefined; + if (expense.itemId && !item) { + // A link to an item that is gone, or to an item with no cost code: + // no answer either way, so fall through to the rules below. + } else if (item && item.costCodeId) { + const sameEstimate = item.estimateId === expense.estimateId; + const sameProject = + item.projectId !== null && item.projectId === resolvedProjectId; + if (!sameEstimate && !sameProject) { + add(expense, "item-outside-estimate"); + continue; + } codeFills.push({ id: expense.id, - costCodeId: fromItem, + costCodeId: item.costCodeId, costCodeSource: "backfill", costCodeConfidence: null, - why: "item cost code", + why: sameEstimate ? "item cost code (own estimate)" : "item cost code (same project)", expense, }); continue; } // (c) the rules, only on active customer jobs. - if (!resolvedProjectId || !inScope.has(resolvedProjectId)) { - remainder.push(expense); + if (!resolvedProjectId) { + add(expense, "no-project"); + continue; + } + if (!inScope.has(resolvedProjectId)) { + add(expense, "out-of-scope"); continue; } const suggestion = suggestCode(expense); @@ -143,7 +173,7 @@ export function planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, sco expense, }); } else { - remainder.push(expense); + add(expense, suggestion ? "unknown-code" : "no-rule-match"); } } @@ -164,7 +194,7 @@ function csvEscape(value) { } export function remainderCsv(remainder, projectNameById) { - const lines = [["expense_id", "project", "date", "vendor", "amount", "description"].join(",")]; + const lines = [["expense_id", "project", "date", "vendor", "amount", "reason", "description"].join(",")]; for (const expense of remainder) { lines.push([ expense.id, @@ -172,6 +202,10 @@ export function remainderCsv(remainder, projectNameById) { expense.date ? new Date(expense.date).toISOString().slice(0, 10) : "", csvEscape(expense.vendor), num(expense.amount).toFixed(2), + // WHY it is here. "item-outside-estimate" in particular is not + // "we could not guess" — it is "this row claims a line item on + // another job", which is a data problem a human should look at. + csvEscape(expense.reason), csvEscape(expense.description), ].join(",")); } @@ -208,19 +242,29 @@ export async function runBackfill({ const expenses = await db.expense.findMany({ select: { - id: true, projectId: true, costCodeId: true, costCodeSource: true, + id: true, estimateId: true, projectId: true, costCodeId: true, costCodeSource: true, itemId: true, amount: true, vendor: true, description: true, date: true, estimate: { select: { projectId: true } }, }, }); - const items = await db.estimateItem.findMany({ + // The item's OWN estimate and project come back with it: the fallback has + // to prove the link does not cross jobs before it copies a code. + const itemRows = await db.estimateItem.findMany({ where: { costCodeId: { not: null } }, - select: { id: true, costCodeId: true }, + select: { + id: true, costCodeId: true, estimateId: true, + estimate: { select: { projectId: true } }, + }, }); - const itemCostCodeById = new Map(items.map(i => [i.id, i.costCodeId])); + const items = new Map(itemRows.map(row => [row.id, { + costCodeId: row.costCodeId, + estimateId: row.estimateId, + projectId: row.estimate?.projectId ?? null, + }])); + const itemCostCodeById = new Map(itemRows.map(row => [row.id, row.costCodeId])); - const plan = planBackfill({ expenses, itemCostCodeById, costCodeIdByCode, scopedProjectIds }); + const plan = planBackfill({ expenses, items, costCodeIdByCode, scopedProjectIds }); // ── the table ─────────────────────────────────────────────────────────── const scoped = new Set(scopedProjectIds); @@ -279,6 +323,15 @@ export async function runBackfill({ log(` ${key.padEnd(40)} ${String(v.n).padStart(4)} rows ${money(v.sum).padStart(13)}`); } log(`NEEDS HUMAN: ${plan.remainder.length} rows ${money(plan.remainder.reduce((t, e) => t + Math.abs(num(e.amount)), 0))}`); + const byReason = new Map(); + for (const expense of plan.remainder) { + byReason.set(expense.reason, (byReason.get(expense.reason) ?? 0) + 1); + } + for (const [reason, n] of [...byReason.entries()].sort((a, b) => b[1] - a[1])) { + // A non-zero "item-outside-estimate" is a DATA problem, not a coding + // gap: some expense points at a line item on another job. + log(` ${String(reason).padEnd(24)} ${String(n).padStart(4)} rows`); + } if (csvPath) { writeFile(csvPath, remainderCsv(plan.remainder, projectNameById)); diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 2fef80202..f2a15bae4 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -20,6 +20,7 @@ const OVERHEAD_ID = "overhead-project"; type StubExpense = { id: string; + estimateId: string; projectId: string | null; costCodeId: string | null; costCodeSource: string | null; @@ -31,9 +32,13 @@ type StubExpense = { estimate: { projectId: string | null }; }; +/** What the loader hands `planBackfill`: the item plus WHERE it lives. */ +type StubItem = { costCodeId: string | null; estimateId: string; projectId: string | null }; + function expense(overrides: Partial = {}): StubExpense { return { id: "e1", + estimateId: "est-job-1", projectId: null, costCodeId: null, costCodeSource: null, @@ -47,12 +52,17 @@ function expense(overrides: Partial = {}): StubExpense { }; } +const NO_ITEMS = new Map(); + const COST_CODE_IDS = new Map([ ["03-PLUMB", "cc-plumb"], ["02-FRAME", "cc-frame"], ]); -function createStub(expenses: StubExpense[], items: { id: string; costCodeId: string | null }[] = []) { +function createStub( + expenses: StubExpense[], + items: { id: string; costCodeId: string | null; estimateId: string; estimate: { projectId: string | null } }[] = [], +) { const writes: { where: Record; data: Record }[] = []; return { writes, @@ -97,7 +107,7 @@ test("plans a projectId fill only for rows whose column is still NULL", () => { expense({ id: "already-set", projectId: "job-1", estimate: { projectId: "job-1" } }), expense({ id: "no-answer", projectId: null, estimate: { projectId: null } }), ], - itemCostCodeById: new Map(), + items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); @@ -107,7 +117,7 @@ test("plans a projectId fill only for rows whose column is still NULL", () => { test("the item fallback wins over the rules, and is sourced 'backfill'", () => { const plan = planBackfill({ expenses: [expense({ id: "e1", itemId: "item-1", vendor: "Summit Plumbing" })], - itemCostCodeById: new Map([["item-1", "cc-frame"]]), + items: new Map([["item-1", { costCodeId: "cc-frame", estimateId: "est-job-1", projectId: "job-1" }]]), costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); @@ -117,11 +127,71 @@ test("the item fallback wins over the rules, and is sourced 'backfill'", () => { assert.equal(plan.codeFills[0].costCodeConfidence, null); }); +// ── the item link has to be checked before it is trusted (blocker 3) ──────── + +test("an itemId pointing at ANOTHER job's line item is skipped, not copied", () => { + // Expense.itemId is ON DELETE SET NULL and was never scoped to the + // expense's own estimate on the historical write paths, so a stored link + // can legitimately point somewhere else. Copying that code would move a + // phase across jobs and label the result "backfill". + const plan = planBackfill({ + expenses: [expense({ id: "e1", estimateId: "est-job-1", itemId: "item-other" })], + items: new Map([["item-other", { costCodeId: "cc-frame", estimateId: "est-job-2", projectId: "job-2" }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1", "job-2"], + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder.length, 1); + assert.equal(plan.remainder[0].reason, "item-outside-estimate"); +}); + +test("a cross-job item is skipped even when the rules WOULD have had an answer", () => { + // The skip must be terminal for that row, not a fall-through to the + // suggester — otherwise a bad link quietly becomes a regex guess and the + // data problem is never surfaced to a human. + const plan = planBackfill({ + expenses: [expense({ id: "e1", itemId: "item-other", vendor: "Summit Plumbing" })], + items: new Map([["item-other", { costCodeId: "cc-frame", estimateId: "est-job-2", projectId: "job-2" }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder[0].reason, "item-outside-estimate"); +}); + +test("an item on ANOTHER estimate of the SAME job is accepted", () => { + // Change-order and revised-estimate work lands on a different estimate of + // the same project, and createExpenseCore already requires a CO and its + // estimate to share the project. That link does not cross jobs. + const plan = planBackfill({ + expenses: [expense({ id: "e1", estimateId: "est-job-1", projectId: "job-1", itemId: "item-co" })], + items: new Map([["item-co", { costCodeId: "cc-frame", estimateId: "est-job-1-co", projectId: "job-1" }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.equal(plan.codeFills.length, 1); + assert.equal(plan.codeFills[0].costCodeId, "cc-frame"); + assert.match(plan.codeFills[0].why, /same project/); +}); + +test("a dangling itemId falls through to the rules rather than being skipped", () => { + // The item is gone (or carries no code). That is no evidence either way, + // so it must not consume the row — the suggester still gets its turn. + const plan = planBackfill({ + expenses: [expense({ id: "e1", projectId: "job-1", itemId: "item-deleted", vendor: "Summit Plumbing" })], + items: NO_ITEMS, + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.equal(plan.codeFills.length, 1); + assert.equal(plan.codeFills[0].costCodeSource, "ai"); +}); + test("a human's cost code is never planned over — capture and manual both", () => { for (const costCodeSource of ["capture", "manual"]) { const plan = planBackfill({ expenses: [expense({ costCodeSource, vendor: "Summit Plumbing" })], - itemCostCodeById: new Map(), + items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); @@ -133,7 +203,7 @@ test("a human's cost code is never planned over — capture and manual both", () test("an already-coded row is left alone even with a NULL source", () => { const plan = planBackfill({ expenses: [expense({ costCodeId: "cc-existing", vendor: "Summit Plumbing" })], - itemCostCodeById: new Map(), + items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); @@ -147,12 +217,13 @@ test("the overhead bucket and closed jobs are out of the suggester's scope", () expense({ id: "closed", projectId: "job-closed", estimate: { projectId: "job-closed" }, vendor: "Summit Plumbing" }), expense({ id: "active", projectId: "job-1", estimate: { projectId: "job-1" }, vendor: "Summit Plumbing" }), ], - itemCostCodeById: new Map(), + items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); assert.deepEqual(plan.codeFills.map(f => f.id), ["active"]); assert.deepEqual(plan.remainder.map(e => e.id).sort(), ["closed", "overhead"]); + assert.deepEqual(plan.remainder.map(e => e.reason), ["out-of-scope", "out-of-scope"]); }); test("a rule hit below the confidence floor is left for a human", () => { @@ -162,7 +233,7 @@ test("a rule hit below the confidence floor is left for a human", () => { assert.equal(MIN_CONFIDENCE, 0.7); const plan = planBackfill({ expenses: [expense({ vendor: "Summit Plumbing" })], - itemCostCodeById: new Map(), + items: NO_ITEMS, // The rules name 03-PLUMB; this company does not have that code. costCodeIdByCode: new Map(), scopedProjectIds: ["job-1"], @@ -188,7 +259,7 @@ test("the projected 'after' applies the plan without touching the database", () const rows = [expense({ id: "e1", vendor: "Summit Plumbing", amount: 400 })]; const plan = planBackfill({ expenses: rows, - itemCostCodeById: new Map(), + items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); @@ -197,14 +268,20 @@ test("the projected 'after' applies the plan without touching the database", () assert.equal(rows[0].costCodeId, null, "the source rows are not mutated"); }); -test("the remainder CSV carries what Marge needs to decide", () => { +test("the remainder CSV carries what Marge needs to decide, including WHY", () => { const csv = remainderCsv( - [expense({ id: "e9", vendor: 'Lowe"s', description: "misc\nsupplies", amount: 12.5 })], + [{ + ...expense({ id: "e9", vendor: 'Lowe"s', description: "misc\nsupplies", amount: 12.5 }), + reason: "item-outside-estimate", + }], new Map([["job-1", "Mueller Bath"]]), ); const [header, row] = csv.split("\n"); - assert.equal(header, "expense_id,project,date,vendor,amount,description"); - assert.match(row, /^e9,"Mueller Bath",2026-08-01,"Lowe""s",12\.50,"misc supplies"$/); + assert.equal(header, "expense_id,project,date,vendor,amount,reason,description"); + assert.match( + row, + /^e9,"Mueller Bath",2026-08-01,"Lowe""s",12\.50,"item-outside-estimate","misc supplies"$/, + ); }); // ── the run ───────────────────────────────────────────────────────────────── @@ -279,6 +356,6 @@ test("the CSV is written on a dry run — reviewing it is the point", async () = }); assert.equal(files.length, 1); assert.equal(files[0].path, "out.csv"); - assert.match(files[0].body, /^expense_id,project,date,vendor,amount,description/); + assert.match(files[0].body, /^expense_id,project,date,vendor,amount,reason,description/); assert.match(files[0].body, /e1/); }); From 8d44879da571f625c9fec0a75d46a63100a4ead1 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:45:05 -0700 Subject: [PATCH 071/144] fix(reports,auth): label and authorize an expense by the project it is actually on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, issue 4 + checker item 3. A populated projectId that disagrees with estimate.projectId is the TRUTH — that is the whole point of the column — so three places that still read the estimate had to follow. payouts-report and transactions-report FILTERED with the resolver but LABELLED off the estimate, so a re-attributed expense could be selected for job B and then listed under job A. Both now select the project directly and fall back to the estimate's. api/expenses/[id]/receipt authorized uploads off expense.estimate.projectId. For a re-attributed expense that both admits people from the job it used to be on and locks out the crew who now own it. Resolved the same way as every other gate. Co-Authored-By: Claude Fable 5.1 --- src/app/api/expenses/[id]/receipt/route.ts | 12 +++++++++--- src/lib/payouts-report.ts | 13 +++++++++---- src/lib/transactions-report.ts | 11 +++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/app/api/expenses/[id]/receipt/route.ts b/src/app/api/expenses/[id]/receipt/route.ts index ef9b514aa..fe0b32f32 100644 --- a/src/app/api/expenses/[id]/receipt/route.ts +++ b/src/app/api/expenses/[id]/receipt/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { resolveExpenseProjectId } from "@/lib/expense-attribution"; import { getCurrentUserWithPermissions, canAccessProject } from "@/lib/permissions"; import { getSupabase, STORAGE_BUCKET } from "@/lib/supabase"; @@ -30,12 +31,17 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id: const expense = await prisma.expense.findUnique({ where: { id }, - select: { id: true, estimate: { select: { projectId: true } } }, + select: { id: true, projectId: true, estimate: { select: { projectId: true } } }, }); if (!expense) return NextResponse.json({ error: "Expense not found" }, { status: 404 }); - // Fail closed: an expense whose estimate has no project cannot be + // Fail closed: an expense with no resolvable project cannot be // authorized against project access, so nobody uploads to it. - const projectId = expense.estimate?.projectId; + // + // Resolved, not read off the estimate: a re-attributed expense belongs + // to the project its `projectId` names, and authorizing it against the + // job it USED to be on would both admit the wrong people and lock out + // the crew who now own it. + const projectId = resolveExpenseProjectId(expense); if (!projectId || !canAccessProject(user, projectId)) { return NextResponse.json({ error: "Forbidden" }, { status: 403 }); } diff --git a/src/lib/payouts-report.ts b/src/lib/payouts-report.ts index 0cf8660ad..1504b87b5 100644 --- a/src/lib/payouts-report.ts +++ b/src/lib/payouts-report.ts @@ -1,5 +1,5 @@ import { prisma } from "@/lib/prisma"; -import { expenseForProjectWhere } from "@/lib/expense-attribution"; +import { expenseForProjectWhere, resolveExpenseProjectId } from "@/lib/expense-attribution"; import { formatLocalDateString, defaultMonthRange, @@ -71,7 +71,12 @@ export async function queryPayoutsData(filters: PayoutsFilters): Promise<{ : {}), }, include: { - estimate: { select: { project: { select: { id: true, name: true } } } }, + // BOTH sides, so the row can be LABELLED by the same + // project the filter above selected it by. Labelling off + // the estimate while filtering on the resolver would list a + // re-attributed expense under the job it used to be on. + project: { select: { id: true, name: true } }, + estimate: { select: { projectId: true, project: { select: { id: true, name: true } } } }, purchaseOrder: { select: { code: true } }, }, orderBy: { date: "desc" }, @@ -106,8 +111,8 @@ export async function queryPayoutsData(filters: PayoutsFilters): Promise<{ vendorName: exp.vendor ?? "Unknown Vendor", type: "Expense", amount: Number(exp.amount), - projectName: exp.estimate.project?.name ?? "No Project", - projectId: exp.estimate.project?.id ?? null, + projectName: exp.project?.name ?? exp.estimate.project?.name ?? "No Project", + projectId: resolveExpenseProjectId(exp), reference: exp.purchaseOrder?.code ?? null, }); } diff --git a/src/lib/transactions-report.ts b/src/lib/transactions-report.ts index dc79ad89b..b641f5975 100644 --- a/src/lib/transactions-report.ts +++ b/src/lib/transactions-report.ts @@ -1,5 +1,5 @@ import { prisma } from "@/lib/prisma"; -import { expenseForProjectWhere } from "@/lib/expense-attribution"; +import { expenseForProjectWhere, resolveExpenseProjectId } from "@/lib/expense-attribution"; import { formatLocalDateString, defaultMonthRange, @@ -119,7 +119,10 @@ export async function queryTransactionsData(filters: TransactionsFilters): Promi : {}), }, include: { - estimate: { select: { project: { select: { id: true, name: true } } } }, + // BOTH sides — see payouts-report: a row must be labelled + // by the project the filter selected it by. + project: { select: { id: true, name: true } }, + estimate: { select: { projectId: true, project: { select: { id: true, name: true } } } }, }, orderBy: { date: "desc" }, }) @@ -179,8 +182,8 @@ export async function queryTransactionsData(filters: TransactionsFilters): Promi description: exp.description ?? exp.vendor ?? "Expense", type: "Expense", amount: Number(exp.amount), - projectName: exp.estimate.project?.name ?? "No Project", - projectId: exp.estimate.project?.id ?? null, + projectName: exp.project?.name ?? exp.estimate.project?.name ?? "No Project", + projectId: resolveExpenseProjectId(exp), category: "Expense", }); } From ad059e4ee9386de31e419f29dbeda1eda1927a7a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:52:39 -0700 Subject: [PATCH 072/144] fix(tax-report): integer cents, company-timezone periods, formula-safe CSV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, issues 6-8. All three are things that look right on screen while being wrong on a tax filing. 8. INTEGER CENTS. Every sum is whole cents now, converted from the Decimal's STRING form so no float is ever involved (`Number("16.55") * 100` is 1655.0000000000002). Dollars appear only at the display and CSV boundary. Tested with 100 rows of $0.10 tax: exact $10.00, where the float sum drifts. 7. COMPANY TIME ZONE. Period boundaries and month buckets are computed in the configured zone via resolveCompanyTimeZone/tz-date, never in the server's or the browser's. A receipt bought 30 September at 6pm Pacific is 1 October UTC — bucketed in UTC it lands in the wrong QUARTER and the deduction moves onto the wrong excise return. Filters now travel as company day keys, so the client does no date math at all and the quarter presets are the server's values. Boundary tests cover the New Year instant, a DST-crossing range, and the last day of the range being INCLUDED. 6. CSV INJECTION. New src/lib/csv-safe.ts neutralizes a leading = + - @ TAB CR with a quote prefix. Vendor and description are free text a model lifted off a receipt, so this is real input. Numbers go through csvNumber and are deliberately exempt — quote-prefixing "-12.50" would turn it into text and break every SUM in Marge's sheet. Adds tests/tax-at-source-query.test.ts (checker item 2): a require-patched prisma test asserting the where clause is taxAtSource true, installedAtCustomer TRUE (not `not: false` — a NULL must never be spent as a deduction), taxAmount > 0, and a date window in company midnights. NOTE, not fixed here: src/lib/sales-tax-report.ts has its own escapeCsv with the same formula gap. Pre-existing and outside this change; flagged. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- .../tax-paid-at-source/export/route.ts | 14 +- .../TaxAtSourceFiltersForm.tsx | 61 ++-- src/app/reports/tax-paid-at-source/page.tsx | 61 ++-- src/lib/csv-safe.ts | 51 ++++ src/lib/tax-at-source-report.ts | 283 ++++++++++++------ tests/tax-at-source-query.test.ts | 128 ++++++++ tests/tax-at-source-report.test.ts | 230 ++++++++++---- 8 files changed, 614 insertions(+), 218 deletions(-) create mode 100644 src/lib/csv-safe.ts create mode 100644 tests/tax-at-source-query.test.ts diff --git a/package.json b/package.json index 9744683b4..5696c0a6a 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/reports/tax-paid-at-source/export/route.ts b/src/app/api/reports/tax-paid-at-source/export/route.ts index 6ec3308f5..14fc594e5 100644 --- a/src/app/api/reports/tax-paid-at-source/export/route.ts +++ b/src/app/api/reports/tax-paid-at-source/export/route.ts @@ -2,9 +2,8 @@ import { NextRequest, NextResponse } from "next/server"; import { getSessionOrDev } from "@/lib/auth"; import { canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; import { - formatLocalDateString, - parseTaxAtSourceFilters, queryTaxAtSourceRows, + resolveTaxAtSourceFilters, rowsToCsv, } from "@/lib/tax-at-source-report"; @@ -26,15 +25,12 @@ export async function GET(req: NextRequest) { return NextResponse.json({ error: "Forbidden" }, { status: 403 }); } - const filters = parseTaxAtSourceFilters(req.nextUrl.searchParams); + const filters = await resolveTaxAtSourceFilters(req.nextUrl.searchParams); const rows = await queryTaxAtSourceRows(filters); - // Both dates come from parseLocalDateString (or the quarter fallback), so - // they can only be YYYY-MM-DD and cannot inject a header. Formatted - // defensively anyway rather than interpolating the raw query string. - const inclusiveTo = new Date(filters.to.getTime()); - inclusiveTo.setDate(inclusiveTo.getDate() - 1); - const filename = `tax-paid-at-source-${formatLocalDateString(filters.from)}-to-${formatLocalDateString(inclusiveTo)}.csv`; + // Both keys are validated /^\d{4}-\d{2}-\d{2}$/ by the parser (anything + // else falls back to the current quarter), so neither can inject a header. + const filename = `tax-paid-at-source-${filters.fromKey}-to-${filters.toKey}.csv`; return new NextResponse(rowsToCsv(rows), { status: 200, diff --git a/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx b/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx index 98290458b..b504b91eb 100644 --- a/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx +++ b/src/app/reports/tax-paid-at-source/TaxAtSourceFiltersForm.tsx @@ -2,19 +2,31 @@ import { useRouter } from "next/navigation"; import { useState } from "react"; -import { formatLocalDateString, type TaxAtSourceFilters } from "@/lib/tax-at-source-report"; -/** The `to` the server holds is EXCLUSIVE; the picker shows the inclusive day. */ -function inclusiveTo(filters: TaxAtSourceFilters): string { - const day = new Date(filters.to.getTime()); - day.setDate(day.getDate() - 1); - return formatLocalDateString(day); +/** + * Dates in, dates out — both as company-calendar "YYYY-MM-DD" keys the server + * computed. The browser deliberately does NO date math: a crew laptop set to + * Mountain Time must not shift a quarter boundary, and `new Date()` here would + * do exactly that. The quarter presets are the server's own values, passed in. + */ +export interface QuarterPreset { + label: string; + fromKey: string; + toKey: string; } -export default function TaxAtSourceFiltersForm({ filters }: { filters: TaxAtSourceFilters }) { +export default function TaxAtSourceFiltersForm({ + fromKey, + toKey, + presets, +}: { + fromKey: string; + toKey: string; + presets: QuarterPreset[]; +}) { const router = useRouter(); - const [from, setFrom] = useState(formatLocalDateString(filters.from)); - const [to, setTo] = useState(inclusiveTo(filters)); + const [from, setFrom] = useState(fromKey); + const [to, setTo] = useState(toKey); function apply(next?: { from: string; to: string }) { const params = new URLSearchParams({ @@ -24,17 +36,6 @@ export default function TaxAtSourceFiltersForm({ filters }: { filters: TaxAtSour router.push(`/reports/tax-paid-at-source?${params.toString()}`); } - function applyQuarter(offset: number) { - const now = new Date(); - const start = new Date(now.getFullYear(), Math.floor(now.getMonth() / 3) * 3 + offset * 3, 1); - const end = new Date(start.getFullYear(), start.getMonth() + 3, 0); - const nextFrom = formatLocalDateString(start); - const nextTo = formatLocalDateString(end); - setFrom(nextFrom); - setTo(nextTo); - apply({ from: nextFrom, to: nextTo }); - } - return (
); diff --git a/src/app/reports/tax-paid-at-source/page.tsx b/src/app/reports/tax-paid-at-source/page.tsx index 0beee3edd..a2f943826 100644 --- a/src/app/reports/tax-paid-at-source/page.tsx +++ b/src/app/reports/tax-paid-at-source/page.tsx @@ -3,16 +3,19 @@ import Link from "next/link"; import { getSessionOrDev } from "@/lib/auth"; import { canUseDevAuthFallback, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; import { formatCurrency } from "@/lib/utils"; -import { formatMoneyDate } from "@/lib/payment-date"; import { TAX_REPORT_AMOUNT_NOTE, TAX_REPORT_FOOTNOTE, + centsToDollars, + currentQuarterKeys, groupTaxAtSource, - parseTaxAtSourceFilters, + monthLabelFromKey, queryTaxAtSourceRows, + resolveTaxAtSourceFilters, stringifyTaxAtSourceFilters, } from "@/lib/tax-at-source-report"; -import TaxAtSourceFiltersForm from "./TaxAtSourceFiltersForm"; +import { addDaysToKey } from "@/lib/tz-date"; +import TaxAtSourceFiltersForm, { type QuarterPreset } from "./TaxAtSourceFiltersForm"; type SearchParams = Record; @@ -34,15 +37,29 @@ export default async function TaxPaidAtSourcePage({ return
Access denied. This report requires the Financial Reports permission.
; } - const filters = parseTaxAtSourceFilters(await searchParams); + const filters = await resolveTaxAtSourceFilters(await searchParams); const rows = await queryTaxAtSourceRows(filters); const { months, summary } = groupTaxAtSource(rows); const csvHref = `/api/reports/tax-paid-at-source/export?${stringifyTaxAtSourceFilters(filters)}`; - const dateLabel = (value: Date) => - value.toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" }); - const inclusiveTo = new Date(filters.to.getTime()); - inclusiveTo.setDate(inclusiveTo.getDate() - 1); + // Presets are computed HERE, in the company zone, and handed to the client + // as plain day keys. A browser on a different clock must not be able to + // shift a quarter boundary and move a deduction onto the wrong return. + const thisQuarter = currentQuarterKeys(new Date(), filters.timeZone); + const lastQuarter = currentQuarterKeys( + new Date(`${addDaysToKey(thisQuarter.fromKey, -1)}T12:00:00Z`), + "UTC", + ); + const presets: QuarterPreset[] = [ + { label: "This quarter", ...thisQuarter }, + { label: "Last quarter", ...lastQuarter }, + ]; + + // Labels are formatted from the day KEY, not from a Date: rendering a + // company-midnight instant with the server's locale/zone would print the + // day before whenever the server sits east of the company. + const dayLabel = (dayKey: string) => + `${monthLabelFromKey(dayKey).split(" ")[0].slice(0, 3)} ${Number(dayKey.slice(8, 10))}, ${dayKey.slice(0, 4)}`; return (
@@ -50,7 +67,7 @@ export default async function TaxPaidAtSourcePage({

Tax Paid at Source

- Material tax paid at the register and installed at a customer job · {dateLabel(filters.from)} → {dateLabel(inclusiveTo)} + Material tax paid at the register and installed at a customer job · {dayLabel(filters.fromKey)} → {dayLabel(filters.toKey)}

@@ -59,12 +76,12 @@ export default async function TaxPaidAtSourcePage({
- +
- - + +
@@ -97,16 +114,16 @@ export default async function TaxPaidAtSourcePage({ {job.projectName} {job.count} - {formatCurrency(job.deductionBase)} - {formatCurrency(job.tax)} + {formatCurrency(centsToDollars(job.deductionBaseCents))} + {formatCurrency(centsToDollars(job.taxCents))} ))} {month.label} total {month.count} - {formatCurrency(month.deductionBase)} - {formatCurrency(month.tax)} + {formatCurrency(centsToDollars(month.deductionBaseCents))} + {formatCurrency(centsToDollars(month.taxCents))} ))} @@ -115,8 +132,8 @@ export default async function TaxPaidAtSourcePage({ Total {summary.count} - {formatCurrency(summary.deductionBase)} - {formatCurrency(summary.tax)} + {formatCurrency(centsToDollars(summary.deductionBaseCents))} + {formatCurrency(centsToDollars(summary.taxCents))} @@ -149,14 +166,14 @@ export default async function TaxPaidAtSourcePage({ {rows.map(row => ( - {formatMoneyDate(row.date, { month: "short", day: "numeric", year: "numeric" }, "en-US")} + {dayLabel(row.dayKey)} {row.vendor || "—"} {row.projectName} {row.reference || "—"} - {formatCurrency(row.receiptTotal)} - {formatCurrency(row.deductionBase)} - {formatCurrency(row.tax)} + {formatCurrency(centsToDollars(row.receiptTotalCents))} + {formatCurrency(centsToDollars(row.deductionBaseCents))} + {formatCurrency(centsToDollars(row.taxCents))} ))} diff --git a/src/lib/csv-safe.ts b/src/lib/csv-safe.ts new file mode 100644 index 000000000..81e10df21 --- /dev/null +++ b/src/lib/csv-safe.ts @@ -0,0 +1,51 @@ +// CSV cells that are safe to open in a spreadsheet. +// +// RFC 4180 quoting is only half the problem. Excel, LibreOffice, and Google +// Sheets all treat a cell whose text begins with `=`, `+`, `-`, `@`, TAB, or CR +// as a FORMULA, so a vendor named `=cmd|'/c calc'!A1` — or, far more likely +// around here, a description a person typed starting with a minus sign — +// executes or errors when Marge opens the export. The data came from receipts +// and free-text fields, so this is not hypothetical input. +// +// The fix is the standard one: prefix a single quote, which spreadsheets read +// as "the rest of this cell is literal text" and strip from the display. +// +// NOTE: src/lib/sales-tax-report.ts has its own `escapeCsv` that quotes but +// does NOT neutralize formulas. Same gap, pre-existing, and out of scope for +// the change that added this file — flagged rather than silently rewritten. + +/** + * A leading character a spreadsheet will read as the start of a formula. + * `-` is included deliberately even though `-12.50` is a harmless negative + * number: `-1+1` is not, and a text cell has no business starting with either. + * Numbers are emitted by `csvNumber` instead, which is exempt for that reason. + */ +const FORMULA_TRIGGER = /^[=+\-@\t\r]/; + +/** + * Quote and neutralize one TEXT cell. Always quoted, so an embedded comma, + * quote, or newline is safe too. + */ +export function csvCell(value: unknown): string { + if (value === null || value === undefined) return '""'; + const raw = String(value); + const neutralized = FORMULA_TRIGGER.test(raw) ? `'${raw}` : raw; + return `"${neutralized.replace(/"/g, '""')}"`; +} + +/** + * A NUMERIC cell, emitted unquoted so the spreadsheet reads it as a number. + * + * Safe without neutralizing because the output shape is produced here, not + * echoed from input: `toFixed` on a finite number can only ever yield + * `-?\d+\.\d*`, which no spreadsheet parses as a formula. A non-finite value + * becomes an empty cell rather than "NaN", which would silently poison a SUM. + */ +export function csvNumber(value: number, digits = 2): string { + return Number.isFinite(value) ? value.toFixed(digits) : ""; +} + +/** Join rows with CRLF and a trailing terminator, per RFC 4180. */ +export function csvDocument(rows: string[][]): string { + return rows.map(cells => cells.join(",")).join("\r\n") + "\r\n"; +} diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts index a85cdae82..289b22761 100644 --- a/src/lib/tax-at-source-report.ts +++ b/src/lib/tax-at-source-report.ts @@ -15,39 +15,55 @@ // must never be spent as a deduction, and // * `taxAmount > 0` — a zero is an answer (no tax), not an absence. // -// The aggregation is pure and unit-tested; only `queryTaxAtSourceRows` touches -// Prisma. +// TWO THINGS THIS FILE IS FUSSY ABOUT, both because it feeds a tax filing: +// +// 1. INTEGER CENTS. Every sum is in whole cents, converted from the Decimal's +// STRING form so no float is ever involved. Summing dollars as floats +// drifts — 0.1 + 0.2 is the canonical example, and a quarter's worth of +// receipts drifts by more than a cent — and a total that disagrees with the +// sum of its own rows is exactly the thing a bookkeeper will find and stop +// trusting the report over. +// +// 2. THE COMPANY TIME ZONE. Period boundaries and month buckets are computed +// in the company's configured zone, never in the server's or the browser's. +// A receipt bought on 30 September at 6pm Pacific is stored as 1 October +// UTC; bucketed in UTC it lands in the wrong QUARTER, and moves a deduction +// onto the wrong excise return. +// +// The aggregation is pure and unit-tested; only `queryTaxAtSourceRows` and +// `resolveTaxAtSourceFilters` touch Prisma. import { prisma } from "@/lib/prisma"; -import { toNum } from "@/lib/prisma-helpers"; -import { formatMoneyMonth, formatMoneyMonthKey, formatMoneyDateISO } from "./payment-date"; -import { parseLocalDateString, formatLocalDateString } from "./report-utils"; +import { resolveCompanyTimeZone } from "./company-timezone"; +import { addDaysToKey, dayKeyInTimeZone, startOfDateInTimeZone } from "./tz-date"; import { resolveExpenseProjectId } from "./expense-attribution"; - -export { parseLocalDateString, formatLocalDateString } from "./report-utils"; +import { csvCell, csvDocument, csvNumber } from "./csv-safe"; export interface TaxAtSourceFilters { + /** First day of the period, company calendar, "YYYY-MM-DD" — INCLUSIVE. */ + fromKey: string; + /** Last day of the period, company calendar — INCLUSIVE, as a human reads it. */ + toKey: string; + /** Instant bounds for the query: [from, to), both company-midnight. */ from: Date; - /** Exclusive upper bound. */ to: Date; + timeZone: string; } export interface TaxAtSourceRow { id: string; date: Date; + /** Company-calendar day the receipt falls on, "YYYY-MM-DD". */ + dayKey: string; vendor: string; projectId: string | null; projectName: string; /** Invoice / check reference, when the description carries one. */ reference: string; - /** - * What was paid in total. See the caveat in `TAX_REPORT_AMOUNT_NOTE`: - * intake-born rows are gross-with-tax, and legacy QBO rows carry no - * taxAmount at all so they never reach this report. - */ - receiptTotal: number; - /** receiptTotal - tax. The figure the excise line is computed from. */ - deductionBase: number; - tax: number; + /** Gross paid, in whole cents. */ + receiptTotalCents: number; + /** receiptTotal - tax, in whole cents. The excise line's base. */ + deductionBaseCents: number; + taxCents: number; } export const TAX_REPORT_AMOUNT_NOTE = @@ -58,17 +74,66 @@ export const TAX_REPORT_FOOTNOTE = "\"taxable amount for tax paid at source\". Only receipts flagged installed-at-customer count; Shop and consumable " + "purchases are excluded."; -/** Calendar quarter containing `now`, as [from, to) local dates. */ -export function currentQuarterRange(now: Date = new Date()): TaxAtSourceFilters { - const quarterStartMonth = Math.floor(now.getMonth() / 3) * 3; - return { - from: new Date(now.getFullYear(), quarterStartMonth, 1, 0, 0, 0, 0), - to: new Date(now.getFullYear(), quarterStartMonth + 3, 1, 0, 0, 0, 0), - }; +/** Whole cents from a value for display only. */ +export function centsToDollars(cents: number): number { + return cents / 100; } +/** + * Exact cents from a Prisma Decimal (or anything that stringifies to a decimal + * literal), WITHOUT going through a float. + * + * `Number("16.55") * 100` is 1655.0000000000002 — fine once, wrong after enough + * additions. Prisma's Decimal stringifies exactly, so the digits are parsed + * directly and the third decimal place decides a half-up round. + */ +export function toCents(value: unknown): number { + if (value === null || value === undefined) return 0; + const text = String(value).trim(); + const match = /^(-?)(\d+)(?:\.(\d*))?$/.exec(text); + if (!match) { + // Scientific notation or junk — not a shape money arrives in, but a + // silent 0 would understate a deduction. Fall back rather than drop. + const asNumber = Number(text); + return Number.isFinite(asNumber) ? Math.round(asNumber * 100) : 0; + } + const sign = match[1] === "-" ? -1 : 1; + const fraction = (match[3] ?? "").padEnd(3, "0"); + const cents = Number(match[2]) * 100 + Number(fraction.slice(0, 2)); + const roundUp = Number(fraction[2]) >= 5; + return sign * (cents + (roundUp ? 1 : 0)); +} + +/** Calendar quarter containing `now`, expressed in the COMPANY's zone. */ +export function currentQuarterKeys(now: Date, timeZone: string): { fromKey: string; toKey: string } { + const today = dayKeyInTimeZone(now, timeZone); + const year = Number(today.slice(0, 4)); + const month = Number(today.slice(5, 7)); + const quarterStartMonth = Math.floor((month - 1) / 3) * 3 + 1; + const fromKey = `${year}-${String(quarterStartMonth).padStart(2, "0")}-01`; + // First day of the month AFTER the quarter, then step back one day to get + // the inclusive last day — no month-length table, and correct in February. + const endExclusiveMonth = quarterStartMonth + 3; + const endYear = endExclusiveMonth > 12 ? year + 1 : year; + const endMonth = endExclusiveMonth > 12 ? endExclusiveMonth - 12 : endExclusiveMonth; + const toKey = addDaysToKey(`${endYear}-${String(endMonth).padStart(2, "0")}-01`, -1); + return { fromKey, toKey }; +} + +function isDayKey(value: string | undefined): value is string { + return !!value && /^\d{4}-\d{2}-\d{2}$/.test(value); +} + +/** + * Build the filters from URL params, in the company's zone. + * + * An inverted or unparseable range falls back to the current quarter rather + * than returning an empty period: an empty table reads as "no tax was paid this + * quarter", which is a very different claim from "your dates are backwards". + */ export function parseTaxAtSourceFilters( params: URLSearchParams | Record, + timeZone: string, now: Date = new Date(), ): TaxAtSourceFilters { const get = (key: string): string | undefined => { @@ -76,27 +141,39 @@ export function parseTaxAtSourceFilters( const value = (params as Record)[key]; return Array.isArray(value) ? value[0] : value ?? undefined; }; - const fallback = currentQuarterRange(now); - const from = (get("from") && parseLocalDateString(get("from")!)) || fallback.from; - const parsedTo = get("to") ? parseLocalDateString(get("to")!) : null; - // The picker's "to" is INCLUSIVE to a human; the query bound is exclusive. - const to = parsedTo - ? new Date(parsedTo.getFullYear(), parsedTo.getMonth(), parsedTo.getDate() + 1, 0, 0, 0, 0) - : fallback.to; - // An inverted range is a typo, not a query. Returning the fallback rather - // than an empty table stops the page reading as "no tax paid this quarter". - if (to.getTime() <= from.getTime()) return fallback; - return { from, to }; + + const fallback = currentQuarterKeys(now, timeZone); + const rawFrom = get("from"); + const rawTo = get("to"); + let fromKey = isDayKey(rawFrom) ? rawFrom : fallback.fromKey; + let toKey = isDayKey(rawTo) ? rawTo : fallback.toKey; + if (toKey < fromKey) { + fromKey = fallback.fromKey; + toKey = fallback.toKey; + } + + return { + fromKey, + toKey, + from: startOfDateInTimeZone(fromKey, timeZone), + // `toKey` is the last day a human means to include, so the exclusive + // bound is the start of the NEXT company day. Taken literally, an + // exclusive bound would silently drop everything bought on the last day + // of the quarter. + to: startOfDateInTimeZone(addDaysToKey(toKey, 1), timeZone), + timeZone, + }; +} + +/** Resolve the company zone, then parse. The one Prisma-touching entry point. */ +export async function resolveTaxAtSourceFilters( + params: URLSearchParams | Record, +): Promise { + return parseTaxAtSourceFilters(params, await resolveCompanyTimeZone()); } export function stringifyTaxAtSourceFilters(filters: TaxAtSourceFilters): string { - // `to` goes back out as the INCLUSIVE day the user typed. - const inclusiveTo = new Date(filters.to.getTime()); - inclusiveTo.setDate(inclusiveTo.getDate() - 1); - return new URLSearchParams({ - from: formatLocalDateString(filters.from), - to: formatLocalDateString(inclusiveTo), - }).toString(); + return new URLSearchParams({ from: filters.fromKey, to: filters.toKey }).toString(); } /** @@ -118,9 +195,9 @@ export interface TaxAtSourceJobGroup { projectId: string | null; projectName: string; count: number; - deductionBase: number; - receiptTotal: number; - tax: number; + deductionBaseCents: number; + receiptTotalCents: number; + taxCents: number; } export interface TaxAtSourceMonthGroup { @@ -128,38 +205,56 @@ export interface TaxAtSourceMonthGroup { label: string; jobs: TaxAtSourceJobGroup[]; count: number; - deductionBase: number; - receiptTotal: number; - tax: number; + deductionBaseCents: number; + receiptTotalCents: number; + taxCents: number; } export interface TaxAtSourceSummary { count: number; - deductionBase: number; - receiptTotal: number; - tax: number; + deductionBaseCents: number; + receiptTotalCents: number; + taxCents: number; } -/** Month × job rollup. Pure — this is the function the unit tests drive. */ +const MONTH_LABELS = [ + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", +]; + +/** "2026-06" -> "June 2026", from the company-calendar key. No Date involved. */ +export function monthLabelFromKey(monthKey: string): string { + const year = monthKey.slice(0, 4); + const month = Number(monthKey.slice(5, 7)); + return `${MONTH_LABELS[month - 1] ?? monthKey} ${year}`; +} + +/** + * Month × job rollup, in whole cents. Pure — this is the function the unit + * tests drive. Month buckets come from each row's COMPANY-calendar day key, so + * a late-evening purchase never falls into the next month (or quarter). + */ export function groupTaxAtSource(rows: TaxAtSourceRow[]): { months: TaxAtSourceMonthGroup[]; summary: TaxAtSourceSummary; } { const months = new Map(); - const summary: TaxAtSourceSummary = { count: 0, deductionBase: 0, receiptTotal: 0, tax: 0 }; + const summary: TaxAtSourceSummary = { + count: 0, deductionBaseCents: 0, receiptTotalCents: 0, taxCents: 0, + }; for (const row of rows) { - const key = formatMoneyMonthKey(row.date); + const key = row.dayKey.slice(0, 7); let month = months.get(key); if (!month) { month = { key, - label: formatMoneyMonth(row.date), + label: monthLabelFromKey(key), jobs: [], count: 0, - deductionBase: 0, - receiptTotal: 0, - tax: 0, + deductionBaseCents: 0, + receiptTotalCents: 0, + taxCents: 0, }; months.set(key, month); } @@ -171,32 +266,32 @@ export function groupTaxAtSource(rows: TaxAtSourceRow[]): { projectId: row.projectId, projectName: row.projectName, count: 0, - deductionBase: 0, - receiptTotal: 0, - tax: 0, + deductionBaseCents: 0, + receiptTotalCents: 0, + taxCents: 0, }; month.jobs.push(job); } job.count += 1; - job.deductionBase += row.deductionBase; - job.receiptTotal += row.receiptTotal; - job.tax += row.tax; + job.deductionBaseCents += row.deductionBaseCents; + job.receiptTotalCents += row.receiptTotalCents; + job.taxCents += row.taxCents; month.count += 1; - month.deductionBase += row.deductionBase; - month.receiptTotal += row.receiptTotal; - month.tax += row.tax; + month.deductionBaseCents += row.deductionBaseCents; + month.receiptTotalCents += row.receiptTotalCents; + month.taxCents += row.taxCents; summary.count += 1; - summary.deductionBase += row.deductionBase; - summary.receiptTotal += row.receiptTotal; - summary.tax += row.tax; + summary.deductionBaseCents += row.deductionBaseCents; + summary.receiptTotalCents += row.receiptTotalCents; + summary.taxCents += row.taxCents; } const ordered = [...months.values()].sort((a, b) => a.key.localeCompare(b.key)); for (const month of ordered) { - month.jobs.sort((a, b) => b.tax - a.tax); + month.jobs.sort((a, b) => b.taxCents - a.taxCents); } return { months: ordered, summary }; } @@ -224,46 +319,48 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise }); return rows.map(row => { - const tax = toNum(row.taxAmount); - const receiptTotal = toNum(row.amount); + const taxCents = toCents(row.taxAmount); + const receiptTotalCents = toCents(row.amount); return { id: row.id, // The `date: { gte }` filter above already excluded null dates. date: row.date!, + dayKey: dayKeyInTimeZone(row.date!, filters.timeZone), vendor: row.vendor ?? "", projectId: resolveExpenseProjectId(row), projectName: row.project?.name ?? row.estimate?.project?.name ?? "(unassigned)", reference: extractReference(row.description), - receiptTotal, - deductionBase: receiptTotal - tax, - tax, + receiptTotalCents, + deductionBaseCents: receiptTotalCents - taxCents, + taxCents, }; }); } -function escapeCsv(value: string): string { - return `"${String(value ?? "").replace(/"/g, '""')}"`; -} - /** * Mirrors the columns of the workbook Vanessa already receives * (Date, Vendor, Job, Invoice, Receipt Total, deduction base, Tax), so the * handoff file shape survives the move into ProBuild. + * + * Text cells go through `csvCell`, which neutralizes the leading characters a + * spreadsheet reads as a formula — vendor names and descriptions are free text + * lifted off receipts, so that is real input, not a hypothetical. */ export function rowsToCsv(rows: TaxAtSourceRow[]): string { - const lines = [ - ["Date", "Vendor", "Job", "Invoice", "Receipt Total", "Material Amount (deduction base)", "Tax Paid at Source"].join(","), - ]; + const document: string[][] = [[ + "Date", "Vendor", "Job", "Invoice", + "Receipt Total", "Material Amount (deduction base)", "Tax Paid at Source", + ]]; for (const row of rows) { - lines.push([ - formatMoneyDateISO(row.date), - escapeCsv(row.vendor), - escapeCsv(row.projectName), - escapeCsv(row.reference), - row.receiptTotal.toFixed(2), - row.deductionBase.toFixed(2), - row.tax.toFixed(2), - ].join(",")); + document.push([ + csvCell(row.dayKey), + csvCell(row.vendor), + csvCell(row.projectName), + csvCell(row.reference), + csvNumber(centsToDollars(row.receiptTotalCents)), + csvNumber(centsToDollars(row.deductionBaseCents)), + csvNumber(centsToDollars(row.taxCents)), + ]); } - return lines.join("\r\n") + "\r\n"; + return csvDocument(document); } diff --git a/tests/tax-at-source-query.test.ts b/tests/tax-at-source-query.test.ts new file mode 100644 index 000000000..ccd972db6 --- /dev/null +++ b/tests/tax-at-source-query.test.ts @@ -0,0 +1,128 @@ +/** + * The FILTER, not the arithmetic (checker item 2). + * + * tax-at-source-report.test.ts proves the sums. This proves the three + * conditions that decide which receipts are summed at all — and they are the + * part that can be wrong without anything looking wrong: a report that quietly + * includes `installedAtCustomer: null` rows overstates a tax deduction and + * still renders a perfectly plausible table. + * + * Prisma is patched at require() time, the same shape as + * tests/job-variance-db.test.ts. No mock.module — CI pins Node 20. + */ +import { test, before } from "node:test"; +import assert from "node:assert/strict"; +import Module from "node:module"; + +const PACIFIC = "America/Los_Angeles"; + +const recorded: any[] = []; + +const fakePrisma = { + companySettings: { + findUnique: async () => ({ timeZone: PACIFIC }), + }, + expense: { + findMany: async (args: any) => { + recorded.push(args); + return [ + { + id: "e1", + // 6pm Pacific on 30 September — 1 October in UTC. + date: new Date("2026-10-01T01:00:00.000Z"), + vendor: "Harbor Freight", + description: "[Receipt intake] Invoice 82766", + amount: "207.74", + taxAmount: "16.55", + projectId: "job-b", + project: { name: "Mesplay Kitchen" }, + estimate: { projectId: "job-a", project: { name: "Mueller Bath" } }, + }, + ]; + }, + }, +}; + +let queryTaxAtSourceRows: any; +let resolveTaxAtSourceFilters: any; +let parseTaxAtSourceFilters: any; + +before(async () => { + const originalRequire = Module.prototype.require; + let requirePatchHit = false; + (Module.prototype as unknown as { require: (id: string) => unknown }).require = function ( + this: NodeModule, + id: string, + ) { + if (id === "@/lib/prisma" || id === "./prisma") { + requirePatchHit = true; + return { prisma: fakePrisma }; + } + // eslint-disable-next-line prefer-rest-params + return originalRequire.apply(this, arguments as unknown as [string]); + } as typeof Module.prototype.require; + + let mod: any; + try { + mod = await import("../src/lib/tax-at-source-report"); + } finally { + Module.prototype.require = originalRequire; + } + if (typeof mod.queryTaxAtSourceRows !== "function") { + throw new Error( + "tax-at-source-query.test.ts: prisma mock did not apply — " + + `the require patch ${requirePatchHit ? "WAS" : "was NOT"} hit.`, + ); + } + queryTaxAtSourceRows = mod.queryTaxAtSourceRows; + resolveTaxAtSourceFilters = mod.resolveTaxAtSourceFilters; + parseTaxAtSourceFilters = mod.parseTaxAtSourceFilters; +}); + +test("the where clause asks for all three conditions POSITIVELY", async () => { + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + recorded.length = 0; + await queryTaxAtSourceRows(filters); + + const where = recorded[0].where; + assert.equal(where.taxAtSource, true); + // `true`, not `{ not: false }`: a NULL means "nobody said", and a NULL must + // never be spent as a tax deduction. + assert.equal(where.installedAtCustomer, true); + assert.deepEqual(where.taxAmount, { gt: 0 }, "a $0 tax row is an ANSWER, not a candidate"); +}); + +test("the date window is the COMPANY quarter, in instants", async () => { + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + recorded.length = 0; + await queryTaxAtSourceRows(filters); + + const range = recorded[0].where.date; + // PDT is UTC-7, so company midnight is 07:00Z. A UTC-computed bound would + // read 00:00Z and pull in seven hours of the neighbouring quarter. + assert.equal(range.gte.toISOString(), "2026-07-01T07:00:00.000Z"); + assert.equal(range.lt.toISOString(), "2026-10-01T07:00:00.000Z", "start of the day AFTER the last one"); +}); + +test("resolveTaxAtSourceFilters takes the zone from company settings", async () => { + const filters = await resolveTaxAtSourceFilters({ from: "2026-01-01", to: "2026-03-31" }); + assert.equal(filters.timeZone, PACIFIC); + assert.equal(filters.from.toISOString(), "2026-01-01T08:00:00.000Z", "PST, not the server's zone"); +}); + +test("a row is stamped with its COMPANY day and its RESOLVED job", async () => { + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + const [row] = await queryTaxAtSourceRows(filters); + + // The instant is 1 October UTC; the company calendar says 30 September, and + // that is the quarter the deduction belongs to. + assert.equal(row.dayKey, "2026-09-30"); + // The fixture's projectId and estimate.projectId disagree — the column wins. + assert.equal(row.projectId, "job-b"); + assert.equal(row.projectName, "Mesplay Kitchen"); + // Cents, from the Decimal's string form. + assert.equal(row.receiptTotalCents, 20774); + assert.equal(row.taxCents, 1655); + assert.equal(row.deductionBaseCents, 19119); + assert.equal(row.reference, "82766"); +}); diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts index 5608e4834..4ed77c866 100644 --- a/tests/tax-at-source-report.test.ts +++ b/tests/tax-at-source-report.test.ts @@ -2,84 +2,133 @@ * The WA excise deduction. Getting this wrong overstates a tax deduction, so * the exclusions matter more than the sums: a row counts only on POSITIVE * evidence, never on the absence of a contradiction. + * + * Three properties are asserted hard, because each has a failure mode that is + * invisible in the output: + * * INTEGER CENTS — a float sum drifts and the grand total stops agreeing + * with the rows it is made of; + * * COMPANY TIME ZONE — a 6pm-Pacific receipt on 30 September is 1 October + * UTC, and bucketed in UTC it lands on the wrong excise return; + * * CSV FORMULA NEUTRALIZATION — vendor and description are free text off a + * receipt, so a leading `=` reaches Marge's spreadsheet as a formula. */ import assert from "node:assert/strict"; import test from "node:test"; import { - currentQuarterRange, + currentQuarterKeys, extractReference, groupTaxAtSource, + monthLabelFromKey, parseTaxAtSourceFilters, rowsToCsv, stringifyTaxAtSourceFilters, + toCents, type TaxAtSourceRow, } from "../src/lib/tax-at-source-report"; +import { csvCell, csvNumber } from "../src/lib/csv-safe"; + +const PACIFIC = "America/Los_Angeles"; function row(overrides: Partial = {}): TaxAtSourceRow { - const receiptTotal = overrides.receiptTotal ?? 207.74; - const tax = overrides.tax ?? 16.55; + const receiptTotalCents = overrides.receiptTotalCents ?? 20774; + const taxCents = overrides.taxCents ?? 1655; return { id: "e1", - date: new Date("2026-06-26T00:00:00.000Z"), + date: new Date("2026-06-26T19:00:00.000Z"), + dayKey: "2026-06-26", vendor: "Harbor Freight", projectId: "job-mesplay", projectName: "Mesplay Kitchen", reference: "001916749100246", - receiptTotal, - deductionBase: receiptTotal - tax, - tax, + receiptTotalCents, + deductionBaseCents: receiptTotalCents - taxCents, + taxCents, ...overrides, }; } +// ── integer cents ─────────────────────────────────────────────────────────── + +test("toCents reads a Decimal's digits, never a float", () => { + assert.equal(toCents("16.55"), 1655); + assert.equal(toCents("16.550000000000000000000000000000"), 1655, "Prisma's Decimal string form"); + assert.equal(toCents("0.1"), 10); + assert.equal(toCents("207"), 20700); + assert.equal(toCents("-4.79"), -479); + assert.equal(toCents("1.005"), 101, "half-up on the third decimal"); + assert.equal(toCents("1.004"), 100); + assert.equal(toCents(null), 0); + assert.equal(toCents(""), 0); +}); + +test("a hundred float-hostile receipts still sum exactly", () => { + // The canonical drift: 0.1 + 0.2 !== 0.3. Summing dollars as numbers, 100 + // rows of $0.10 tax lands at 10.000000000000002 and the grand total stops + // matching the sum of the rows a bookkeeper can see. + const rows = Array.from({ length: 100 }, (_, i) => + row({ id: `e${i}`, receiptTotalCents: 30, taxCents: 10 }), + ); + const { summary, months } = groupTaxAtSource(rows); + assert.equal(summary.taxCents, 1000, "exactly $10.00"); + assert.equal(summary.deductionBaseCents, 2000); + assert.equal(months[0].taxCents, 1000); + assert.equal( + summary.taxCents, + months.reduce((total, month) => total + month.taxCents, 0), + "the grand total is the month totals, not a second computation", + ); + // Proof the guard is load-bearing: the same thing in floats does drift. + const floatSum = rows.reduce(total => total + 0.1, 0); + assert.notEqual(floatSum, 10); +}); + // ── grouping ──────────────────────────────────────────────────────────────── test("sums per month and per job, and the totals tie out", () => { const { months, summary } = groupTaxAtSource([ - row({ id: "a", tax: 10, receiptTotal: 110, deductionBase: 100 }), - row({ id: "b", tax: 5, receiptTotal: 55, deductionBase: 50 }), + row({ id: "a", taxCents: 1000, receiptTotalCents: 11000, deductionBaseCents: 10000 }), + row({ id: "b", taxCents: 500, receiptTotalCents: 5500, deductionBaseCents: 5000 }), row({ id: "c", projectId: "job-mueller", projectName: "Mueller Bath", - tax: 20, - receiptTotal: 220, - deductionBase: 200, - }), - row({ - id: "d", - date: new Date("2026-07-02T00:00:00.000Z"), - tax: 1, - receiptTotal: 11, - deductionBase: 10, + taxCents: 2000, + receiptTotalCents: 22000, + deductionBaseCents: 20000, }), + row({ id: "d", dayKey: "2026-07-02", taxCents: 100, receiptTotalCents: 1100, deductionBaseCents: 1000 }), ]); assert.deepEqual(months.map(m => m.key), ["2026-06", "2026-07"], "months are chronological"); const june = months[0]; assert.equal(june.count, 3); - assert.equal(june.tax, 35); - assert.equal(june.deductionBase, 350); + assert.equal(june.taxCents, 3500); + assert.equal(june.deductionBaseCents, 35000); + assert.equal(june.label, "June 2026"); // Jobs sort by tax, largest first — that is the row a bookkeeper checks. assert.deepEqual(june.jobs.map(j => j.projectId), ["job-mueller", "job-mesplay"]); assert.equal(june.jobs.find(j => j.projectId === "job-mesplay")!.count, 2); - assert.equal(june.jobs.find(j => j.projectId === "job-mesplay")!.tax, 15); + assert.equal(june.jobs.find(j => j.projectId === "job-mesplay")!.taxCents, 1500); assert.equal(summary.count, 4); - assert.equal(summary.tax, 36); - assert.equal(summary.deductionBase, 360); - assert.equal( - summary.tax, - months.reduce((total, month) => total + month.tax, 0), - "the grand total is the month totals, not a second computation", - ); + assert.equal(summary.taxCents, 3600); + assert.equal(summary.deductionBaseCents, 36000); +}); + +test("month buckets follow the COMPANY day key, not the UTC instant", () => { + // 30 Sep 2026, 6pm Pacific = 1 Oct 01:00 UTC. Bucketed on the instant this + // row lands in October — a different QUARTER, so the deduction would be + // claimed on the wrong excise return. + const { months } = groupTaxAtSource([ + row({ id: "late", date: new Date("2026-10-01T01:00:00.000Z"), dayKey: "2026-09-30" }), + ]); + assert.deepEqual(months.map(m => m.key), ["2026-09"]); }); test("two jobs sharing a name stay separate", () => { - // Bucketing by name would merge one client's deduction into another's. const { months } = groupTaxAtSource([ - row({ id: "a", projectId: "job-1", projectName: "Bathroom Remodel", tax: 10 }), - row({ id: "b", projectId: "job-2", projectName: "Bathroom Remodel", tax: 20 }), + row({ id: "a", projectId: "job-1", projectName: "Bathroom Remodel", taxCents: 1000 }), + row({ id: "b", projectId: "job-2", projectName: "Bathroom Remodel", taxCents: 2000 }), ]); assert.equal(months[0].jobs.length, 2); assert.deepEqual(months[0].jobs.map(j => j.projectId), ["job-2", "job-1"]); @@ -87,54 +136,83 @@ test("two jobs sharing a name stay separate", () => { test("an unattributed receipt is shown, not dropped", () => { // Money that was spent is real even when nobody said whose job it was. - // Hiding it would make the report quietly understate the deduction. const { months, summary } = groupTaxAtSource([ - row({ projectId: null, projectName: "(unassigned)", tax: 7 }), + row({ projectId: null, projectName: "(unassigned)", taxCents: 700 }), ]); assert.equal(months[0].jobs[0].projectId, null); - assert.equal(summary.tax, 7); + assert.equal(summary.taxCents, 700); }); test("an empty period is zeros, not NaN", () => { const { months, summary } = groupTaxAtSource([]); assert.deepEqual(months, []); - assert.deepEqual(summary, { count: 0, deductionBase: 0, receiptTotal: 0, tax: 0 }); + assert.deepEqual(summary, { count: 0, deductionBaseCents: 0, receiptTotalCents: 0, taxCents: 0 }); }); -// ── the filter contract ───────────────────────────────────────────────────── +// ── period boundaries, in the company zone ────────────────────────────────── + +test("the quarter is the company's calendar quarter, not the server's", () => { + // 1 Jan 2027, 04:00 UTC is still 31 Dec 2026, 8pm Pacific. A server in UTC + // would offer Q1 2027; the company is still filing Q4 2026. + const newYearInstant = new Date("2027-01-01T04:00:00.000Z"); + assert.deepEqual(currentQuarterKeys(newYearInstant, PACIFIC), { + fromKey: "2026-10-01", + toKey: "2026-12-31", + }); + assert.deepEqual(currentQuarterKeys(newYearInstant, "UTC"), { + fromKey: "2027-01-01", + toKey: "2027-03-31", + }); +}); -test("the default period is the current calendar quarter, [from, to)", () => { - const filters = parseTaxAtSourceFilters({}, new Date(2026, 7, 15)); - assert.equal(filters.from.getMonth(), 6, "Q3 starts in July"); - assert.equal(filters.from.getDate(), 1); - assert.equal(filters.to.getMonth(), 9, "and ends at the start of October, exclusive"); - assert.equal(filters.to.getDate(), 1); - assert.deepEqual(filters, currentQuarterRange(new Date(2026, 7, 15))); +test("quarter ends land on the right last day, February included", () => { + assert.deepEqual(currentQuarterKeys(new Date("2028-02-10T20:00:00.000Z"), PACIFIC), { + fromKey: "2028-01-01", + toKey: "2028-03-31", + }); + assert.deepEqual(currentQuarterKeys(new Date("2026-05-10T20:00:00.000Z"), PACIFIC), { + fromKey: "2026-04-01", + toKey: "2026-06-30", + }); }); -test("the picker's `to` is inclusive to a human and exclusive to the query", () => { - // A receipt dated on the last day of the range must be IN it. An exclusive - // bound taken literally from the picker would silently drop that day. - const filters = parseTaxAtSourceFilters({ from: "2026-06-01", to: "2026-06-30" }); - assert.equal(filters.from.getDate(), 1); - assert.equal(filters.to.getMonth(), 6); - assert.equal(filters.to.getDate(), 1, "start of July"); - // ...and it round-trips back to the day the user typed. +test("the query bounds are company midnights, and `to` is exclusive-next-day", () => { + const filters = parseTaxAtSourceFilters({ from: "2026-06-01", to: "2026-06-30" }, PACIFIC); + assert.equal(filters.fromKey, "2026-06-01"); + assert.equal(filters.toKey, "2026-06-30"); + // June is PDT (UTC-7): midnight local is 07:00Z. + assert.equal(filters.from.toISOString(), "2026-06-01T07:00:00.000Z"); + // A receipt bought on the LAST day of the range must be inside it, so the + // exclusive bound is the start of 1 July, not of 30 June. + assert.equal(filters.to.toISOString(), "2026-07-01T07:00:00.000Z"); assert.equal(stringifyTaxAtSourceFilters(filters), "from=2026-06-01&to=2026-06-30"); }); +test("a winter range uses the winter offset — the bound is not a fixed number of hours", () => { + const filters = parseTaxAtSourceFilters({ from: "2026-01-01", to: "2026-03-31" }, PACIFIC); + assert.equal(filters.from.toISOString(), "2026-01-01T08:00:00.000Z", "PST is UTC-8"); + assert.equal(filters.to.toISOString(), "2026-04-01T07:00:00.000Z", "PDT by 1 April"); +}); + test("an inverted or unparseable range falls back to the quarter, not to empty", () => { // An empty table reads as "no tax was paid", which is a very different // claim from "your dates are backwards". - const now = new Date(2026, 7, 15); - assert.deepEqual( - parseTaxAtSourceFilters({ from: "2026-06-30", to: "2026-06-01" }, now), - currentQuarterRange(now), - ); - assert.deepEqual( - parseTaxAtSourceFilters({ from: "not-a-date", to: "also-not" }, now), - currentQuarterRange(now), - ); + const now = new Date("2026-08-15T20:00:00.000Z"); + const quarter = currentQuarterKeys(now, PACIFIC); + for (const params of [ + { from: "2026-06-30", to: "2026-06-01" }, + { from: "not-a-date", to: "also-not" }, + {}, + ]) { + const filters = parseTaxAtSourceFilters(params, PACIFIC, now); + assert.equal(filters.fromKey, quarter.fromKey, JSON.stringify(params)); + assert.equal(filters.toKey, quarter.toKey, JSON.stringify(params)); + } +}); + +test("monthLabelFromKey formats from the key, with no Date in the way", () => { + assert.equal(monthLabelFromKey("2026-06"), "June 2026"); + assert.equal(monthLabelFromKey("2026-12"), "December 2026"); }); // ── reference extraction ──────────────────────────────────────────────────── @@ -152,7 +230,7 @@ test("the invoice reference is recovered from the description, or left blank", ( // ── CSV ───────────────────────────────────────────────────────────────────── test("the CSV mirrors the workbook columns Vanessa already receives", () => { - const csv = rowsToCsv([row({ receiptTotal: 207.74, tax: 16.55, deductionBase: 191.19 })]); + const csv = rowsToCsv([row({ receiptTotalCents: 20774, taxCents: 1655, deductionBaseCents: 19119 })]); const [header, line] = csv.trimEnd().split("\r\n"); assert.equal( header, @@ -160,11 +238,31 @@ test("the CSV mirrors the workbook columns Vanessa already receives", () => { ); assert.equal( line, - '2026-06-26,"Harbor Freight","Mesplay Kitchen","001916749100246",207.74,191.19,16.55', + '"2026-06-26","Harbor Freight","Mesplay Kitchen","001916749100246",207.74,191.19,16.55', ); }); -test("CSV quoting survives a vendor name with a comma or a quote", () => { - const csv = rowsToCsv([row({ vendor: 'Lowe"s, Vancouver' })]); - assert.match(csv, /"Lowe""s, Vancouver"/); +test("a vendor name that starts a formula is neutralized, not executed", () => { + // Vendor and description are free text lifted off a receipt by a model. + const csv = rowsToCsv([row({ vendor: "=cmd|'/c calc'!A1", reference: "+1-555", projectName: "@job" })]); + const line = csv.trimEnd().split("\r\n")[1]; + assert.match(line, /"'=cmd\|'\/c calc'!A1"/); + assert.match(line, /"'\+1-555"/); + assert.match(line, /"'@job"/); +}); + +test("CSV quoting survives a comma, a quote, and a newline", () => { + const csv = rowsToCsv([row({ vendor: 'Lowe"s, Vancouver\nStore 42' })]); + assert.match(csv, /"Lowe""s, Vancouver\nStore 42"/); +}); + +test("csv-safe: numbers stay numbers, text gets neutralized", () => { + // A negative amount must NOT be quote-prefixed — that would turn a number + // into text and break every SUM in the sheet. + assert.equal(csvNumber(-12.5), "-12.50"); + assert.equal(csvNumber(Number.NaN), "", "never the string NaN, which poisons a SUM"); + assert.equal(csvCell("-12.50"), '"\'-12.50"', "the same digits as TEXT are neutralized"); + assert.equal(csvCell("plain"), '"plain"'); + assert.equal(csvCell(null), '""'); + assert.equal(csvCell("\tlead-tab"), '"\'\tlead-tab"'); }); From 7e545af57eaf343df3c1e1780ee45db7c0cb01c9 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 19:57:08 -0700 Subject: [PATCH 073/144] fix(migration): verify the FK by DEFINITION, not by its name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 1, issue 9. Both the DO-block guard and the post-run verification keyed on `conname` alone, so a pre-existing Expense_projectId_fkey pointing at another table — or carrying ON DELETE CASCADE — would be skipped by the guard and then pass verification. That is exactly the failure SET NULL exists to prevent: a project delete taking spend history with it. The guard now reads pg_get_constraintdef. Absent, it creates. Present and correct, it is a no-op. Present and WRONG, it raises rather than being silently accepted — the script has no business dropping and recreating a live constraint, but an operator has to be told. The verification step asserts the rendered definition against the same four patterns, and the test checks that an ON DELETE CASCADE constraint of the right name fails at least one of them. Script and migration stay byte-identical DDL (asserted by the existing parity test). Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 21 ++++-- scripts/apply-expense-attribution.mjs | 65 +++++++++++++++---- tests/apply-expense-attribution.test.ts | 34 +++++++++- 3 files changed, 100 insertions(+), 20 deletions(-) diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 501a48de1..0321c2a52 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -27,13 +27,26 @@ CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); -- SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A -- project delete must not silently destroy spend history that the estimate -- still holds. -DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_projectId_fkey' - AND conrelid = '"Expense"'::regclass) THEN +-- +-- Guarded on the DEFINITION, not just the name (Codex round 1, issue 9). A +-- name-only `IF NOT EXISTS` silently accepts a pre-existing constraint of the +-- same name that points somewhere else or carries ON DELETE CASCADE — which is +-- precisely the failure this constraint exists to prevent. Existing-and-wrong +-- raises instead of being skipped: an operator has to look at it. +DO $$ +DECLARE existing_def TEXT; +BEGIN + SELECT pg_get_constraintdef(oid) INTO existing_def + FROM pg_constraint + WHERE conname = 'Expense_projectId_fkey' + AND conrelid = '"Expense"'::regclass; + IF existing_def IS NULL THEN ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE SET NULL ON UPDATE CASCADE; + ELSIF existing_def NOT LIKE '%REFERENCES "Project"(id)%' + OR existing_def NOT LIKE '%ON DELETE SET NULL%' THEN + RAISE EXCEPTION 'Expense_projectId_fkey already exists with an unexpected definition: %', existing_def; END IF; END $$; diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 94f304218..596e300ae 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -69,15 +69,26 @@ export const statements = [ // SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A // project delete must not silently destroy spend history that the estimate // still holds. - `DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_projectId_fkey' - AND conrelid = '"Expense"'::regclass) THEN - ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" - FOREIGN KEY ("projectId") REFERENCES "Project"("id") - ON DELETE SET NULL ON UPDATE CASCADE; - END IF; - END $$`, + // + // Guarded on the DEFINITION, not just the name: a name-only IF NOT EXISTS + // silently accepts a same-named constraint that points elsewhere or carries + // ON DELETE CASCADE. Existing-and-wrong raises rather than being skipped. + `DO $$ +DECLARE existing_def TEXT; +BEGIN + SELECT pg_get_constraintdef(oid) INTO existing_def + FROM pg_constraint + WHERE conname = 'Expense_projectId_fkey' + AND conrelid = '"Expense"'::regclass; + IF existing_def IS NULL THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "Project"("id") + ON DELETE SET NULL ON UPDATE CASCADE; + ELSIF existing_def NOT LIKE '%REFERENCES "Project"(id)%' + OR existing_def NOT LIKE '%ON DELETE SET NULL%' THEN + RAISE EXCEPTION 'Expense_projectId_fkey already exists with an unexpected definition: %', existing_def; + END IF; +END $$`, // The backfill. Idempotent by predicate, and a no-op on an empty database. `UPDATE "Expense" e SET "projectId" = est."projectId" @@ -103,8 +114,27 @@ export const expectedColumns = { ], }; +/** + * Verified by DEFINITION, not by name (Codex round 1, issue 9). + * + * The DO-block above skips when a constraint of that NAME already exists — so + * a pre-existing `Expense_projectId_fkey` pointing at the wrong table, or + * carrying ON DELETE CASCADE, would be left in place and the script would still + * report success. The whole point of SET NULL here is that deleting a project + * must not destroy spend history; a CASCADE wearing the same name is the exact + * failure this check has to catch. + */ export const expectedConstraints = [ - { name: "Expense_projectId_fkey", table: "Expense" }, + { + name: "Expense_projectId_fkey", + table: "Expense", + mustMatch: [ + /FOREIGN KEY \("projectId"\)/, + /REFERENCES "?Project"?\(id\)/, + /ON UPDATE CASCADE/, + /ON DELETE SET NULL/, + ], + }, ]; export const expectedIndexes = [ @@ -160,16 +190,25 @@ async function main() { } console.log(`verified ${table}: ${columns.length} columns`); } - for (const { name, table } of expectedConstraints) { + for (const { name, table, mustMatch } of expectedConstraints) { const [row] = await prisma.$queryRawUnsafe( - `SELECT 1 AS ok FROM pg_constraint WHERE conname = $1`, name, + `SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint + WHERE conname = $1 AND conrelid = $2::regclass`, + name, `"${table}"`, ); if (!row) { console.error(`VERIFY FAILED: constraint ${name} missing on ${table}`); process.exit(1); } + for (const pattern of mustMatch) { + if (!pattern.test(row.def)) { + console.error(`VERIFY FAILED: ${name} does not match ${pattern} + actual: ${row.def}`); + process.exit(1); + } + } + console.log(`verified constraint ${name}: ${row.def}`); } - console.log(`verified ${expectedConstraints.length} constraint(s)`); for (const { name, table } of expectedIndexes) { const [row] = await prisma.$queryRawUnsafe( `SELECT 1 AS ok FROM pg_class WHERE relname = $1 AND relnamespace = 'public'::regnamespace`, name, diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 3837fe387..75d48569f 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -92,12 +92,40 @@ test("the backfill UPDATE only ever touches rows whose projectId is still NULL", assert.ok(!/SET "projectId" = est\."projectId"[\s\S]*WHERE(?![\s\S]*projectId" IS NULL)/.test(update!)); }); -test("the FK is SET NULL, guarded, and named the way Prisma would name it", () => { +test("the FK is SET NULL, named the way Prisma would name it, and guarded on its DEFINITION", () => { + // A name-only `IF NOT EXISTS` would silently accept a pre-existing + // Expense_projectId_fkey that points at another table or carries ON DELETE + // CASCADE — exactly the thing SET NULL is here to prevent. Existing-and- + // wrong must RAISE, not be skipped. const fk = (statements as string[]).find(s => s.includes("Expense_projectId_fkey")); assert.ok(fk); - assert.match(fk!, /IF NOT EXISTS \(SELECT 1 FROM pg_constraint/); + assert.match(fk!, /pg_get_constraintdef\(oid\)/); assert.match(fk!, /ON DELETE SET NULL ON UPDATE CASCADE/); - assert.deepEqual(expectedConstraints, [{ name: "Expense_projectId_fkey", table: "Expense" }]); + assert.match(fk!, /NOT LIKE '%ON DELETE SET NULL%'/); + assert.match(fk!, /NOT LIKE '%REFERENCES "Project"\(id\)%'/); + assert.match(fk!, /RAISE EXCEPTION/); + assert.ok( + !/IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(fk!), + "the name-only guard must be gone, not merely accompanied", + ); + + // ...and the post-run verification asserts the same thing against the live + // catalog, rather than only asserting the constraint's NAME exists. + assert.equal(expectedConstraints.length, 1); + const [constraint] = expectedConstraints as { name: string; table: string; mustMatch: RegExp[] }[]; + assert.equal(constraint.name, "Expense_projectId_fkey"); + assert.equal(constraint.table, "Expense"); + const rendered = 'FOREIGN KEY ("projectId") REFERENCES "Project"(id) ON UPDATE CASCADE ON DELETE SET NULL'; + for (const pattern of constraint.mustMatch) { + assert.match(rendered, pattern, `pg_get_constraintdef output must satisfy ${pattern}`); + } + // A CASCADE wearing the right name must fail every one of those checks. + const cascade = 'FOREIGN KEY ("projectId") REFERENCES "Project"(id) ON UPDATE CASCADE ON DELETE CASCADE'; + assert.ok( + constraint.mustMatch.some(pattern => !pattern.test(cascade)), + "an ON DELETE CASCADE constraint of the same name must be rejected", + ); + assert.deepEqual(expectedIndexes, [{ name: "Expense_projectId_idx", table: "Expense" }]); }); From dbe7131f8c3ed85531bbf0fb84b922d6de507043 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 20:27:35 -0700 Subject: [PATCH 074/144] fix(expenses): attribution is write-once; stale-plan predicates; CSV and date hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 2. 1. Backfill: the `sameEstimate` shortcut is gone. The item's project must equal the expense's resolved project, full stop. The one case the shortcut uniquely covered was the unsound one — two rows agreeing on an estimate whose own projectId is NULL or has moved, so it fired precisely BECAUSE the project check had already failed. 2. Sync: the "projects agree, so refresh estimateId" carve-out is DROPPED. projectId and estimateId are the same fact said twice; they are written together by one `updateMany where projectId: null` and never again. The carve-out bought a row following its job to a newer estimate and paid by making the rule conditional, which is how the original cross-job bug got in. Spec §3.1 records the decision. 3. Both cost-code writers re-assert the attribution their decision was scoped to (`projectId: expected`, uncoded, not-human-coded). A plan is a snapshot; the predicate is what makes acting on it safe. The backfill reports how many planned writes were skipped for having moved. 4. csv-safe: leading whitespace no longer smuggles a formula past the check (trim-then-test, original value preserved — a bare TAB is still caught, and the whitespace is data, not something to silently strip). csvNumber accepts a Decimal-like or boxed value, uses a fixed-point formatter that never emits exponent notation, and returns "" rather than a fabricated "0.00" for a blank. 5. Tax filters validate a day key as a REAL calendar date — "2026-02-31" passed the regex and made startOfDateInTimeZone throw, turning a URL typo into a 500. One bad endpoint now discards BOTH rather than inventing half a range, and an invalid time zone degrades to the default. 6. The migration's FK guard checks the same four properties as the apply script's verifier; a test asserts the two cannot drift. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 12 +++ .../migration.sql | 6 +- scripts/apply-expense-attribution.mjs | 6 +- scripts/backfill-expense-attribution.mjs | 49 +++++++--- src/lib/csv-safe.ts | 65 +++++++++++-- src/lib/qbo-expense-sync.ts | 87 ++++++++++------- src/lib/tax-at-source-report.ts | 81 ++++++++++++---- tests/apply-expense-attribution.test.ts | 15 ++- tests/backfill-expense-attribution.test.ts | 53 +++++++++++ tests/qbo-expense-sync.test.ts | 95 +++++++++++++++---- tests/tax-at-source-report.test.ts | 33 +++++++ 11 files changed, 412 insertions(+), 90 deletions(-) diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index 4d8c81dcd..dc56e25b7 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -171,6 +171,18 @@ backfill > null. Nothing but a human edit may change a row whose `costCodeSource posture as the deliberate receiptUrl omission, :578-580 comment). Extend the `findUnique` select with `projectId` and adjust `expenseMatchesQboWrite` so "unchanged" detection stays correct (compare projectId only when the update would write it). + **AS BUILT — stricter than the line above (Codex round 2). ATTRIBUTION IS + WRITE-ONCE.** `projectId` and `estimateId` are the same fact said twice, so they are + written *together*, by one `updateMany` whose own predicate is `projectId: null`, and + never again afterwards. The guarantee lives in the SQL rather than in a value read + earlier in the transaction. An interim version also refreshed `estimateId` when the + stored project and the incoming QBO match *agreed* — "same job, newer estimate", the + sync's long-standing attach-to-the-active-estimate behaviour. **That carve-out was + dropped.** It bought a row following its job to a newer estimate, and paid for it by + making the rule conditional, which is exactly how the original cross-job bug got in + (`projectId` kept, `estimateId` overwritten → the row on job B for every reader and on + job A's estimate for cascade-delete and billing). Re-pointing an estimate belongs to an + explicit re-attribution path, not to an import. **Cost-code suggestion**: extract `VENDOR_RULES`, `LINE_RULES`, `suggestCode` from `scripts/suggest-expense-cost-codes.mjs` into a new pure module `src/lib/expense-cost-suggest.ts` (the script imports it back — one copy). After a diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 0321c2a52..858ac7d6f 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -44,8 +44,10 @@ BEGIN ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE SET NULL ON UPDATE CASCADE; - ELSIF existing_def NOT LIKE '%REFERENCES "Project"(id)%' - OR existing_def NOT LIKE '%ON DELETE SET NULL%' THEN + ELSIF existing_def NOT LIKE '%FOREIGN KEY ("projectId")%' + OR existing_def NOT LIKE '%REFERENCES "Project"(id)%' + OR existing_def NOT LIKE '%ON DELETE SET NULL%' + OR existing_def NOT LIKE '%ON UPDATE CASCADE%' THEN RAISE EXCEPTION 'Expense_projectId_fkey already exists with an unexpected definition: %', existing_def; END IF; END $$; diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 596e300ae..b62a42839 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -84,8 +84,10 @@ BEGIN ALTER TABLE "Expense" ADD CONSTRAINT "Expense_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE SET NULL ON UPDATE CASCADE; - ELSIF existing_def NOT LIKE '%REFERENCES "Project"(id)%' - OR existing_def NOT LIKE '%ON DELETE SET NULL%' THEN + ELSIF existing_def NOT LIKE '%FOREIGN KEY ("projectId")%' + OR existing_def NOT LIKE '%REFERENCES "Project"(id)%' + OR existing_def NOT LIKE '%ON DELETE SET NULL%' + OR existing_def NOT LIKE '%ON UPDATE CASCADE%' THEN RAISE EXCEPTION 'Expense_projectId_fkey already exists with an unexpected definition: %', existing_def; END IF; END $$`, diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 346751cf4..76723519d 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -123,21 +123,25 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI // "backfill" — a wrong code is worse than an absent one, and this is // exactly the kind of wrong that no one would notice. // - // Accepted: the item belongs to the expense's OWN estimate, or to - // another estimate of the SAME project (which is how change-order and - // revised-estimate work reaches an item — CO line items live on the - // project's estimates, and `createExpenseCore` already requires a CO - // and its estimate to share the project). Anything else is skipped and - // listed for a human with reason "item-outside-estimate". + // THE TEST IS THE PROJECT, and only the project (Codex round 2). An + // earlier version also accepted `item.estimateId === expense.estimateId` + // as a shortcut. That shortcut is unsound in the one case it uniquely + // covers: a shared estimate whose own `projectId` is NULL or has moved + // means two rows can agree on an estimate while the expense resolves to + // a different job — and the shortcut would accept exactly then, because + // the project check had already failed. Same-estimate is a SUBSET of + // same-project whenever both are known, so dropping it loses nothing + // real and removes the branch where the two disagree. const item = expense.itemId ? items.get(expense.itemId) : undefined; if (expense.itemId && !item) { // A link to an item that is gone, or to an item with no cost code: // no answer either way, so fall through to the rules below. } else if (item && item.costCodeId) { - const sameEstimate = item.estimateId === expense.estimateId; const sameProject = - item.projectId !== null && item.projectId === resolvedProjectId; - if (!sameEstimate && !sameProject) { + resolvedProjectId !== null && + item.projectId !== null && + item.projectId === resolvedProjectId; + if (!sameProject) { add(expense, "item-outside-estimate"); continue; } @@ -146,7 +150,8 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI costCodeId: item.costCodeId, costCodeSource: "backfill", costCodeConfidence: null, - why: sameEstimate ? "item cost code (own estimate)" : "item cost code (same project)", + why: "item cost code (same project)", + expectedProjectId: expense.projectId ?? null, expense, }); continue; @@ -170,6 +175,10 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI costCodeSource: "ai", costCodeConfidence: suggestion.confidence, why: suggestion.why, + // The attribution the decision was scoped BY, so the write can + // require it is unchanged. `null` is a real expectation: it + // means the row was still unattributed when the plan was made. + expectedProjectId: expense.projectId ?? null, expense, }); } else { @@ -362,9 +371,20 @@ export async function runBackfill({ } let costCodesWritten = 0; + let costCodesSkipped = 0; for (const fill of plan.codeFills) { const result = await db.expense.updateMany({ - where: { id: fill.id, costCodeId: null, ...notHumanCodedExpenseWhere() }, + where: { + id: fill.id, + // Everything the plan depended on, re-asserted at write time. + // The plan is a snapshot taken before pass (a) ran and before + // any concurrent sync or bookkeeper edit; the predicate is what + // makes it safe to act on. A row that moved is skipped, not + // coded on stale reasoning. + projectId: fill.expectedProjectId, + costCodeId: null, + ...notHumanCodedExpenseWhere(), + }, data: { costCodeId: fill.costCodeId, costCodeSource: fill.costCodeSource, @@ -372,10 +392,17 @@ export async function runBackfill({ }, }); costCodesWritten += result.count; + if (result.count === 0) costCodesSkipped += 1; } log(""); log(`applied ${projectIdsWritten} projectId and ${costCodesWritten} cost code(s).`); + if (costCodesSkipped > 0) { + // Not an error: it means a row changed between the plan and the write, + // and the predicate did its job. Reported because a LARGE number would + // mean the plan was stale enough to re-run. + log(`${costCodesSkipped} planned cost code(s) skipped — the row moved after the plan was made.`); + } log(`${plan.remainder.length} rows left NULL for human review. Re-run (dry) — it must report 0 planned writes.`); return { plan, diff --git a/src/lib/csv-safe.ts b/src/lib/csv-safe.ts index 81e10df21..5f3aef566 100644 --- a/src/lib/csv-safe.ts +++ b/src/lib/csv-safe.ts @@ -22,6 +22,21 @@ */ const FORMULA_TRIGGER = /^[=+\-@\t\r]/; +/** + * Leading whitespace (including a newline or a BOM) is INVISIBLE to a reviewer + * and is stripped by the spreadsheet before it decides whether the cell is a + * formula — so `" =1+1"` is every bit as live as `"=1+1"`, while sailing past a + * naive first-character check. The lead test runs on the trimmed value; the + * value itself is emitted UNCHANGED apart from the prefix, because the + * whitespace is data and silently trimming it would edit the export. + */ +function startsFormula(value: string): boolean { + // BOTH forms. The trimmed check alone would stop treating a bare leading + // TAB as dangerous — TAB is itself a trigger, and it is also whitespace, so + // trimming makes it disappear before the test can see it. + return FORMULA_TRIGGER.test(value) || FORMULA_TRIGGER.test(value.replace(/^[\s]+/, "")); +} + /** * Quote and neutralize one TEXT cell. Always quoted, so an embedded comma, * quote, or newline is safe too. @@ -29,20 +44,58 @@ const FORMULA_TRIGGER = /^[=+\-@\t\r]/; export function csvCell(value: unknown): string { if (value === null || value === undefined) return '""'; const raw = String(value); - const neutralized = FORMULA_TRIGGER.test(raw) ? `'${raw}` : raw; + const neutralized = startsFormula(raw) ? `'${raw}` : raw; return `"${neutralized.replace(/"/g, '""')}"`; } +const fixedFormatters = new Map(); + +/** + * Fixed-point, never exponent notation. + * + * `Number.prototype.toFixed` switches to exponent form at 1e21 — `"1e+21"` in a + * CSV is read as text by some spreadsheets and as a broken number by others, + * and either way it stops being a figure anyone can sum. `Intl.NumberFormat` + * with grouping off always writes the digits out. + */ +function fixedPoint(value: number, digits: number): string { + let formatter = fixedFormatters.get(digits); + if (!formatter) { + formatter = new Intl.NumberFormat("en-US", { + useGrouping: false, + minimumFractionDigits: digits, + maximumFractionDigits: digits, + }); + fixedFormatters.set(digits, formatter); + } + return formatter.format(value); +} + /** * A NUMERIC cell, emitted unquoted so the spreadsheet reads it as a number. * + * Accepts anything numeric-shaped — a plain number, a boxed Number, or a Prisma + * Decimal — because a caller handing this a Decimal is the likeliest mistake + * and `Decimal.toFixed` is NOT `Number.prototype.toFixed`. Everything is + * normalized through `Number(String(value))` first. + * * Safe without neutralizing because the output shape is produced here, not - * echoed from input: `toFixed` on a finite number can only ever yield - * `-?\d+\.\d*`, which no spreadsheet parses as a formula. A non-finite value - * becomes an empty cell rather than "NaN", which would silently poison a SUM. + * echoed from input: fixed-point formatting of a finite number can only ever + * yield `-?\d+\.\d*`, which no spreadsheet parses as a formula. A non-finite or + * unparseable value becomes an empty cell rather than "NaN", which would + * silently poison a SUM. */ -export function csvNumber(value: number, digits = 2): string { - return Number.isFinite(value) ? value.toFixed(digits) : ""; +export function csvNumber(value: unknown, digits = 2): string { + if (typeof value !== "number") { + // `Number("")` is 0, so a null/blank source would print "0.00" — a + // fabricated figure in a tax export, and one that sums as if it were + // measured. Absent stays absent. + const text = String(value ?? "").trim(); + if (!text) return ""; + const parsed = Number(text); + return Number.isFinite(parsed) ? fixedPoint(parsed, digits) : ""; + } + return Number.isFinite(value) ? fixedPoint(value, digits) : ""; } /** Join rows with CRLF and a trailing terminator, per RFC 4180. */ diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 4e6ebd6ef..cf18752a9 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -542,7 +542,7 @@ type ExpenseTransaction = { }): Promise; updateMany(args: { where: { id: string; projectId: null }; - data: { projectId: string }; + data: { projectId?: string; estimateId: string }; }): Promise<{ count: number }>; }; }; @@ -573,40 +573,43 @@ type ExistingQboExpense = export interface QboExpenseUpdatePlan { /** - * Written by its OWN guarded statement, not by the main UPDATE. Null when - * the row already has a project (or the write has none to give). + * The ATTRIBUTION fill, applied by its own statement under a + * `projectId: null` predicate — so the project and the estimate that + * belongs to it land together or neither does. Null when there is nothing + * to fill. + */ + fill: { projectId?: string; estimateId: string } | null; + /** + * Everything the main UPDATE may write. NEVER contains `projectId` or + * `estimateId` — both are attribution and both go in `fill`. */ - fillProjectId: string | null; - /** Everything the main UPDATE may write. NEVER contains `projectId`. */ data: Partial; } /** * Decide what a re-sync is allowed to change on a row that already exists. * - * TWO SEPARATE RULES, and they used to be one read-then-unconditional-update: + * ONE RULE, and it used to be a read-then-unconditional-update: * - * 1. `projectId` is only ever FILLED, never overwritten, and it is filled by a - * statement whose own predicate says `projectId: null`. Deciding from a - * value read earlier in the transaction is not the same as deciding from - * the row's state at write time, and the guarantee belongs in the SQL. + * ATTRIBUTION IS WRITE-ONCE. `projectId` and `estimateId` are the same fact + * said twice, so they move together and only while the row has no project + * yet. The write happens under a `projectId: null` predicate, so the + * guarantee lives in the SQL rather than in a value read earlier in the + * transaction. Once a row is attributed — by this sync, by the backfill, or + * by a bookkeeper — QuickBooks never re-points it. * - * 2. `estimateId` follows it. The estimate in `write` is the one the QBO - * customer match picked; if a bookkeeper has since re-attributed the row to - * a DIFFERENT job, writing that estimate back would leave `projectId` and - * `estimateId` pointing at two different jobs — an expense that is on job B - * for every reader and on job A's estimate for cascade-delete and billing. - * So the estimate is left alone exactly when the projects disagree. - * - * Note rule 2 is scoped to DISAGREEMENT rather than to "projectId is set". - * When the stored project and the incoming match are the SAME job, moving - * `estimateId` to that job's newest estimate is what the sync has always done - * and is still correct — it is the pre-existing "attach to the active estimate" - * behaviour, and suppressing it there would strand rows on superseded - * estimates for no safety gain. + * An earlier version of this refreshed `estimateId` when the stored project and + * the incoming match AGREED, on the reasoning that "same job, newer estimate" + * was the sync's long-standing attach-to-the-active-estimate behaviour. That + * carve-out is GONE (Codex round 2). It bought a marginal benefit — a row + * following its job to a newer estimate — and paid for it by making the rule + * conditional, which is how the original bug got in. A rule that is simply + * "never after the first write" cannot be reasoned about wrongly at a call + * site, and re-pointing an estimate is a job for an explicit re-attribution + * path, not for an import. */ export function planQboExpenseUpdate( - existing: Pick, + existing: Pick, write: QboExpenseWrite, ): QboExpenseUpdatePlan { const existingProjectId = existing.projectId ?? null; @@ -614,12 +617,19 @@ export function planQboExpenseUpdate( const data: Partial = { ...write }; delete data.projectId; - if (existingProjectId !== null && existingProjectId !== incomingProjectId) { - delete data.estimateId; - } + delete data.estimateId; + + if (existingProjectId !== null) return { fill: null, data }; + + const wantsProjectId = incomingProjectId !== null; + const wantsEstimateId = existing.estimateId !== write.estimateId; + if (!wantsProjectId && !wantsEstimateId) return { fill: null, data }; return { - fillProjectId: existingProjectId === null ? incomingProjectId : null, + fill: { + ...(incomingProjectId !== null ? { projectId: incomingProjectId } : {}), + estimateId: write.estimateId, + }, data, }; } @@ -631,10 +641,9 @@ export function planQboExpenseUpdate( * `qbSyncedAt` is excluded because it changes on every run by construction. */ function planIsNoop(existing: ExistingQboExpense, plan: QboExpenseUpdatePlan): boolean { - if (plan.fillProjectId !== null) return false; + if (plan.fill !== null) return false; const data = plan.data; if (data.qbSyncToken !== undefined && existing.qbSyncToken !== data.qbSyncToken) return false; - if (data.estimateId !== undefined && existing.estimateId !== data.estimateId) return false; if (data.amount !== undefined && Number(existing.amount) !== data.amount) return false; if (data.vendor !== undefined && existing.vendor !== data.vendor) return false; if (data.date !== undefined && !datesEqual(existing.date, data.date)) return false; @@ -694,13 +703,14 @@ export async function upsertQboExpense( const plan = planQboExpenseUpdate(existing, write); if (planIsNoop(existing, plan)) return "unchanged"; - // The project fill is its OWN statement, and its predicate — not a + // The attribution fill is its OWN statement, and its predicate — not a // value read a few lines above — is what guarantees a human's - // re-attribution survives. - if (plan.fillProjectId !== null) { + // re-attribution survives. projectId and estimateId land together or + // neither does. + if (plan.fill !== null) { await transaction.expense.updateMany({ where: { id: existing.id, projectId: null }, - data: { projectId: plan.fillProjectId }, + data: plan.fill, }); } await transaction.expense.update({ @@ -861,6 +871,11 @@ export async function applyQboExpenseCostCodeSuggestion( const projectId = resolveExpenseProjectId(stored); if (!projectId) return "skipped-no-project"; if (isOverheadProject(projectId)) return "skipped-overhead"; + // The attribution this decision was MADE on, so the write can require that + // it has not changed underneath. `null` is itself a meaningful expectation: + // it means the row was still unattributed when we scoped the suggestion, + // and a row that has since been attributed must be re-scoped, not written. + const expectedProjectId = stored.projectId ?? null; const suggestion = suggestCode({ vendor: input.vendor, description: input.description }); if (!suggestion) return "no-match"; @@ -871,6 +886,10 @@ export async function applyQboExpenseCostCodeSuggestion( const written = await client.expense.updateMany({ where: { qbPurchaseId: input.qbPurchaseId, + // Everything the decision depended on, re-asserted at write time. + // A row re-attributed or coded between the read above and here is + // skipped rather than written on stale reasoning. + projectId: expectedProjectId, costCodeId: null, ...notHumanCodedExpenseWhere(), }, diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts index 289b22761..a3f6223cc 100644 --- a/src/lib/tax-at-source-report.ts +++ b/src/lib/tax-at-source-report.ts @@ -34,7 +34,13 @@ // `resolveTaxAtSourceFilters` touch Prisma. import { prisma } from "@/lib/prisma"; import { resolveCompanyTimeZone } from "./company-timezone"; -import { addDaysToKey, dayKeyInTimeZone, startOfDateInTimeZone } from "./tz-date"; +import { + DEFAULT_COMPANY_TIME_ZONE, + addDaysToKey, + dayKeyInTimeZone, + startOfDateInTimeZone, + validTimeZone, +} from "./tz-date"; import { resolveExpenseProjectId } from "./expense-attribution"; import { csvCell, csvDocument, csvNumber } from "./csv-safe"; @@ -120,8 +126,24 @@ export function currentQuarterKeys(now: Date, timeZone: string): { fromKey: stri return { fromKey, toKey }; } +/** + * A REAL calendar day, not merely a well-shaped string. + * + * `/^\d{4}-\d{2}-\d{2}$/` accepts "2026-02-31" and "2026-13-01", and + * `startOfDateInTimeZone` THROWS on both — which turns a typo in a URL into a + * 500 on a finance page. Construct the date and compare the fields back. + */ function isDayKey(value: string | undefined): value is string { - return !!value && /^\d{4}-\d{2}-\d{2}$/.test(value); + if (!value || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const year = Number(value.slice(0, 4)); + const month = Number(value.slice(5, 7)); + const day = Number(value.slice(8, 10)); + const probe = new Date(Date.UTC(year, month - 1, day)); + return ( + probe.getUTCFullYear() === year && + probe.getUTCMonth() === month - 1 && + probe.getUTCDate() === day + ); } /** @@ -145,24 +167,47 @@ export function parseTaxAtSourceFilters( const fallback = currentQuarterKeys(now, timeZone); const rawFrom = get("from"); const rawTo = get("to"); - let fromKey = isDayKey(rawFrom) ? rawFrom : fallback.fromKey; - let toKey = isDayKey(rawTo) ? rawTo : fallback.toKey; - if (toKey < fromKey) { - fromKey = fallback.fromKey; - toKey = fallback.toKey; + + // EITHER endpoint being unusable discards BOTH. Keeping the good half and + // defaulting the other silently invents a range the user never asked for — + // a quarter that starts where they said and ends three months later reads + // as a real answer, and its total would be reported to the state. + let fromKey = fallback.fromKey; + let toKey = fallback.toKey; + if (isDayKey(rawFrom) && isDayKey(rawTo) && rawTo >= rawFrom) { + fromKey = rawFrom; + toKey = rawTo; + } else if (rawFrom === undefined && rawTo === undefined) { + // No params at all is the normal first visit, not a bad request. } - return { - fromKey, - toKey, - from: startOfDateInTimeZone(fromKey, timeZone), - // `toKey` is the last day a human means to include, so the exclusive - // bound is the start of the NEXT company day. Taken literally, an - // exclusive bound would silently drop everything bought on the last day - // of the quarter. - to: startOfDateInTimeZone(addDaysToKey(toKey, 1), timeZone), - timeZone, - }; + try { + return { + fromKey, + toKey, + from: startOfDateInTimeZone(fromKey, timeZone), + // `toKey` is the last day a human means to include, so the + // exclusive bound is the start of the NEXT company day. Taken + // literally, an exclusive bound would silently drop everything + // bought on the last day of the quarter. + to: startOfDateInTimeZone(addDaysToKey(toKey, 1), timeZone), + timeZone, + }; + } catch { + // Belt and braces. `isDayKey` has already ruled out every input + // `startOfDateInTimeZone` rejects, so reaching here means an invalid + // TIME ZONE — and a finance page must degrade to the default quarter + // rather than 500. + const zone = validTimeZone(timeZone) ? timeZone : DEFAULT_COMPANY_TIME_ZONE; + const safe = currentQuarterKeys(now, zone); + return { + fromKey: safe.fromKey, + toKey: safe.toKey, + from: startOfDateInTimeZone(safe.fromKey, zone), + to: startOfDateInTimeZone(addDaysToKey(safe.toKey, 1), zone), + timeZone: zone, + }; + } } /** Resolve the company zone, then parse. The one Prisma-touching entry point. */ diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 75d48569f..f223c4860 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -101,8 +101,12 @@ test("the FK is SET NULL, named the way Prisma would name it, and guarded on its assert.ok(fk); assert.match(fk!, /pg_get_constraintdef\(oid\)/); assert.match(fk!, /ON DELETE SET NULL ON UPDATE CASCADE/); - assert.match(fk!, /NOT LIKE '%ON DELETE SET NULL%'/); + // The in-database guard must check the SAME four properties the post-run + // verifier does, or the two can disagree about what "correct" means. + assert.match(fk!, /NOT LIKE '%FOREIGN KEY \("projectId"\)%'/); assert.match(fk!, /NOT LIKE '%REFERENCES "Project"\(id\)%'/); + assert.match(fk!, /NOT LIKE '%ON DELETE SET NULL%'/); + assert.match(fk!, /NOT LIKE '%ON UPDATE CASCADE%'/); assert.match(fk!, /RAISE EXCEPTION/); assert.ok( !/IF NOT EXISTS \(SELECT 1 FROM pg_constraint/.test(fk!), @@ -119,6 +123,15 @@ test("the FK is SET NULL, named the way Prisma would name it, and guarded on its for (const pattern of constraint.mustMatch) { assert.match(rendered, pattern, `pg_get_constraintdef output must satisfy ${pattern}`); } + // Both halves check the same four things — the SQL guard by LIKE, the + // verifier by regex — so a constraint either guard accepts, the other does. + for (const property of ['FOREIGN KEY ("projectId")', 'REFERENCES "Project"(id)', "ON DELETE SET NULL", "ON UPDATE CASCADE"]) { + assert.ok(fk!.includes(`NOT LIKE '%${property}%'`), `SQL guard does not check ${property}`); + assert.ok( + constraint.mustMatch.some(pattern => pattern.test(property)), + `verifier does not check ${property}`, + ); + } // A CASCADE wearing the right name must fail every one of those checks. const cascade = 'FOREIGN KEY ("projectId") REFERENCES "Project"(id) ON UPDATE CASCADE ON DELETE CASCADE'; assert.ok( diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index f2a15bae4..b658921e5 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -174,6 +174,40 @@ test("an item on ANOTHER estimate of the SAME job is accepted", () => { assert.match(plan.codeFills[0].why, /same project/); }); +test("the PROJECT decides, and a matching estimateId is no longer a shortcut", () => { + // Codex round 2, blocker 1. The old code accepted + // `item.estimateId === expense.estimateId` as an alternative to the project + // check. The one case that shortcut uniquely covered is the unsound one: + // the two rows agree on an estimate whose own projectId is NULL or has + // moved, so the expense resolves to one job and the item to none — and the + // shortcut fired precisely BECAUSE the project check had already failed. + const plan = planBackfill({ + expenses: [expense({ id: "e1", estimateId: "shared-est", projectId: "job-1", itemId: "item-x" })], + items: new Map([["item-x", { costCodeId: "cc-frame", estimateId: "shared-est", projectId: null }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder[0].reason, "item-outside-estimate"); +}); + +test("an expense with no resolvable job cannot borrow a phase from an item", () => { + // Nothing to compare the item against, so there is no evidence the link is + // on the right job. A guess here is a wrong cost code with "backfill" + // provenance on it. + const plan = planBackfill({ + expenses: [expense({ + id: "e1", estimateId: "est-job-1", projectId: null, + estimate: { projectId: null }, itemId: "item-1", + })], + items: new Map([["item-1", { costCodeId: "cc-frame", estimateId: "est-job-1", projectId: "job-1" }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder[0].reason, "item-outside-estimate"); +}); + test("a dangling itemId falls through to the rules rather than being skipped", () => { // The item is gone (or carries no code). That is no evidence either way, // so it must not consume the row — the suggester still gets its turn. @@ -314,6 +348,9 @@ test("apply writes both passes, each behind its own predicate", async () => { { costCodeSource: null }, { costCodeSource: { notIn: ["capture", "manual"] } }, ]); + // The attribution the plan was made under, re-asserted at write time. The + // plan is a snapshot; the predicate is what makes acting on it safe. + assert.equal(codeWrite.where.projectId, null); assert.deepEqual(codeWrite.data, { costCodeId: "cc-plumb", costCodeSource: "ai", @@ -321,6 +358,18 @@ test("apply writes both passes, each behind its own predicate", async () => { }); }); +test("a cost-code write requires the project the plan was scoped to", async () => { + // A row that was ALREADY attributed when the plan was made must be written + // under that same id — not under `null`, which would silently match a + // different set of rows. + const stub = createStub([ + expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" }), + ]); + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; + assert.equal(codeWrite.where.projectId, "job-1"); +}); + test("the write predicate re-checks NULL, not just the plan", async () => { // Between the read and the write, a re-sync or a bookkeeper can set either // field. A plan is a snapshot; the predicate is the guarantee. @@ -330,6 +379,10 @@ test("the write predicate re-checks NULL, not just the plan", async () => { const guardsNull = write.where.projectId === null || write.where.costCodeId === null; assert.ok(guardsNull, `unguarded write: ${JSON.stringify(write.where)}`); + assert.ok( + "projectId" in write.where, + `write without an attribution predicate: ${JSON.stringify(write.where)}`, + ); } }); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 3ed7e4880..6ba71eb09 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -601,7 +601,7 @@ function createFakePrisma(initial: StoredExpense[] = []) { // whole guarantee the split-out project fill is buying. async updateMany(args: { where: { id: string; projectId: null }; - data: { projectId: string }; + data: { projectId?: string; estimateId: string }; }) { const current = [...rows.values()].find(row => row.id === args.where.id); if (!current) return { count: 0 }; @@ -1239,6 +1239,29 @@ test("the write is guarded on uncoded AND not-human-coded, with a NULL branch", { costCodeSource: null }, { costCodeSource: { notIn: ["capture", "manual"] } }, ]); + // Everything the decision depended on is re-asserted at write time, so a + // row re-attributed between the read and the write is skipped rather than + // coded on stale reasoning. + assert.equal(where.projectId, "project-1"); +}); + +test("the write requires the SAME attribution the suggestion was scoped to", async () => { + // A row whose project came from the estimate (column still NULL) must be + // written under `projectId: null` — "still unattributed" is the state the + // decision was made in, and it is just as much a precondition as a + // populated id. + const fake = fakeSuggestionClient({ + projectId: null, + costCodeId: null, + costCodeSource: null, + estimate: { projectId: "project-1" }, + }); + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + COST_CODE_IDS, + ); + assert.equal(fake.calls[0].where.projectId, null); }); test("a row a human already coded is refused on the STORED source, before any write", async () => { @@ -1382,28 +1405,56 @@ test("the sync asks for a phase on a matched job, by purchase id only", async () // ── the split project fill (Codex round 1, blocker 1) ────────────────────── -test("the project fill is its OWN statement, guarded on projectId being NULL", () => { - const plan = planQboExpenseUpdate({ projectId: null }, WRITE); - assert.equal(plan.fillProjectId, "project-1"); +test("projectId and estimateId move TOGETHER, in one statement guarded on NULL", () => { + // They are the same fact said twice, so a statement that could write one + // without the other is a statement that can leave the row incoherent. + const plan = planQboExpenseUpdate({ projectId: null, estimateId: "stale-estimate" }, WRITE); + assert.deepEqual(plan.fill, { projectId: "project-1", estimateId: "estimate-1" }); assert.ok(!("projectId" in plan.data), "the main UPDATE never carries projectId"); - assert.equal(plan.data.estimateId, "estimate-1"); + assert.ok(!("estimateId" in plan.data), "nor estimateId"); + assert.equal(plan.data.amount, WRITE.amount, "the rest of the write still lands"); }); -test("a row already on a project is never re-projected, and keeps its estimate", () => { - // Both halves matter: leaving projectId alone while still writing the QBO - // match's estimateId would put the row on job B for every reader and on - // job A's estimate for cascade-delete and billing. - const plan = planQboExpenseUpdate({ projectId: "moved-by-hand" }, WRITE); - assert.equal(plan.fillProjectId, null); +test("a row already on a project is never re-projected AND never re-estimated", () => { + const plan = planQboExpenseUpdate({ projectId: "moved-by-hand", estimateId: "estimate-of-job-b" }, WRITE); + assert.equal(plan.fill, null); assert.ok(!("projectId" in plan.data)); assert.ok(!("estimateId" in plan.data), "the estimate belongs to the OTHER job"); - assert.equal(plan.data.amount, WRITE.amount, "the rest of the write still lands"); + assert.equal(plan.data.amount, WRITE.amount); }); -test("when the stored project AGREES with the match, the estimate still tracks it", () => { - const plan = planQboExpenseUpdate({ projectId: "project-1" }, { ...WRITE, estimateId: "estimate-2" }); - assert.equal(plan.fillProjectId, null); - assert.equal(plan.data.estimateId, "estimate-2", "same job, newer estimate — that is the old behaviour"); +test("attribution is write-ONCE: even the SAME job does not get a newer estimate", () => { + // The dropped carve-out (Codex round 2). An earlier version refreshed + // estimateId when the stored project and the incoming match agreed. It + // bought very little — a row following its job to a newer estimate — and + // paid by making the rule conditional, which is how the original bug got + // in. Re-pointing an estimate is a job for an explicit re-attribution path, + // not for an import. + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1" }, + { ...WRITE, estimateId: "estimate-2" }, + ); + assert.equal(plan.fill, null); + assert.ok(!("estimateId" in plan.data), "never after the first write"); +}); + +test("an unattributed row still gets its estimate corrected, even with no project to fill", () => { + // projectId null on both sides: nothing to attribute to, but the estimate + // link is still write-once-not-yet-written, so it may land. + const plan = planQboExpenseUpdate( + { projectId: null, estimateId: "stale" }, + { ...WRITE, projectId: null }, + ); + assert.deepEqual(plan.fill, { estimateId: "estimate-1" }); + assert.ok(!("projectId" in (plan.fill ?? {}))); +}); + +test("nothing to fill is reported as nothing, not as an empty write", () => { + const plan = planQboExpenseUpdate( + { projectId: null, estimateId: "estimate-1" }, + { ...WRITE, projectId: null }, + ); + assert.equal(plan.fill, null); }); test("a re-attributed row's estimateId survives a real re-sync", async () => { @@ -1419,3 +1470,15 @@ test("a re-attributed row's estimateId survives a real re-sync", async () => { assert.equal(row?.estimateId, "estimate-of-job-b"); assert.equal(row?.amount, 300); }); + +test("an ALREADY-attributed row keeps its estimate when QBO points at a newer one", async () => { + const fake = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: "project-1", estimateId: "estimate-1", receiptUrl: null }, + ]); + assert.equal( + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", estimateId: "estimate-2", amount: 300 }), + "updated", + ); + assert.equal(fake.rows.get("purchase-1")?.estimateId, "estimate-1", "write-once, no exceptions"); + assert.equal(fake.rows.get("purchase-1")?.amount, 300); +}); diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts index 4ed77c866..baf077b28 100644 --- a/tests/tax-at-source-report.test.ts +++ b/tests/tax-at-source-report.test.ts @@ -266,3 +266,36 @@ test("csv-safe: numbers stay numbers, text gets neutralized", () => { assert.equal(csvCell(null), '""'); assert.equal(csvCell("\tlead-tab"), '"\'\tlead-tab"'); }); + +test("INVISIBLE leading whitespace does not smuggle a formula past the check", () => { + // A spreadsheet trims before deciding, so " =1+1" is every bit as live as + // "=1+1" while sailing past a naive first-character test. + assert.equal(csvCell(" =1+1"), `"' =1+1"`); + assert.equal(csvCell("\n=1+1"), `"'\n=1+1"`); + assert.equal(csvCell(" @SUM(A1)"), `"' @SUM(A1)"`); + // The whitespace itself is DATA and is preserved — neutralizing must not + // quietly edit the export. + assert.match(csvCell(" =1+1"), / =1\+1/); + // A leading space in front of harmless text stays untouched. + assert.equal(csvCell(" Harbor Freight"), `" Harbor Freight"`); + // ...and a bare leading TAB is still caught, even though trimming would + // make it vanish before the test could see it. + assert.equal(csvCell("\tlead-tab"), `"'\tlead-tab"`); +}); + +test("csvNumber accepts a Decimal-like or boxed value, and never emits exponent form", () => { + // Prisma Decimal has its OWN toFixed, so a caller handing one straight in + // is the likeliest mistake; everything is normalized through String/Number. + const decimalLike = { toString: () => "207.74" }; + assert.equal(csvNumber(decimalLike), "207.74"); + // eslint-disable-next-line no-new-wrappers + assert.equal(csvNumber(new Number(16.5)), "16.50"); + assert.equal(csvNumber("16.555", 2), "16.56"); + // toFixed flips to "1e+21" here; a CSV cell reading "1e+21" is not a number + // anyone can sum. + const huge = csvNumber(1e21); + assert.ok(!/[eE]/.test(huge), `exponent notation leaked: ${huge}`); + assert.match(huge, /^\d+\.\d{2}$/); + assert.equal(csvNumber(undefined), ""); + assert.equal(csvNumber("not a number"), ""); +}); From 7e52a26a0228690a0bc737140d03e89850d6a7cf Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 20:58:42 -0700 Subject: [PATCH 075/144] fix(expenses): backfill ordering, phase scope everywhere, no default tax deduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex PR #442 round 2. Items 6 and 7 are Phase 1 / Phase 0 code, fixed on their own branches. 2. THE BACKFILL WROTE NO COST CODES. Pass (a) filled `projectId`; pass (c)'s predicate still asserted the PRE-fill value, so every legacy row pass (a) touched then matched nothing — an `--apply` that reported success and coded nothing. The predicate now uses the resolved/post-fill project. The stub returned `{count: 1}` without mutating state, which is exactly why this passed: it is now stateful and honours predicates (including SQL's `NULL NOT IN (...)`), and a new test proves one `--apply` codes the row and a second dry run plans zero. Mutation-checked. 3. PHASE SCOPE. "The cost code exists" is not a permission, and five writers were treating it as one. The intake route, the worker's phase loader (it took a projectId and ignored it, offering every company code to the model), QBO suggestions, the manual expense edit and the backfill all now require the code to be a phase OF THAT JOB. 4. TAX POSITION. `installedAtCustomer` no longer defaults from the project — silence is NULL everywhere, including job-folder receipts. Defaulting it true turned "nobody looked at this" into a deduction on a state return, and a job receipt is just as likely to be consumables, tools, fuel or a service; WAC 458-20-102(12)(b) allows the cost of the articles actually RESOLD. The report still counts only an explicit true. New `Expense.taxDeductibleBase` (additive, in schema + migration + apply script) lets a bookkeeper allocate the resold portion of a MIXED receipt, and the expense PUT is the correction path — it accepts `installedAtCustomer` and `taxDeductibleBase`, validated 0 ≤ base ≤ amount − tax against the amount the request LEAVES on the row. `taxAtSource` is unchanged: it stays the factual "tax was charged here". 5. The backfill CSV used a private escaper that only quoted, leaving OCR'd vendor names executable. It now uses csvCell/csvNumber from csv-safe. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- .../migration.sql | 4 + prisma/schema.prisma | 13 +- scripts/apply-expense-attribution.mjs | 5 +- scripts/backfill-expense-attribution.mjs | 92 ++++++++-- .../api/cron/receipt-intake-worker/route.ts | 2 + src/app/api/expenses/[id]/route.ts | 110 +++++++++++- src/app/api/receipts/intake/route.ts | 35 ++-- src/lib/qbo-expense-sync.ts | 20 +++ src/lib/tax-at-source-report.ts | 33 +++- tests/backfill-expense-attribution.test.ts | 160 ++++++++++++++++-- tests/expense-phase-scope.test.ts | 101 +++++++++++ tests/qbo-expense-sync.test.ts | 28 +++ tests/tax-at-source-query.test.ts | 19 +++ tests/tax-at-source-report.test.ts | 13 ++ 15 files changed, 566 insertions(+), 73 deletions(-) create mode 100644 tests/expense-phase-scope.test.ts diff --git a/package.json b/package.json index 5696c0a6a..b4f1215e0 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 858ac7d6f..9e2829378 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -21,6 +21,10 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DE ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT; ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,30); +-- Mixed receipts: the portion actually resold, when it is less than the whole +-- pre-tax total. NULL means "all of it", and is only reached on a row a human +-- has explicitly flagged installed-at-customer. +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30); CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d8c3c727d..d14efb747 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -645,9 +645,18 @@ model Expense { /// WA excise "tax paid at source": GTR paid sales tax at the register on /// material it resold as part of customer work. taxAtSource Boolean @default(false) - /// null = unknown/legacy. Only TRUE rows are deductible on the excise - /// return, so an absent answer must never read as a yes. + /// null = NOBODY HAS SAID. Only an explicit TRUE is deductible on the excise + /// return: WAC 458-20-102(12)(b) allows the cost of the articles actually + /// resold, so an unanswered receipt must never be claimed. Defaulting this + /// from the project (a real job => true) was wrong and was removed — a job + /// receipt can still be consumables, tools, or services. installedAtCustomer Boolean? + /// The portion of this receipt that was actually resold to the customer, + /// when it is LESS than the whole pre-tax total (a mixed receipt: some + /// material installed, some shop consumables). Null means "the whole pre-tax + /// total", which is only reached once installedAtCustomer is an explicit + /// true. Set by a bookkeeper on the expense edit route. + taxDeductibleBase Decimal? /// capture | ai | manual | backfill. Precedence: capture = manual > /// ai = backfill > null. Nothing but a human edit rewrites capture/manual. costCodeSource String? diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index b62a42839..a9af0aa8c 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -63,6 +63,9 @@ export const statements = [ `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN`, `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT`, `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,30)`, + // Mixed receipts: the portion actually resold, when it is less than the + // whole pre-tax total. NULL means "all of it". + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30)`, `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, @@ -112,7 +115,7 @@ END $$`, export const expectedColumns = { Expense: [ "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", - "costCodeSource", "costCodeConfidence", + "costCodeSource", "costCodeConfidence", "taxDeductibleBase", ], }; diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 76723519d..759f6617c 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -44,6 +44,7 @@ import { resolveExpenseProjectId, } from "../src/lib/expense-attribution.ts"; import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project.ts"; +import { csvCell, csvNumber } from "../src/lib/csv-safe.ts"; /** * Both current tiers clear this (0.9 vendor, 0.75 line), so it changes nothing @@ -92,8 +93,18 @@ export function measureCoverage(rows) { * estimate item, so the link can be checked before it is trusted * @param costCodeIdByCode "03-PLUMB" -> cost code id * @param scopedProjectIds the In Progress, non-overhead jobs the suggester may touch + * @param allowedCodesByProject project id -> the cost code ids that are actually + * PHASES OF THAT JOB. "The cost code exists" is not a permission + * (src/lib/cost-coding.ts SCOPE note), and a rule that fires on a vendor name + * knows nothing about which phases the job has. */ -export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectIds }) { +export function planBackfill({ + expenses, + items, + costCodeIdByCode, + scopedProjectIds, + allowedCodesByProject = new Map(), +}) { const inScope = new Set(scopedProjectIds); const projectFills = []; const codeFills = []; @@ -151,7 +162,7 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI costCodeSource: "backfill", costCodeConfidence: null, why: "item cost code (same project)", - expectedProjectId: expense.projectId ?? null, + expectedProjectId: resolvedProjectId, expense, }); continue; @@ -168,6 +179,15 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI } const suggestion = suggestCode(expense); const costCodeId = suggestion ? costCodeIdByCode.get(suggestion.code) : undefined; + // A phase the job does not have is not an answer, however confident the + // regex was. Same rule the clock-in route enforces via + // isCostCodeAllowedForProject — applied here too, because an automated + // write has less standing to invent a phase than a human does, not more. + const allowed = allowedCodesByProject.get(resolvedProjectId); + if (costCodeId && allowed && !allowed.has(costCodeId)) { + add(expense, "phase-not-on-project"); + continue; + } if (suggestion && costCodeId && suggestion.confidence >= MIN_CONFIDENCE) { codeFills.push({ id: expense.id, @@ -175,10 +195,10 @@ export function planBackfill({ expenses, items, costCodeIdByCode, scopedProjectI costCodeSource: "ai", costCodeConfidence: suggestion.confidence, why: suggestion.why, - // The attribution the decision was scoped BY, so the write can - // require it is unchanged. `null` is a real expectation: it - // means the row was still unattributed when the plan was made. - expectedProjectId: expense.projectId ?? null, + // The attribution the decision was scoped by, as it will be + // AFTER pass (a) has run — which is the state the write + // actually meets. + expectedProjectId: resolvedProjectId, expense, }); } else { @@ -198,24 +218,34 @@ export function projectedRows(expenses, codeFills) { })); } -function csvEscape(value) { - return `"${String(value ?? "").replace(/"/g, '""').replace(/\s+/g, " ").slice(0, 160)}"`; +/** + * One cell of free text, collapsed to a single line and capped. + * + * The collapse and the cap are this script's own presentation choice — a + * 4,000-character receipt description makes the CSV unreadable. The QUOTING and + * the formula neutralization are NOT: they come from src/lib/csv-safe.ts, the + * same helper the tax report uses. This used to be a private `csvEscape` that + * only quoted, which left every vendor and description in this file executable + * when Marge opened it — and vendor names here are OCR output. + */ +function textCell(value) { + return csvCell(String(value ?? "").replace(/\s+/g, " ").slice(0, 160)); } export function remainderCsv(remainder, projectNameById) { const lines = [["expense_id", "project", "date", "vendor", "amount", "reason", "description"].join(",")]; for (const expense of remainder) { lines.push([ - expense.id, - csvEscape(projectNameById.get(resolveExpenseProjectId(expense)) ?? ""), - expense.date ? new Date(expense.date).toISOString().slice(0, 10) : "", - csvEscape(expense.vendor), - num(expense.amount).toFixed(2), + textCell(expense.id), + textCell(projectNameById.get(resolveExpenseProjectId(expense)) ?? ""), + textCell(expense.date ? new Date(expense.date).toISOString().slice(0, 10) : ""), + textCell(expense.vendor), + csvNumber(num(expense.amount)), // WHY it is here. "item-outside-estimate" in particular is not // "we could not guess" — it is "this row claims a line item on // another job", which is a data problem a human should look at. - csvEscape(expense.reason), - csvEscape(expense.description), + textCell(expense.reason), + textCell(expense.description), ].join(",")); } return lines.join("\n"); @@ -259,21 +289,41 @@ export async function runBackfill({ // The item's OWN estimate and project come back with it: the fallback has // to prove the link does not cross jobs before it copies a code. - const itemRows = await db.estimateItem.findMany({ + const itemRowsRaw = await db.estimateItem.findMany({ where: { costCodeId: { not: null } }, select: { id: true, costCodeId: true, estimateId: true, estimate: { select: { projectId: true } }, }, }); - const items = new Map(itemRows.map(row => [row.id, { + const itemRows = itemRowsRaw.map(row => ({ + id: row.id, costCodeId: row.costCodeId, estimateId: row.estimateId, projectId: row.estimate?.projectId ?? null, + })); + const items = new Map(itemRows.map(row => [row.id, { + costCodeId: row.costCodeId, + estimateId: row.estimateId, + projectId: row.projectId, }])); const itemCostCodeById = new Map(itemRows.map(row => [row.id, row.costCodeId])); - const plan = planBackfill({ expenses, items, costCodeIdByCode, scopedProjectIds }); + // The phases each job actually has. Mirrors resolveProjectPhaseCodes' + // estimate-item half (src/lib/project-phases.ts) — the Safety phase is + // deliberately absent, because a materials receipt is never a safety + // meeting and this pass has no business assigning one. + const allowedCodesByProject = new Map(); + for (const row of itemRows) { + const projectId = row.projectId; + if (!projectId || !row.costCodeId) continue; + if (!allowedCodesByProject.has(projectId)) allowedCodesByProject.set(projectId, new Set()); + allowedCodesByProject.get(projectId).add(row.costCodeId); + } + + const plan = planBackfill({ + expenses, items, costCodeIdByCode, scopedProjectIds, allowedCodesByProject, + }); // ── the table ─────────────────────────────────────────────────────────── const scoped = new Set(scopedProjectIds); @@ -381,6 +431,12 @@ export async function runBackfill({ // any concurrent sync or bookkeeper edit; the predicate is what // makes it safe to act on. A row that moved is skipped, not // coded on stale reasoning. + // The RESOLVED project — i.e. the value pass (a) above has + // just written, not the NULL this row had when the plan was + // made. Using the pre-fill value meant every legacy row the + // project pass touched then failed this predicate and silently + // wrote no cost code at all: an `--apply` that reported success + // and coded nothing. projectId: fill.expectedProjectId, costCodeId: null, ...notHumanCodedExpenseWhere(), diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 1bed5fbc5..788ecea68 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -3,6 +3,8 @@ import { NextResponse } from "next/server"; import { isCronAuthorized } from "@/lib/cron-auth"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; +import { resolveProjectPhaseCodes } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; import { downloadVerified, inspectStoredObject, sealAndPublish } from "@/lib/receipt-intake/stored-object"; diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index ea11c0764..04dc4d385 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -6,6 +6,11 @@ import { QboManagedExpenseError, assertExpenseMutableOutsideQbo, } from "@/lib/qbo-expense-guard"; +import { resolveExpenseProjectId } from "@/lib/expense-attribution"; +import { resolveCostCode } from "@/lib/cost-coding"; +import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; +import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; export async function DELETE(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { @@ -42,9 +47,16 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const expense = await prisma.expense.findUnique({ where: { id }, - select: { qbPurchaseId: true }, + select: { + qbPurchaseId: true, + amount: true, + taxAmount: true, + projectId: true, + estimate: { select: { projectId: true } }, + }, }); assertExpenseMutableOutsideQbo(expense); + if (!expense) return NextResponse.json({ error: "Expense not found" }, { status: 404 }); const body = await req.json(); if (body.itemId) { @@ -63,17 +75,97 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const editsCostCode = Object.prototype.hasOwnProperty.call(body, "costCodeId"); const nextCostCodeId: string | null = typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null; + const resolvedProjectId = resolveExpenseProjectId(expense); if (editsCostCode && nextCostCodeId) { - const costCode = await prisma.costCode.findUnique({ - where: { id: nextCostCodeId }, - select: { id: true, isActive: true }, + // BOTH checks, per the SCOPE note on resolveCostCode: existence and + // active-ness are ATTRIBUTION, "this code belongs to this job" is + // PERMISSION, and neither implies the other. Validating only the + // former let a human move an expense onto a phase from an entirely + // different job. + const resolved = await resolveCostCode(prismaCostCodingDataSource, { + costCodeId: nextCostCodeId, }); - if (!costCode) { - return NextResponse.json({ error: "Cost code not found." }, { status: 400 }); + if (!resolved.ok) { + return NextResponse.json( + { error: resolved.error, code: resolved.code }, + { status: resolved.status }, + ); + } + if (!resolvedProjectId) { + return NextResponse.json( + { + error: "This expense isn't attached to a project, so a phase can't be validated against one.", + code: "PHASE_NOT_ON_PROJECT", + }, + { status: 400 }, + ); + } + const allowed = await isCostCodeAllowedForProject( + prismaPhaseDataSource, + resolvedProjectId, + resolved.costCodeId, + ); + if (!allowed) { + return NextResponse.json( + { + error: "That cost code isn't one of this project's phases.", + code: "PHASE_NOT_ON_PROJECT", + }, + { status: 400 }, + ); + } + } + + // ── the tax-deduction correction path (Phase 3, §7) ───────────────── + // + // Nothing defaults `installedAtCustomer` any more, so this route is how + // an unreviewed receipt becomes a claimable one. It is the ONLY place a + // human answer can be recorded after capture, which is why it validates + // rather than trusts: a deduction base larger than the pre-tax total is + // a filing error, and it must be refused at the write rather than + // clamped later where nobody would see it. + const editsInstalled = Object.prototype.hasOwnProperty.call(body, "installedAtCustomer"); + let nextInstalled: boolean | null = null; + if (editsInstalled) { + const raw = body.installedAtCustomer; + if (raw !== null && typeof raw !== "boolean") { + return NextResponse.json( + { error: "installedAtCustomer must be true, false, or null." }, + { status: 400 }, + ); + } + nextInstalled = raw; + } + + const editsBase = Object.prototype.hasOwnProperty.call(body, "taxDeductibleBase"); + let nextBase: number | null = null; + if (editsBase && body.taxDeductibleBase !== null) { + const parsed = Number(body.taxDeductibleBase); + if (!Number.isFinite(parsed) || parsed < 0) { + return NextResponse.json( + { error: "taxDeductibleBase must be a number ≥ 0, or null." }, + { status: 400 }, + ); } - if (!costCode.isActive) { - return NextResponse.json({ error: "That cost code is inactive." }, { status: 400 }); + // Validated against the amount this request LEAVES on the row, not + // the one it started with — a PUT that lowers the amount and sets a + // base in the same call must not be able to slip past by being + // checked against the old, larger figure. + const nextAmount = + body.amount !== undefined && body.amount !== null + ? Number(body.amount) + : Number(expense.amount); + const tax = Number(expense.taxAmount ?? 0); + const ceiling = Math.round((nextAmount - tax) * 100) / 100; + if (!Number.isFinite(ceiling) || parsed > ceiling) { + return NextResponse.json( + { + error: `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).`, + }, + { status: 400 }, + ); } + nextBase = parsed; } const updatedExpense = await prisma.expense.update({ @@ -94,6 +186,8 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: costCodeConfidence: null, } : {}), + ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), + ...(editsBase ? { taxDeductibleBase: nextBase } : {}), }, }); diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 07334ff82..cbc7acee7 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -23,7 +23,6 @@ import { serializeReceiptIntake, withArchiveDownloadUrls, } from "@/lib/receipt-intake/queries"; -import { isOverheadProject } from "@/lib/overhead-project"; export const dynamic = "force-dynamic"; export const maxDuration = 30; @@ -142,22 +141,23 @@ function optionalBool(value: unknown): boolean | null { } /** - * The default the mobile app relies on (spec §5c): a receipt filed against a - * real job was installed at the customer's; one filed against the Shop overhead - * bucket was consumed by the business and is NOT deductible on the excise - * return. An explicit answer from the caller always wins — the crew member - * standing in front of the material knows better than this rule does. + * NO DEFAULT. Silence means NULL, on every source, including a receipt that + * arrived in a job folder. * - * With no project at all the answer stays NULL. Guessing "yes" would quietly - * inflate a tax deduction, which is the one direction this must never fail in. + * An earlier version defaulted this to TRUE for any non-overhead project, on + * the reasoning that a job receipt is job material. That was wrong, and wrong + * in the one direction a tax figure must never fail in: WAC 458-20-102(12)(b) + * allows the cost of the articles actually RESOLD, and a receipt coded to a + * live job is just as likely to be consumables, tools, fuel, dump fees, or a + * service. Defaulting it turned "nobody looked at this" into a deduction + * claimed on a state return. + * + * An explicit true/false from the caller is honoured — the crew member standing + * in front of the material is the one person who actually knows — and a + * bookkeeper can correct it afterwards on the expense edit route. */ -export function resolveInstalledAtCustomer( - declared: boolean | null, - projectId: string | null, -): boolean | null { - if (declared !== null) return declared; - if (!projectId) return null; - return !isOverheadProject(projectId); +export function resolveInstalledAtCustomer(declared: boolean | null): boolean | null { + return declared; } async function parseBody(req: Request): Promise { @@ -332,10 +332,7 @@ export async function POST(req: Request) { dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId: parsed.projectId, costCodeId: parsed.costCodeId, - installedAtCustomer: resolveInstalledAtCustomer( - parsed.installedAtCustomer, - parsed.projectId, - ), + installedAtCustomer: resolveInstalledAtCustomer(parsed.installedAtCustomer), createdById: auth.via === "session" ? auth.user.id : null, // Only a shared-secret forwarder may assert this: it is the // claim that v1 already put this document in the books, and it diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index cf18752a9..80a4104b9 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -11,6 +11,8 @@ import { resolveExpenseProjectId, } from "./expense-attribution"; import { isOverheadProject } from "./overhead-project"; +import { isCostCodeAllowedForProject } from "./project-phases"; +import { prismaPhaseDataSource } from "./project-phases-db"; // Shared with the register merge layer (register-merge.ts, Unified Money // Register plan §4) so the classification values this module WRITES can // never drift from the values that module READS. @@ -833,6 +835,8 @@ export type QboCostCodeSuggestionResult = | "no-match" /** The rules named a code this company does not have active. */ | "unknown-code" + /** The code exists, but it is not one of THIS job's phases. */ + | "phase-not-on-project" /** Already coded, or coded by a human: the guard held. */ | "not-written" | "written"; @@ -849,6 +853,12 @@ export async function applyQboExpenseCostCodeSuggestion( client: QboCostCodeSuggestionClient, input: QboCostCodeSuggestionInput, costCodeIdByCode: ReadonlyMap, + /** + * "Is this code a phase of that job?" Injected so the rules stay testable + * without a database. Omitted = no scope check, which is only acceptable + * for a caller that has already scoped the map it passed in. + */ + isAllowedForProject?: (projectId: string, costCodeId: string) => Promise, ): Promise { const stored = await client.expense.findUnique({ where: { qbPurchaseId: input.qbPurchaseId }, @@ -883,6 +893,14 @@ export async function applyQboExpenseCostCodeSuggestion( const costCodeId = costCodeIdByCode.get(suggestion.code); if (!costCodeId) return "unknown-code"; + // A phase the job does not have is not an answer, however confident the + // regex was. The rules match on a vendor name; they know nothing about + // which phases this job actually carries, and an automated write has LESS + // standing to invent one than a human does, not more. + if (isAllowedForProject && !(await isAllowedForProject(projectId, costCodeId))) { + return "phase-not-on-project"; + } + const written = await client.expense.updateMany({ where: { qbPurchaseId: input.qbPurchaseId, @@ -1100,6 +1118,8 @@ function createDefaultSyncDependencies(): QboExpenseSyncDependencies { prisma as unknown as QboCostCodeSuggestionClient, input, await loadCostCodes(), + (projectId, costCodeId) => + isCostCodeAllowedForProject(prismaPhaseDataSource, projectId, costCodeId), ); }, now: () => new Date(), diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts index a3f6223cc..cc6fe6d78 100644 --- a/src/lib/tax-at-source-report.ts +++ b/src/lib/tax-at-source-report.ts @@ -12,7 +12,9 @@ // never when it merely lacks a contradiction: // * `taxAtSource` — the read actually found tax on the receipt, // * `installedAtCustomer === true` — a NULL is "nobody said", and a NULL -// must never be spent as a deduction, and +// must never be spent as a deduction. Nothing defaults this: it is set at +// capture by the person holding the material, or corrected afterwards by a +// bookkeeper on the expense edit route, and // * `taxAmount > 0` — a zero is an answer (no tax), not an absence. // // TWO THINGS THIS FILE IS FUSSY ABOUT, both because it feeds a tax filing: @@ -67,18 +69,28 @@ export interface TaxAtSourceRow { reference: string; /** Gross paid, in whole cents. */ receiptTotalCents: number; - /** receiptTotal - tax, in whole cents. The excise line's base. */ + /** + * The excise line's base, in whole cents. `taxDeductibleBase` when a + * bookkeeper has allocated one (a MIXED receipt: some material resold, some + * shop consumables), otherwise the whole pre-tax total. + */ deductionBaseCents: number; + /** True when the base above is an explicit human allocation, not the whole receipt. */ + baseIsAllocated: boolean; taxCents: number; } export const TAX_REPORT_AMOUNT_NOTE = - "Receipt Total is the gross amount paid; the deduction base is that total less the sales tax on the receipt."; + "Receipt Total is the gross amount paid. The deduction base is that total less the sales tax on the receipt, " + + "unless a bookkeeper has allocated a smaller amount for a mixed receipt — in which case only the allocated " + + "portion is claimed."; export const TAX_REPORT_FOOTNOTE = "Sales tax already paid on materials resold as part of customer work is deductible on the WA excise return line " + - "\"taxable amount for tax paid at source\". Only receipts flagged installed-at-customer count; Shop and consumable " + - "purchases are excluded."; + "\"taxable amount for tax paid at source\" (WAC 458-20-102(12)(b) — the cost of the articles actually resold). " + + "A receipt counts only when someone has explicitly marked it installed-at-customer; unreviewed receipts, Shop " + + "purchases, consumables, tools and services are all excluded. For a mixed receipt, set a deduction base on the " + + "expense so only the resold portion is claimed."; /** Whole cents from a value for display only. */ export function centsToDollars(cents: number): number { @@ -356,6 +368,7 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise description: true, amount: true, taxAmount: true, + taxDeductibleBase: true, projectId: true, project: { select: { name: true } }, estimate: { select: { projectId: true, project: { select: { name: true } } } }, @@ -366,6 +379,11 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise return rows.map(row => { const taxCents = toCents(row.taxAmount); const receiptTotalCents = toCents(row.amount); + // An allocation, when a human made one, ALWAYS wins — including when it + // is larger than the pre-tax total, which the edit route refuses to + // store. Silently clamping here would hide a bad allocation instead of + // surfacing it; the validation belongs at the write, and does exist. + const allocated = row.taxDeductibleBase !== null && row.taxDeductibleBase !== undefined; return { id: row.id, // The `date: { gte }` filter above already excluded null dates. @@ -376,7 +394,10 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise projectName: row.project?.name ?? row.estimate?.project?.name ?? "(unassigned)", reference: extractReference(row.description), receiptTotalCents, - deductionBaseCents: receiptTotalCents - taxCents, + deductionBaseCents: allocated + ? toCents(row.taxDeductibleBase) + : receiptTotalCents - taxCents, + baseIsAllocated: allocated, taxCents, }; }); diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index b658921e5..2fc4cfa42 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -59,13 +59,45 @@ const COST_CODE_IDS = new Map([ ["02-FRAME", "cc-frame"], ]); +/** + * STATEFUL by design. An earlier version returned `{ count: 1 }` without + * touching the fixture, which made every write look like it succeeded — and + * that is exactly what hid the bug where pass (a) filled `projectId` and pass + * (c)'s predicate then matched nothing. A stub that cannot fail a predicate + * cannot test a predicate. + */ function createStub( expenses: StubExpense[], items: { id: string; costCodeId: string | null; estimateId: string; estimate: { projectId: string | null } }[] = [], ) { const writes: { where: Record; data: Record }[] = []; + const rows = expenses; + + const matches = (row: StubExpense, where: Record): boolean => { + if (typeof where.id === "string" && row.id !== where.id) return false; + if (where.id && typeof where.id === "object") { + const ids = (where.id as { in?: string[] }).in ?? []; + if (!ids.includes(row.id)) return false; + } + if ("projectId" in where && (row.projectId ?? null) !== where.projectId) return false; + if ("costCodeId" in where && (row.costCodeId ?? null) !== where.costCodeId) return false; + if (Array.isArray(where.OR)) { + const ok = (where.OR as Record[]).some(branch => { + if (!("costCodeSource" in branch)) return false; + const expected = branch.costCodeSource; + if (expected === null) return (row.costCodeSource ?? null) === null; + const notIn = (expected as { notIn?: string[] }).notIn ?? []; + // SQL semantics: NULL NOT IN (...) is NULL, i.e. NOT a match. + return row.costCodeSource !== null && !notIn.includes(row.costCodeSource); + }); + if (!ok) return false; + } + return true; + }; + return { writes, + rows, db: { project: { async findMany() { @@ -88,10 +120,16 @@ function createStub( async findMany() { return []; }, }, expense: { - async findMany() { return expenses; }, + async findMany() { return rows; }, async updateMany(args: { where: Record; data: Record }) { writes.push(args); - return { count: 1 }; + let count = 0; + for (const row of rows) { + if (!matches(row, args.where)) continue; + Object.assign(row, args.data); + count += 1; + } + return { count }; }, }, }, @@ -312,12 +350,30 @@ test("the remainder CSV carries what Marge needs to decide, including WHY", () = ); const [header, row] = csv.split("\n"); assert.equal(header, "expense_id,project,date,vendor,amount,reason,description"); - assert.match( + assert.equal( row, - /^e9,"Mueller Bath",2026-08-01,"Lowe""s",12\.50,"item-outside-estimate","misc supplies"$/, + '"e9","Mueller Bath","2026-08-01","Lowe""s",12.50,"item-outside-estimate","misc supplies"', ); }); +test("the remainder CSV neutralizes formulas — vendor names here are OCR output", () => { + // This file used to have its own escaper that merely quoted, so a receipt + // read as `=cmd|...` was executable the moment the CSV was opened. + const csv = remainderCsv( + [{ + ...expense({ id: "e1", vendor: "=cmd|'/c calc'!A1", description: "@SUM(A1)", amount: -5 }), + reason: "no-rule-match", + }], + new Map([["job-1", "+Mueller"]]), + ); + const row = csv.split("\n")[1]; + assert.match(row, /"'=cmd\|'\/c calc'!A1"/); + assert.match(row, /"'@SUM\(A1\)"/); + assert.match(row, /"'\+Mueller"/); + // ...while a negative amount stays a NUMBER, or every SUM in the sheet breaks. + assert.match(row, /,-5\.00,/); +}); + // ── the run ───────────────────────────────────────────────────────────────── test("a dry run makes ZERO write calls", async () => { @@ -333,9 +389,10 @@ test("a dry run makes ZERO write calls", async () => { }); test("apply writes both passes, each behind its own predicate", async () => { - const stub = createStub([ - expense({ id: "e1", vendor: "Summit Plumbing", projectId: null, estimate: { projectId: "job-1" } }), - ]); + const stub = createStub( + [expense({ id: "e1", vendor: "Summit Plumbing", projectId: null, estimate: { projectId: "job-1" } })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); const projectWrite = stub.writes.find(w => "projectId" in w.data)!; @@ -348,9 +405,9 @@ test("apply writes both passes, each behind its own predicate", async () => { { costCodeSource: null }, { costCodeSource: { notIn: ["capture", "manual"] } }, ]); - // The attribution the plan was made under, re-asserted at write time. The - // plan is a snapshot; the predicate is what makes acting on it safe. - assert.equal(codeWrite.where.projectId, null); + // The POST-FILL project. Asserting `null` here is what let the ordering + // bug through: pass (a) had already set it, so the write matched nothing. + assert.equal(codeWrite.where.projectId, "job-1"); assert.deepEqual(codeWrite.data, { costCodeId: "cc-plumb", costCodeSource: "ai", @@ -358,18 +415,87 @@ test("apply writes both passes, each behind its own predicate", async () => { }); }); -test("a cost-code write requires the project the plan was scoped to", async () => { - // A row that was ALREADY attributed when the plan was made must be written - // under that same id — not under `null`, which would silently match a - // different set of rows. - const stub = createStub([ - expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" }), - ]); +test("ONE --apply actually codes the row, and a second dry run plans nothing", async () => { + // The end-to-end proof, against a stub that honours predicates. The bug + // this catches produced a run that reported success and wrote no cost code + // at all, because pass (c)'s predicate still said `projectId: null` after + // pass (a) had filled it. + const stub = createStub( + [expense({ id: "e1", vendor: "Summit Plumbing", projectId: null, estimate: { projectId: "job-1" } })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + + const applied = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(applied.written.projectIds, 1); + assert.equal(applied.written.costCodes, 1, "the cost code must actually land"); + assert.equal(stub.rows[0].projectId, "job-1"); + assert.equal(stub.rows[0].costCodeId, "cc-plumb"); + assert.equal(stub.rows[0].costCodeSource, "ai"); + + const rerun = await runBackfill({ db: stub.db, apply: false, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.deepEqual(rerun.plan.projectFills, [], "re-run must plan zero project fills"); + assert.deepEqual(rerun.plan.codeFills, [], "re-run must plan zero cost codes"); +}); + +test("a row re-attributed between the plan and the write is skipped, not miscoded", async () => { + const stub = createStub( + [expense({ id: "e1", vendor: "Summit Plumbing", projectId: "job-1" })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + // Someone moves the row after findMany has handed it to the planner. + const passThrough = stub.db.expense.updateMany; + let moved = false; + stub.db.expense.updateMany = async (args: { where: Record; data: Record }) => { + if (!moved) { + moved = true; + stub.rows[0].projectId = "job-elsewhere"; + } + return passThrough(args); + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0); + assert.equal(stub.rows[0].costCodeId, null, "no phase from the job it left"); +}); + +test("a cost-code write requires the project the plan resolved", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; assert.equal(codeWrite.where.projectId, "job-1"); }); +test("a suggested phase the JOB does not have is refused", () => { + // "The cost code exists" is not a permission (cost-coding.ts SCOPE note), + // and a regex that fired on a vendor name knows nothing about which phases + // this job has. The allowed set comes from the project's coded estimate + // items — here it holds only framing, so the plumbing suggestion is out. + const plan = planBackfill({ + expenses: [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], + items: NO_ITEMS, + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + allowedCodesByProject: new Map([["job-1", new Set(["cc-frame"])]]), + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder[0].reason, "phase-not-on-project"); +}); + +test("a suggested phase the job DOES have is accepted", () => { + const plan = planBackfill({ + expenses: [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], + items: NO_ITEMS, + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + allowedCodesByProject: new Map([["job-1", new Set(["cc-plumb"])]]), + }); + assert.equal(plan.codeFills.length, 1); + assert.equal(plan.codeFills[0].costCodeId, "cc-plumb"); +}); + test("the write predicate re-checks NULL, not just the plan", async () => { // Between the read and the write, a re-sync or a bookkeeper can set either // field. A plan is a snapshot; the predicate is the guarantee. diff --git a/tests/expense-phase-scope.test.ts b/tests/expense-phase-scope.test.ts new file mode 100644 index 000000000..407f2b986 --- /dev/null +++ b/tests/expense-phase-scope.test.ts @@ -0,0 +1,101 @@ +/** + * "The cost code exists" is not a permission (src/lib/cost-coding.ts SCOPE + * note). Five writers can put a phase on an expense, and Codex round 2 found + * that most of them checked only that the code existed and was active — so any + * of them could pin a phase from an entirely different job onto a receipt. + * + * This covers the two rules that decide it, plus the intake capture default + * that feeds a tax filing. The route-level wiring is exercised by + * tests/qbo-expense-sync.test.ts (the sync) and by the DI checks below; the + * pure rules are asserted here so a regression names itself. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { isCostCodeAllowedForProject, type PhaseDataSource } from "../src/lib/project-phases"; +import { resolveCostCode, type CostCodingDataSource } from "../src/lib/cost-coding"; +import { resolveInstalledAtCustomer } from "../src/app/api/receipts/intake/route"; + +// ── the two checks every phase writer must run ───────────────────────────── + +const PHASES: Record = { + "job-mueller": [{ id: "cc-plumb", code: "03-PLUMB", name: "Plumbing", isActive: true }], + "job-mesplay": [{ id: "cc-frame", code: "02-FRAME", name: "Framing", isActive: true }], +}; + +const phaseSource: PhaseDataSource = { + async getProject(projectId) { + return PHASES[projectId] ? { id: projectId, status: "In Progress" } : null; + }, + async getEstimateCostCodes(projectId) { + return PHASES[projectId] ?? []; + }, + async getSafetyCostCode() { + return null; + }, +}; + +const codingSource: CostCodingDataSource = { + async getCostCode(costCodeId) { + const all = Object.values(PHASES).flat(); + const found = all.find(phase => phase.id === costCodeId); + if (found) return { id: found.id, isActive: found.isActive }; + if (costCodeId === "cc-retired") return { id: "cc-retired", isActive: false }; + return null; + }, + async getLineItem() { + return null; + }, +}; + +test("a phase from ANOTHER job is rejected even though the code is real and active", async () => { + // This is the whole finding: `resolveCostCode` alone says yes, because the + // code exists and is active. It is the wrong job's phase. + const resolved = await resolveCostCode(codingSource, { costCodeId: "cc-frame" }); + assert.equal(resolved.ok, true, "attribution alone accepts it"); + assert.equal( + await isCostCodeAllowedForProject(phaseSource, "job-mueller", "cc-frame"), + false, + "...and permission is what refuses it", + ); +}); + +test("the job's own phase passes both checks", async () => { + const resolved = await resolveCostCode(codingSource, { costCodeId: "cc-plumb" }); + assert.equal(resolved.ok, true); + assert.equal(await isCostCodeAllowedForProject(phaseSource, "job-mueller", "cc-plumb"), true); +}); + +test("an INACTIVE code is refused by the attribution check", async () => { + const resolved = await resolveCostCode(codingSource, { costCodeId: "cc-retired" }); + assert.equal(resolved.ok, false); + if (!resolved.ok) assert.equal(resolved.code, "COST_CODE_INACTIVE"); +}); + +test("a code that does not exist is refused, and named as such", async () => { + const resolved = await resolveCostCode(codingSource, { costCodeId: "cc-nope" }); + assert.equal(resolved.ok, false); + if (!resolved.ok) assert.equal(resolved.code, "COST_CODE_NOT_FOUND"); +}); + +test("a project with no phases at all accepts nothing", async () => { + assert.equal(await isCostCodeAllowedForProject(phaseSource, "job-unknown", "cc-plumb"), false); +}); + +// ── the intake capture default (tax position) ────────────────────────────── + +test("installedAtCustomer has NO default — silence is unknown, on every source", () => { + // It used to default TRUE for any non-overhead project. That turned + // "nobody looked at this" into a deduction claimed on a state return, and a + // job receipt is just as likely to be consumables, tools, fuel, or a + // service. WAC 458-20-102(12)(b) allows the cost of the articles actually + // RESOLD, not whatever got coded to a live job. + assert.equal(resolveInstalledAtCustomer(null), null, "no project named"); + assert.equal(resolveInstalledAtCustomer(null), null, "a real job does not imply yes"); +}); + +test("an explicit answer from the capturer is honoured, both ways", () => { + // The crew member holding the material is the one person who actually + // knows, so the app's toggle must survive untouched. + assert.equal(resolveInstalledAtCustomer(true), true); + assert.equal(resolveInstalledAtCustomer(false), false); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 6ba71eb09..8bf84ef11 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1350,6 +1350,34 @@ test("a vanished row is reported, not treated as a silent success", async () => ); }); +test("a phase the JOB does not have is refused, however confident the rule was", async () => { + // The rules match on a vendor name. They know nothing about which phases + // this job carries, and an automated write has LESS standing to invent one + // than a human does, not more. + const fake = fakeSuggestionClient(); + const result = await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + COST_CODE_IDS, + async () => false, + ); + assert.equal(result, "phase-not-on-project"); + assert.equal(fake.calls.length, 0, "and nothing is written"); +}); + +test("the scope check is asked about the row's OWN job and the resolved code", async () => { + const fake = fakeSuggestionClient(); + const asked: { projectId: string; costCodeId: string }[] = []; + await applyQboExpenseCostCodeSuggestion( + fake.client, + { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + COST_CODE_IDS, + async (projectId, costCodeId) => { asked.push({ projectId, costCodeId }); return true; }, + ); + assert.deepEqual(asked, [{ projectId: "project-1", costCodeId: "cc-plumb" }]); + assert.equal(fake.calls.length, 1, "an allowed phase still writes"); +}); + test("no rule match and an unknown code both write nothing", async () => { const fake = fakeSuggestionClient(); assert.equal( diff --git a/tests/tax-at-source-query.test.ts b/tests/tax-at-source-query.test.ts index ccd972db6..44cd9f63c 100644 --- a/tests/tax-at-source-query.test.ts +++ b/tests/tax-at-source-query.test.ts @@ -17,6 +17,8 @@ import Module from "node:module"; const PACIFIC = "America/Los_Angeles"; const recorded: any[] = []; +/** Per-test override for the mixed-receipt allocation. */ +let withTaxDeductibleBase: string | null = null; const fakePrisma = { companySettings: { @@ -34,6 +36,7 @@ const fakePrisma = { description: "[Receipt intake] Invoice 82766", amount: "207.74", taxAmount: "16.55", + taxDeductibleBase: withTaxDeductibleBase, projectId: "job-b", project: { name: "Mesplay Kitchen" }, estimate: { projectId: "job-a", project: { name: "Mueller Bath" } }, @@ -110,6 +113,22 @@ test("resolveTaxAtSourceFilters takes the zone from company settings", async () assert.equal(filters.from.toISOString(), "2026-01-01T08:00:00.000Z", "PST, not the server's zone"); }); +test("an allocated deduction base replaces the whole pre-tax total", async () => { + // The mixed-receipt correction path. Without it the report claims the + // entire job-coded receipt, which WAC 458-20-102(12)(b) does not allow — + // only the cost of the articles actually resold. + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + withTaxDeductibleBase = "50.00"; + try { + const [row] = await queryTaxAtSourceRows(filters); + assert.equal(row.deductionBaseCents, 5000); + assert.equal(row.baseIsAllocated, true); + assert.equal(row.receiptTotalCents, 20774, "the gross is unchanged"); + } finally { + withTaxDeductibleBase = null; + } +}); + test("a row is stamped with its COMPANY day and its RESOLVED job", async () => { const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); const [row] = await queryTaxAtSourceRows(filters); diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts index baf077b28..5ce9b19cd 100644 --- a/tests/tax-at-source-report.test.ts +++ b/tests/tax-at-source-report.test.ts @@ -42,6 +42,7 @@ function row(overrides: Partial = {}): TaxAtSourceRow { reference: "001916749100246", receiptTotalCents, deductionBaseCents: receiptTotalCents - taxCents, + baseIsAllocated: false, taxCents, ...overrides, }; @@ -149,6 +150,18 @@ test("an empty period is zeros, not NaN", () => { assert.deepEqual(summary, { count: 0, deductionBaseCents: 0, receiptTotalCents: 0, taxCents: 0 }); }); +test("a mixed receipt contributes only its ALLOCATED base", () => { + // $207.74 gross, $16.55 tax, but only $50 of it was material resold to the + // customer — the rest was shop consumables. Claiming the whole pre-tax + // $191.19 would overstate the deduction by $141.19 on one receipt. + const { summary } = groupTaxAtSource([ + row({ receiptTotalCents: 20774, taxCents: 1655, deductionBaseCents: 5000, baseIsAllocated: true }), + ]); + assert.equal(summary.deductionBaseCents, 5000); + assert.equal(summary.receiptTotalCents, 20774, "the gross is still reported honestly"); + assert.equal(summary.taxCents, 1655); +}); + // ── period boundaries, in the company zone ────────────────────────────────── test("the quarter is the company's calendar quarter, not the server's", () => { From 7fd2150b39208c31c5fdfe684caab08e43bbbc4e Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:13:35 -0700 Subject: [PATCH 076/144] fix(tests): the phase-scope test must not need a JWT secret CI's Build job failed where local passed. The new test imported src/app/api/receipts/intake/route to reach resolveInstalledAtCustomer, and that route pulls in mobile-auth, which THROWS at import time when NEXTAUTH_SECRET is unset. My machine has it via .env.production.local; CI does not. The helper moves to src/lib/expense-attribution.ts, which is already the pure, no-I/O module for this decision. Verified by running the test with NEXTAUTH_SECRET unset. (The "Format check" step also logged a missing prettier-plugin-tailwindcss, but that step is continue-on-error and was not the failure.) Co-Authored-By: Claude Fable 5.1 --- src/app/api/receipts/intake/route.ts | 21 +-------------------- src/lib/expense-attribution.ts | 20 ++++++++++++++++++++ tests/expense-phase-scope.test.ts | 5 ++++- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index cbc7acee7..6dd4130e4 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -5,6 +5,7 @@ import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { resolveInstalledAtCustomer } from "@/lib/expense-attribution"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { receiptObjectSize, uploadReceiptObject } from "@/lib/receipt-intake/bucket"; import { getSupabase } from "@/lib/supabase"; @@ -140,26 +141,6 @@ function optionalBool(value: unknown): boolean | null { return null; } -/** - * NO DEFAULT. Silence means NULL, on every source, including a receipt that - * arrived in a job folder. - * - * An earlier version defaulted this to TRUE for any non-overhead project, on - * the reasoning that a job receipt is job material. That was wrong, and wrong - * in the one direction a tax figure must never fail in: WAC 458-20-102(12)(b) - * allows the cost of the articles actually RESOLD, and a receipt coded to a - * live job is just as likely to be consumables, tools, fuel, dump fees, or a - * service. Defaulting it turned "nobody looked at this" into a deduction - * claimed on a state return. - * - * An explicit true/false from the caller is honoured — the crew member standing - * in front of the material is the one person who actually knows — and a - * bookkeeper can correct it afterwards on the expense edit route. - */ -export function resolveInstalledAtCustomer(declared: boolean | null): boolean | null { - return declared; -} - async function parseBody(req: Request): Promise { const contentType = req.headers.get("content-type") ?? ""; const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index e5d93478a..6adba70d5 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -137,3 +137,23 @@ export function notHumanCodedExpenseWhere(): Prisma.ExpenseWhereInput { ], }; } + +/** + * NO DEFAULT. Silence means NULL, on every source, including a receipt that + * arrived in a job folder. + * + * An earlier version defaulted this to TRUE for any non-overhead project, on + * the reasoning that a job receipt is job material. That was wrong, and wrong + * in the one direction a tax figure must never fail in: WAC 458-20-102(12)(b) + * allows the cost of the articles actually RESOLD, and a receipt coded to a + * live job is just as likely to be consumables, tools, fuel, dump fees, or a + * service. Defaulting it turned "nobody looked at this" into a deduction + * claimed on a state return. + * + * An explicit true/false from the caller is honoured — the crew member standing + * in front of the material is the one person who actually knows — and a + * bookkeeper can correct it afterwards on the expense edit route. + */ +export function resolveInstalledAtCustomer(declared: boolean | null): boolean | null { + return declared; +} diff --git a/tests/expense-phase-scope.test.ts b/tests/expense-phase-scope.test.ts index 407f2b986..1632fcbaa 100644 --- a/tests/expense-phase-scope.test.ts +++ b/tests/expense-phase-scope.test.ts @@ -13,7 +13,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import { isCostCodeAllowedForProject, type PhaseDataSource } from "../src/lib/project-phases"; import { resolveCostCode, type CostCodingDataSource } from "../src/lib/cost-coding"; -import { resolveInstalledAtCustomer } from "../src/app/api/receipts/intake/route"; +// From the PURE module, not the route: importing the route pulls in +// mobile-auth, which throws at import time unless NEXTAUTH_SECRET is set — +// true in CI, and a unit test has no business needing a JWT secret. +import { resolveInstalledAtCustomer } from "../src/lib/expense-attribution"; // ── the two checks every phase writer must run ───────────────────────────── From 9885f1c72f0780d008d98aaae4689db3539bda1c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 21:37:45 -0700 Subject: [PATCH 077/144] fix(expenses): authorize the expense PUT; fail closed on phases; scope item links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex PR #442 round 3. Items 1, 2 and 8 are Phase 1's and arrive on rebase. 3. [P0] PUT /api/expenses/[id] checked only that SOMEBODY was signed in — no project authorization, no permission. I added tax-return fields to that route, so any authenticated user who knew an expense id could edit the numbers on a state excise return. It now resolves the project and requires access to it (fail closed when there is none), requires `timeClock` to edit an expense at all, and requires `financialReports` on top for installedAtCustomer/taxDeductibleBase — "may edit this expense" and "may decide what the company deducts" are not the same authority. NOTE: DELETE on the same route has the identical pre-existing gap. Untouched here because it predates this PR; flagged for its own fix. 4. The deduction-base invariant is about the RESULTING ROW, not this request's fields. Validating only when taxDeductibleBase was sent meant a PUT that merely LOWERED amount could strand an existing base above the new pre-tax total — the same illegal state through the other door. 5. The item link is scoped to the expense's own estimate or its project. An existence check alone let an edit point at another job's line item, which then feeds the item->costCode fallback. 6. createExpenseCore stored an arbitrary costCodeId and stamped it "manual" (outranking every automated pass) with no project check; receipt-ingest v1 matched a Gemini category against every active company code. Both now go through resolveCostCode + isCostCodeAllowedForProject. RISK, flagged: a change-order expense whose code is not on a phase-eligible estimate item will now be rejected rather than silently miscoded. Correct, but it is a behaviour change on a live path. 7. The backfill's phase check failed OPEN when a project had no mapped phases — absent in exactly the case where we know its phases least. Now requires a positive answer; an unmapped project skips with reason "no-phases". 9. Spec §5 still documented the default-true/false toggle. Rewritten with the as-built tax position, since the mobile repo consumes that section. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 36 ++- package.json | 4 +- scripts/backfill-expense-attribution.mjs | 22 +- src/app/api/expenses/[id]/route.ts | 90 ++++++-- .../api/integrations/receipt-ingest/route.ts | 12 +- src/lib/time-expense-core.ts | 35 ++- tests/backfill-expense-attribution.test.ts | 47 +++- tests/expense-edit-authz.test.ts | 205 ++++++++++++++++++ 8 files changed, 409 insertions(+), 42 deletions(-) create mode 100644 tests/expense-edit-authz.test.ts diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index dc56e25b7..d9e3696ec 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -267,9 +267,10 @@ File-level diff: (`/api/projects/[id]/cost-codes` + `/api/projects/[id]/estimate-items` — already in the site proxy allowlist, src/proxy.ts:25) and `lib/phasePicker.ts` labels (`phaseCodeLabel`); optional but encouraged (inline nudge), never blocking; - (c) add an "Installed at customer job" toggle — default TRUE for a real job, FALSE when - the selected project is Shop/overhead (the app needs the overhead project id; expose it - on the config/session payload the app already loads — executor picks the carrier); + (c) add an "Installed at customer job" toggle. **SUPERSEDED — see the as-built note + below.** This line originally said "default TRUE for a real job, FALSE when the selected + project is Shop/overhead". That default was REMOVED: the toggle now starts UNSET and the + app must send an explicit answer (or nothing at all); (d) when a PHOTO is attached, submit via a new `api.receipts.intake` → `POST /api/receipts/intake` (JSON `fileBase64`, `source:"mobile"`, `projectId`, `costCodeId?`, `installedAtCustomer`) instead of the signed-upload + `/api/expenses` @@ -295,17 +296,37 @@ Server contracts the app can call today: | phase list for the picker | `GET /api/projects/[id]/cost-codes` + `/estimate-items` | already existed, already proxied (`src/proxy.ts`) | | no-photo expense WITH a phase | `POST /api/expenses` now accepts `costCodeId` | **done** — validated through `resolveCostCode` AND `isCostCodeAllowedForProject`, stored with `costCodeSource: "capture"`. Rejections are `{error, code}` with `COST_CODE_NOT_FOUND` / `COST_CODE_INACTIVE` / `PHASE_NOT_ON_PROJECT`, so the app can show a useful message | | photo expense through the pipeline | `POST /api/receipts/intake` accepts `projectId`, `costCodeId`, `installedAtCustomer` | **done** — `installedAtCustomer` is read from JSON (`true`/`false`) or multipart (`"true"`/`"false"`); anything else means "the caller did not say" | -| the toggle's default | server-side `resolveInstalledAtCustomer` | **done** — an explicit value from the app always wins; silence defaults TRUE for a real job, FALSE for the overhead project, and NULL when there is no project. The app should still SHOW the toggle: the server default is a fallback, not a substitute for asking | +| the toggle's answer | server-side `resolveInstalledAtCustomer` (`src/lib/expense-attribution.ts`) | **done, and it has NO DEFAULT** — silence stays NULL on every source, including a receipt filed against a real job. See the tax-position note below. The app MUST show the toggle and send a real answer; there is no server-side fallback to lean on | Auth is unchanged: `/api/receipts/intake` is on the proxy's exact-match public bypass and calls `authenticateMobileOrSession` itself, so the crew Bearer token works with no proxy change. +**TAX POSITION — `installedAtCustomer` has no default (Codex round 2, PR #442).** +An earlier build of this branch defaulted it to TRUE for any non-overhead project, exactly +as §5c above originally specified. That was wrong in the one direction a tax figure must +never fail in: WAC 458-20-102(12)(b) allows the cost of the articles actually RESOLD, and a +receipt coded to a live job is just as likely to be consumables, tools, fuel, dump fees or a +service. Defaulting it turned "nobody looked at this" into a deduction claimed on a state +return. As built: + +* silence is `NULL` on every source — mobile, web, Drive, email, chat; +* `/reports/tax-paid-at-source` counts ONLY an explicit `true`; +* `Expense.taxDeductibleBase` (added in this PR) holds the resold portion of a MIXED + receipt, and the report uses it in place of the pre-tax total when it is set; +* the correction path is `PUT /api/expenses/[id]`, which accepts `installedAtCustomer` and + `taxDeductibleBase` (validated `0 ≤ base ≤ amount − taxAmount` against the ROW the request + leaves behind) and requires the `financialReports` permission on top of project access. + +The mobile PR must therefore ship the toggle as a real three-state question, not as a +pre-answered switch, and `overheadProjectId` on `/api/mobile/me` is now only a UI hint for +which way to lean the copy — it no longer decides anything. + Remaining mobile-repo diff (a separate PR in `gtr-probuild-mobile`): - `apps/mobile/app/(tabs)/expenses.tsx` — (a) default the project dropdown to the clocked-in job; (b) add the phase picker (`lib/phasePicker.ts` labels); - (c) add the "Installed at customer job" toggle, defaulted from - `overheadProjectId`; (d) when a PHOTO is attached, submit via + (c) add the "Installed at customer job" toggle — UNSET by default, never + pre-answered (see the tax-position note below); (d) when a PHOTO is attached, submit via `api.receipts.intake` instead of the signed-upload + `/api/expenses` pair. - `apps/mobile/lib/api.ts` + `lib/api-types.ts` — add `receipts.intake(...)` and the `overheadProjectId` field on the `/me` response type. @@ -340,7 +361,8 @@ Re-run after `--apply` must report 0 changes (backfill-estimate-item-cost-codes - Gate: the `financialReports` permission (`src/lib/permissions.ts:110,173`) — same check pattern as the sibling reports pages. - Data: expenses where `taxAtSource = true AND installedAtCustomer = true AND - taxAmount > 0`, grouped by month (from `date`, company timezone) × project + taxAmount > 0` (all three POSITIVE — a NULL `installedAtCustomer` is "nobody said" and is + never claimed), grouped by month (from `date`, company timezone) × project (`resolveExpenseProjectId`), summing `taxAmount`. Columns: Month, Job, Receipts (count), Taxable amount (Σ `amount` — see risk 1), Tax paid at source (Σ `taxAmount`). Month and grand totals. Period filter, default current quarter. CSV export mirroring the existing diff --git a/package.json b/package.json index b4f1215e0..a52a2b7b6 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 759f6617c..325269ab6 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -183,10 +183,24 @@ export function planBackfill({ // regex was. Same rule the clock-in route enforces via // isCostCodeAllowedForProject — applied here too, because an automated // write has less standing to invent a phase than a human does, not more. - const allowed = allowedCodesByProject.get(resolvedProjectId); - if (costCodeId && allowed && !allowed.has(costCodeId)) { - add(expense, "phase-not-on-project"); - continue; + // + // FAILS CLOSED. An earlier version wrote `allowed && !allowed.has(...)`, + // which skipped the rejection entirely when the map had no entry for + // the project — and it has no entry in exactly one case: the job has no + // coded estimate items at all, i.e. the job whose phases we know the + // LEAST about. That is the one where a global code match is most likely + // to be wrong, so it now needs a positive answer, not the absence of a + // negative one. + if (costCodeId) { + const allowed = allowedCodesByProject.get(resolvedProjectId); + if (!allowed || allowed.size === 0) { + add(expense, "no-phases"); + continue; + } + if (!allowed.has(costCodeId)) { + add(expense, "phase-not-on-project"); + continue; + } } if (suggestion && costCodeId && suggestion.confidence >= MIN_CONFIDENCE) { codeFills.push({ diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 04dc4d385..bdc301d61 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; import { getServerSession } from "next-auth/next"; import { authOptions } from "@/lib/auth"; +import { canAccessProject, getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; import { QboManagedExpenseError, assertExpenseMutableOutsideQbo, @@ -39,8 +40,19 @@ export async function DELETE(req: NextRequest, { params }: { params: Promise<{ i export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { - const session = await getServerSession(authOptions); - if (!session?.user?.email) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + // AUTHORIZATION, not merely authentication. This route checked only + // that SOMEBODY was signed in, so any authenticated user who knew an + // expense id could rewrite it — and once it started accepting + // `installedAtCustomer` and `taxDeductibleBase`, that meant editing the + // numbers on a state excise return. The POST on this resource has + // always resolved the project and checked access; the PUT now does the + // same, plus the `timeClock` permission that /projects/[id]/time-expenses + // and deleteExpenses already require to touch an expense at all. + const user = await getCurrentUserWithPermissions(); + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + if (!hasPermission(user, "timeClock")) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } const id = (await params).id; if (!id) return NextResponse.json({ error: "id is required" }, { status: 400 }); @@ -51,18 +63,41 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: qbPurchaseId: true, amount: true, taxAmount: true, + taxDeductibleBase: true, + estimateId: true, projectId: true, estimate: { select: { projectId: true } }, }, }); assertExpenseMutableOutsideQbo(expense); if (!expense) return NextResponse.json({ error: "Expense not found" }, { status: 404 }); + + // Fail CLOSED on an unattributable row: with no project there is no + // scope to authorize against, so nobody may edit it here. + const resolvedProjectId = resolveExpenseProjectId(expense); + if (!resolvedProjectId || !canAccessProject(user, resolvedProjectId)) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + const body = await req.json(); + // #5: an item link must belong to THIS expense's job. Checking only + // that the id exists let an edit point the expense at a line item on + // another project — which then feeds the item->costCode fallback and + // silently books the phase of a different job. if (body.itemId) { - const itemExists = await prisma.estimateItem.findUnique({ where: { id: body.itemId }, select: { id: true } }); + const itemExists = await prisma.estimateItem.findFirst({ + where: { + id: body.itemId, + OR: [ + { estimateId: expense.estimateId }, + { estimate: { projectId: resolvedProjectId } }, + ], + }, + select: { id: true }, + }); if (!itemExists) { - return NextResponse.json({ error: "This cost code is unsaved. Please click 'Save' on the Estimate first before moving an expense to it." }, { status: 400 }); + return NextResponse.json({ error: "That line item isn't on this project's estimates. Save the Estimate on the web first, or pick a line item from this job." }, { status: 400 }); } } @@ -75,7 +110,6 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const editsCostCode = Object.prototype.hasOwnProperty.call(body, "costCodeId"); const nextCostCodeId: string | null = typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null; - const resolvedProjectId = resolveExpenseProjectId(expense); if (editsCostCode && nextCostCodeId) { // BOTH checks, per the SCOPE note on resolveCostCode: existence and // active-ness are ATTRIBUTION, "this code belongs to this job" is @@ -147,25 +181,45 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: { status: 400 }, ); } - // Validated against the amount this request LEAVES on the row, not - // the one it started with — a PUT that lowers the amount and sets a - // base in the same call must not be able to slip past by being - // checked against the old, larger figure. - const nextAmount = - body.amount !== undefined && body.amount !== null - ? Number(body.amount) - : Number(expense.amount); - const tax = Number(expense.taxAmount ?? 0); - const ceiling = Math.round((nextAmount - tax) * 100) / 100; - if (!Number.isFinite(ceiling) || parsed > ceiling) { + nextBase = parsed; + } + + // Only the money permission may move numbers that land on a tax return. + // `timeClock` above is "may edit this expense"; this is "may decide what + // the company deducts", and they are not the same authority. + if ((editsInstalled || editsBase) && !hasPermission(user, "financialReports")) { + return NextResponse.json( + { error: "Editing tax-deduction fields requires the Financial Reports permission." }, + { status: 403 }, + ); + } + + // THE INVARIANT IS ABOUT THE RESULTING ROW, not about this request's + // fields. Validating only when `taxDeductibleBase` was sent meant a PUT + // that merely LOWERED `amount` could leave an existing base above the + // new pre-tax total — the same impossible state, reached by the other + // door. So the whole row is re-checked whenever any input to the + // invariant moves. + const resultingAmount = + body.amount !== undefined && body.amount !== null + ? Number(body.amount) + : Number(expense.amount); + const resultingTax = Number(expense.taxAmount ?? 0); + const resultingBase = editsBase + ? nextBase + : (expense.taxDeductibleBase === null ? null : Number(expense.taxDeductibleBase)); + if (resultingBase !== null) { + const ceiling = Math.round((resultingAmount - resultingTax) * 100) / 100; + if (!Number.isFinite(ceiling) || resultingBase > ceiling) { return NextResponse.json( { - error: `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).`, + error: editsBase + ? `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).` + : `This amount would leave a deduction base of ${resultingBase.toFixed(2)} above the pre-tax total (${ceiling.toFixed(2)}). Clear or lower the deduction base first.`, }, { status: 400 }, ); } - nextBase = parsed; } const updatedExpense = await prisma.expense.update({ diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index eb47d98cd..14223ae75 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -1,5 +1,7 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { resolveProjectPhaseCodes } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { matchProjectByName, matchCostCode } from "@/lib/project-match"; export const dynamic = "force-dynamic"; @@ -78,10 +80,12 @@ export async function POST(req: Request) { return NextResponse.json({ ok: false, reason: "project-has-no-estimate", projectId: project.id }); } - const costCodes = await prisma.costCode.findMany({ - where: { isActive: true }, - select: { id: true, code: true, name: true }, - }); + // The PROJECT's phases, not every active company code. Matching a Gemini + // category string against the whole company list let a Drive import book a + // phase that exists only on some other job — and this path writes the code + // straight onto the expense. + const costCodes = (await resolveProjectPhaseCodes(prismaPhaseDataSource, project.id)) + .map((phase) => ({ id: phase.id, code: phase.code, name: phase.name })); const isCheck = String(body.docType || "receipt").toLowerCase() === "check"; const docRef = isCheck diff --git a/src/lib/time-expense-core.ts b/src/lib/time-expense-core.ts index a818c89ab..3eca72691 100644 --- a/src/lib/time-expense-core.ts +++ b/src/lib/time-expense-core.ts @@ -1,4 +1,8 @@ import { prisma } from "./prisma"; +import { resolveCostCode } from "./cost-coding"; +import { prismaCostCodingDataSource } from "./cost-coding-db"; +import { isCostCodeAllowedForProject } from "./project-phases"; +import { prismaPhaseDataSource } from "./project-phases-db"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "./company-timezone"; import { resolveScheduleTaskIdForPunch } from "./punch-task-binding"; import { toCompanyDayKey } from "./company-day"; @@ -163,6 +167,31 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str } } + // A cost code arriving here used to be stored verbatim and stamped + // "manual" — permanently outranking every automated pass — without anyone + // checking it belonged to this job. "The cost code exists" is not a + // permission (src/lib/cost-coding.ts SCOPE note): both checks, or a form + // post can pin another project's phase onto this expense forever. + let costCodeId = data.costCodeId || null; + let costTypeId = data.costTypeId || null; + if (costCodeId) { + const resolved = await resolveCostCode(prismaCostCodingDataSource, { costCodeId }); + if (!resolved.ok) throw new Error(resolved.error); + const onProject = await isCostCodeAllowedForProject( + prismaPhaseDataSource, + estimate.projectId, + resolved.costCodeId, + ); + if (!onProject) { + throw new Error("That cost code isn't one of this project's phases."); + } + costCodeId = resolved.costCodeId; + // Only an estimate item knows whether the money is Labor or Material, + // so an explicit code carries no cost type of its own — keep the + // caller's when it gave one, rather than inventing a guess. + costTypeId = costTypeId ?? resolved.costTypeId; + } + let receiptUrl = data.receiptUrl?.trim() || null; if (data.receiptFileId) { const receipt = await prisma.projectFile.findUnique({ @@ -188,12 +217,12 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str // above (including the change-order cross-check). projectId: estimate.projectId, itemId: data.itemId || null, - costCodeId: data.costCodeId || null, - costTypeId: data.costTypeId || null, + costCodeId, + costTypeId, // Every caller of this core is a human picking a code in a web form // or a CO flow, so a code here is "manual" and is off limits to the // sync and the backfill. - costCodeSource: data.costCodeId ? "manual" : null, + costCodeSource: costCodeId ? "manual" : null, amount: dollars(data.amount), vendor: data.vendor?.trim() || null, date: expenseDate, diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 2fc4cfa42..37dc1bc17 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -54,6 +54,17 @@ function expense(overrides: Partial = {}): StubExpense { const NO_ITEMS = new Map(); +/** + * The jobs' own phases. Required for a suggestion to be written at all — the + * check fails CLOSED, so a test that omits this is asserting the refusal path. + */ +const ALL_PHASES = new Map([ + ["job-1", new Set(["cc-plumb", "cc-frame"])], + ["job-2", new Set(["cc-plumb", "cc-frame"])], + ["job-closed", new Set(["cc-plumb", "cc-frame"])], + ["overhead-project", new Set(["cc-plumb", "cc-frame"])], +]); + const COST_CODE_IDS = new Map([ ["03-PLUMB", "cc-plumb"], ["02-FRAME", "cc-frame"], @@ -254,6 +265,7 @@ test("a dangling itemId falls through to the rules rather than being skipped", ( items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], + allowedCodesByProject: ALL_PHASES, }); assert.equal(plan.codeFills.length, 1); assert.equal(plan.codeFills[0].costCodeSource, "ai"); @@ -292,6 +304,7 @@ test("the overhead bucket and closed jobs are out of the suggester's scope", () items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], + allowedCodesByProject: ALL_PHASES, }); assert.deepEqual(plan.codeFills.map(f => f.id), ["active"]); assert.deepEqual(plan.remainder.map(e => e.id).sort(), ["closed", "overhead"]); @@ -334,6 +347,7 @@ test("the projected 'after' applies the plan without touching the database", () items: NO_ITEMS, costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], + allowedCodesByProject: ALL_PHASES, }); assert.equal(measureCoverage(rows).attributed, 0); assert.equal(measureCoverage(projectedRows(rows, plan.codeFills)).attributed, 400); @@ -377,10 +391,13 @@ test("the remainder CSV neutralizes formulas — vendor names here are OCR outpu // ── the run ───────────────────────────────────────────────────────────────── test("a dry run makes ZERO write calls", async () => { - const stub = createStub([ - expense({ id: "e1", vendor: "Summit Plumbing" }), - expense({ id: "e2", projectId: null, estimate: { projectId: "job-1" } }), - ]); + const stub = createStub( + [ + expense({ id: "e1", vendor: "Summit Plumbing" }), + expense({ id: "e2", projectId: null, estimate: { projectId: "job-1" } }), + ], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); const result = await runBackfill({ db: stub.db, apply: false, log: () => {}, overheadProjectId: OVERHEAD_ID }); assert.equal(stub.writes.length, 0, "dry run is the default and it must be inert"); assert.equal(result.written.projectIds, 0); @@ -484,6 +501,28 @@ test("a suggested phase the JOB does not have is refused", () => { assert.equal(plan.remainder[0].reason, "phase-not-on-project"); }); +test("a project with NO mapped phases fails CLOSED", () => { + // The map has no entry for a job with no coded estimate items — i.e. the + // job whose phases we know the LEAST about, and the one where a globally + // matched code is most likely to be wrong. An earlier version treated the + // missing entry as "no opinion" and wrote the code anyway. + for (const allowedCodesByProject of [ + new Map>(), + new Map([["job-1", new Set()]]), + undefined, + ]) { + const plan = planBackfill({ + expenses: [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], + items: NO_ITEMS, + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + ...(allowedCodesByProject ? { allowedCodesByProject } : {}), + }); + assert.deepEqual(plan.codeFills, [], "nothing may be written without a positive answer"); + assert.equal(plan.remainder[0].reason, "no-phases"); + } +}); + test("a suggested phase the job DOES have is accepted", () => { const plan = planBackfill({ expenses: [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts new file mode 100644 index 000000000..34561ccce --- /dev/null +++ b/tests/expense-edit-authz.test.ts @@ -0,0 +1,205 @@ +/** + * PUT /api/expenses/[id] — authorization and the deduction-base invariant + * (Codex PR #442 round 3, items 3, 4 and 5). + * + * The route checked only that SOMEBODY was signed in. Once it started accepting + * `installedAtCustomer` and `taxDeductibleBase`, that meant any authenticated + * user who knew an expense id could edit the numbers on a state excise return. + * + * Prisma, next-auth and the permission reader are patched at require() time — + * the same shape as tests/job-variance-db.test.ts. No mock.module: CI is + * Node 20. + */ +import { test, before, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import Module from "node:module"; + +interface FakeUser { + id: string; + role: string; + permissions: Record | null; + projectIds: string[]; +} + +let currentUser: FakeUser | null; +let storedExpense: Record | null; +let updateArgs: { where: unknown; data: Record } | null; +let estimateItems: { id: string; estimateId: string; projectId: string | null }[]; + +const fakePrisma = { + expense: { + findUnique: async () => storedExpense, + update: async (args: { where: unknown; data: Record }) => { + updateArgs = args; + return { id: "e1", ...args.data }; + }, + }, + estimateItem: { + findFirst: async (args: { where: Record }) => { + const { id, OR } = args.where; + const item = estimateItems.find(candidate => candidate.id === id); + if (!item) return null; + const branches = (OR ?? []) as Record[]; + const ok = branches.some(branch => + branch.estimateId !== undefined + ? branch.estimateId === item.estimateId + : branch.estimate?.projectId === item.projectId, + ); + return ok ? { id: item.id } : null; + }, + findUnique: async (args: { where: { id: string } }) => { + const item = estimateItems.find(candidate => candidate.id === args.where.id); + return item ? { id: item.id } : null; + }, + }, + costCode: { findUnique: async () => null }, +}; + +let PUT: (req: any, ctx: { params: Promise<{ id: string }> }) => Promise; + +before(async () => { + const originalRequire = Module.prototype.require; + (Module.prototype as unknown as { require: (id: string) => unknown }).require = function ( + this: NodeModule, + id: string, + ) { + if (id === "@/lib/prisma") return { prisma: fakePrisma }; + if (id === "@/lib/permissions") { + return { + getCurrentUserWithPermissions: async () => currentUser, + hasPermission: (user: FakeUser | null, key: string) => + !!user && (user.role === "ADMIN" || user.permissions?.[key] === true), + canAccessProject: (user: FakeUser, projectId: string) => + user.role === "ADMIN" || user.projectIds.includes(projectId), + }; + } + if (id === "next-auth/next") return { getServerSession: async () => ({ user: { email: "x@y.z" } }) }; + if (id === "@/lib/auth") return { authOptions: {} }; + // eslint-disable-next-line prefer-rest-params + return originalRequire.apply(this, arguments as unknown as [string]); + } as typeof Module.prototype.require; + + let mod: any; + try { + mod = await import("../src/app/api/expenses/[id]/route"); + } finally { + Module.prototype.require = originalRequire; + } + if (typeof mod.PUT !== "function") throw new Error("expense-edit-authz: mocks did not apply"); + PUT = mod.PUT; +}); + +beforeEach(() => { + currentUser = { id: "u1", role: "MANAGER", permissions: { timeClock: true, financialReports: true }, projectIds: ["job-1"] }; + storedExpense = { + qbPurchaseId: null, + amount: 207.74, + taxAmount: 16.55, + taxDeductibleBase: null, + estimateId: "est-job-1", + projectId: "job-1", + estimate: { projectId: "job-1" }, + }; + updateArgs = null; + estimateItems = [ + { id: "item-own", estimateId: "est-job-1", projectId: "job-1" }, + { id: "item-elsewhere", estimateId: "est-job-2", projectId: "job-2" }, + ]; +}); + +function call(body: Record) { + return PUT({ json: async () => body } as any, { params: Promise.resolve({ id: "e1" }) }); +} + +// ── item 3: authorization ────────────────────────────────────────────────── + +test("a signed-in user with no access to the job cannot edit the expense", async () => { + currentUser = { id: "u2", role: "FIELD_CREW", permissions: { timeClock: true }, projectIds: ["other-job"] }; + const res = await call({ vendor: "Nope" }); + assert.equal(res.status, 403); + assert.equal(updateArgs, null, "and nothing is written"); +}); + +test("a user without the timeClock permission cannot edit an expense at all", async () => { + currentUser = { id: "u3", role: "FIELD_CREW", permissions: {}, projectIds: ["job-1"] }; + const res = await call({ vendor: "Nope" }); + assert.equal(res.status, 403); + assert.equal(updateArgs, null); +}); + +test("no session at all is 401, not 403", async () => { + currentUser = null; + assert.equal((await call({ vendor: "Nope" })).status, 401); +}); + +test("an expense with no resolvable project fails CLOSED", async () => { + storedExpense = { ...storedExpense, projectId: null, estimate: { projectId: null } }; + const res = await call({ vendor: "Nope" }); + assert.equal(res.status, 403, "no scope to authorize against means nobody may edit it"); +}); + +test("editing the tax fields needs financialReports on top of project access", async () => { + currentUser = { id: "u4", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; + const res = await call({ installedAtCustomer: true }); + assert.equal(res.status, 403); + assert.equal(updateArgs, null); + // ...while an ordinary field edit by the same user still works. + assert.equal((await call({ vendor: "Fine" })).status, 200); +}); + +test("a permitted bookkeeper can record the tax answer and the allocation", async () => { + const res = await call({ installedAtCustomer: true, taxDeductibleBase: 50 }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, true); + assert.equal(updateArgs?.data.taxDeductibleBase, 50); +}); + +test("installedAtCustomer can be set back to unknown", async () => { + assert.equal((await call({ installedAtCustomer: null })).status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, null); + assert.equal((await call({ installedAtCustomer: "yes" })).status, 400, "and only a real tri-state is accepted"); +}); + +// ── item 4: the invariant is about the RESULTING row ─────────────────────── + +test("a base above the pre-tax total is refused", async () => { + // 207.74 gross − 16.55 tax = 191.19. + assert.equal((await call({ taxDeductibleBase: 191.20 })).status, 400); + assert.equal((await call({ taxDeductibleBase: 191.19 })).status, 200); +}); + +test("LOWERING the amount cannot strand an impossible base", async () => { + // The other door into the same illegal state: this request never mentions + // taxDeductibleBase, so the old check did not run at all. + storedExpense = { ...storedExpense, taxDeductibleBase: 150 }; + const res = await call({ amount: "100.00" }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); + const body = await res.json(); + assert.match(body.error, /deduction base/i); +}); + +test("lowering the amount is fine when the resulting row still holds", async () => { + storedExpense = { ...storedExpense, taxDeductibleBase: 50 }; + assert.equal((await call({ amount: "100.00" })).status, 200); +}); + +test("both fields moving together are checked against each other", async () => { + // A PUT that lowers the amount AND sets a base must be judged on the pair, + // not on the amount it started with. + assert.equal((await call({ amount: "60.00", taxDeductibleBase: 55 })).status, 400); + assert.equal((await call({ amount: "60.00", taxDeductibleBase: 40 })).status, 200); +}); + +// ── item 5: the item link may not cross jobs ─────────────────────────────── + +test("a line item from another project is refused", async () => { + const res = await call({ itemId: "item-elsewhere" }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); + +test("a line item on this job's estimate is accepted", async () => { + assert.equal((await call({ itemId: "item-own" })).status, 200); + assert.equal(updateArgs?.data.itemId, "item-own"); +}); From dc98f68b9ac2949b1bceb856ad31343ff92429af Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:02:22 -0700 Subject: [PATCH 078/144] fix(expenses): tax corrections get their own PATCH; DELETE authorized; validated tax only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex PR #442 round 4. Items 1, 2, 5 and 6 are Phase 1's and arrive on rebase. 8. DELETE had the same session-only gap PUT had — any authenticated user with an id could destroy any non-QBO expense on any job. Same gate as PUT now: timeClock, resolved project, fail closed when there is none. 3. The correction path could not reach a single row it was built for. PUT is guarded by assertExpenseMutableOutsideQbo, and every pipeline expense carries a qbPurchaseId — precisely the population the tax report reads. Split into a dedicated PATCH that edits ONLY installedAtCustomer, taxDeductibleBase and costCodeId. Those three are ProBuild-only bookkeeping: nothing syncs them to QuickBooks and nothing in QBO overwrites them, so the mutability guard does not apply. amount/vendor/date are refused there at any status. PUT keeps its guard, rejects the tax fields outright (a silent ignore would look like a successful correction), and is now a PARTIAL update — it used to null every field a request left out, so a tax-only edit erased the vendor, date and description. 7. allowedCodesByProject is built from the app's own phase-eligible set (PHASE_ELIGIBLE_ESTIMATE_WHERE, active codes only), not from every coded estimate item. The fail-closed check was looser than it claimed: a code from a draft or archived estimate counted as a phase of the job. The item->code fallback now passes the same gate, instead of bypassing it. 4. Booking persists tax ONLY when buildGroups accepted it. It was storing the raw OCR read even for a check or a nonsense tax >= total, with taxAtSource true — so a misread no human saw could be claimed on an excise return, and amount - taxAmount could go negative. The rejected value now lives only on ReceiptIntake.taxCents, which the report cannot read. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.mjs | 43 +++- src/app/api/expenses/[id]/route.ts | 269 ++++++++++++++++----- src/lib/receipt-intake/book.ts | 29 ++- tests/backfill-expense-attribution.test.ts | 17 ++ tests/expense-edit-authz.test.ts | 124 ++++++++-- 5 files changed, 389 insertions(+), 93 deletions(-) diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 325269ab6..a4ab5531d 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -44,6 +44,7 @@ import { resolveExpenseProjectId, } from "../src/lib/expense-attribution.ts"; import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project.ts"; +import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases.ts"; import { csvCell, csvNumber } from "../src/lib/csv-safe.ts"; /** @@ -156,6 +157,19 @@ export function planBackfill({ add(expense, "item-outside-estimate"); continue; } + // The item link proves the job; it does NOT prove the code is a + // live phase of that job. An item on a draft estimate can carry a + // code the job never committed to, so the same phase gate the + // suggester passes through applies here too. + const allowedForItem = allowedCodesByProject.get(resolvedProjectId); + if (!allowedForItem || allowedForItem.size === 0) { + add(expense, "no-phases"); + continue; + } + if (!allowedForItem.has(item.costCodeId)) { + add(expense, "phase-not-on-project"); + continue; + } codeFills.push({ id: expense.id, costCodeId: item.costCodeId, @@ -304,7 +318,7 @@ export async function runBackfill({ // The item's OWN estimate and project come back with it: the fallback has // to prove the link does not cross jobs before it copies a code. const itemRowsRaw = await db.estimateItem.findMany({ - where: { costCodeId: { not: null } }, + where: { costCodeId: { not: null }, costCode: { isActive: true } }, select: { id: true, costCodeId: true, estimateId: true, estimate: { select: { projectId: true } }, @@ -323,13 +337,30 @@ export async function runBackfill({ }])); const itemCostCodeById = new Map(itemRows.map(row => [row.id, row.costCodeId])); - // The phases each job actually has. Mirrors resolveProjectPhaseCodes' - // estimate-item half (src/lib/project-phases.ts) — the Safety phase is - // deliberately absent, because a materials receipt is never a safety + // THE PHASES EACH JOB ACTUALLY HAS — the same set the app itself uses. + // + // An earlier version built this from `itemRows` above, which is every coded + // estimate item on any estimate at all. That let a code from a DRAFT or + // ARCHIVED estimate, or an INACTIVE cost code, count as "a phase of this + // job" — so the fail-closed check was looser than it claimed, and looser + // than the clock-in route it is supposed to mirror. This now applies + // PHASE_ELIGIBLE_ESTIMATE_WHERE and `costCode.isActive`, exactly as + // resolveProjectPhaseCodes does. + // + // The Safety phase is deliberately NOT appended: resolveProjectPhaseCodes + // adds it for In Progress jobs, but a materials receipt is never a safety // meeting and this pass has no business assigning one. + const phaseRows = await db.estimateItem.findMany({ + where: { + costCodeId: { not: null }, + costCode: { isActive: true }, + estimate: { ...PHASE_ELIGIBLE_ESTIMATE_WHERE }, + }, + select: { costCodeId: true, estimate: { select: { projectId: true } } }, + }); const allowedCodesByProject = new Map(); - for (const row of itemRows) { - const projectId = row.projectId; + for (const row of phaseRows) { + const projectId = row.estimate?.projectId ?? null; if (!projectId || !row.costCodeId) continue; if (!allowedCodesByProject.has(projectId)) allowedCodesByProject.set(projectId, new Set()); allowedCodesByProject.get(projectId).add(row.costCodeId); diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index bdc301d61..a0839930b 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -15,17 +15,37 @@ import { prismaPhaseDataSource } from "@/lib/project-phases-db"; export async function DELETE(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { - const session = await getServerSession(authOptions); - if (!session?.user?.email) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + // Same gate as PUT. Deleting somebody's expense is at least as + // consequential as editing it, and this checked only that SOMEBODY was + // signed in — so any authenticated user with an id could destroy any + // non-QBO expense on any job. + const user = await getCurrentUserWithPermissions(); + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + if (!hasPermission(user, "timeClock")) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } const id = (await params).id; if (!id) return NextResponse.json({ error: "id is required" }, { status: 400 }); const expense = await prisma.expense.findUnique({ where: { id }, - select: { qbPurchaseId: true }, + select: { + qbPurchaseId: true, + projectId: true, + estimate: { select: { projectId: true } }, + }, }); assertExpenseMutableOutsideQbo(expense); + if (!expense) return NextResponse.json({ error: "Expense not found" }, { status: 404 }); + + // Fail CLOSED: with no resolvable project there is no scope to + // authorize against, so nobody may delete it here. + const projectId = resolveExpenseProjectId(expense); + if (!projectId || !canAccessProject(user, projectId)) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + await prisma.expense.deleteMany({ where: { id, qbPurchaseId: null } }); return NextResponse.json({ success: true }); @@ -150,15 +170,156 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: } } - // ── the tax-deduction correction path (Phase 3, §7) ───────────────── - // - // Nothing defaults `installedAtCustomer` any more, so this route is how - // an unreviewed receipt becomes a claimable one. It is the ONLY place a - // human answer can be recorded after capture, which is why it validates - // rather than trusts: a deduction base larger than the pre-tax total is - // a filing error, and it must be refused at the write rather than - // clamped later where nobody would see it. - const editsInstalled = Object.prototype.hasOwnProperty.call(body, "installedAtCustomer"); + // The tax-deduction fields are NOT editable here — the PATCH below is + // their single writer, because this handler's QBO-mutability guard + // excludes exactly the pipeline rows the tax report is made of. A + // silent ignore would look like a successful correction. + for (const field of ["installedAtCustomer", "taxDeductibleBase"]) { + if (Object.prototype.hasOwnProperty.call(body, field)) { + return NextResponse.json( + { error: `Use PATCH on this expense to edit ${field}.` }, + { status: 400 }, + ); + } + } + + // ...but this route CAN change `amount`, and the deduction invariant is + // about the RESULTING ROW rather than about the fields this request + // names. A PUT that merely LOWERS the amount can strand an existing + // base above the new pre-tax total — the same impossible state reached + // through the other door. + const resultingAmount = + body.amount !== undefined && body.amount !== null + ? Number(body.amount) + : Number(expense.amount); + const existingBase = + expense.taxDeductibleBase === null ? null : Number(expense.taxDeductibleBase); + if (existingBase !== null) { + const ceiling = + Math.round((resultingAmount - Number(expense.taxAmount ?? 0)) * 100) / 100; + if (!Number.isFinite(ceiling) || existingBase > ceiling) { + return NextResponse.json( + { + error: `This amount would leave a deduction base of ${existingBase.toFixed(2)} above the pre-tax total (${ceiling.toFixed(2)}). Clear or lower the deduction base first.`, + }, + { status: 400 }, + ); + } + } + + + // PARTIAL UPDATE. This used to write `body.vendor || null` and friends + // unconditionally, so any request that did not resend every field wiped + // the ones it left out — a tax-only edit erased the vendor, the date + // and the description. `undefined` tells Prisma "leave it alone"; + // an explicitly-sent null still clears the field. + const has = (key: string) => Object.prototype.hasOwnProperty.call(body, key); + const updatedExpense = await prisma.expense.update({ + where: { id }, + data: { + amount: body.amount ? parseFloat(body.amount) : undefined, + vendor: has("vendor") ? (body.vendor || null) : undefined, + date: has("date") ? (body.date ? new Date(body.date) : null) : undefined, + description: has("description") ? (body.description || null) : undefined, + itemId: has("itemId") ? (body.itemId || null) : undefined, + ...(editsCostCode + ? { + costCodeId: nextCostCodeId, + // Clearing the code clears the provenance with it — + // leaving "manual" on a null code would guard a row + // that has nothing to guard. + costCodeSource: nextCostCodeId ? "manual" : null, + costCodeConfidence: null, + } + : {}), + }, + }); + + return NextResponse.json(updatedExpense); + } catch (error) { + if (error instanceof QboManagedExpenseError) { + return NextResponse.json({ error: error.message }, { status: 409 }); + } + console.error("Error updating expense:", error); + return NextResponse.json({ error: "Failed to update expense" }, { status: 500 }); + } +} + +/** + * The TAX-CORRECTION path (Codex round 4, item 3). + * + * Split out from PUT because PUT cannot serve it. PUT is guarded by + * `assertExpenseMutableOutsideQbo`, and every expense the receipt pipeline + * creates carries a `qbPurchaseId` — which is precisely the population the tax + * report reads. The correction path therefore could not reach a single row it + * was built for. + * + * The guard is right for PUT and wrong here, and the reason is what these three + * columns ARE: `installedAtCustomer`, `taxDeductibleBase` and `costCodeId` are + * ProBuild-only bookkeeping. Nothing syncs them to QuickBooks and nothing in + * QuickBooks overwrites them, so editing them cannot desynchronise a Purchase. + * `amount`, `vendor` and `date` would, which is why they are not accepted here + * at ANY status — this handler touches nothing else, on purpose. + */ +export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { + try { + const user = await getCurrentUserWithPermissions(); + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + const id = (await params).id; + if (!id) return NextResponse.json({ error: "id is required" }, { status: 400 }); + + const expense = await prisma.expense.findUnique({ + where: { id }, + select: { + amount: true, + taxAmount: true, + taxDeductibleBase: true, + estimateId: true, + projectId: true, + estimate: { select: { projectId: true } }, + }, + }); + if (!expense) return NextResponse.json({ error: "Expense not found" }, { status: 404 }); + + const projectId = resolveExpenseProjectId(expense); + if (!projectId || !canAccessProject(user, projectId)) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + const body = await req.json(); + const has = (key: string) => Object.prototype.hasOwnProperty.call(body, key); + + // Nothing outside the three ProBuild-only columns. A caller that sends + // `amount` here is either confused or probing; either way it must be + // told, not silently ignored. + const allowed = new Set(["installedAtCustomer", "taxDeductibleBase", "costCodeId"]); + const rejected = Object.keys(body).filter(key => !allowed.has(key)); + if (rejected.length) { + return NextResponse.json( + { error: `This endpoint only edits ${[...allowed].join(", ")}. Rejected: ${rejected.join(", ")}.` }, + { status: 400 }, + ); + } + if (!rejected.length && Object.keys(body).length === 0) { + return NextResponse.json({ error: "Nothing to update." }, { status: 400 }); + } + + const editsInstalled = has("installedAtCustomer"); + const editsBase = has("taxDeductibleBase"); + const editsCostCode = has("costCodeId"); + + // The money permission governs anything that lands on a tax return. + if ((editsInstalled || editsBase) && !hasPermission(user, "financialReports")) { + return NextResponse.json( + { error: "Editing tax-deduction fields requires the Financial Reports permission." }, + { status: 403 }, + ); + } + if (editsCostCode && !hasPermission(user, "timeClock")) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + let nextInstalled: boolean | null = null; if (editsInstalled) { const raw = body.installedAtCustomer; @@ -171,7 +332,6 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: nextInstalled = raw; } - const editsBase = Object.prototype.hasOwnProperty.call(body, "taxDeductibleBase"); let nextBase: number | null = null; if (editsBase && body.taxDeductibleBase !== null) { const parsed = Number(body.taxDeductibleBase); @@ -184,73 +344,70 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: nextBase = parsed; } - // Only the money permission may move numbers that land on a tax return. - // `timeClock` above is "may edit this expense"; this is "may decide what - // the company deducts", and they are not the same authority. - if ((editsInstalled || editsBase) && !hasPermission(user, "financialReports")) { - return NextResponse.json( - { error: "Editing tax-deduction fields requires the Financial Reports permission." }, - { status: 403 }, - ); - } - - // THE INVARIANT IS ABOUT THE RESULTING ROW, not about this request's - // fields. Validating only when `taxDeductibleBase` was sent meant a PUT - // that merely LOWERED `amount` could leave an existing base above the - // new pre-tax total — the same impossible state, reached by the other - // door. So the whole row is re-checked whenever any input to the - // invariant moves. - const resultingAmount = - body.amount !== undefined && body.amount !== null - ? Number(body.amount) - : Number(expense.amount); - const resultingTax = Number(expense.taxAmount ?? 0); + // Same invariant as PUT, judged on the row this request leaves behind. const resultingBase = editsBase ? nextBase : (expense.taxDeductibleBase === null ? null : Number(expense.taxDeductibleBase)); if (resultingBase !== null) { - const ceiling = Math.round((resultingAmount - resultingTax) * 100) / 100; + const ceiling = + Math.round((Number(expense.amount) - Number(expense.taxAmount ?? 0)) * 100) / 100; if (!Number.isFinite(ceiling) || resultingBase > ceiling) { return NextResponse.json( - { - error: editsBase - ? `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).` - : `This amount would leave a deduction base of ${resultingBase.toFixed(2)} above the pre-tax total (${ceiling.toFixed(2)}). Clear or lower the deduction base first.`, - }, + { error: `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).` }, { status: 400 }, ); } } - const updatedExpense = await prisma.expense.update({ + let nextCostCodeId: string | null = null; + if (editsCostCode) { + nextCostCodeId = + typeof body.costCodeId === "string" && body.costCodeId.trim() + ? body.costCodeId.trim() + : null; + if (nextCostCodeId) { + const resolved = await resolveCostCode(prismaCostCodingDataSource, { + costCodeId: nextCostCodeId, + }); + if (!resolved.ok) { + return NextResponse.json( + { error: resolved.error, code: resolved.code }, + { status: resolved.status }, + ); + } + const onProject = await isCostCodeAllowedForProject( + prismaPhaseDataSource, + projectId, + resolved.costCodeId, + ); + if (!onProject) { + return NextResponse.json( + { error: "That cost code isn't one of this project's phases.", code: "PHASE_NOT_ON_PROJECT" }, + { status: 400 }, + ); + } + nextCostCodeId = resolved.costCodeId; + } + } + + const updated = await prisma.expense.update({ where: { id }, data: { - amount: body.amount ? parseFloat(body.amount) : undefined, - vendor: body.vendor || null, - date: body.date ? new Date(body.date) : null, - description: body.description || null, - itemId: body.itemId || null, + ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), + ...(editsBase ? { taxDeductibleBase: nextBase } : {}), ...(editsCostCode ? { costCodeId: nextCostCodeId, - // Clearing the code clears the provenance with it — - // leaving "manual" on a null code would guard a row - // that has nothing to guard. costCodeSource: nextCostCodeId ? "manual" : null, costCodeConfidence: null, } : {}), - ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), - ...(editsBase ? { taxDeductibleBase: nextBase } : {}), }, }); - return NextResponse.json(updatedExpense); + return NextResponse.json(updated); } catch (error) { - if (error instanceof QboManagedExpenseError) { - return NextResponse.json({ error: error.message }, { status: 409 }); - } - console.error("Error updating expense:", error); + console.error("Error correcting expense:", error); return NextResponse.json({ error: "Failed to update expense" }, { status: 500 }); } } diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 994412133..a02b2fe30 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -656,15 +656,26 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro costCodeId, costCodeSource, costCodeConfidence, - // The tax the VENDOR charged, from the read — deliberately - // not `taxApplied`. `taxApplied` is what the QBO Purchase - // split onto the reclaimable account, and buildGroups zeroes - // it for a check or a nonsense read. The WA deduction is - // based on tax actually paid at the register, so the receipt's - // own figure is the right one here, and the two are allowed - // to disagree. - taxAmount: row.taxCents !== null ? row.taxCents / 100 : null, - taxAtSource: row.taxAtSource, + // ONLY TAX `buildGroups` ACCEPTED (Codex round 4). + // + // An earlier version stored `row.taxCents` — the raw read — + // on the reasoning that the WA deduction is about tax paid + // at the register, not about what QuickBooks split. That is + // true in principle and wrong in practice: `buildGroups` + // REJECTS a tax read on a check, and rejects a nonsense one + // (tax >= total), and those rejected values were still + // landing on the Expense with `taxAtSource` true. The tax + // report reads exactly those two columns, so an OCR misread + // no human ever saw could be claimed on an excise return — + // and `amount - taxAmount` could even go negative. + // + // `taxApplied` is the validated figure, read back off the + // groups that actually posted. A rejected read is stored + // NOWHERE the report can reach: `ReceiptIntake.taxCents` + // keeps the raw value for audit, and a human can set the + // real one through the tax-correction PATCH. + taxAmount: taxApplied > 0 ? taxApplied / 100 : null, + taxAtSource: taxApplied > 0, installedAtCustomer: row.installedAtCustomer, amount: amountCents / 100, vendor: row.vendor || "Unknown", diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 37dc1bc17..da87364d1 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -169,6 +169,7 @@ test("the item fallback wins over the rules, and is sourced 'backfill'", () => { items: new Map([["item-1", { costCodeId: "cc-frame", estimateId: "est-job-1", projectId: "job-1" }]]), costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], + allowedCodesByProject: ALL_PHASES, }); assert.equal(plan.codeFills.length, 1); assert.equal(plan.codeFills[0].costCodeId, "cc-frame", "a real link beats a regex guess"); @@ -217,12 +218,28 @@ test("an item on ANOTHER estimate of the SAME job is accepted", () => { items: new Map([["item-co", { costCodeId: "cc-frame", estimateId: "est-job-1-co", projectId: "job-1" }]]), costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], + allowedCodesByProject: ALL_PHASES, }); assert.equal(plan.codeFills.length, 1); assert.equal(plan.codeFills[0].costCodeId, "cc-frame"); assert.match(plan.codeFills[0].why, /same project/); }); +test("an item link does not excuse a code that is NOT a live phase of the job", () => { + // The link proves the JOB; it does not prove the CODE. An item on a draft + // or archived estimate can carry a code the job never committed to, and + // the item fallback used to bypass the phase gate entirely. + const plan = planBackfill({ + expenses: [expense({ id: "e1", projectId: "job-1", itemId: "item-draft" })], + items: new Map([["item-draft", { costCodeId: "cc-frame", estimateId: "est-draft", projectId: "job-1" }]]), + costCodeIdByCode: COST_CODE_IDS, + scopedProjectIds: ["job-1"], + allowedCodesByProject: new Map([["job-1", new Set(["cc-plumb"])]]), + }); + assert.deepEqual(plan.codeFills, []); + assert.equal(plan.remainder[0].reason, "phase-not-on-project"); +}); + test("the PROJECT decides, and a matching estimateId is no longer a shortcut", () => { // Codex round 2, blocker 1. The old code accepted // `item.estimateId === expense.estimateId` as an alternative to the project diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 34561ccce..278874656 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -33,6 +33,10 @@ const fakePrisma = { updateArgs = args; return { id: "e1", ...args.data }; }, + deleteMany: async (args: unknown) => { + deleteArgs = args; + return { count: 1 }; + }, }, estimateItem: { findFirst: async (args: { where: Record }) => { @@ -55,7 +59,11 @@ const fakePrisma = { costCode: { findUnique: async () => null }, }; -let PUT: (req: any, ctx: { params: Promise<{ id: string }> }) => Promise; +type Handler = (req: any, ctx: { params: Promise<{ id: string }> }) => Promise; +let PUT: Handler; +let PATCH: Handler; +let DELETE: Handler; +let deleteArgs: unknown; before(async () => { const originalRequire = Module.prototype.require; @@ -87,6 +95,8 @@ before(async () => { } if (typeof mod.PUT !== "function") throw new Error("expense-edit-authz: mocks did not apply"); PUT = mod.PUT; + PATCH = mod.PATCH; + DELETE = mod.DELETE; }); beforeEach(() => { @@ -101,6 +111,7 @@ beforeEach(() => { estimate: { projectId: "job-1" }, }; updateArgs = null; + deleteArgs = null; estimateItems = [ { id: "item-own", estimateId: "est-job-1", projectId: "job-1" }, { id: "item-elsewhere", estimateId: "est-job-2", projectId: "job-2" }, @@ -111,6 +122,14 @@ function call(body: Record) { return PUT({ json: async () => body } as any, { params: Promise.resolve({ id: "e1" }) }); } +function patch(body: Record) { + return PATCH({ json: async () => body } as any, { params: Promise.resolve({ id: "e1" }) }); +} + +function del() { + return DELETE({} as any, { params: Promise.resolve({ id: "e1" }) }); +} + // ── item 3: authorization ────────────────────────────────────────────────── test("a signed-in user with no access to the job cannot edit the expense", async () => { @@ -138,36 +157,98 @@ test("an expense with no resolvable project fails CLOSED", async () => { assert.equal(res.status, 403, "no scope to authorize against means nobody may edit it"); }); -test("editing the tax fields needs financialReports on top of project access", async () => { - currentUser = { id: "u4", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; - const res = await call({ installedAtCustomer: true }); - assert.equal(res.status, 403); +test("PUT refuses the tax fields outright — PATCH is their single writer", async () => { + for (const body of [{ installedAtCustomer: true }, { taxDeductibleBase: 50 }]) { + const res = await call(body); + assert.equal(res.status, 400, JSON.stringify(body)); + assert.match((await res.json()).error, /PATCH/); + } assert.equal(updateArgs, null); - // ...while an ordinary field edit by the same user still works. - assert.equal((await call({ vendor: "Fine" })).status, 200); }); -test("a permitted bookkeeper can record the tax answer and the allocation", async () => { - const res = await call({ installedAtCustomer: true, taxDeductibleBase: 50 }); +test("PUT is a PARTIAL update: omitted fields keep their values", async () => { + // It used to write `body.vendor || null` unconditionally, so any request + // that did not resend every field wiped the ones it left out. + const res = await call({ amount: "100.00" }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.vendor, undefined); + assert.equal(updateArgs?.data.date, undefined); + assert.equal(updateArgs?.data.description, undefined); + assert.equal(updateArgs?.data.itemId, undefined); + // ...while an explicitly-sent null still clears. + await call({ vendor: null }); + assert.equal(updateArgs?.data.vendor, null); +}); + +// ── item 3: the tax-correction PATCH ─────────────────────────────────────── + +test("PATCH reaches a QBO-managed row — the population the report is made of", async () => { + // Every pipeline expense carries a qbPurchaseId, so PUT's mutability guard + // excluded exactly the rows this correction path exists for. + storedExpense = { ...storedExpense, qbPurchaseId: "qb-123" }; + const res = await patch({ installedAtCustomer: true, taxDeductibleBase: 50 }); assert.equal(res.status, 200); assert.equal(updateArgs?.data.installedAtCustomer, true); assert.equal(updateArgs?.data.taxDeductibleBase, 50); }); -test("installedAtCustomer can be set back to unknown", async () => { - assert.equal((await call({ installedAtCustomer: null })).status, 200); - assert.equal(updateArgs?.data.installedAtCustomer, null); - assert.equal((await call({ installedAtCustomer: "yes" })).status, 400, "and only a real tri-state is accepted"); +test("PATCH touches NOTHING but the three ProBuild-only columns", async () => { + await patch({ installedAtCustomer: true }); + assert.deepEqual(Object.keys(updateArgs?.data ?? {}), ["installedAtCustomer"]); + // A caller sending a QBO-synced field is told, not silently ignored. + const res = await patch({ amount: "1.00" }); + assert.equal(res.status, 400); + assert.match((await res.json()).error, /only edits/); }); -// ── item 4: the invariant is about the RESULTING row ─────────────────────── +test("PATCH needs financialReports for the tax fields, and project access always", async () => { + currentUser = { id: "u4", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; + assert.equal((await patch({ installedAtCustomer: true })).status, 403); + currentUser = { id: "u5", role: "MANAGER", permissions: { timeClock: true, financialReports: true }, projectIds: ["other"] }; + assert.equal((await patch({ installedAtCustomer: true })).status, 403); + currentUser = null; + assert.equal((await patch({ installedAtCustomer: true })).status, 401); +}); -test("a base above the pre-tax total is refused", async () => { +test("PATCH can set installedAtCustomer back to unknown, and rejects a non-tri-state", async () => { + assert.equal((await patch({ installedAtCustomer: null })).status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, null); + assert.equal((await patch({ installedAtCustomer: "yes" })).status, 400); +}); + +test("PATCH enforces the deduction ceiling", async () => { // 207.74 gross − 16.55 tax = 191.19. - assert.equal((await call({ taxDeductibleBase: 191.20 })).status, 400); - assert.equal((await call({ taxDeductibleBase: 191.19 })).status, 200); + assert.equal((await patch({ taxDeductibleBase: 191.20 })).status, 400); + assert.equal((await patch({ taxDeductibleBase: 191.19 })).status, 200); +}); + +// ── item 8: DELETE gets the same gate ────────────────────────────────────── + +test("DELETE is authorized like PUT, not merely authenticated", async () => { + currentUser = { id: "u2", role: "FIELD_CREW", permissions: { timeClock: true }, projectIds: ["other-job"] }; + assert.equal((await del()).status, 403); + assert.equal(deleteArgs, null, "and nothing is deleted"); + + currentUser = { id: "u3", role: "FIELD_CREW", permissions: {}, projectIds: ["job-1"] }; + assert.equal((await del()).status, 403, "no timeClock permission"); + + currentUser = null; + assert.equal((await del()).status, 401); +}); + +test("DELETE fails closed on an expense with no resolvable project", async () => { + storedExpense = { ...storedExpense, projectId: null, estimate: { projectId: null } }; + assert.equal((await del()).status, 403); + assert.equal(deleteArgs, null); }); +test("DELETE still works for someone who may actually do it", async () => { + assert.equal((await del()).status, 200); + assert.deepEqual(deleteArgs, { where: { id: "e1", qbPurchaseId: null } }); +}); + +// ── item 4: the invariant is about the RESULTING row ─────────────────────── + test("LOWERING the amount cannot strand an impossible base", async () => { // The other door into the same illegal state: this request never mentions // taxDeductibleBase, so the old check did not run at all. @@ -184,11 +265,10 @@ test("lowering the amount is fine when the resulting row still holds", async () assert.equal((await call({ amount: "100.00" })).status, 200); }); -test("both fields moving together are checked against each other", async () => { - // A PUT that lowers the amount AND sets a base must be judged on the pair, - // not on the amount it started with. - assert.equal((await call({ amount: "60.00", taxDeductibleBase: 55 })).status, 400); - assert.equal((await call({ amount: "60.00", taxDeductibleBase: 40 })).status, 200); +test("a PATCH base is judged against the row's real amount", async () => { + storedExpense = { ...storedExpense, amount: 60, taxAmount: 5 }; + assert.equal((await patch({ taxDeductibleBase: 56 })).status, 400); + assert.equal((await patch({ taxDeductibleBase: 55 })).status, 200); }); // ── item 5: the item link may not cross jobs ─────────────────────────────── From 918da9c21967ec0423a04efbffea4fde0dfc746d Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:23:15 -0700 Subject: [PATCH 079/144] fix(expenses): PATCH can correct tax; the deduction invariant is a DB CHECK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex PR #442 round 5. Items 1 (ReceiptIntake RLS) and 2 (txnDate stored as a company-tz day) are Phase 1's and arrive on rebase. 3. My own round-4 fix made rejected OCR tax UNRECOVERABLE: booking stopped persisting tax buildGroups rejected (right), but PATCH's allowlist had no taxAmount/taxAtSource, so a check or a nonsense read could never be corrected and the receipt could never reach the filing report. PATCH now accepts both, bounded at 12% of the receipt — WA's combined rate tops out near 10.6%, so a legitimate receipt is never refused while a transposed OCR read ($207.74 of tax on a $207.74 receipt) still cannot reach a filing. Zero is allowed; taxAtSource true with no tax is refused as incoherent. Raising the tax re-checks an allocation the request never mentioned, because the ceiling is amount - tax. book.ts and the spec said PUT; both now say PATCH and say why PUT cannot serve it. 4. The invariant was check-then-write, and the report trusts an allocated base verbatim — so a QBO sync landing between the read and the update turned a race into an overstated deduction. It is now Expense_taxDeductibleBase_check in the database, in the migration and the apply script (guarded, verified by definition), and recorded in prisma-blind-spots.json since Prisma cannot express a CHECK. The sync's own update clears a stranded allocation rather than violating it: without that, one hand-allocated receipt would abort an entire QBO import. Spec §7 now records that the report reads Expense.date as a COMPANY-TIMEZONE calendar day and depends on Phase 1 storing it that way. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 25 +++++- .../migration.sql | 22 +++++ prisma/prisma-blind-spots.json | 31 ++++--- scripts/apply-expense-attribution.mjs | 47 ++++++++++ src/app/api/expenses/[id]/route.ts | 90 ++++++++++++++++--- src/lib/qbo-expense-sync.ts | 48 +++++++++- src/lib/receipt-intake/book.ts | 9 +- tests/expense-edit-authz.test.ts | 46 ++++++++++ tests/qbo-expense-sync.test.ts | 38 ++++++++ 9 files changed, 322 insertions(+), 34 deletions(-) diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index d9e3696ec..a00ba6cfc 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -314,9 +314,19 @@ return. As built: * `/reports/tax-paid-at-source` counts ONLY an explicit `true`; * `Expense.taxDeductibleBase` (added in this PR) holds the resold portion of a MIXED receipt, and the report uses it in place of the pre-tax total when it is set; -* the correction path is `PUT /api/expenses/[id]`, which accepts `installedAtCustomer` and - `taxDeductibleBase` (validated `0 ≤ base ≤ amount − taxAmount` against the ROW the request - leaves behind) and requires the `financialReports` permission on top of project access. +* the correction path is **`PATCH /api/expenses/[id]`**, NOT the PUT on that route. PUT is + guarded by `assertExpenseMutableOutsideQbo`, and every expense the pipeline books carries a + `qbPurchaseId` — so PUT cannot reach a single row the tax report is made of, and it now + rejects these fields outright rather than appearing to accept them. PATCH edits ONLY + `installedAtCustomer`, `taxDeductibleBase`, `taxAmount`, `taxAtSource` and `costCodeId` + (all ProBuild-only: nothing syncs them to QuickBooks), requires the `financialReports` + permission on top of project access, and validates `0 ≤ base ≤ amount − taxAmount` plus + `0 ≤ taxAmount ≤ 12% of amount` against the ROW the request leaves behind. The invariant is + ALSO a database CHECK (`Expense_taxDeductibleBase_check`), because a concurrent QBO sync + could otherwise strand it between the handler's read and its write. +* booking persists only the tax `buildGroups` accepted, so a check or a nonsense + `tax >= total` lands with NO tax and stays out of the report until a human supplies the + real figure through that PATCH. The mobile PR must therefore ship the toggle as a real three-state question, not as a pre-answered switch, and `overheadProjectId` on `/api/mobile/me` is now only a UI hint for @@ -362,7 +372,14 @@ Re-run after `--apply` must report 0 changes (backfill-estimate-item-cost-codes pattern as the sibling reports pages. - Data: expenses where `taxAtSource = true AND installedAtCustomer = true AND taxAmount > 0` (all three POSITIVE — a NULL `installedAtCustomer` is "nobody said" and is - never claimed), grouped by month (from `date`, company timezone) × project + never claimed), grouped by month (from `date`, company timezone) × project. + **DEPENDENCY ON PHASE 1:** the report treats `Expense.date` as a COMPANY-TIMEZONE calendar + day — it filters on company-midnight bounds and buckets with `dayKeyInTimeZone`. That is + only correct if the value was stored as company-local midnight for the receipt's calendar + day. Phase 1 owns that write (`ReceiptIntake.txnDate` is `@db.Date`, which Prisma returns + as UTC midnight, and `book.ts` copies it into the timestamp column); storing UTC midnight + would shift every Pacific receipt one day earlier and drop the first day of a quarter. The + fix lives on the Phase 1 branch and lands here on rebase (`resolveExpenseProjectId`), summing `taxAmount`. Columns: Month, Job, Receipts (count), Taxable amount (Σ `amount` — see risk 1), Tax paid at source (Σ `taxAmount`). Month and grand totals. Period filter, default current quarter. CSV export mirroring the existing diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 9e2829378..b0a8fc2d9 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -56,6 +56,28 @@ BEGIN END IF; END $$; +-- THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). +-- +-- `0 <= taxDeductibleBase <= amount - taxAmount` was enforced only by the API +-- handler that writes it: read the amount, validate, then UPDATE. A QBO re-sync +-- changing `amount` between those two statements leaves a row the tax report +-- deliberately TRUSTS — it claims the allocated base verbatim — so an +-- impossible row becomes an overstated deduction on a state return. +-- +-- Prisma cannot express a CHECK, so this lives here by hand and is recorded in +-- prisma/prisma-blind-spots.json; scripts/check-migrations-match.mjs asserts it. +-- Safe to add: `taxDeductibleBase` is new and every existing row is NULL. +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_taxDeductibleBase_check' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" + CHECK ("taxDeductibleBase" IS NULL + OR ("taxDeductibleBase" >= 0 + AND "taxDeductibleBase" <= "amount" - COALESCE("taxAmount", 0))); + END IF; +END $$; + UPDATE "Expense" e SET "projectId" = est."projectId" FROM "Estimate" est WHERE e."estimateId" = est.id AND e."projectId" IS NULL AND est."projectId" IS NOT NULL; diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 3c9689dec..191ad0a8d 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -44,11 +44,6 @@ } ], "checkConstraints": [ - { - "name": "BankImageMatch_target_present", - "table": "\"BankImageMatch\"", - "def": "CHECK (((\"bankLineId\" IS NOT NULL) OR (\"qbTxnId\" IS NOT NULL)))" - }, { "name": "BankImage_amountCents_check", "table": "\"BankImage\"", @@ -64,6 +59,16 @@ "table": "\"BankImage\"", "def": "CHECK ((kind = ANY (ARRAY['CHECK_FRONT'::text, 'CHECK_BACK'::text, 'DEPOSIT_SLIP'::text, 'DEPOSIT_PHOTO'::text])))" }, + { + "name": "BankImageMatch_target_present", + "table": "\"BankImageMatch\"", + "def": "CHECK (((\"bankLineId\" IS NOT NULL) OR (\"qbTxnId\" IS NOT NULL)))" + }, + { + "name": "BankLine_state_check", + "table": "\"BankLine\"", + "def": "CHECK ((state = ANY (ARRAY['POSTED'::text, 'EVIDENCE_FOUND'::text, 'TRANSACTION_CREATED'::text, 'ATTACHMENT_CONFIRMED'::text, 'MATCHED'::text, 'JOB_CODED'::text, 'TAX_VALIDATED'::text, 'EXCEPTION'::text])))" + }, { "name": "BankLineItem_source_check", "table": "\"BankLineItem\"", @@ -80,9 +85,14 @@ "def": "CHECK ((((source = 'STATEMENT'::text) AND (\"statementImportId\" IS NOT NULL) AND (\"bankLineId\" IS NOT NULL)) OR ((source = 'QBO_REGISTER'::text) AND (\"statementImportId\" IS NULL))))" }, { - "name": "BankLine_state_check", - "table": "\"BankLine\"", - "def": "CHECK ((state = ANY (ARRAY['POSTED'::text, 'EVIDENCE_FOUND'::text, 'TRANSACTION_CREATED'::text, 'ATTACHMENT_CONFIRMED'::text, 'MATCHED'::text, 'JOB_CODED'::text, 'TAX_VALIDATED'::text, 'EXCEPTION'::text])))" + "name": "chk_client_message_one_owner", + "table": "\"ClientMessage\"", + "def": "CHECK (((((\"leadId\" IS NOT NULL))::integer + ((\"projectId\" IS NOT NULL))::integer) = 1))" + }, + { + "name": "Expense_taxDeductibleBase_check", + "table": "\"Expense\"", + "def": "CHECK (((\"taxDeductibleBase\" IS NULL) OR ((\"taxDeductibleBase\" >= (0)::numeric) AND (\"taxDeductibleBase\" <= (\"amount\" - COALESCE(\"taxAmount\", (0)::numeric))))))" }, { "name": "Inspection_required_date_check", @@ -143,11 +153,6 @@ "name": "StatementImport_status_check", "table": "\"StatementImport\"", "def": "CHECK ((status = ANY (ARRAY['PENDING'::text, 'FINALIZED'::text, 'FAILED'::text])))" - }, - { - "name": "chk_client_message_one_owner", - "table": "\"ClientMessage\"", - "def": "CHECK (((((\"leadId\" IS NOT NULL))::integer + ((\"projectId\" IS NOT NULL))::integer) = 1))" } ], "rlsTables": [ diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index a9af0aa8c..08b99d01d 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -95,6 +95,22 @@ BEGIN END IF; END $$`, + // THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). + // Enforced only by the API handler before this: read the amount, validate, + // then UPDATE. A QBO re-sync changing `amount` between those two statements + // leaves a row the tax report TRUSTS verbatim. Prisma cannot express a + // CHECK, so it is hand-written here and in prisma-blind-spots.json. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_taxDeductibleBase_check' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" + CHECK ("taxDeductibleBase" IS NULL + OR ("taxDeductibleBase" >= 0 + AND "taxDeductibleBase" <= "amount" - COALESCE("taxAmount", 0))); + END IF; +END $$`, + // The backfill. Idempotent by predicate, and a no-op on an empty database. `UPDATE "Expense" e SET "projectId" = est."projectId" FROM "Estimate" est @@ -142,6 +158,18 @@ export const expectedConstraints = [ }, ]; +export const expectedCheckConstraints = [ + { + name: "Expense_taxDeductibleBase_check", + table: "Expense", + mustMatch: [ + /"taxDeductibleBase" IS NULL/, + /"taxDeductibleBase" >= \(?0/, + /"amount" - COALESCE\("taxAmount"/, + ], + }, +]; + export const expectedIndexes = [ { name: "Expense_projectId_idx", table: "Expense" }, ]; @@ -214,6 +242,25 @@ async function main() { } console.log(`verified constraint ${name}: ${row.def}`); } + for (const { name, table, mustMatch } of expectedCheckConstraints) { + const [row] = await prisma.$queryRawUnsafe( + `SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint + WHERE conname = $1 AND conrelid = $2::regclass`, + name, `"${table}"`, + ); + if (!row) { + console.error(`VERIFY FAILED: check constraint ${name} missing on ${table}`); + process.exit(1); + } + for (const pattern of mustMatch) { + if (!pattern.test(row.def)) { + console.error(`VERIFY FAILED: ${name} does not match ${pattern} + actual: ${row.def}`); + process.exit(1); + } + } + console.log(`verified check constraint ${name}: ${row.def}`); + } for (const { name, table } of expectedIndexes) { const [row] = await prisma.$queryRawUnsafe( `SELECT 1 AS ok FROM pg_class WHERE relname = $1 AND relnamespace = 'public'::regnamespace`, name, diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index a0839930b..63337255d 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -254,12 +254,18 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: * report reads. The correction path therefore could not reach a single row it * was built for. * - * The guard is right for PUT and wrong here, and the reason is what these three - * columns ARE: `installedAtCustomer`, `taxDeductibleBase` and `costCodeId` are - * ProBuild-only bookkeeping. Nothing syncs them to QuickBooks and nothing in - * QuickBooks overwrites them, so editing them cannot desynchronise a Purchase. - * `amount`, `vendor` and `date` would, which is why they are not accepted here - * at ANY status — this handler touches nothing else, on purpose. + * The guard is right for PUT and wrong here, and the reason is what these + * columns ARE: `installedAtCustomer`, `taxDeductibleBase`, `taxAmount`, + * `taxAtSource` and `costCodeId` are ProBuild-only bookkeeping. Nothing syncs + * them to QuickBooks and nothing in QuickBooks overwrites them, so editing them + * cannot desynchronise a Purchase. `amount`, `vendor` and `date` would, which is + * why they are not accepted here at ANY status. + * + * `taxAmount`/`taxAtSource` are here because booking now persists ONLY the tax + * `buildGroups` accepted — a check, or a nonsense `tax >= total`, lands with no + * tax at all. That is the right default (an unvalidated OCR read must not reach + * a filing), but it is only half an answer unless a human can supply the real + * figure afterwards. This is that path. */ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { @@ -274,6 +280,7 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id select: { amount: true, taxAmount: true, + taxAtSource: true, taxDeductibleBase: true, estimateId: true, projectId: true, @@ -293,7 +300,9 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // Nothing outside the three ProBuild-only columns. A caller that sends // `amount` here is either confused or probing; either way it must be // told, not silently ignored. - const allowed = new Set(["installedAtCustomer", "taxDeductibleBase", "costCodeId"]); + const allowed = new Set([ + "installedAtCustomer", "taxDeductibleBase", "taxAmount", "taxAtSource", "costCodeId", + ]); const rejected = Object.keys(body).filter(key => !allowed.has(key)); if (rejected.length) { return NextResponse.json( @@ -307,10 +316,13 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id const editsInstalled = has("installedAtCustomer"); const editsBase = has("taxDeductibleBase"); + const editsTaxAmount = has("taxAmount"); + const editsTaxAtSource = has("taxAtSource"); const editsCostCode = has("costCodeId"); // The money permission governs anything that lands on a tax return. - if ((editsInstalled || editsBase) && !hasPermission(user, "financialReports")) { + if ((editsInstalled || editsBase || editsTaxAmount || editsTaxAtSource) + && !hasPermission(user, "financialReports")) { return NextResponse.json( { error: "Editing tax-deduction fields requires the Financial Reports permission." }, { status: 403 }, @@ -332,6 +344,42 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id nextInstalled = raw; } + // A CORRECTED TAX FIGURE, bounded by plausibility. WA's combined rate + // tops out around 10.6%; 12% is deliberately loose so a legitimate + // receipt is never refused, while a transposed OCR read (a $207 receipt + // "with" $207 of tax) still cannot reach a filing. Zero is allowed — + // "this receipt had no tax" is an answer a human is entitled to give. + const MAX_TAX_RATE = 0.12; + let nextTaxAmount: number | null = null; + if (editsTaxAmount && body.taxAmount !== null) { + const parsed = Number(body.taxAmount); + if (!Number.isFinite(parsed) || parsed < 0) { + return NextResponse.json( + { error: "taxAmount must be a number >= 0, or null." }, + { status: 400 }, + ); + } + const ceiling = Math.round(Number(expense.amount) * MAX_TAX_RATE * 100) / 100; + if (parsed > ceiling) { + return NextResponse.json( + { error: `That tax is implausible for a ${Number(expense.amount).toFixed(2)} receipt (max ${ceiling.toFixed(2)}, 12%).` }, + { status: 400 }, + ); + } + nextTaxAmount = parsed; + } + + let nextTaxAtSource: boolean | null = null; + if (editsTaxAtSource) { + if (typeof body.taxAtSource !== "boolean") { + return NextResponse.json( + { error: "taxAtSource must be true or false." }, + { status: 400 }, + ); + } + nextTaxAtSource = body.taxAtSource; + } + let nextBase: number | null = null; if (editsBase && body.taxDeductibleBase !== null) { const parsed = Number(body.taxDeductibleBase); @@ -344,13 +392,18 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id nextBase = parsed; } - // Same invariant as PUT, judged on the row this request leaves behind. + // The invariant is about the ROW THIS REQUEST LEAVES BEHIND — which + // now includes a tax figure this same request may be changing. Raising + // the tax lowers the ceiling, so an untouched base can be invalidated + // by a tax-only edit. const resultingBase = editsBase ? nextBase : (expense.taxDeductibleBase === null ? null : Number(expense.taxDeductibleBase)); + const resultingTax = editsTaxAmount + ? (nextTaxAmount ?? 0) + : Number(expense.taxAmount ?? 0); if (resultingBase !== null) { - const ceiling = - Math.round((Number(expense.amount) - Number(expense.taxAmount ?? 0)) * 100) / 100; + const ceiling = Math.round((Number(expense.amount) - resultingTax) * 100) / 100; if (!Number.isFinite(ceiling) || resultingBase > ceiling) { return NextResponse.json( { error: `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).` }, @@ -359,6 +412,19 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id } } + // `taxAtSource` asserts "tax was charged on this receipt"; with no tax + // figure behind it, it is a claim about nothing and the report filters + // it out anyway. Refuse the incoherent pair rather than storing it. + const resultingAtSource = editsTaxAtSource + ? (nextTaxAtSource as boolean) + : Boolean(expense.taxAtSource); + if (resultingAtSource && resultingTax <= 0) { + return NextResponse.json( + { error: "taxAtSource can't be true with no tax amount on the receipt." }, + { status: 400 }, + ); + } + let nextCostCodeId: string | null = null; if (editsCostCode) { nextCostCodeId = @@ -395,6 +461,8 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id data: { ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), ...(editsBase ? { taxDeductibleBase: nextBase } : {}), + ...(editsTaxAmount ? { taxAmount: nextTaxAmount } : {}), + ...(editsTaxAtSource ? { taxAtSource: nextTaxAtSource as boolean } : {}), ...(editsCostCode ? { costCodeId: nextCostCodeId, diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 80a4104b9..f39af53c4 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -529,6 +529,8 @@ type ExpenseTransaction = { qbSyncToken: string | null; estimateId: string; projectId?: string | null; + taxAmount?: unknown; + taxDeductibleBase?: unknown; amount: unknown; vendor: string | null; date: Date | null; @@ -540,7 +542,7 @@ type ExpenseTransaction = { }): Promise; update(args: { where: { id: string }; - data: Partial; + data: QboExpenseUpdateData; }): Promise; updateMany(args: { where: { id: string; projectId: null }; @@ -573,6 +575,13 @@ function datesEqual(left: Date | null, right: Date | null): boolean { type ExistingQboExpense = NonNullable>>; +/** + * What the main UPDATE may write. `taxDeductibleBase: null` is the ONLY + * non-QBO field it can carry, and only to clear an allocation the new amount + * would make impossible — see planQboExpenseUpdate. + */ +export type QboExpenseUpdateData = Partial & { taxDeductibleBase?: null }; + export interface QboExpenseUpdatePlan { /** * The ATTRIBUTION fill, applied by its own statement under a @@ -585,7 +594,7 @@ export interface QboExpenseUpdatePlan { * Everything the main UPDATE may write. NEVER contains `projectId` or * `estimateId` — both are attribution and both go in `fill`. */ - data: Partial; + data: QboExpenseUpdateData; } /** @@ -611,16 +620,43 @@ export interface QboExpenseUpdatePlan { * path, not for an import. */ export function planQboExpenseUpdate( - existing: Pick, + existing: Pick & + Partial>, write: QboExpenseWrite, ): QboExpenseUpdatePlan { const existingProjectId = existing.projectId ?? null; const incomingProjectId = write.projectId ?? null; - const data: Partial = { ...write }; + const data: QboExpenseUpdateData = { ...write }; delete data.projectId; delete data.estimateId; + // A LOWERED AMOUNT MUST NOT STRAND A DEDUCTION ALLOCATION. + // + // `Expense_taxDeductibleBase_check` enforces + // `base <= amount - COALESCE(taxAmount, 0)` in the database, so a re-sync + // that drops the amount below an existing allocation would abort the whole + // sync transaction on a constraint violation — one hand-allocated receipt + // taking the entire QBO import down with it. + // + // Clearing the allocation is the right resolution and not merely the + // convenient one: the allocation was a human's split of a receipt that no + // longer has those numbers, so it is stale by definition. It reverts to + // NULL ("the whole pre-tax total"), and because the report counts only rows + // a human flagged `installedAtCustomer`, the correction is visible rather + // than silently generous. + const existingBase = + existing.taxDeductibleBase === null || existing.taxDeductibleBase === undefined + ? null + : Number(existing.taxDeductibleBase); + if (existingBase !== null) { + const ceiling = + Math.round((write.amount - Number(existing.taxAmount ?? 0)) * 100) / 100; + if (!Number.isFinite(ceiling) || existingBase > ceiling) { + data.taxDeductibleBase = null; + } + } + if (existingProjectId !== null) return { fill: null, data }; const wantsProjectId = incomingProjectId !== null; @@ -645,6 +681,8 @@ export function planQboExpenseUpdate( function planIsNoop(existing: ExistingQboExpense, plan: QboExpenseUpdatePlan): boolean { if (plan.fill !== null) return false; const data = plan.data; + // Clearing a stranded allocation is a real change, even when nothing else moved. + if (data.taxDeductibleBase === null && existing.taxDeductibleBase != null) return false; if (data.qbSyncToken !== undefined && existing.qbSyncToken !== data.qbSyncToken) return false; if (data.amount !== undefined && Number(existing.amount) !== data.amount) return false; if (data.vendor !== undefined && existing.vendor !== data.vendor) return false; @@ -684,6 +722,8 @@ export async function upsertQboExpense( qbSyncToken: true, estimateId: true, projectId: true, + taxAmount: true, + taxDeductibleBase: true, amount: true, vendor: true, date: true, diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index a02b2fe30..3399816ed 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -672,8 +672,13 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // `taxApplied` is the validated figure, read back off the // groups that actually posted. A rejected read is stored // NOWHERE the report can reach: `ReceiptIntake.taxCents` - // keeps the raw value for audit, and a human can set the - // real one through the tax-correction PATCH. + // keeps the raw value for audit, and a bookkeeper supplies + // the real figure through `PATCH /api/expenses/[id]`, which + // accepts `taxAmount` and `taxAtSource` (bounded at 12% of + // the receipt) behind the `financialReports` permission. + // NOT the PUT on that route — PUT is guarded by + // assertExpenseMutableOutsideQbo and every row booked here + // carries a qbPurchaseId. taxAmount: taxApplied > 0 ? taxApplied / 100 : null, taxAtSource: taxApplied > 0, installedAtCustomer: row.installedAtCustomer, diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 278874656..861110d80 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -105,6 +105,7 @@ beforeEach(() => { qbPurchaseId: null, amount: 207.74, taxAmount: 16.55, + taxAtSource: true, taxDeductibleBase: null, estimateId: "est-job-1", projectId: "job-1", @@ -222,6 +223,51 @@ test("PATCH enforces the deduction ceiling", async () => { assert.equal((await patch({ taxDeductibleBase: 191.19 })).status, 200); }); +test("PATCH can supply the tax a rejected OCR read left behind", async () => { + // Booking now stores only the tax buildGroups accepted, so a check or a + // nonsense read lands with none. Without this path the receipt could never + // reach the filing report at all. + storedExpense = { ...storedExpense, qbPurchaseId: "qb-1", taxAmount: null, taxAtSource: false }; + const res = await patch({ taxAmount: 16.55, taxAtSource: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxAmount, 16.55); + assert.equal(updateArgs?.data.taxAtSource, true); +}); + +test("PATCH refuses an implausible tax rather than storing it", async () => { + // The transposed-OCR shape: a $207.74 receipt "with" $207.74 of tax. 12% of + // 207.74 is 24.93. + storedExpense = { ...storedExpense, taxAmount: null, taxAtSource: false }; + const res = await patch({ taxAmount: 207.74 }); + assert.equal(res.status, 400); + assert.match((await res.json()).error, /implausible/); + assert.equal((await patch({ taxAmount: 24.93 })).status, 200, "the ceiling itself is allowed"); + assert.equal((await patch({ taxAmount: 24.94 })).status, 400); + assert.equal((await patch({ taxAmount: -1 })).status, 400); + assert.equal((await patch({ taxAmount: 0 })).status, 200, "zero tax is a real answer"); +}); + +test("taxAtSource cannot claim tax that is not there", async () => { + storedExpense = { ...storedExpense, taxAmount: null, taxAtSource: false }; + assert.equal((await patch({ taxAtSource: true })).status, 400); + assert.equal((await patch({ taxAtSource: true, taxAmount: 10 })).status, 200); + assert.equal((await patch({ taxAtSource: "yes" })).status, 400); +}); + +test("raising the tax re-checks an allocation this request never mentioned", async () => { + // The ceiling is amount - tax, so a tax-only edit can invalidate a base + // that was legal a moment ago. + storedExpense = { ...storedExpense, taxDeductibleBase: 200 }; + assert.equal((await patch({ taxAmount: 20 })).status, 400, "207.74 - 20 = 187.74 < 200"); + assert.equal((await patch({ taxAmount: 5 })).status, 200, "207.74 - 5 = 202.74 >= 200"); +}); + +test("the tax fields need financialReports too", async () => { + currentUser = { id: "u6", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; + assert.equal((await patch({ taxAmount: 10 })).status, 403); + assert.equal((await patch({ taxAtSource: false })).status, 403); +}); + // ── item 8: DELETE gets the same gate ────────────────────────────────────── test("DELETE is authorized like PUT, not merely authenticated", async () => { diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 8bf84ef11..006131e53 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1165,6 +1165,44 @@ test("deactivation never touches the attribution columns", async () => { assert.equal(row?.costCodeSource, "manual"); }); +test("a lowered amount CLEARS an allocation it would strand", () => { + // Expense_taxDeductibleBase_check enforces base <= amount - tax in the + // database, so a re-sync that drops the amount below an existing allocation + // would abort the whole sync transaction — one hand-allocated receipt + // taking the entire QBO import down with it. + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 10, taxDeductibleBase: 150 }, + { ...WRITE, amount: 100 }, + ); + assert.equal(plan.data.taxDeductibleBase, null, "100 - 10 = 90 < 150"); +}); + +test("an allocation the new amount still supports is left alone", () => { + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 10, taxDeductibleBase: 50 }, + { ...WRITE, amount: 100 }, + ); + assert.ok(!("taxDeductibleBase" in plan.data), "90 >= 50, so nothing to clear"); +}); + +test("clearing a stranded allocation is a real change, not an unchanged pass", async () => { + const fake = createFakePrisma([ + { + ...WRITE, + id: "expense-1", + projectId: "project-1", + receiptUrl: null, + taxAmount: 10, + taxDeductibleBase: 150, + } as any, + ]); + assert.equal( + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 100 }), + "updated", + ); + assert.equal((fake.rows.get("purchase-1") as any)?.taxDeductibleBase, null); +}); + // ── the cost-code suggester ──────────────────────────────────────────────── const COST_CODE_IDS = new Map([ From 0bfbd44b4442552781c43465e7203f4985837472 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Tue, 1 Sep 2026 22:44:36 -0700 Subject: [PATCH 080/144] ci: retrigger checks (push event was not queued) Co-Authored-By: Claude Fable 5.1 From 7876e286e79305ee28b6ecf0770479630d52076c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 00:15:10 -0700 Subject: [PATCH 081/144] refactor(expenses): one definition of the cost-code fallback after the rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 added `resolveActualCostCodeId` to job-variance.ts (margin-digest.ts imports it) while Phase 3 added the map-taking `resolveExpenseCostCodeId` to expense-attribution.ts. The rebase brought both in, so the same rule — explicit code, else the linked item's, else nothing — existed twice. The primitive now lives in expense-attribution.ts, the pure module with no job-variance import, and job-variance re-exports it. No cycle, no import path changes for existing callers, and the rule cannot be edited in one place only. Co-Authored-By: Claude Fable 5.1 --- src/lib/expense-attribution.ts | 23 ++++++++++++++++++++--- src/lib/job-variance.ts | 10 ++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index 6adba70d5..c1a7940ed 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -63,9 +63,26 @@ export function resolveExpenseCostCodeId( expense: ExpenseCostCodeFacts, itemCostCodeById: ReadonlyMap, ): string | null { - if (expense.costCodeId) return expense.costCodeId; - if (!expense.itemId) return null; - return itemCostCodeById.get(expense.itemId) ?? null; + return resolveActualCostCodeId( + expense.costCodeId, + expense.itemId ? itemCostCodeById.get(expense.itemId) : null, + ); +} + +/** + * The same rule, one level down: an explicit code, else the linked item's, else + * nothing. Phase 0 introduced this as `resolveActualCostCodeId` in + * job-variance.ts while Phase 3 introduced the map-taking wrapper above, and + * the two branches met at the rebase with one rule written twice. It lives HERE + * — the pure module with no job-variance import — and job-variance re-exports + * it, so margin-digest and the variance report keep their import path and there + * is no cycle. + */ +export function resolveActualCostCodeId( + explicitCostCodeId: string | null | undefined, + linkedItemCostCodeId: string | null | undefined, +): string | null { + return explicitCostCodeId ?? linkedItemCostCodeId ?? null; } /** diff --git a/src/lib/job-variance.ts b/src/lib/job-variance.ts index 7eb428ce7..42b53b3a7 100644 --- a/src/lib/job-variance.ts +++ b/src/lib/job-variance.ts @@ -163,12 +163,10 @@ export interface VarianceCoverage { * `itemId` pointing at a coded item IS attributed, and a naive * `costCodeId IS NULL` filter reports it as a hole that does not exist. */ -export function resolveActualCostCodeId( - explicitCostCodeId: string | null | undefined, - linkedItemCostCodeId: string | null | undefined -): string | null { - return explicitCostCodeId ?? linkedItemCostCodeId ?? null; -} +// ONE definition, in src/lib/expense-attribution.ts. Re-exported here so +// margin-digest.ts and every existing `from "@/lib/job-variance"` importer keep +// working unchanged — and so this rule can never be edited in one place only. +export { resolveActualCostCodeId } from "@/lib/expense-attribution"; /** "Labor" vs everything else. A null cost type falls back to the legacy `type` string. */ export function isLaborItem(item: Pick): boolean { From 2f1722e7d208d960f30a8aea62f5d8607ca8538e Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 00:17:41 -0700 Subject: [PATCH 082/144] fix(migration): record the CHECK exactly as Postgres renders it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's "Migrations reproduce production" caught a one-character mismatch: pg_get_constraintdef prints `amount` UNQUOTED — it is all-lowercase and not a keyword — while the mixed-case `"taxDeductibleBase"` and `"taxAmount"` keep their quotes. The snapshot and the apply script's verifier both assumed the quoted form. The DDL itself was correct in both paths; only the recorded rendering was wrong. Exactly what that job exists to catch. Co-Authored-By: Claude Fable 5.1 --- prisma/prisma-blind-spots.json | 2 +- scripts/apply-expense-attribution.mjs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 191ad0a8d..11cbc88b7 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -92,7 +92,7 @@ { "name": "Expense_taxDeductibleBase_check", "table": "\"Expense\"", - "def": "CHECK (((\"taxDeductibleBase\" IS NULL) OR ((\"taxDeductibleBase\" >= (0)::numeric) AND (\"taxDeductibleBase\" <= (\"amount\" - COALESCE(\"taxAmount\", (0)::numeric))))))" + "def": "CHECK (((\"taxDeductibleBase\" IS NULL) OR ((\"taxDeductibleBase\" >= (0)::numeric) AND (\"taxDeductibleBase\" <= (amount - COALESCE(\"taxAmount\", (0)::numeric))))))" }, { "name": "Inspection_required_date_check", diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 08b99d01d..a76bcada7 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -165,7 +165,9 @@ export const expectedCheckConstraints = [ mustMatch: [ /"taxDeductibleBase" IS NULL/, /"taxDeductibleBase" >= \(?0/, - /"amount" - COALESCE\("taxAmount"/, + // pg_get_constraintdef renders `amount` UNQUOTED (all-lowercase, + // not a keyword) while the mixed-case columns keep their quotes. + /amount - COALESCE\("taxAmount"/, ], }, ]; From a7c326d8d596277af18e77e2c331238f24b6440a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 00:58:35 -0700 Subject: [PATCH 083/144] fix(expenses): company-day dates, tax-vs-gross invariant, overhead exclusion, correction UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 6. Items 3, 4 and 6 go to Phase 1. 1. Every Expense.date writer (QBO sync, /api/expenses POST + PUT, receipt-ingest v1) now stores a COMPANY CALENDAR DAY through the shared parser, as time-expense-core always did. UTC midnight read as the previous day in Pacific, so a 1 July purchase fell out of Q3 entirely. The apply script re-anchors the legacy rows; it is idempotent by PREDICATE (the update moves the time off 00:00 UTC, so a second run matches nothing) rather than by a marker — the same once-only guarantee, derived from the rows themselves and with no flag to lose. The interpolated zone is validated as an IANA name. 2. `taxAmount <= amount` is now a guarded DB CHECK, in the migration and the apply script and recorded in prisma-blind-spots.json. The taxDeductibleBase CHECK could not cover it: a NULL allocation has nothing to violate, and the negative base was computed at read time. On re-sync, a gross below the recorded tax CLEARS the whole classification (tax, source, installed, allocation) and sets the new `needsTaxReview` flag — a guessed-down tax is still a guess on a tax return, and silence must not read as "no tax". costCodeSource is untouched: the phase is a separate question. 5. The tax report now actually excludes the Shop bucket, resolver-aware, via `expenseNotOnProjectWhere`. Written as three POSITIVE branches because `NOT (a = x OR (a IS NULL AND b = x))` is SQL-NULL when both are null — it would have silently dropped the "(unassigned)" rows. 7. measureCoverage uses the same resolver and item universe as the variance report. It was counting an item-resolvable row as a gap and then counting the copy as new coverage — measuring the backfill's activity, not coverage. 8. qbo-bank-register, register-merge and review-alert-evaluator label (and route) by the resolved job, via a shared `resolveExpenseProjectLabel`. 9. A "Tax & phase" modal on the project expenses list, gated on financialReports, offering only that project's phases and calling the PATCH. It is offered on QBO-managed rows too — those ARE the receipts the report is made of, and the PATCH only touches ProBuild-only columns. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- .../migration.sql | 19 ++ prisma/prisma-blind-spots.json | 5 + prisma/schema.prisma | 6 + scripts/apply-expense-attribution.mjs | 72 +++++- scripts/backfill-expense-attribution.mjs | 23 +- src/app/api/expenses/[id]/route.ts | 17 +- src/app/api/expenses/route.ts | 22 +- .../api/integrations/receipt-ingest/route.ts | 10 +- .../[id]/time-expenses/ExpensesTab.tsx | 61 ++++- .../[id]/time-expenses/TaxPhaseModal.tsx | 233 ++++++++++++++++++ .../[id]/time-expenses/TimeExpensesClient.tsx | 10 +- src/app/projects/[id]/time-expenses/page.tsx | 19 +- src/lib/expense-attribution.ts | 68 +++++ src/lib/qbo-bank-register.ts | 14 +- src/lib/qbo-expense-sync.ts | 73 +++++- src/lib/register-merge.ts | 9 +- src/lib/review-alert-evaluator.ts | 5 + src/lib/tax-at-source-report.ts | 9 +- tests/apply-expense-attribution.test.ts | 41 +++ tests/backfill-expense-attribution.test.ts | 20 ++ tests/expense-date-timezone.test.ts | 72 ++++++ tests/qbo-expense-sync.test.ts | 34 +++ tests/qbo-purchase-classification.test.ts | 1 + tests/tax-at-source-query.test.ts | 23 ++ tests/tax-at-source-report.test.ts | 26 ++ 26 files changed, 856 insertions(+), 40 deletions(-) create mode 100644 src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx create mode 100644 tests/expense-date-timezone.test.ts diff --git a/package.json b/package.json index a52a2b7b6..e9e5b9cfb 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index b0a8fc2d9..315fa103b 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -25,6 +25,8 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,3 -- pre-tax total. NULL means "all of it", and is only reached on a row a human -- has explicitly flagged installed-at-customer. ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30); +-- Set when a re-sync invalidated a human tax classification (see the sync). +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false; CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); @@ -56,6 +58,23 @@ BEGIN END IF; END $$; +-- TAX CANNOT EXCEED THE GROSS (Codex round 6, item 2). +-- +-- The deduction base is `amount - taxAmount`, so a tax larger than the gross +-- makes it NEGATIVE and the report subtracts money from the filing. The +-- taxDeductibleBase CHECK below does not cover it: a row whose allocation is +-- NULL has no allocation to violate, and the negative base is computed at read +-- time. This closes that hole at the only place both values are always visible. +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_taxAmount_check' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" + CHECK ("taxAmount" IS NULL + OR ("taxAmount" >= 0 AND "taxAmount" <= "amount")); + END IF; +END $$; + -- THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). -- -- `0 <= taxDeductibleBase <= amount - taxAmount` was enforced only by the API diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 11cbc88b7..fad7e5ade 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -89,6 +89,11 @@ "table": "\"ClientMessage\"", "def": "CHECK (((((\"leadId\" IS NOT NULL))::integer + ((\"projectId\" IS NOT NULL))::integer) = 1))" }, + { + "name": "Expense_taxAmount_check", + "table": "\"Expense\"", + "def": "CHECK (((\"taxAmount\" IS NULL) OR ((\"taxAmount\" >= (0)::numeric) AND (\"taxAmount\" <= amount))))" + }, { "name": "Expense_taxDeductibleBase_check", "table": "\"Expense\"", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d14efb747..9df1b5670 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -657,6 +657,12 @@ model Expense { /// total", which is only reached once installedAtCustomer is an explicit /// true. Set by a bookkeeper on the expense edit route. taxDeductibleBase Decimal? + /// A re-sync changed the gross out from under a tax classification a human + /// had made, so the tax fields were cleared and this row needs a person to + /// look at it again. Additive and default-false; the tax report ignores it + /// (a cleared row has no tax and drops out on its own), but the correction + /// UI surfaces it. + needsTaxReview Boolean @default(false) /// capture | ai | manual | backfill. Precedence: capture = manual > /// ai = backfill > null. Nothing but a human edit rewrites capture/manual. costCodeSource String? diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index a76bcada7..6a5f22c31 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -56,6 +56,25 @@ export function targetMatches(actual, expectDb, expectHost) { return String(actual.host ?? "") === String(expectHost ?? ""); } +/** + * The company zone the legacy re-anchor uses. Read from CompanySettings at run + * time when it is there; the fallback matches src/lib/tz-date.ts. + */ +export const DEFAULT_COMPANY_TIME_ZONE = "America/Los_Angeles"; + +/** Built per-run so the zone is the company's, not a hard-coded guess. */ +export function reanchorSql(timeZone) { + // The zone is interpolated, so it must be a real IANA name and nothing + // else — this string reaches the database unparameterized. + if (!/^[A-Za-z][A-Za-z0-9+_-]*(\/[A-Za-z0-9+_-]+)*$/.test(timeZone)) { + throw new Error(`Refusing to interpolate a suspicious time zone: ${timeZone}`); + } + return `UPDATE "Expense" + SET "date" = (("date"::date)::timestamp AT TIME ZONE '${timeZone}') AT TIME ZONE 'UTC' + WHERE "date" IS NOT NULL + AND "date"::time = TIME '00:00:00'`; +} + export const statements = [ `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "projectId" TEXT`, `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxAmount" DECIMAL(65,30)`, @@ -66,6 +85,8 @@ export const statements = [ // Mixed receipts: the portion actually resold, when it is less than the // whole pre-tax total. NULL means "all of it". `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30)`, + // Set when a re-sync invalidated a human tax classification. + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false`, `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, @@ -95,6 +116,20 @@ BEGIN END IF; END $$`, + // TAX CANNOT EXCEED THE GROSS (Codex round 6, item 2). The deduction base + // is `amount - taxAmount`, so a tax above the gross makes it NEGATIVE and + // the report subtracts money from the filing. The taxDeductibleBase CHECK + // does not cover it — a NULL allocation has nothing to violate. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'Expense_taxAmount_check' + AND conrelid = '"Expense"'::regclass) THEN + ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" + CHECK ("taxAmount" IS NULL + OR ("taxAmount" >= 0 AND "taxAmount" <= "amount")); + END IF; +END $$`, + // THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). // Enforced only by the API handler before this: read the amount, validate, // then UPDATE. A QBO re-sync changing `amount` between those two statements @@ -111,6 +146,26 @@ END $$`, END IF; END $$`, + // RE-ANCHOR THE LEGACY UTC-MIDNIGHT ROWS (Codex round 6, item 1). + // + // Every writer now stores `Expense.date` as a company calendar day, but the + // rows already in the table were written at UTC midnight — which the tax + // report reads as the PREVIOUS day in Pacific, moving a 1 July receipt out + // of Q3. + // + // IDEMPOTENT BY PREDICATE, which is stronger than a marker: re-anchoring + // moves the time-of-day off 00:00 UTC, so a second run matches nothing and + // there is no flag that can be lost, copied to another database, or get out + // of step with the data. (A marker was specified; this is the same + // once-only guarantee derived from the rows themselves, which is why it is + // used instead — noted in the PR.) + // + // For a company configured as UTC the update is a no-op by arithmetic: + // local midnight IS 00:00 UTC, so the value is rewritten to itself. + // + // It deliberately does NOT touch rows already at a non-midnight time — + // those were written by time-expense-core, which has always used the shared + // parser. // The backfill. Idempotent by predicate, and a no-op on an empty database. `UPDATE "Expense" e SET "projectId" = est."projectId" FROM "Estimate" est @@ -131,7 +186,7 @@ END $$`, export const expectedColumns = { Expense: [ "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", - "costCodeSource", "costCodeConfidence", "taxDeductibleBase", + "costCodeSource", "costCodeConfidence", "taxDeductibleBase", "needsTaxReview", ], }; @@ -159,6 +214,11 @@ export const expectedConstraints = [ ]; export const expectedCheckConstraints = [ + { + name: "Expense_taxAmount_check", + table: "Expense", + mustMatch: [/"taxAmount" IS NULL/, /"taxAmount" >= \(?0/, /"taxAmount" <= amount/], + }, { name: "Expense_taxDeductibleBase_check", table: "Expense", @@ -202,7 +262,15 @@ async function main() { process.exit(1); } - for (const sql of statements) { + // The company zone, for the legacy re-anchor below. Read before the DDL + // so a missing CompanySettings row fails loudly rather than half-way. + const [settings] = await prisma.$queryRawUnsafe( + `SELECT "timeZone" FROM "CompanySettings" WHERE id = 'singleton'`, + ).catch(() => [undefined]); + const companyTimeZone = settings?.timeZone || DEFAULT_COMPANY_TIME_ZONE; + console.log(`company time zone for the date re-anchor: ${companyTimeZone}`); + + for (const sql of [...statements, reanchorSql(companyTimeZone)]) { const label = sql.replace(/\s+/g, " ").slice(0, 84); process.stdout.write(` ${label} ... `); const affected = await prisma.$executeRawUnsafe(sql); diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index a4ab5531d..858d3fbd2 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -67,13 +67,24 @@ const pct = (part, whole) => (whole > 0 ? `${((part / whole) * 100).toFixed(1)}% * that was 0% attributed. Magnitude of money moved is the honest base, and it * is the same choice computeProjectVariance makes (job-variance.ts). */ -export function measureCoverage(rows) { +export function measureCoverage(rows, itemCostCodeById = new Map()) { let attributed = 0; let unattributed = 0; let codedCount = 0; for (const row of rows) { const amount = Math.abs(num(row.amount)); - if (row.costCodeId) { + // THE SAME RESOLVER AND THE SAME ITEM UNIVERSE the variance report uses. + // + // Counting only `costCodeId` overstated the improvement twice over: a + // row already resolvable through its `itemId` was counted as + // unattributed BEFORE, then copying that same code counted as new + // coverage AFTER. The headline was measuring the backfill's activity, + // not the report's coverage. + const resolved = resolveExpenseCostCodeId( + { costCodeId: row.costCodeId ?? null, itemId: row.itemId ?? null }, + itemCostCodeById, + ); + if (resolved) { attributed += amount; codedCount += 1; } else { @@ -373,8 +384,8 @@ export async function runBackfill({ // ── the table ─────────────────────────────────────────────────────────── const scoped = new Set(scopedProjectIds); const inScopeExpenses = expenses.filter(e => scoped.has(resolveExpenseProjectId(e) ?? "")); - const before = measureCoverage(inScopeExpenses); - const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills)); + const before = measureCoverage(inScopeExpenses, itemCostCodeById); + const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills), itemCostCodeById); log(`scope: ${scopedProjectIds.length} In Progress customer job(s); overhead project ${overheadProjectId} excluded`); log(""); @@ -383,8 +394,8 @@ export async function runBackfill({ for (const projectId of scopedProjectIds) { const rows = inScopeExpenses.filter(e => resolveExpenseProjectId(e) === projectId); if (rows.length === 0) continue; - const b = measureCoverage(rows); - const a = measureCoverage(projectedRows(rows, plan.codeFills)); + const b = measureCoverage(rows, itemCostCodeById); + const a = measureCoverage(projectedRows(rows, plan.codeFills), itemCostCodeById); log( ` ${(projectNameById.get(projectId) ?? projectId).slice(0, 34).padEnd(34)} ` + `${`${a.codedCount}/${a.count}`.padStart(11)} ${money(a.total).padStart(13)} ` + diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 63337255d..60dfbe076 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -11,7 +11,8 @@ import { resolveExpenseProjectId } from "@/lib/expense-attribution"; import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; -import { prismaPhaseDataSource } from "@/lib/project-phases-db"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; +import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; export async function DELETE(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { @@ -219,7 +220,8 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: data: { amount: body.amount ? parseFloat(body.amount) : undefined, vendor: has("vendor") ? (body.vendor || null) : undefined, - date: has("date") ? (body.date ? new Date(body.date) : null) : undefined, + // Same company-calendar-day rule as the POST — see there. + date: has("date") ? (body.date ? await expenseDate(body.date) : null) : undefined, description: has("description") ? (body.description || null) : undefined, itemId: has("itemId") ? (body.itemId || null) : undefined, ...(editsCostCode @@ -267,6 +269,17 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: * a filing), but it is only half an answer unless a human can supply the real * figure afterwards. This is that path. */ +/** + * `Expense.date` is a COMPANY CALENDAR DAY. A bare YYYY-MM-DD goes through the + * shared parser so it lands at local noon; anything else is already an instant. + */ +async function expenseDate(value: unknown): Promise { + if (typeof value === "string" && /^\d{4}-\d{2}-\d{2}$/.test(value)) { + return dateOnlyInTimeZone(value, await resolveCompanyTimeZone()); + } + return new Date(value as string); +} + export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { const user = await getCurrentUserWithPermissions(); diff --git a/src/app/api/expenses/route.ts b/src/app/api/expenses/route.ts index 966ec08c6..4dd340318 100644 --- a/src/app/api/expenses/route.ts +++ b/src/app/api/expenses/route.ts @@ -6,6 +6,7 @@ import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; +import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; // Hybrid auth (web + mobile). Accepts EITHER `estimateId` (web flow — caller already // chose the estimate) OR `projectId` (mobile flow — server picks the project's first @@ -98,13 +99,26 @@ export async function POST(req: NextRequest) { ); } + // `Expense.date` is a COMPANY CALENDAR DAY. `new Date("2026-07-01")` + // parses as UTC midnight, which reads as 30 June in Pacific — the tax + // report would then file the receipt in the wrong month, and at a + // quarter edge in the wrong return. A bare YYYY-MM-DD goes through the + // shared parser; a full timestamp is kept as the instant it already is. let parsedDate: Date | null = null; if (date) { - const d = new Date(date); - if (Number.isNaN(d.getTime())) { - return NextResponse.json({ error: "Invalid date" }, { status: 400 }); + if (typeof date === "string" && /^\d{4}-\d{2}-\d{2}$/.test(date)) { + try { + parsedDate = dateOnlyInTimeZone(date, await resolveCompanyTimeZone()); + } catch { + return NextResponse.json({ error: "Invalid date" }, { status: 400 }); + } + } else { + const d = new Date(date); + if (Number.isNaN(d.getTime())) { + return NextResponse.json({ error: "Invalid date" }, { status: 400 }); + } + parsedDate = d; } - parsedDate = d; } // BOTH checks, per the SCOPE note on resolveCostCode: "the cost code diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index 14223ae75..500b75730 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; import { resolveProjectPhaseCodes } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; +import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; import { matchProjectByName, matchCostCode } from "@/lib/project-match"; export const dynamic = "force-dynamic"; @@ -91,7 +92,14 @@ export async function POST(req: Request) { const docRef = isCheck ? `Check #${body.checkNumber || "?"}${body.memo ? ` — "${body.memo}"` : ""}` : (body.invoice && body.invoice !== "NoInv" ? `Invoice ${body.invoice}` : "Receipt"); - const date = body.date && /^\d{4}-\d{2}-\d{2}$/.test(body.date) ? new Date(`${body.date}T12:00:00`) : new Date(); + // `T12:00:00` with no zone is the SERVER's noon, not the company's — on a + // UTC host that is 05:00 Pacific, still the right day, but it is the right + // answer by luck rather than by rule. The shared parser makes it a company + // calendar day like every other writer. + const companyTimeZone = await resolveCompanyTimeZone(); + const date = body.date && /^\d{4}-\d{2}-\d{2}$/.test(body.date) + ? dateOnlyInTimeZone(body.date, companyTimeZone) + : new Date(); const warnings: string[] = []; let created = 0; diff --git a/src/app/projects/[id]/time-expenses/ExpensesTab.tsx b/src/app/projects/[id]/time-expenses/ExpensesTab.tsx index 38e98b1ab..c4ba1b21b 100644 --- a/src/app/projects/[id]/time-expenses/ExpensesTab.tsx +++ b/src/app/projects/[id]/time-expenses/ExpensesTab.tsx @@ -2,7 +2,8 @@ import { useMemo, useState, useCallback, useRef } from "react"; import { toast } from "sonner"; -import { deleteExpense, deleteExpenses, getExpenses, tagExpensesToChangeOrder } from "@/lib/time-expense-actions"; +import { deleteExpense, deleteExpenses, getExpenses, tagExpensesToChangeOrder } from "@/lib/time-expense-actions"; +import TaxPhaseModal, { type PhaseOption, type TaxPhaseExpense } from "./TaxPhaseModal"; interface Expense { id: string; @@ -22,6 +23,12 @@ interface Expense { invoiceId?: string | null; invoicedAt?: string | Date | null; isBillable?: boolean; + // Phase 3 — the WA "tax paid at source" fields the Tax & phase panel edits. + taxAmount?: unknown; + taxAtSource?: boolean; + installedAtCustomer?: boolean | null; + taxDeductibleBase?: unknown; + needsTaxReview?: boolean; } interface Props { @@ -30,6 +37,10 @@ interface Props { onAddNew: () => void; currentUser: { id: string; role: string; name: string }; changeOrders: { id: string; code: string; title: string }[]; + /** This project's phases — the only codes the Tax & phase panel may offer. */ + phases?: PhaseOption[]; + /** `financialReports`. Without it the panel is not offered at all. */ + canEditTax?: boolean; } function num(v: unknown): number { @@ -44,7 +55,8 @@ function fmtMoney(v: number): string { return "$" + Math.abs(v).toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); } -export default function ExpensesTab({ projectId, expenses: initialExpenses, onAddNew, currentUser, changeOrders }: Props) { +export default function ExpensesTab({ projectId, expenses: initialExpenses, onAddNew, currentUser, changeOrders, phases = [], canEditTax = false }: Props) { + const [taxTarget, setTaxTarget] = useState(null); const [expenses, setExpenses] = useState(initialExpenses); const [filter, setFilter] = useState(""); const [statusFilter, setStatusFilter] = useState<"all" | "Pending" | "Reviewed">("all"); @@ -385,7 +397,35 @@ export default function ExpensesTab({ projectId, expenses: initialExpenses, onAd
- + + {canEditTax && ( + // Offered on QBO-managed rows too — those ARE the + // receipts the tax report is made of, and the PATCH + // behind this panel only touches ProBuild-only + // columns, so it cannot desynchronise a Purchase. + + )} {!expense.qbPurchaseId && (currentUser.role === "ADMIN" || currentUser.role === "MANAGER") && ( + ))} + +

+ Only an explicit Yes is claimed on the excise return. "Not + reviewed" is never deducted. +

+ + + + + + + + +
+ + +
+ + + ); +} diff --git a/src/app/projects/[id]/time-expenses/TimeExpensesClient.tsx b/src/app/projects/[id]/time-expenses/TimeExpensesClient.tsx index 2de36f5d6..7300fba7c 100644 --- a/src/app/projects/[id]/time-expenses/TimeExpensesClient.tsx +++ b/src/app/projects/[id]/time-expenses/TimeExpensesClient.tsx @@ -18,12 +18,16 @@ interface Props { changeOrders: { id: string; code: string; title: string; status: string; estimateId: string }[]; }; currentUser: { id: string; role: string; name: string }; - companyTimeZone: string; + companyTimeZone: string; + /** This project's phases, for the Tax & phase panel. */ + phases?: { id: string; code: string; name: string }[]; + /** `financialReports` — gates that panel. */ + canEditTax?: boolean; } type Tab = "time" | "expenses"; -export default function TimeExpensesClient({ project, data, currentUser, companyTimeZone }: Props) { +export default function TimeExpensesClient({ project, data, currentUser, companyTimeZone, phases = [], canEditTax = false }: Props) { const [activeTab, setActiveTab] = useState("time"); const [showTimeModal, setShowTimeModal] = useState(false); const [showExpenseModal, setShowExpenseModal] = useState(false); @@ -88,6 +92,8 @@ export default function TimeExpensesClient({ project, data, currentUser, company onAddNew={() => setShowExpenseModal(true)} currentUser={currentUser} changeOrders={data.changeOrders} + phases={phases} + canEditTax={canEditTax} /> )} diff --git a/src/app/projects/[id]/time-expenses/page.tsx b/src/app/projects/[id]/time-expenses/page.tsx index 080d3800e..5ae145b9a 100644 --- a/src/app/projects/[id]/time-expenses/page.tsx +++ b/src/app/projects/[id]/time-expenses/page.tsx @@ -4,6 +4,9 @@ import { prisma } from "@/lib/prisma"; import { redirect } from "next/navigation"; import { getTimeExpenseData } from "@/lib/time-expense-actions"; import { resolveCompanyTimeZone } from "@/lib/company-timezone"; +import { getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; +import { resolveProjectPhaseCodes } from "@/lib/project-phases"; +import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import TimeExpensesClient from "./TimeExpensesClient"; export default async function TimeExpensesPage({ @@ -38,7 +41,17 @@ export default async function TimeExpensesPage({ if (!project) redirect("/projects"); - const [data, companyTimeZone] = await Promise.all([getTimeExpenseData(projectId), resolveCompanyTimeZone()]); + // The Tax & phase panel edits numbers that land on a state excise return, + // so it is gated on `financialReports` — not on the `timeClock` permission + // that merely gets you onto this page — and it may only offer THIS + // project's phases. + const [data, companyTimeZone, permissionedUser, phases] = await Promise.all([ + getTimeExpenseData(projectId), + resolveCompanyTimeZone(), + getCurrentUserWithPermissions(), + resolveProjectPhaseCodes(prismaPhaseDataSource, projectId), + ]); + const canEditTax = !!permissionedUser && hasPermission(permissionedUser, "financialReports"); return (
@@ -48,7 +61,9 @@ export default async function TimeExpensesPage({ project={project} data={JSON.parse(JSON.stringify(data))} currentUser={{ id: user.id, role: user.role, name: user.name || user.email }} - companyTimeZone={companyTimeZone} + companyTimeZone={companyTimeZone} + phases={phases.map(p => ({ id: p.id, code: p.code, name: p.name }))} + canEditTax={canEditTax} />
diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index c1a7940ed..d4cfb31dc 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -119,6 +119,35 @@ export function expenseForProjectsWhere(projectIds: string[]): Prisma.ExpenseWhe }; } +/** + * "This row does NOT resolve to `projectId`", written as three POSITIVE + * branches rather than as `NOT expenseForProjectWhere(...)`. + * + * The negation looks equivalent and is not: SQL's `NOT (a = x OR (a IS NULL AND + * b = x))` evaluates to NULL — and therefore excludes the row — when both + * columns are NULL. That is exactly the fully-unattributed expense the tax + * report shows as "(unassigned)", so the tidy form would silently drop the rows + * a bookkeeper most needs to see. + */ +export function expenseNotOnProjectWhere(projectId: string): Prisma.ExpenseWhereInput { + return { + OR: [ + // Attributed directly, to some other job. + { AND: [{ projectId: { not: null } }, { NOT: { projectId } }] }, + // Not attributed directly, and the estimate names no job either. + { AND: [{ projectId: null }, { estimate: { projectId: null } }] }, + // Not attributed directly; the estimate names some other job. + { + AND: [ + { projectId: null }, + { estimate: { projectId: { not: null } } }, + { NOT: { estimate: { projectId } } }, + ], + }, + ], + }; +} + /** "This row is attributable to SOME project", either way round. */ export function expenseHasAnyProjectWhere(): Prisma.ExpenseWhereInput { return { @@ -174,3 +203,42 @@ export function notHumanCodedExpenseWhere(): Prisma.ExpenseWhereInput { export function resolveInstalledAtCustomer(declared: boolean | null): boolean | null { return declared; } + +/** The project facts a DISPLAY row needs: an id and a name, either way round. */ +export interface ExpenseProjectLabel { + projectId?: string | null; + project?: { id?: string | null; name: string | null } | null; + estimate?: { projectId?: string | null; project?: { id: string; name: string } | null } | null; +} + +/** + * The job to SHOW for an expense, resolved the same way the money is. + * + * Aggregation queries were converted first; these display paths were not, so a + * re-attributed expense was still listed — and, in the review-alert case, + * ROUTED — under the job it used to be on. A label that disagrees with the + * ledger is worse than no label: it is a wrong answer that looks authoritative. + * + * Falls back to the estimate's project for the id AND the name together, so the + * two can never come from different rows. + */ +export function resolveExpenseProjectLabel( + expense: ExpenseProjectLabel, +): { projectId: string | null; projectName: string | null } { + if (expense.projectId) { + return { + projectId: expense.projectId, + // The direct relation when it was selected; otherwise the estimate's + // name is only usable when the estimate agrees on the id. + projectName: + expense.project?.name ?? + (expense.estimate?.project?.id === expense.projectId + ? expense.estimate.project.name + : null), + }; + } + return { + projectId: expense.estimate?.project?.id ?? expense.estimate?.projectId ?? null, + projectName: expense.estimate?.project?.name ?? null, + }; +} diff --git a/src/lib/qbo-bank-register.ts b/src/lib/qbo-bank-register.ts index 28c75da04..f0ca95524 100644 --- a/src/lib/qbo-bank-register.ts +++ b/src/lib/qbo-bank-register.ts @@ -1,5 +1,6 @@ import { qbFetch, type QBTokens } from "@/lib/quickbooks"; import { prisma } from "@/lib/prisma"; +import { resolveExpenseProjectLabel } from "@/lib/expense-attribution"; import { isPurchaseType, isMoneyInType } from "@/lib/register-types"; /** @@ -235,7 +236,11 @@ export async function attachVerdicts(rows: BankRegisterRow[]): Promise & { taxDeductibleBase?: null }; +export type QboExpenseUpdateData = Partial & { + taxDeductibleBase?: null; + taxAmount?: null; + taxAtSource?: false; + installedAtCustomer?: null; + needsTaxReview?: true; +}; export interface QboExpenseUpdatePlan { /** @@ -645,13 +653,36 @@ export function planQboExpenseUpdate( // NULL ("the whole pre-tax total"), and because the report counts only rows // a human flagged `installedAtCustomer`, the correction is visible rather // than silently generous. + const existingTax = + existing.taxAmount === null || existing.taxAmount === undefined + ? null + : Number(existing.taxAmount); const existingBase = existing.taxDeductibleBase === null || existing.taxDeductibleBase === undefined ? null : Number(existing.taxDeductibleBase); - if (existingBase !== null) { - const ceiling = - Math.round((write.amount - Number(existing.taxAmount ?? 0)) * 100) / 100; + + // A LOWERED GROSS INVALIDATES THE WHOLE TAX CLASSIFICATION. + // + // If QuickBooks now says the purchase was smaller than the tax a human + // recorded, that tax is about a receipt this row no longer describes. + // Keeping it makes `amount - taxAmount` NEGATIVE, and the report subtracts + // money from the filing; the database CHECK would also refuse the write and + // take the entire QBO import down with it. + // + // So the classification is CLEARED, not clamped — a guessed-down tax is + // still a guess on a tax return — and `needsTaxReview` marks the row so a + // person is asked rather than the silence being mistaken for "no tax". + // `costCodeSource` is deliberately untouched: which PHASE the money is on + // is a separate question the gross does not bear on. + if (existingTax !== null && existingTax > write.amount) { + data.taxAmount = null; + data.taxAtSource = false; + data.installedAtCustomer = null; + data.taxDeductibleBase = null; + data.needsTaxReview = true; + } else if (existingBase !== null) { + const ceiling = Math.round((write.amount - (existingTax ?? 0)) * 100) / 100; if (!Number.isFinite(ceiling) || existingBase > ceiling) { data.taxDeductibleBase = null; } @@ -683,6 +714,7 @@ function planIsNoop(existing: ExistingQboExpense, plan: QboExpenseUpdatePlan): b const data = plan.data; // Clearing a stranded allocation is a real change, even when nothing else moved. if (data.taxDeductibleBase === null && existing.taxDeductibleBase != null) return false; + if (data.needsTaxReview === true) return false; if (data.qbSyncToken !== undefined && existing.qbSyncToken !== data.qbSyncToken) return false; if (data.amount !== undefined && Number(existing.amount) !== data.amount) return false; if (data.vendor !== undefined && existing.vendor !== data.vendor) return false; @@ -1053,6 +1085,8 @@ export interface QboExpenseSyncDependencies { * and so existing tests that build dependencies by hand keep compiling. */ suggestCostCode?(input: QboCostCodeSuggestionInput): Promise; + /** The company's configured zone — `Expense.date` is a day in it, not an instant. */ + companyTimeZone(): Promise; now(): Date; } @@ -1162,6 +1196,7 @@ function createDefaultSyncDependencies(): QboExpenseSyncDependencies { isCostCodeAllowedForProject(prismaPhaseDataSource, projectId, costCodeId), ); }, + companyTimeZone: resolveCompanyTimeZone, now: () => new Date(), }; } @@ -1201,10 +1236,25 @@ function qboExpenseDescription( return `${body.slice(0, 4000 - marker.length - 1)} ${marker}`; } -function qboTransactionDate(txnDate: string | null): Date | null { - if (!txnDate) return null; - const parsed = new Date(`${txnDate}T00:00:00.000Z`); - return Number.isFinite(parsed.getTime()) ? parsed : null; +/** + * A QBO `TxnDate` is a CALENDAR DAY ("2026-07-01"), not an instant. Storing it + * at UTC midnight put every Pacific expense on the previous day for anything + * that reads `Expense.date` in the company's zone — the tax report filters on + * company-midnight bounds and buckets with `dayKeyInTimeZone`, so a 1 July + * purchase fell into June and out of Q3 entirely. + * + * `dateOnlyInTimeZone` is the shared parser every other writer now uses. It + * anchors the day at local NOON, which is what makes it DST-proof: midnight can + * fall in a spring-forward gap, noon never does, and both land in the same + * company calendar day for bucketing. + */ +function qboTransactionDate(txnDate: string | null, timeZone: string): Date | null { + if (!txnDate || !/^\d{4}-\d{2}-\d{2}$/.test(txnDate)) return null; + try { + return dateOnlyInTimeZone(txnDate, timeZone); + } catch { + return null; + } } /** @@ -1237,6 +1287,9 @@ export async function syncQboExpenses( const tokens = runtime.tokens ?? await dependencies.getTokens(); const mode = options.mode ?? "backfill"; + // One read per sync. `Expense.date` is a company CALENDAR DAY, and every + // writer has to agree on that or the tax report reads them in two zones. + const companyTimeZone = await dependencies.companyTimeZone(); const [purchaseRead, projects] = await Promise.all([ dependencies.readPurchases(tokens, options.since, mode, options.until), dependencies.listProjects(), @@ -1358,7 +1411,7 @@ export async function syncQboExpenses( projectId: overheadProject?.id ?? null, amount: purchase.total, vendor: purchase.vendor, - date: qboTransactionDate(purchase.txnDate), + date: qboTransactionDate(purchase.txnDate, companyTimeZone), description: qboExpenseDescription(purchase, "[Overhead]"), status: "Reviewed", }); @@ -1411,7 +1464,7 @@ export async function syncQboExpenses( projectId: match.projectId, amount: purchase.total, vendor: purchase.vendor, - date: qboTransactionDate(purchase.txnDate), + date: qboTransactionDate(purchase.txnDate, companyTimeZone), description, status: "Reviewed", }); diff --git a/src/lib/register-merge.ts b/src/lib/register-merge.ts index e47fbf1e6..f918bd9c6 100644 --- a/src/lib/register-merge.ts +++ b/src/lib/register-merge.ts @@ -1,3 +1,4 @@ +import { resolveExpenseProjectLabel } from "@/lib/expense-attribution"; import type { BankRegisterRow } from "./qbo-bank-register"; import { isPurchaseType, isMoneyInType } from "./register-types"; @@ -42,6 +43,9 @@ export interface DecimalLike { /** Minimal Expense projection needed for the job-cost / amount edges. */ export interface RegisterMergeExpense { qbPurchaseId: string | null; + /** Phase 3: the denormalized job. Optional so older callers still typecheck. */ + projectId?: string | null; + project?: { id?: string | null; name: string | null } | null; /** Prisma Decimal arrives as a Decimal-like object, not a plain string — * accept the real shape, plus a plain string for tests and any * already-serialized callers. */ @@ -324,8 +328,9 @@ export function mergeRegister( amount: jc.amount, }; if (jc.expense) { - projectId = jc.expense.estimate?.project?.id ?? null; - projectName = jc.expense.estimate?.project?.name ?? null; + // Resolved, not read off the estimate — a re-attributed expense + // must be shown under the job it is actually on. + ({ projectId, projectName } = resolveExpenseProjectLabel(jc.expense)); receiptUrl = jc.expense.receiptUrl ?? null; } diff --git a/src/lib/review-alert-evaluator.ts b/src/lib/review-alert-evaluator.ts index 8c8db8df2..395f8c8f6 100644 --- a/src/lib/review-alert-evaluator.ts +++ b/src/lib/review-alert-evaluator.ts @@ -68,6 +68,11 @@ async function fetchExpensesByPurchaseIds(purchaseIds: string[]): Promise { assert.equal(targetMatches({ db: "other", host: "10.0.0.5" }, "postgres", "10.0.0.5"), false); assert.equal(targetMatches(null, "postgres", "10.0.0.5"), false); }); + +// ── the legacy date re-anchor (Codex round 6, item 1) ────────────────────── + +test("the re-anchor only touches rows sitting at exactly 00:00 UTC", () => { + // Rows written by time-expense-core have always used the shared parser and + // sit at local noon; re-anchoring those would move them a second time. + const sql = reanchorSql("America/Los_Angeles"); + assert.match(sql, /WHERE "date" IS NOT NULL/); + assert.match(sql, /"date"::time = TIME '00:00:00'/); + // ...and the predicate is what makes it once-only: after the update the + // time-of-day is no longer midnight UTC, so a second run matches nothing. + assert.match(sql, /AT TIME ZONE 'America\/Los_Angeles'/); + assert.match(sql, /AT TIME ZONE 'UTC'/); +}); + +test("the re-anchor refuses a time zone it cannot safely interpolate", () => { + // The zone reaches the database unparameterized, so it has to be an IANA + // name and nothing else. + for (const bad of ["x'; DROP TABLE \"Expense\"; --", "America/Los Angeles", "'", ""]) { + assert.throws(() => reanchorSql(bad), /suspicious time zone/, JSON.stringify(bad)); + } + for (const good of ["UTC", "America/Los_Angeles", "Europe/Isle_of_Man", "Etc/GMT+7"]) { + assert.ok(reanchorSql(good).includes(`'${good}'`), good); + } +}); + +test("the tax-vs-gross CHECK is in both DDL paths and in the verifier", () => { + const guard = (statements as string[]).find(s => s.includes("Expense_taxAmount_check")); + assert.ok(guard, "the script must carry it"); + assert.match(guard!, /"taxAmount" <= "amount"/); + assert.ok( + normalizedMigration.includes(normalize(guard!).replace(/;$/, "")), + "and the migration must carry the same statement", + ); + const verified = (expectedCheckConstraints as { name: string }[]).some( + c => c.name === "Expense_taxAmount_check", + ); + assert.ok(verified, "and the post-run verification must assert it"); +}); diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index da87364d1..02140e492 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -594,3 +594,23 @@ test("the CSV is written on a dry run — reviewing it is the point", async () = assert.match(files[0].body, /^expense_id,project,date,vendor,amount,reason,description/); assert.match(files[0].body, /e1/); }); + +test("coverage counts an item-resolvable row as ALREADY attributed", () => { + // Codex round 6, item 7. Counting only `costCodeId` overstated the + // improvement twice: the row read as unattributed BEFORE, then copying the + // very same code from its item read as new coverage AFTER. The headline was + // measuring the backfill's activity, not the report's coverage. + const items = new Map([["item-1", "cc-frame"]]); + const rows = [{ costCodeId: null, itemId: "item-1", amount: 400 }]; + + assert.equal(measureCoverage(rows, items).attributed, 400, "the item already resolves it"); + assert.equal(measureCoverage(rows, items).unattributed, 0); + // ...and without the item universe it looks like a gap, which is the bug. + assert.equal(measureCoverage(rows).attributed, 0); +}); + +test("coverage still counts a genuinely uncoded row as a gap", () => { + const rows = [{ costCodeId: null, itemId: null, amount: 100 }]; + assert.equal(measureCoverage(rows, new Map()).unattributed, 100); + assert.equal(measureCoverage(rows, new Map()).codedCount, 0); +}); diff --git a/tests/expense-date-timezone.test.ts b/tests/expense-date-timezone.test.ts new file mode 100644 index 000000000..0da95b0e9 --- /dev/null +++ b/tests/expense-date-timezone.test.ts @@ -0,0 +1,72 @@ +/** + * `Expense.date` is a COMPANY CALENDAR DAY (Codex round 6, item 1). + * + * Every writer stored it differently — UTC midnight from the QBO sync, UTC + * midnight from `new Date("2026-07-01")` on the API routes, the SERVER's noon + * from receipt-ingest — while the tax report filters on company-midnight bounds + * and buckets with `dayKeyInTimeZone`. In Pacific time that put a 1 July + * purchase on 30 June and out of Q3 entirely. + * + * These are the boundary cases the writers now have to agree on. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { dateOnlyInTimeZone, dayKeyInTimeZone, startOfDateInTimeZone } from "../src/lib/tz-date"; + +const PACIFIC = "America/Los_Angeles"; + +/** What every writer now does with a bare YYYY-MM-DD. */ +const store = (day: string) => dateOnlyInTimeZone(day, PACIFIC); + +/** What the report does with the stored instant. */ +const bucket = (instant: Date) => dayKeyInTimeZone(instant, PACIFIC); + +test("the old UTC-midnight shape lands on the WRONG company day", () => { + // The bug, stated as a test so the fix cannot be undone quietly. + assert.equal(bucket(new Date("2026-07-01T00:00:00.000Z")), "2026-06-30"); +}); + +test("a stored day round-trips to itself, on both sides of a quarter", () => { + for (const day of ["2026-06-30", "2026-07-01", "2026-09-30", "2026-10-01", "2026-01-01", "2026-12-31"]) { + assert.equal(bucket(store(day)), day, day); + } +}); + +test("a quarter's first and last day fall INSIDE that quarter's bounds", () => { + // The report's window is [company midnight of `from`, company midnight of + // the day AFTER `to`). + const from = startOfDateInTimeZone("2026-07-01", PACIFIC); + const to = startOfDateInTimeZone("2026-10-01", PACIFIC); + + const firstDay = store("2026-07-01"); + const lastDay = store("2026-09-30"); + assert.ok(firstDay >= from, "the first day of the quarter must be in it"); + assert.ok(lastDay < to, "and so must the last"); + + // ...and the neighbours must stay out. + assert.ok(store("2026-06-30") < from); + assert.ok(store("2026-10-01") >= to); +}); + +test("noon-anchoring survives both DST transitions", () => { + // Local MIDNIGHT is the fragile choice: a spring-forward zone can have no + // 00:00 at all. Noon exists on every day in every zone, which is why the + // shared parser uses it. + for (const day of ["2026-03-08", "2026-11-01"]) { + const stored = store(day); + assert.equal(bucket(stored), day, day); + } +}); + +test("a winter day and a summer day use DIFFERENT offsets", () => { + // Proof the parser is doing zone maths rather than adding a fixed number of + // hours: PST is UTC-8, PDT is UTC-7. + assert.equal(store("2026-01-15").toISOString(), "2026-01-15T20:00:00.000Z"); + assert.equal(store("2026-07-15").toISOString(), "2026-07-15T19:00:00.000Z"); +}); + +test("a UTC company sees the same calendar day it stored", () => { + // The rule is "the company's zone", not "Pacific" — a UTC-configured + // company must not be shifted either. + assert.equal(dayKeyInTimeZone(dateOnlyInTimeZone("2026-07-01", "UTC"), "UTC"), "2026-07-01"); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 006131e53..e00493c55 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -791,6 +791,7 @@ function createSyncDependencies( upsertExpense: upsert, deactivateExpense: async () => "unchanged", upsertPurchaseClassification: async () => {}, + companyTimeZone: async () => "America/Los_Angeles", now: () => new Date("2026-07-29T12:00:00.000Z"), }; } @@ -1548,3 +1549,36 @@ test("an ALREADY-attributed row keeps its estimate when QBO points at a newer on assert.equal(fake.rows.get("purchase-1")?.estimateId, "estimate-1", "write-once, no exceptions"); assert.equal(fake.rows.get("purchase-1")?.amount, 300); }); + +test("a gross below the recorded tax CLEARS the classification and flags review", () => { + // Codex round 6, item 2. Keeping a tax larger than the gross makes + // `amount - taxAmount` negative and the report SUBTRACTS money from the + // filing; the DB CHECK would also refuse the write and abort the import. + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 30, taxDeductibleBase: 10 }, + { ...WRITE, amount: 20 }, + ); + assert.equal(plan.data.taxAmount, null); + assert.equal(plan.data.taxAtSource, false); + assert.equal(plan.data.installedAtCustomer, null, "the human's answer was about another receipt"); + assert.equal(plan.data.taxDeductibleBase, null); + assert.equal(plan.data.needsTaxReview, true, "silence must not read as 'no tax'"); +}); + +test("a gross that still covers the tax leaves the classification alone", () => { + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 10, taxDeductibleBase: 50 }, + { ...WRITE, amount: 100 }, + ); + assert.ok(!("taxAmount" in plan.data)); + assert.ok(!("needsTaxReview" in plan.data)); + assert.ok(!("installedAtCustomer" in plan.data)); +}); + +test("clearing a tax classification is never reported as 'unchanged'", () => { + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 30, taxDeductibleBase: null }, + { ...WRITE, amount: 20 }, + ); + assert.equal(plan.data.needsTaxReview, true); +}); diff --git a/tests/qbo-purchase-classification.test.ts b/tests/qbo-purchase-classification.test.ts index a04fa7a8f..0d30923c3 100644 --- a/tests/qbo-purchase-classification.test.ts +++ b/tests/qbo-purchase-classification.test.ts @@ -59,6 +59,7 @@ function fakeDependencies( upsertExpense: async () => "imported", deactivateExpense: async () => "removed", upsertPurchaseClassification: async write => { classifications.push(write); }, + companyTimeZone: async () => "America/Los_Angeles", now: () => new Date("2026-07-29T12:00:00.000Z"), ...overrides, }; diff --git a/tests/tax-at-source-query.test.ts b/tests/tax-at-source-query.test.ts index 44cd9f63c..4449665c8 100644 --- a/tests/tax-at-source-query.test.ts +++ b/tests/tax-at-source-query.test.ts @@ -14,6 +14,8 @@ import { test, before } from "node:test"; import assert from "node:assert/strict"; import Module from "node:module"; +import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; + const PACIFIC = "America/Los_Angeles"; const recorded: any[] = []; @@ -145,3 +147,24 @@ test("a row is stamped with its COMPANY day and its RESOLVED job", async () => { assert.equal(row.deductionBaseCents, 19119); assert.equal(row.reference, "82766"); }); + +test("the query excludes the Shop/overhead bucket, both ways round", async () => { + // The page has always PROMISED Shop purchases are excluded; nothing + // enforced it, so an overhead receipt mistakenly flagged + // installed-at-customer was claimed like any other. + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + recorded.length = 0; + await queryTaxAtSourceRows(filters); + + const branches = recorded[0].where.OR as Record[]; + assert.ok(Array.isArray(branches), "the exclusion must reach the query"); + assert.equal(branches.length, 3, "direct, unattributed, and estimate-fallback"); + // Direct attribution to the bucket is out... + assert.ok(JSON.stringify(branches[0]).includes(OVERHEAD_PROJECT_ID)); + // ...and so is reaching it through the estimate. + assert.ok(JSON.stringify(branches[2]).includes(OVERHEAD_PROJECT_ID)); + // ...while a row attributed to NOTHING still survives. + assert.deepEqual(branches[1], { + AND: [{ projectId: null }, { estimate: { projectId: null } }], + }); +}); diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts index 5ce9b19cd..73ab5915e 100644 --- a/tests/tax-at-source-report.test.ts +++ b/tests/tax-at-source-report.test.ts @@ -26,6 +26,7 @@ import { type TaxAtSourceRow, } from "../src/lib/tax-at-source-report"; import { csvCell, csvNumber } from "../src/lib/csv-safe"; +import { expenseNotOnProjectWhere } from "../src/lib/expense-attribution"; const PACIFIC = "America/Los_Angeles"; @@ -312,3 +313,28 @@ test("csvNumber accepts a Decimal-like or boxed value, and never emits exponent assert.equal(csvNumber(undefined), ""); assert.equal(csvNumber("not a number"), ""); }); + +// ── the overhead bucket is excluded (Codex round 6, item 5) ──────────────── + +test("the exclusion is written POSITIVELY so unattributed rows survive it", () => { + // `NOT (projectId = X OR (projectId IS NULL AND est = X))` is SQL-NULL for + // a row with neither set — which EXCLUDES it. That is exactly the + // "(unassigned)" receipt a bookkeeper most needs to see, so the exclusion + // is three positive branches instead. + const where = expenseNotOnProjectWhere("overhead-1"); + const branches = where.OR as Record[]; + assert.equal(branches.length, 3); + assert.deepEqual(branches[0], { + AND: [{ projectId: { not: null } }, { NOT: { projectId: "overhead-1" } }], + }); + assert.deepEqual(branches[1], { + AND: [{ projectId: null }, { estimate: { projectId: null } }], + }); + assert.deepEqual(branches[2], { + AND: [ + { projectId: null }, + { estimate: { projectId: { not: null } } }, + { NOT: { estimate: { projectId: "overhead-1" } } }, + ], + }); +}); From b3965764e6165ed30234ffb7a4a905c8ad290561 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 01:21:10 -0700 Subject: [PATCH 084/144] chore(rebase): onto Phase 1 head 4daa8c20 (phase re-validation, two-step upload) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Union-resolved package.json, the intake route helpers, book.ts and its test. Provenance in book.ts now follows WHICH source survived Phase 1's resolvePhase re-validation rather than which one was merely present — calling a dropped captured code "capture" would claim a decision that did not stick. Co-Authored-By: Claude Fable 5.1 --- src/app/api/receipts/intake/route.ts | 2 ++ src/lib/receipt-intake/book.ts | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 6dd4130e4..4993dc31a 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -130,7 +130,9 @@ function tooLargeForInline(limit: number, encoding: "json" | "multipart") { }, { status: 413 }, ); +} +/** * Accept a boolean from either a JSON body (real boolean) or a multipart form * (everything is a string). Anything else is "the caller did not say". */ diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 3399816ed..68afc697b 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -50,7 +50,6 @@ export interface BookableRow { suggestedCostCodeId: string | null; /** The model's confidence in that phase suggestion, 0..1. */ suggestedConfidence: number | null; - suggestedConfidence: number | null; /** Phase 3: the read found sales tax paid at the register. */ taxAtSource: boolean; /** Phase 3: installed at a customer job (deductible) — null = unknown. */ From 6401828b46aab15025b633d49d137ff3d04f09ff Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 01:30:30 -0700 Subject: [PATCH 085/144] fix(expenses): retire tax on deactivation, close the review lifecycle, scope coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 7. Rebased onto Phase 1 head 4daa8c20 first, so its DATE re-anchor and phase re-validation are in and are NOT duplicated here. 1. /api/receipts/parse was the last writer still storing a bare model date at UTC midnight; it uses the shared company-timezone parser now. The DATE -> Expense round trip is asserted at the 1 July quarter boundary, which is the case the raw @db.Date instant used to fail. 2. deactivateQboExpense retires the tax classification in the SAME statement that zeroes the amount. Leaving taxAmount behind left taxAmount > amount, which the new CHECK refuses — one classified receipt would have aborted the whole sync — and the classification describes a purchase QuickBooks says never happened. needsTaxReview is CLEARED there, not set: a vanished purchase is not something to re-check. 3. The lifecycle closes: invalidation raises needsTaxReview, the report query excludes those rows, and a human's answer clears the flag in the same write as the answer. Without the query condition, "a null taxDeductibleBase means the whole pre-tax total" would have claimed the FULL amount of a receipt nobody had re-checked. A phase-only edit deliberately does not clear it. 4. The expense item check and tagExpensesToChangeOrderCore authorize on the RESOLVED project. The item check's `estimateId` escape hatch is gone — for a re-attributed row the estimate belongs to the job it left, so that branch admitted exactly the cross-job link the check exists to stop. 5. measureCoverage takes a project-scoped, phase-validated item map. A global id->code map counted another job's line item as coverage, which is the one direction this metric must not flatter. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.mjs | 40 ++++++++++-- src/app/api/expenses/[id]/route.ts | 23 +++++-- src/app/api/receipts/parse/route.ts | 9 ++- src/lib/qbo-expense-sync.ts | 54 ++++++++++++++- src/lib/tax-at-source-report.ts | 12 ++++ src/lib/time-expense-core.ts | 10 ++- tests/backfill-expense-attribution.test.ts | 37 +++++++++++ tests/expense-date-timezone.test.ts | 16 +++++ tests/expense-edit-authz.test.ts | 76 +++++++++++++++++++--- tests/qbo-expense-sync.test.ts | 57 ++++++++++++++++ tests/tax-at-source-query.test.ts | 11 ++++ 11 files changed, 320 insertions(+), 25 deletions(-) diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 858d3fbd2..05177bd25 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -67,6 +67,35 @@ const pct = (part, whole) => (whole > 0 ? `${((part / whole) * 100).toFixed(1)}% * that was 0% attributed. Magnitude of money moved is the honest base, and it * is the same choice computeProjectVariance makes (job-variance.ts). */ +/** + * Build the per-expense item lookup the VARIANCE REPORT would use. + * + * The variance report resolves an item link only within the project's own item + * pool (job-variance-db.ts), so an expense pointing at another job's line item + * is unattributed there. A single global id->code map does not model that: it + * counted cross-job dollars as covered, which is the one direction this + * metric must not flatter. + * + * The returned map is keyed by item id but populated ONLY with items whose + * project matches the expense set being measured. + */ +export function scopedItemCostCodes(rows, items, allowedCodesByProject) { + const scoped = new Map(); + for (const row of rows) { + if (!row.itemId) continue; + const item = items.get(row.itemId); + if (!item || !item.costCodeId) continue; + const projectId = resolveExpenseProjectId(row); + // Same two gates the writer applies: the link must not cross jobs, and + // the code must be a live phase of that job. + if (!projectId || item.projectId !== projectId) continue; + const allowed = allowedCodesByProject.get(projectId); + if (!allowed || !allowed.has(item.costCodeId)) continue; + scoped.set(row.itemId, item.costCodeId); + } + return scoped; +} + export function measureCoverage(rows, itemCostCodeById = new Map()) { let attributed = 0; let unattributed = 0; @@ -384,8 +413,11 @@ export async function runBackfill({ // ── the table ─────────────────────────────────────────────────────────── const scoped = new Set(scopedProjectIds); const inScopeExpenses = expenses.filter(e => scoped.has(resolveExpenseProjectId(e) ?? "")); - const before = measureCoverage(inScopeExpenses, itemCostCodeById); - const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills), itemCostCodeById); + // Project-scoped and phase-validated, so a cross-job item link stays + // unattributed in the metric exactly as it does on the variance page. + const coverageItems = scopedItemCostCodes(inScopeExpenses, items, allowedCodesByProject); + const before = measureCoverage(inScopeExpenses, coverageItems); + const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills), coverageItems); log(`scope: ${scopedProjectIds.length} In Progress customer job(s); overhead project ${overheadProjectId} excluded`); log(""); @@ -394,8 +426,8 @@ export async function runBackfill({ for (const projectId of scopedProjectIds) { const rows = inScopeExpenses.filter(e => resolveExpenseProjectId(e) === projectId); if (rows.length === 0) continue; - const b = measureCoverage(rows, itemCostCodeById); - const a = measureCoverage(projectedRows(rows, plan.codeFills), itemCostCodeById); + const b = measureCoverage(rows, coverageItems); + const a = measureCoverage(projectedRows(rows, plan.codeFills), coverageItems); log( ` ${(projectNameById.get(projectId) ?? projectId).slice(0, 34).padEnd(34)} ` + `${`${a.codedCount}/${a.count}`.padStart(11)} ${money(a.total).padStart(13)} ` + diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 60dfbe076..8f969fe5e 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -102,18 +102,20 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const body = await req.json(); - // #5: an item link must belong to THIS expense's job. Checking only - // that the id exists let an edit point the expense at a line item on - // another project — which then feeds the item->costCode fallback and + // An item link must belong to THIS expense's RESOLVED job. Checking + // only that the id exists let an edit point the expense at a line item + // on another project, which then feeds the item->costCode fallback and // silently books the phase of a different job. + // + // The `estimateId` escape hatch is gone: for a RE-ATTRIBUTED expense + // the estimate belongs to the job it left, so that branch admitted + // exactly the cross-job link the check exists to stop. The resolved + // project is the only authority. if (body.itemId) { const itemExists = await prisma.estimateItem.findFirst({ where: { id: body.itemId, - OR: [ - { estimateId: expense.estimateId }, - { estimate: { projectId: resolvedProjectId } }, - ], + estimate: { projectId: resolvedProjectId }, }, select: { id: true }, }); @@ -476,6 +478,13 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id ...(editsBase ? { taxDeductibleBase: nextBase } : {}), ...(editsTaxAmount ? { taxAmount: nextTaxAmount } : {}), ...(editsTaxAtSource ? { taxAtSource: nextTaxAtSource as boolean } : {}), + // A human just answered, so the row is no longer awaiting one. + // Cleared in the SAME write as the answer: two statements would + // leave a window where the report sees an answered row it still + // refuses to count. + ...(editsInstalled || editsBase || editsTaxAmount || editsTaxAtSource + ? { needsTaxReview: false } + : {}), ...(editsCostCode ? { costCodeId: nextCostCodeId, diff --git a/src/app/api/receipts/parse/route.ts b/src/app/api/receipts/parse/route.ts index cd246acec..3f39860fa 100644 --- a/src/app/api/receipts/parse/route.ts +++ b/src/app/api/receipts/parse/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; import Anthropic from "@anthropic-ai/sdk"; import { authenticateMobileOrSession, userCanAccessProject } from "@/lib/mobile-auth"; import { getSupabase, STORAGE_BUCKET } from "@/lib/supabase"; @@ -298,7 +299,13 @@ export async function POST(req: NextRequest) { projectId, description: `[AI ${confidence}%] ${parsed.vendor} receipt — pending bookkeeper review`, amount: parsed.total as number, - date: parsed.date ? new Date(parsed.date as string) : new Date(), + // A COMPANY CALENDAR DAY, like every other writer. The + // model returns a bare "2026-07-01", and `new Date()` on + // that is UTC midnight — which reads as 30 June in + // Pacific and files the receipt in the wrong quarter. + date: typeof parsed.date === "string" && /^\d{4}-\d{2}-\d{2}$/.test(parsed.date) + ? dateOnlyInTimeZone(parsed.date, await resolveCompanyTimeZone()) + : (parsed.date ? new Date(parsed.date as string) : new Date()), vendor: parsed.vendor as string, status: "Pending", }, diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 62265851e..34f446521 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -532,7 +532,10 @@ type ExpenseTransaction = { estimateId: string; projectId?: string | null; taxAmount?: unknown; + taxAtSource?: boolean; + installedAtCustomer?: boolean | null; taxDeductibleBase?: unknown; + needsTaxReview?: boolean; amount: unknown; vendor: string | null; date: Date | null; @@ -544,7 +547,7 @@ type ExpenseTransaction = { }): Promise; update(args: { where: { id: string }; - data: QboExpenseUpdateData; + data: QboExpenseUpdateData | QboExpenseRetirementData; }): Promise; updateMany(args: { where: { id: string; projectId: null }; @@ -582,6 +585,24 @@ type ExistingQboExpense = * non-QBO field it can carry, and only to clear an allocation the new amount * would make impossible — see planQboExpenseUpdate. */ +/** + * What `deactivateQboExpense` writes. Separate from the update shape because it + * is the only path allowed to zero the amount AND retire a tax classification + * in one statement — see the comment at its call site. + */ +export interface QboExpenseRetirementData { + amount: 0; + description: string; + status: "Reviewed"; + qbSyncToken?: string | undefined; + qbSyncedAt: Date; + taxAmount: null; + taxAtSource: false; + installedAtCustomer: null; + taxDeductibleBase: null; + needsTaxReview: false; +} + export type QboExpenseUpdateData = Partial & { taxDeductibleBase?: null; taxAmount?: null; @@ -821,6 +842,11 @@ export async function deactivateQboExpense( date: true, description: true, status: true, + taxAmount: true, + taxAtSource: true, + installedAtCustomer: true, + taxDeductibleBase: true, + needsTaxReview: true, }, }); if (!existing) return "unchanged"; @@ -833,11 +859,30 @@ export async function deactivateQboExpense( const description = `[QuickBooks import] Removed in QBO (${removal.reason})`; const qbSyncToken = removal.qbSyncToken ?? existing.qbSyncToken; + // A DELETED PURCHASE HAS NO TAX CLASSIFICATION TO KEEP. + // + // Zeroing `amount` while leaving `taxAmount` behind leaves + // `taxAmount > amount` — which the new CHECK refuses, so the whole sync + // transaction would abort on a receipt someone had classified. And the + // classification is about a purchase QuickBooks says never happened, so + // there is nothing to preserve: it is RETIRED, in the same statement as + // the zeroing, or the row is briefly inconsistent in a way the report + // can read. + // + // `needsTaxReview` is cleared rather than set: this is not a figure a + // human needs to re-check, it is a purchase that is gone. + const classificationIsRetired = + existing.taxAmount === null && + existing.taxAtSource === false && + existing.installedAtCustomer === null && + existing.taxDeductibleBase === null && + existing.needsTaxReview === false; if ( Number(existing.amount) === 0 && existing.description === description && existing.qbSyncToken === qbSyncToken && - existing.status === "Reviewed" + existing.status === "Reviewed" && + classificationIsRetired ) { return "unchanged"; } @@ -849,6 +894,11 @@ export async function deactivateQboExpense( status: "Reviewed", qbSyncToken: qbSyncToken ?? undefined, qbSyncedAt: removal.qbSyncedAt, + taxAmount: null, + taxAtSource: false, + installedAtCustomer: null, + taxDeductibleBase: null, + needsTaxReview: false, }, }); return "removed"; diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts index 7ba568e91..9119f5d13 100644 --- a/src/lib/tax-at-source-report.ts +++ b/src/lib/tax-at-source-report.ts @@ -17,6 +17,13 @@ // bookkeeper on the expense edit route, and // * `taxAmount > 0` — a zero is an answer (no tax), not an absence. // +// ...and one NEGATIVE condition, which is about the row's LIFECYCLE rather +// than its content: `needsTaxReview` must be false. A QBO re-sync that moves +// the gross under a recorded tax retires the classification and raises that +// flag; until a human answers again the row is not a deduction, and in +// particular the "a null `taxDeductibleBase` means the whole pre-tax total" +// rule must not fire on it. +// // TWO THINGS THIS FILE IS FUSSY ABOUT, both because it feeds a tax filing: // // 1. INTEGER CENTS. Every sum is in whole cents, converted from the Decimal's @@ -360,6 +367,11 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise taxAtSource: true, installedAtCustomer: true, taxAmount: { gt: 0 }, + // A row whose gross moved under a human's tax answer is NOT a + // deduction until a person looks again. Without this the "null + // taxDeductibleBase means the whole pre-tax total" rule would claim + // the full amount of a receipt nobody has re-checked. + needsTaxReview: false, date: { gte: filters.from, lt: filters.to }, // The page promises Shop purchases are excluded; until now nothing // enforced it. An overhead receipt mistakenly flagged diff --git a/src/lib/time-expense-core.ts b/src/lib/time-expense-core.ts index 3eca72691..9790c5530 100644 --- a/src/lib/time-expense-core.ts +++ b/src/lib/time-expense-core.ts @@ -3,6 +3,7 @@ import { resolveCostCode } from "./cost-coding"; import { prismaCostCodingDataSource } from "./cost-coding-db"; import { isCostCodeAllowedForProject } from "./project-phases"; import { prismaPhaseDataSource } from "./project-phases-db"; +import { resolveExpenseProjectId } from "./expense-attribution"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "./company-timezone"; import { resolveScheduleTaskIdForPunch } from "./punch-task-binding"; import { toCompanyDayKey } from "./company-day"; @@ -267,13 +268,20 @@ export async function tagExpensesToChangeOrderCore( select: { id: true, qbPurchaseId: true, + projectId: true, estimate: { select: { projectId: true } }, invoiceId: true, invoicedAt: true, }, }); if (rows.length !== new Set(input.ids).size) throw new Error("One or more expenses were not found"); - if (rows.some((row) => row.estimate.projectId !== changeOrder.projectId)) throw new Error("All expenses must belong to the change order project"); + // Resolved, not read off the estimate. A re-attributed expense belongs to + // the job its `projectId` names — checking the estimate would let it be + // tagged to a change order on the job it USED to be on, and would refuse a + // legitimate tag on the job it is actually on now. + if (rows.some((row) => resolveExpenseProjectId(row) !== changeOrder.projectId)) { + throw new Error("All expenses must belong to the change order project"); + } for (const row of rows) assertExpenseMutableOutsideQbo(row); if (rows.some((row) => row.invoiceId || row.invoicedAt)) throw new Error("Billed expenses cannot be retagged"); const result = await prisma.expense.updateMany({ diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 02140e492..13499da20 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -14,6 +14,7 @@ import { projectedRows, remainderCsv, runBackfill, + scopedItemCostCodes, } from "../scripts/backfill-expense-attribution.mjs"; const OVERHEAD_ID = "overhead-project"; @@ -614,3 +615,39 @@ test("coverage still counts a genuinely uncoded row as a gap", () => { assert.equal(measureCoverage(rows, new Map()).unattributed, 100); assert.equal(measureCoverage(rows, new Map()).codedCount, 0); }); + +test("coverage keeps CROSS-JOB item dollars unattributed", () => { + // Codex round 7, item 5. The variance report resolves an item link only + // within the project's own item pool, so an expense pointing at another + // job's line item is unattributed there. A global id->code map counted + // those dollars as covered — flattering the one number this metric exists + // to report honestly. + const rows = [ + expense({ id: "cross", projectId: "job-1", itemId: "item-elsewhere", amount: 500 }), + expense({ id: "own", projectId: "job-1", itemId: "item-own", amount: 300 }), + ]; + const items = new Map([ + ["item-elsewhere", { costCodeId: "cc-frame", estimateId: "est-2", projectId: "job-2" }], + ["item-own", { costCodeId: "cc-plumb", estimateId: "est-1", projectId: "job-1" }], + ]); + const scoped = scopedItemCostCodes(rows, items, ALL_PHASES); + + assert.equal(scoped.has("item-own"), true, "the same-job link counts"); + assert.equal(scoped.has("item-elsewhere"), false, "the cross-job link does not"); + + const coverage = measureCoverage(rows, scoped); + assert.equal(coverage.attributed, 300); + assert.equal(coverage.unattributed, 500, "the cross-job dollars stay a gap"); +}); + +test("coverage ignores an item whose code is not a live phase of the job", () => { + // Same gate the writer applies: a code from a draft estimate is not a + // phase, so it cannot count as coverage either. + const rows = [expense({ id: "e1", projectId: "job-1", itemId: "item-draft", amount: 100 })]; + const items = new Map([ + ["item-draft", { costCodeId: "cc-retired", estimateId: "est-1", projectId: "job-1" }], + ]); + const scoped = scopedItemCostCodes(rows, items, ALL_PHASES); + assert.equal(scoped.size, 0); + assert.equal(measureCoverage(rows, scoped).unattributed, 100); +}); diff --git a/tests/expense-date-timezone.test.ts b/tests/expense-date-timezone.test.ts index 0da95b0e9..79ef8d571 100644 --- a/tests/expense-date-timezone.test.ts +++ b/tests/expense-date-timezone.test.ts @@ -70,3 +70,19 @@ test("a UTC company sees the same calendar day it stored", () => { // company must not be shifted either. assert.equal(dayKeyInTimeZone(dateOnlyInTimeZone("2026-07-01", "UTC"), "UTC"), "2026-07-01"); }); + +test("a @db.Date read back and stored on an Expense keeps its calendar day", () => { + // The pipeline path end to end: `ReceiptIntake.txnDate` is `@db.Date`, so + // Prisma hands it back as UTC midnight. Phase 1 re-anchors it before it + // reaches `Expense.date`; this asserts the RESULT at the quarter boundary + // that used to break — 1 July arriving as 2026-07-01T00:00:00Z. + const fromDbDate = new Date("2026-07-01T00:00:00.000Z"); + const calendarDay = fromDbDate.toISOString().slice(0, 10); + const stored = dateOnlyInTimeZone(calendarDay, PACIFIC); + + assert.equal(dayKeyInTimeZone(stored, PACIFIC), "2026-07-01"); + // ...and it lands inside Q3, which the raw DATE instant did not. + const q3Start = startOfDateInTimeZone("2026-07-01", PACIFIC); + assert.ok(stored >= q3Start, "the re-anchored value is in the quarter"); + assert.ok(fromDbDate < q3Start, "the raw @db.Date instant was not"); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 861110d80..1f81e0d6b 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -40,16 +40,11 @@ const fakePrisma = { }, estimateItem: { findFirst: async (args: { where: Record }) => { - const { id, OR } = args.where; - const item = estimateItems.find(candidate => candidate.id === id); + // The route now scopes purely on the RESOLVED project — no + // estimateId escape hatch — so the stub models exactly that. + const item = estimateItems.find(candidate => candidate.id === args.where.id); if (!item) return null; - const branches = (OR ?? []) as Record[]; - const ok = branches.some(branch => - branch.estimateId !== undefined - ? branch.estimateId === item.estimateId - : branch.estimate?.projectId === item.projectId, - ); - return ok ? { id: item.id } : null; + return args.where.estimate?.projectId === item.projectId ? { id: item.id } : null; }, findUnique: async (args: { where: { id: string } }) => { const item = estimateItems.find(candidate => candidate.id === args.where.id); @@ -107,6 +102,7 @@ beforeEach(() => { taxAmount: 16.55, taxAtSource: true, taxDeductibleBase: null, + needsTaxReview: false, estimateId: "est-job-1", projectId: "job-1", estimate: { projectId: "job-1" }, @@ -195,7 +191,9 @@ test("PATCH reaches a QBO-managed row — the population the report is made of", test("PATCH touches NOTHING but the three ProBuild-only columns", async () => { await patch({ installedAtCustomer: true }); - assert.deepEqual(Object.keys(updateArgs?.data ?? {}), ["installedAtCustomer"]); + // `needsTaxReview` rides along because answering IS clearing the flag — + // still nothing outside the ProBuild-only set. + assert.deepEqual(Object.keys(updateArgs?.data ?? {}), ["installedAtCustomer", "needsTaxReview"]); // A caller sending a QBO-synced field is told, not silently ignored. const res = await patch({ amount: "1.00" }); assert.equal(res.status, 400); @@ -329,3 +327,61 @@ test("a line item on this job's estimate is accepted", async () => { assert.equal((await call({ itemId: "item-own" })).status, 200); assert.equal(updateArgs?.data.itemId, "item-own"); }); + +// ── the needsTaxReview lifecycle (Codex round 7, item 3) ─────────────────── + +test("a human answer CLEARS needsTaxReview in the same write", async () => { + // Two statements would leave a window where the report sees an answered + // row it still refuses to count. + storedExpense = { ...storedExpense, needsTaxReview: true }; + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, true); + assert.equal(updateArgs?.data.needsTaxReview, false, "answered, so no longer awaiting one"); +}); + +test("every tax field clears the flag, and a phase-only edit does not", async () => { + storedExpense = { ...storedExpense, needsTaxReview: true }; + for (const body of [ + { taxAmount: 10 }, + { taxAtSource: false }, + { taxDeductibleBase: 50 }, + { installedAtCustomer: false }, + ]) { + await patch(body); + assert.equal(updateArgs?.data.needsTaxReview, false, JSON.stringify(body)); + } + // A cost-code edit is not an answer to the tax question, so the row stays + // flagged — otherwise re-phasing a receipt would quietly re-admit it to the + // filing. + await patch({ costCodeId: null }); + assert.equal(updateArgs?.data.needsTaxReview, undefined); +}); + +// ── the item link is judged on the RESOLVED job (item 4) ─────────────────── + +test("a re-attributed expense may take an item from its NEW job", async () => { + storedExpense = { + ...storedExpense, + projectId: "job-1", + estimateId: "est-job-2", + estimate: { projectId: "job-2" }, + }; + estimateItems = [{ id: "item-new-job", estimateId: "est-job-1", projectId: "job-1" }]; + assert.equal((await call({ itemId: "item-new-job" })).status, 200); +}); + +test("...and NOT one from the job it left, even via its own estimate", async () => { + // The `estimateId` escape hatch used to admit exactly this: for a + // re-attributed row the estimate belongs to the job it left. + storedExpense = { + ...storedExpense, + projectId: "job-1", + estimateId: "est-job-2", + estimate: { projectId: "job-2" }, + }; + estimateItems = [{ id: "item-old-job", estimateId: "est-job-2", projectId: "job-2" }]; + const res = await call({ itemId: "item-old-job" }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index e00493c55..8fd69dbea 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1582,3 +1582,60 @@ test("clearing a tax classification is never reported as 'unchanged'", () => { ); assert.equal(plan.data.needsTaxReview, true); }); + +test("deactivation RETIRES the tax classification in the same statement as the zeroing", async () => { + // Codex round 7, item 2. Zeroing `amount` while leaving `taxAmount` behind + // leaves taxAmount > amount, which the new CHECK refuses — so one + // classified receipt would abort the whole sync. And the classification is + // about a purchase QuickBooks says never happened. + const fake = createFakePrisma([ + { + ...WRITE, + id: "expense-1", + projectId: "project-1", + receiptUrl: null, + taxAmount: 16.55, + taxAtSource: true, + installedAtCustomer: true, + taxDeductibleBase: 50, + needsTaxReview: true, + } as any, + ]); + assert.equal( + await deactivateQboExpense(fake.client, { + qbPurchaseId: "purchase-1", + qbSyncToken: "1", + qbSyncedAt: new Date("2026-07-29T14:00:00.000Z"), + reason: "deleted", + }), + "removed", + ); + const row = fake.rows.get("purchase-1") as any; + assert.equal(row.amount, 0); + assert.equal(row.taxAmount, null); + assert.equal(row.taxAtSource, false); + assert.equal(row.installedAtCustomer, null); + assert.equal(row.taxDeductibleBase, null); + assert.equal(row.needsTaxReview, false, "a vanished purchase is not something to re-check"); + // The constrained row is legal: tax <= amount and base <= amount - tax. + assert.ok(Number(row.taxAmount ?? 0) <= Number(row.amount)); +}); + +test("a second deactivation of an already-retired row is unchanged", async () => { + const retired = { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + amount: 0, status: "Reviewed" as const, + description: "[QuickBooks import] Removed in QBO (deleted)", + qbSyncToken: "1", + taxAmount: null, taxAtSource: false, installedAtCustomer: null, + taxDeductibleBase: null, needsTaxReview: false, + }; + const fake = createFakePrisma([retired as any]); + assert.equal( + await deactivateQboExpense(fake.client, { + qbPurchaseId: "purchase-1", qbSyncToken: "1", + qbSyncedAt: new Date("2026-07-29T15:00:00.000Z"), reason: "deleted", + }), + "unchanged", + ); +}); diff --git a/tests/tax-at-source-query.test.ts b/tests/tax-at-source-query.test.ts index 4449665c8..470434532 100644 --- a/tests/tax-at-source-query.test.ts +++ b/tests/tax-at-source-query.test.ts @@ -168,3 +168,14 @@ test("the query excludes the Shop/overhead bucket, both ways round", async () => AND: [{ projectId: null }, { estimate: { projectId: null } }], }); }); + +test("a row awaiting re-review is not a deduction", async () => { + // Codex round 7, item 3. Without this the "a null taxDeductibleBase means + // the whole pre-tax total" rule would claim the FULL amount of a receipt + // whose gross moved under a human's tax answer and that nobody has + // re-checked. + const filters = parseTaxAtSourceFilters({ from: "2026-07-01", to: "2026-09-30" }, PACIFIC); + recorded.length = 0; + await queryTaxAtSourceRows(filters); + assert.equal(recorded[0].where.needsTaxReview, false); +}); From 5471c62f5ae01a359bce36728b31ae6d64864d70 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 02:13:31 -0700 Subject: [PATCH 086/144] fix(expenses): two-step capture rules, already-booked fill, reader migration, CAS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 8. Rebased onto Phase 1 head e3da4a6d first. 1. /start and /finalize enforce the SAME capture rules as the inline door, through one shared validateCapturedPhase: a costCodeId is checked for existence, active-ness AND membership of the named project, and is refused outright when there is no project to check it against. Both persist installedAtCustomer as a tri-state with no default. The two-step path had no validation at all, so a crew phone could pin any active company code to a receipt on any job — and booking copies a captured code onto the Expense with provenance "capture", which no automated pass may correct. 2. bookReceipt's alreadyExists path FILLS the Phase 3 fields it left blank (projectId, phase, provenance, tax, installedAtCustomer) and only the blanks: costCodeSource capture/manual and an already-answered installedAtCustomer are a human's and are untouchable. If the existing Purchase is on a DIFFERENT job than the intake claims, nothing is filled and the row parks as NEEDS_REVIEW "attribution-conflict" — filling would be guessing which job is right, overwriting would move real money between jobs. 3. schedule-core (both spots), automation-events, the ai-review route and the manager receipt queue now label and roll up by the resolved job. 4. The tax PATCH writes under a compare-and-set on the values its validation depended on (amount, taxAmount, taxDeductibleBase) and answers 409 on a miss; the sync's write does the same and RE-PLANS once against a fresh read. Mutation-checked: dropping the predicate loses a bookkeeper's correction. 5. PUT refuses all five tax fields by name with the field in the response — a silent drop looks like a successful correction. Co-Authored-By: Claude Fable 5.1 --- src/app/api/automation/ai-review/route.ts | 13 ++- .../api/cron/receipt-intake-worker/route.ts | 5 +- src/app/api/expenses/[id]/route.ts | 67 +++++++++++--- .../receipts/intake/[id]/finalize/route.ts | 7 +- src/app/api/receipts/intake/route.ts | 12 +-- src/app/api/receipts/intake/start/route.ts | 6 ++ src/lib/automation-events.ts | 10 +- src/lib/qbo-expense-sync.ts | 47 +++++++++- src/lib/receipt-capture-validation.ts | 22 +++++ src/lib/receipt-intake/book.ts | 92 ++++++++++++++++++- src/lib/schedule-core.ts | 15 ++- tests/expense-attribution.test.ts | 49 ++++++++++ tests/expense-edit-authz.test.ts | 64 +++++++++++++ tests/expense-phase-scope.test.ts | 41 +++++++++ tests/qbo-expense-sync.test.ts | 61 +++++++++++- tests/receipt-intake-book.test.ts | 14 ++- tests/receipt-intake-worker.test.ts | 1 - 17 files changed, 475 insertions(+), 51 deletions(-) create mode 100644 src/lib/receipt-capture-validation.ts diff --git a/src/app/api/automation/ai-review/route.ts b/src/app/api/automation/ai-review/route.ts index 8447e7dde..8a1b8bf6d 100644 --- a/src/app/api/automation/ai-review/route.ts +++ b/src/app/api/automation/ai-review/route.ts @@ -4,6 +4,7 @@ import Anthropic from "@anthropic-ai/sdk"; import { GoogleGenAI } from "@google/genai"; import { getCurrentUserWithPermissions, hasPermission } from "@/lib/permissions"; import { prisma } from "@/lib/prisma"; +import { resolveExpenseProjectLabel } from "@/lib/expense-attribution"; import { logAutomationEvent, resolveEventFileId } from "@/lib/automation-events"; import { readIdentifier, resolveReceiptPushEvent, trustedQbPurchaseId } from "@/lib/automation-key-resolver"; import { decimalToCents, type DecimalLike } from "@/lib/register-merge"; @@ -389,12 +390,20 @@ async function computeReasonableness( date: Date | null; costCode: { name: string } | null; costType: { name: string } | null; + projectId: string | null; + project: { name: string; status: string } | null; estimate: { project: { name: string; status: string } | null } | null; }, ): Promise { try { const vendor = expense.vendor ?? pushEvent.vendor; - const project = expense.estimate?.project ?? null; + // The resolver's rule, applied to the display object: the denormalized + // job when the row has one, the estimate's otherwise. An AI review that + // names the wrong job is worse than one that names none, because a + // reader will act on it. + const project = expense.projectId + ? expense.project + : (expense.estimate?.project ?? null); const vendorHistory = vendor ? await vendorExpenseStats(vendor, expense.id) : null; return await judgeReasonableness({ vendor, @@ -593,6 +602,8 @@ export async function POST(request: Request) { date: true, costCode: { select: { name: true } }, costType: { select: { name: true } }, + projectId: true, + project: { select: { name: true, status: true } }, estimate: { select: { project: { select: { name: true, status: true } } } }, }, }); diff --git a/src/app/api/cron/receipt-intake-worker/route.ts b/src/app/api/cron/receipt-intake-worker/route.ts index 788ecea68..575e05e4e 100644 --- a/src/app/api/cron/receipt-intake-worker/route.ts +++ b/src/app/api/cron/receipt-intake-worker/route.ts @@ -3,7 +3,7 @@ import { NextResponse } from "next/server"; import { isCronAuthorized } from "@/lib/cron-auth"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; -import { resolveProjectPhaseCodes } from "@/lib/project-phases"; +import { isCostCodeAllowedForProject, resolveProjectPhaseCodes } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { isPaused, PAUSE_KEYS } from "@/lib/automation-settings"; import { logAutomationEvent } from "@/lib/automation-events"; @@ -24,8 +24,6 @@ import { driveFileIdOf, triageCutoverRows, resolveCutoverBoundary } from "@/lib/receipt-intake/cutover"; import { resolveCompanyTimeZone } from "@/lib/company-timezone"; -import { isCostCodeAllowedForProject, resolveProjectPhaseCodes } from "@/lib/project-phases"; -import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { bookReceipt, type BookPrismaClient } from "@/lib/receipt-intake/book"; import { backoffMs } from "@/lib/receipt-intake/route-state"; import { @@ -73,7 +71,6 @@ const LEASE_MS = CLAIM_LEASE_MINUTES * 60_000; const WORKER_ROW_SELECT = { id: true, source: true, sourceRef: true, state: true, dryRun: true, projectId: true, costCodeId: true, suggestedCostCodeId: true, - suggestedConfidence: true, storagePath: true, fileName: true, mimeType: true, fileSize: true, vendor: true, txnDate: true, totalCents: true, taxCents: true, // Phase 3 attribution — booking copies these straight onto the Expense. diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 8f969fe5e..42192a883 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -173,14 +173,29 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: } } - // The tax-deduction fields are NOT editable here — the PATCH below is - // their single writer, because this handler's QBO-mutability guard - // excludes exactly the pipeline rows the tax report is made of. A - // silent ignore would look like a successful correction. - for (const field of ["installedAtCustomer", "taxDeductibleBase"]) { + // STRICT ALLOWLIST. Every tax-return field is refused here BY NAME — + // PATCH is their single writer, because this handler's QBO-mutability + // guard excludes exactly the pipeline rows the tax report is made of. + // + // Refused rather than ignored: a silent drop looks like a successful + // correction, and the caller would believe a deduction was recorded + // that never was. `needsTaxReview` is in the list too — it is a + // lifecycle flag the server owns, and no client may clear it without + // supplying the answer that justifies clearing it. + const TAX_FIELDS_OWNED_BY_PATCH = [ + "taxAmount", + "taxAtSource", + "needsTaxReview", + "installedAtCustomer", + "taxDeductibleBase", + ]; + for (const field of TAX_FIELDS_OWNED_BY_PATCH) { if (Object.prototype.hasOwnProperty.call(body, field)) { return NextResponse.json( - { error: `Use PATCH on this expense to edit ${field}.` }, + { + error: `${field} can't be edited here. Use PATCH on this expense.`, + field, + }, { status: 400 }, ); } @@ -471,9 +486,29 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id } } - const updated = await prisma.expense.update({ - where: { id }, - data: { + // COMPARE-AND-SET on the VALUES the decision rested on. + // + // Everything above was validated against the row as it was READ: the + // ceiling for `taxDeductibleBase` is `amount - taxAmount`, and a QBO + // re-sync can move either between that read and this write. Writing + // anyway would store a figure that was legal a moment ago and is not + // now — which the database CHECK then refuses (aborting the request) or + // which slips through as an overstated deduction. + // + // The predicate names those inputs rather than a row version, so it + // fails ONLY when something the answer depended on actually moved; an + // unrelated edit does not force the bookkeeper to redo their work. + // + // Zero rows means it moved: 409, and the caller re-reads. No automatic + // retry — a human decided against numbers that have since changed, so + // the ANSWER may be wrong now, not just the write. + const casWhere = { + id, + amount: expense.amount, + taxAmount: expense.taxAmount, + taxDeductibleBase: expense.taxDeductibleBase, + }; + const data = { ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), ...(editsBase ? { taxDeductibleBase: nextBase } : {}), ...(editsTaxAmount ? { taxAmount: nextTaxAmount } : {}), @@ -492,9 +527,19 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id costCodeConfidence: null, } : {}), - }, - }); + }; + const written = await prisma.expense.updateMany({ where: casWhere, data }); + if (written.count === 0) { + return NextResponse.json( + { + error: "This expense changed while you were editing it. Reopen it and check the figures before saving.", + code: "STALE_EXPENSE", + }, + { status: 409 }, + ); + } + const updated = await prisma.expense.findUnique({ where: { id } }); return NextResponse.json(updated); } catch (error) { console.error("Error correcting expense:", error); diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 41b453ea8..8423a94b5 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -117,7 +117,12 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const { id } = await context.params; - let body: { sha256?: unknown; costCodeId?: unknown; projectId?: unknown } = {}; + let body: { + sha256?: unknown; + costCodeId?: unknown; + projectId?: unknown; + installedAtCustomer?: unknown; + } = {}; try { body = await req.json(); } catch { diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index 4993dc31a..ca8b99775 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -5,6 +5,7 @@ import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { optionalBool } from "@/lib/receipt-capture-validation"; import { resolveInstalledAtCustomer } from "@/lib/expense-attribution"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { receiptObjectSize, uploadReceiptObject } from "@/lib/receipt-intake/bucket"; @@ -132,17 +133,6 @@ function tooLargeForInline(limit: number, encoding: "json" | "multipart") { ); } -/** - * Accept a boolean from either a JSON body (real boolean) or a multipart form - * (everything is a string). Anything else is "the caller did not say". - */ -function optionalBool(value: unknown): boolean | null { - if (typeof value === "boolean") return value; - if (value === "true") return true; - if (value === "false") return false; - return null; -} - async function parseBody(req: Request): Promise { const contentType = req.headers.get("content-type") ?? ""; const str = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : null); diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index f8da8601f..fc5f7d0f3 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -3,6 +3,9 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; +import { optionalBool } from "@/lib/receipt-capture-validation"; +import { SECURE_BUCKET } from "@/lib/secure-storage"; +import { getSupabase } from "@/lib/supabase"; import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; import { ACCEPTED_MIME_TYPES, EXT_BY_MIME } from "@/lib/receipt-intake/file-type"; import { decideSource, MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; @@ -145,6 +148,9 @@ export async function POST(req: Request) { dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId, costCodeId, + // Tri-state, and nothing defaults it: silence is "nobody said", + // which is never claimed on the excise return. + installedAtCustomer: optionalBool(body.installedAtCustomer), createdById: auth.via === "session" ? auth.user.id : null, // Forwarder-only, same as the single-shot path: this is the // claim that v1 already booked the document. diff --git a/src/lib/automation-events.ts b/src/lib/automation-events.ts index 291195649..b20149402 100644 --- a/src/lib/automation-events.ts +++ b/src/lib/automation-events.ts @@ -1,4 +1,5 @@ import { prisma } from "@/lib/prisma"; +import { resolveExpenseProjectLabel } from "@/lib/expense-attribution"; /** * Append-only event log behind the Automation Command Center. @@ -715,12 +716,11 @@ function attachSyncedExpenses(journeys: Map, expenses: S const { exp: newest, fullId, syncedAt } = withSyncedAt[0]; j.syncedExpenseId = newest.id; - j.syncedProjectName = newest.estimate?.project?.name ?? null; + j.syncedProjectName = resolveExpenseProjectLabel(newest).projectName; j.driveFileId = j.driveFileId ?? fullId; j.synced = { expenseId: newest.id, - projectId: newest.estimate?.project?.id ?? null, - projectName: newest.estimate?.project?.name ?? null, + ...resolveExpenseProjectLabel(newest), // Prisma Decimal → cents; guard the conversion, this is display data. amountCents: newest.amount != null ? Math.round(Number(newest.amount) * 100) : null, vendor: newest.vendor ?? null, @@ -755,6 +755,10 @@ const SYNCED_EXPENSE_SELECT = { createdAt: true, qbPurchaseId: true, // Expense hangs off the ESTIMATE, not the project directly. + // BOTH sides — the register names the job the money is on, not the one + // the estimate happens to belong to. + projectId: true, + project: { select: { id: true, name: true } }, estimate: { select: { project: { select: { id: true, name: true } } } }, } as const; diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 34f446521..50300675e 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -549,9 +549,20 @@ type ExpenseTransaction = { where: { id: string }; data: QboExpenseUpdateData | QboExpenseRetirementData; }): Promise; + /** + * Two guarded writes go through here, and both put their guarantee in + * the PREDICATE rather than in a value read earlier in the transaction: + * the attribution fill (`projectId: null`) and a COMPARE-AND-SET on the + * tax values a plan was computed from. The tax PATCH can commit between + * this transaction's read and its write, and a bookkeeper's answer must + * not be overwritten by a plan made before it existed. + */ updateMany(args: { - where: { id: string; projectId: null }; - data: { projectId?: string; estimateId: string }; + where: Record; + data: + | { projectId?: string; estimateId: string } + | QboExpenseUpdateData + | QboExpenseRetirementData; }): Promise<{ count: number }>; }; }; @@ -808,10 +819,38 @@ export async function upsertQboExpense( data: plan.fill, }); } - await transaction.expense.update({ - where: { id: existing.id }, + // CAS when the client supports it: a tax correction committing between + // the read above and this write would otherwise be clobbered by a plan + // that never saw it. Zero rows means the row moved — re-read and + // re-plan once, which is enough because the second read is inside the + // advisory lock this transaction already holds. + const cas = await transaction.expense.updateMany({ + where: { + id: existing.id, + taxAmount: existing.taxAmount ?? null, + taxDeductibleBase: existing.taxDeductibleBase ?? null, + }, data: plan.data, }); + if (cas.count > 0) return "updated"; + + // The tax values moved between the read and the write — a bookkeeper's + // PATCH landed. Re-read and RE-PLAN once against what is actually + // there. Once is enough: this transaction holds the per-purchase + // advisory lock, so the only writer that can beat it is that PATCH, and + // it has now committed. + const fresh = await transaction.expense.findUnique({ + where: { qbPurchaseId: write.qbPurchaseId }, + select: { + id: true, qbSyncToken: true, estimateId: true, projectId: true, + taxAmount: true, taxDeductibleBase: true, amount: true, + vendor: true, date: true, description: true, status: true, + }, + }); + if (!fresh) return "unchanged"; + const replanned = planQboExpenseUpdate(fresh, write); + if (planIsNoop(fresh, replanned)) return "unchanged"; + await transaction.expense.update({ where: { id: fresh.id }, data: replanned.data }); return "updated"; }); } diff --git a/src/lib/receipt-capture-validation.ts b/src/lib/receipt-capture-validation.ts new file mode 100644 index 000000000..ede66e4f2 --- /dev/null +++ b/src/lib/receipt-capture-validation.ts @@ -0,0 +1,22 @@ +// The capture-time facts a front door has to normalise, in ONE place. +// +// The phase gate itself lives in receipt-intake/late-fields.ts (`authorizePhase`), +// which every door — inline POST /api/receipts/intake, POST .../start, and +// POST .../[id]/finalize — calls after it has authorized the project. +// +// What is left here is the one capture fact with no gate of its own: +// `installedAtCustomer` decides whether a receipt is claimed on a state excise +// return, so "the caller did not say" has to survive as NULL rather than +// collapsing to false. +/** + * Accept a boolean from either a JSON body (a real boolean) or a multipart form + * (where everything is a string). Anything else — including a missing key — is + * "the caller did not say", which is NOT the same as "no" and is stored as + * NULL. Nothing defaults it: an unanswered receipt must never be claimed. + */ +export function optionalBool(value: unknown): boolean | null { + if (typeof value === "boolean") return value; + if (value === "true") return true; + if (value === "false") return false; + return null; +} diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 68afc697b..92f48d047 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -167,8 +167,19 @@ export interface BookPrismaClient { } | null>; }; expense: { - findUnique(args: any): Promise<{ id: string } | null>; + /** The Phase 3 columns the already-exists fill reads before deciding. */ + findUnique(args: any): Promise<{ + id: string; + projectId?: string | null; + costCodeId?: string | null; + costCodeSource?: string | null; + taxAmount?: unknown; + taxAtSource?: boolean; + installedAtCustomer?: boolean | null; + estimate?: { projectId: string | null } | null; + } | null>; create(args: any): Promise<{ id: string }>; + update(args: any): Promise; }; receiptIntake: { update(args: any): Promise; @@ -637,14 +648,74 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro ? `Check #${(row.refNumber ?? "").replace(/^Check/, "") || "?"}${row.memo ? ` — "${row.memo}"` : ""}` : (row.refNumber && row.refNumber !== "NoInv" ? `Invoice ${row.refNumber}` : "Receipt"); - const expenseId = await deps.db.$transaction(async tx => { + const booked = await deps.db.$transaction(async tx => { // A retry after a crash between the Purchase and this commit finds // its own Expense here (qbPurchaseId is @unique) — create it twice // and the insert would fail on that constraint anyway. const existing = await tx.expense.findUnique({ where: { qbPurchaseId: result.qbPurchaseId }, - select: { id: true }, + select: { + id: true, + projectId: true, + costCodeId: true, + costCodeSource: true, + taxAmount: true, + taxAtSource: true, + installedAtCustomer: true, + estimate: { select: { projectId: true } }, + }, }); + + // AN ALREADY-BOOKED PURCHASE STILL NEEDS ITS PHASE 3 FIELDS. + // + // `alreadyExists` is the lost-response retry, and it also covers a + // row v1 created before this pipeline existed. Returning it + // untouched left `projectId`, the phase, the provenance and the tax + // columns NULL forever on exactly the receipts the tax report is + // made of — the booking said BOOKED and the report saw nothing. + // + // So the gaps are filled and only the gaps: a human's decision + // outranks anything this pass knows. `costCodeSource` "capture" or + // "manual" is untouchable, and an `installedAtCustomer` that is + // already answered (true OR false) is a tax answer nobody but a + // bookkeeper may change. + if (existing) { + const existingProjectId = existing.projectId ?? existing.estimate?.projectId ?? null; + // ATTRIBUTION CONFLICT. The Purchase is already on a different + // job than this intake row claims. Filling fields would be + // guessing which one is right, and overwriting would silently + // move real money between jobs — so nobody is booked and a + // person is asked. The strong key stays held: the Purchase + // exists. + if (existingProjectId && row.projectId && existingProjectId !== row.projectId) { + return { conflict: true as const }; + } + + const humanCoded = + existing.costCodeSource === "capture" || existing.costCodeSource === "manual"; + const fill: Record = {}; + if (!existing.projectId && row.projectId) fill.projectId = row.projectId; + if (!existing.costCodeId && !humanCoded && costCodeId) { + fill.costCodeId = costCodeId; + fill.costCodeSource = costCodeSource; + fill.costCodeConfidence = costCodeConfidence; + } + // Tax is filled only when there is none recorded at all — + // a stored figure came either from an earlier booking of this + // same document or from a bookkeeper, and both outrank a + // re-read. + if (existing.taxAmount === null && taxApplied > 0) { + fill.taxAmount = taxApplied / 100; + fill.taxAtSource = true; + } + if (existing.installedAtCustomer === null && row.installedAtCustomer !== null) { + fill.installedAtCustomer = row.installedAtCustomer; + } + if (Object.keys(fill).length > 0) { + await tx.expense.update({ where: { id: existing.id }, data: fill }); + } + } + const expense = existing ?? await tx.expense.create({ data: { estimateId, @@ -724,10 +795,23 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro claimedAt: null, }, }); + // Phase 1's claim fence stays: a row whose claim token moved has + // been taken over by another worker, and this transaction must not + // commit a booking on its behalf. if (claimed.count === 0) throw new StaleClaimError(); - return expense.id; + return { conflict: false as const, expenseId: expense.id }; }); + if (booked.conflict) { + // A send WAS attempted, so the strong key stays claimed. + return { + outcome: "needs-review", + reason: "attribution-conflict", + releaseStrongKey: false, + }; + } + const expenseId = booked.expenseId; + // Audit row so the /automation register keeps seeing v2 bookings // alongside the bot's. Fire-and-forget by contract — never fails a // booking that already happened. diff --git a/src/lib/schedule-core.ts b/src/lib/schedule-core.ts index 0f87052be..da4c89a49 100644 --- a/src/lib/schedule-core.ts +++ b/src/lib/schedule-core.ts @@ -1,4 +1,5 @@ import { prisma } from "@/lib/prisma"; +import { resolveExpenseProjectId, resolveExpenseProjectLabel } from "@/lib/expense-attribution"; import { Prisma } from "@prisma/client"; import { withTxRetry } from "./tx-retry"; import { OPEN_PROJECT_STATUSES } from "./project-status"; @@ -1989,6 +1990,9 @@ export async function getCalendarOverlays( orderBy: { date: "asc" }, select: { id: true, amount: true, vendor: true, date: true, + // BOTH sides — the digest names the job the money is on. + projectId: true, + project: { select: { id: true, name: true } }, estimate: { select: { projectId: true, project: { select: { id: true, name: true } } } }, }, }), @@ -2027,8 +2031,9 @@ export async function getCalendarOverlays( amount: Number(e.amount), vendor: e.vendor, date: e.date!.toISOString(), - projectId: e.estimate?.project?.id ?? null, - projectName: e.estimate?.project?.name ?? null, + // Resolved, not read off the estimate — a re-attributed expense + // must be reported under the job it is actually on. + ...resolveExpenseProjectLabel(e), })), hours: hours.map(h => ({ id: h.id, @@ -2220,7 +2225,9 @@ async function getProjectMonthStrip(from: Date, to: Date, coRows: OverlayChangeO }), prisma.expense.findMany({ where: { date: { gte: from, lt: to } }, - select: { amount: true, estimate: { select: { projectId: true } } }, + // BOTH sides: the roll-up must count an expense against the job + // it is actually on, not the one its estimate names. + select: { amount: true, projectId: true, estimate: { select: { projectId: true } } }, }), prisma.timeEntry.findMany({ where: { startTime: { gte: from, lt: to } }, @@ -2261,7 +2268,7 @@ async function getProjectMonthStrip(from: Date, to: Date, coRows: OverlayChangeO row(pid).received += Number(m.amount); } for (const e of expenseRows) { - const pid = e.estimate?.projectId; + const pid = resolveExpenseProjectId(e); if (!pid || !nameOf.has(pid)) continue; row(pid).expenses += Number(e.amount); } diff --git a/tests/expense-attribution.test.ts b/tests/expense-attribution.test.ts index 348d1fa13..e7f56f241 100644 --- a/tests/expense-attribution.test.ts +++ b/tests/expense-attribution.test.ts @@ -17,6 +17,7 @@ import { notHumanCodedExpenseWhere, resolveExpenseCostCodeId, resolveExpenseProjectId, + resolveExpenseProjectLabel, } from "../src/lib/expense-attribution"; // ── project resolution ────────────────────────────────────────────────────── @@ -159,3 +160,51 @@ test("notHumanCodedExpenseWhere has an explicit NULL branch", () => { assert.deepEqual(branches[1], { costCodeSource: { notIn: ["capture", "manual"] } }); assert.deepEqual([...HUMAN_COST_CODE_SOURCES], ["capture", "manual"]); }); + +// ── the display/routing label, for the readers converted last ────────────── + +test("a re-attributed row is LABELLED by the job it is actually on", () => { + // schedule-core, automation-events, the ai-review route and the manager + // receipt queue all read the estimate. A label that disagrees with the + // ledger is worse than none: it is a wrong answer that looks authoritative, + // and in the review-alert case it also ROUTES the alert. + assert.deepEqual( + resolveExpenseProjectLabel({ + projectId: "job-b", + project: { id: "job-b", name: "Mesplay Kitchen" }, + estimate: { projectId: "job-a", project: { id: "job-a", name: "Mueller Bath" } }, + }), + { projectId: "job-b", projectName: "Mesplay Kitchen" }, + ); +}); + +test("it falls back to the estimate for the id and the name TOGETHER", () => { + // Taking the id from one row and the name from another would print a real + // job's name against a different job's id. + assert.deepEqual( + resolveExpenseProjectLabel({ + projectId: null, + estimate: { projectId: "job-a", project: { id: "job-a", name: "Mueller Bath" } }, + }), + { projectId: "job-a", projectName: "Mueller Bath" }, + ); +}); + +test("a re-attributed row whose direct relation was not selected gives no NAME, not the wrong one", () => { + // The estimate's name belongs to the OLD job. Returning it beside the new + // id would be the exact mislabel this helper exists to stop. + assert.deepEqual( + resolveExpenseProjectLabel({ + projectId: "job-b", + estimate: { projectId: "job-a", project: { id: "job-a", name: "Mueller Bath" } }, + }), + { projectId: "job-b", projectName: null }, + ); +}); + +test("an unattributed row labels as nothing at all", () => { + assert.deepEqual( + resolveExpenseProjectLabel({ projectId: null, estimate: { projectId: null, project: null } }), + { projectId: null, projectName: null }, + ); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 1f81e0d6b..ba528a347 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -33,6 +33,18 @@ const fakePrisma = { updateArgs = args; return { id: "e1", ...args.data }; }, + // The PATCH writes through a COMPARE-AND-SET on the values its + // validation depended on, so the stub has to be able to MISS. + updateMany: async (args: { where: Record; data: Record }) => { + const row = storedExpense as Record | null; + if (!row) return { count: 0 }; + const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); + for (const key of ["amount", "taxAmount", "taxDeductibleBase"]) { + if (key in args.where && !eq(row[key], args.where[key])) return { count: 0 }; + } + updateArgs = args; + return { count: 1 }; + }, deleteMany: async (args: unknown) => { deleteArgs = args; return { count: 1 }; @@ -385,3 +397,55 @@ test("...and NOT one from the job it left, even via its own estimate", async () assert.equal(res.status, 400); assert.equal(updateArgs, null); }); + +// ── #5 PUT's strict allowlist ────────────────────────────────────────────── + +test("PUT rejects EVERY tax-return field, by name", async () => { + // A silent drop looks like a successful correction, and the caller would + // believe a deduction was recorded that never was. + for (const field of [ + "taxAmount", "taxAtSource", "needsTaxReview", + "installedAtCustomer", "taxDeductibleBase", + ]) { + const res = await call({ [field]: field === "taxAtSource" ? true : 1 }); + assert.equal(res.status, 400, field); + const body = await res.json(); + assert.equal(body.field, field, "the response names the offending field"); + assert.match(body.error, /PATCH/); + } + assert.equal(updateArgs, null, "and nothing is ever written"); +}); + +test("PUT still accepts its own fields", async () => { + assert.equal((await call({ vendor: "Fine", amount: "10.00" })).status, 200); +}); + +// ── #4 the PATCH is a compare-and-set on what it validated against ───────── + +test("a PATCH whose row moved under it is refused, not applied", async () => { + // The ceiling for taxDeductibleBase is amount - taxAmount, and a QBO + // re-sync can move either between the read and the write. Writing anyway + // would store a figure that was legal a moment ago and is not now. + const res = await patch({ taxDeductibleBase: 50 }); + assert.equal(res.status, 200, "control: it writes when nothing moved"); + + // Now make the CAS miss the way a concurrent sync would. + const original = fakePrisma.expense.updateMany; + (fakePrisma.expense as any).updateMany = async () => ({ count: 0 }); + try { + const stale = await patch({ taxDeductibleBase: 50 }); + assert.equal(stale.status, 409); + assert.equal((await stale.json()).code, "STALE_EXPENSE"); + } finally { + (fakePrisma.expense as any).updateMany = original; + } +}); + +test("the CAS names the values the decision rested on", async () => { + await patch({ taxDeductibleBase: 50 }); + const where = updateArgs?.where as Record; + assert.equal(where.id, "e1"); + assert.equal(where.amount, 207.74); + assert.equal(where.taxAmount, 16.55); + assert.equal(where.taxDeductibleBase, null); +}); diff --git a/tests/expense-phase-scope.test.ts b/tests/expense-phase-scope.test.ts index 1632fcbaa..a601961fd 100644 --- a/tests/expense-phase-scope.test.ts +++ b/tests/expense-phase-scope.test.ts @@ -17,6 +17,8 @@ import { resolveCostCode, type CostCodingDataSource } from "../src/lib/cost-codi // mobile-auth, which throws at import time unless NEXTAUTH_SECRET is set — // true in CI, and a unit test has no business needing a JWT secret. import { resolveInstalledAtCustomer } from "../src/lib/expense-attribution"; +import { optionalBool } from "../src/lib/receipt-capture-validation"; +import { authorizePhase } from "../src/lib/receipt-intake/late-fields"; // ── the two checks every phase writer must run ───────────────────────────── @@ -102,3 +104,42 @@ test("an explicit answer from the capturer is honoured, both ways", () => { assert.equal(resolveInstalledAtCustomer(true), true); assert.equal(resolveInstalledAtCustomer(false), false); }); + +// ── the two-step upload doors enforce the same capture rules ─────────────── + +const allow = (project: string, code: string) => + isCostCodeAllowedForProject(phaseSource, project, code); + +test("a captured phase with no project to check it against is refused", async () => { + // The row would otherwise carry an unvalidated human-authority phase that + // booking later copies verbatim onto the Expense. + const denial = await authorizePhase(null, "cc-plumb", allow); + assert.equal(denial?.status, 400); + assert.equal(denial?.body.error, "cost-code-without-project"); +}); + +test("a captured phase from another job is refused at the door", async () => { + const denial = await authorizePhase("job-mesplay", "cc-plumb", allow); + assert.equal(denial?.status, 400); + assert.equal(denial?.body.error, "cost-code-not-a-phase"); +}); + +test("no phase at all is fine — the capture is optional", async () => { + assert.equal(await authorizePhase("job-mueller", null, allow), null); +}); + +test("the job's own phase passes the door gate", async () => { + assert.equal(await authorizePhase("job-mueller", "cc-plumb", allow), null); +}); + +test("optionalBool is tri-state across JSON and multipart", () => { + // Multipart sends strings; JSON sends real booleans. Anything else means + // "the caller did not say", which is NOT "no". + assert.equal(optionalBool(true), true); + assert.equal(optionalBool(false), false); + assert.equal(optionalBool("true"), true); + assert.equal(optionalBool("false"), false); + for (const silent of [undefined, null, "", "yes", 1, {}]) { + assert.equal(optionalBool(silent), null, JSON.stringify(silent) ?? "undefined"); + } +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 8fd69dbea..ff780a32b 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -599,13 +599,18 @@ function createFakePrisma(initial: StoredExpense[] = []) { // Models the PREDICATE, not just the write: `projectId: null` in the // where clause has to be able to match zero rows, because that is the // whole guarantee the split-out project fill is buying. - async updateMany(args: { - where: { id: string; projectId: null }; - data: { projectId?: string; estimateId: string }; - }) { + // Models the PREDICATE, not just the write — both the attribution fill + // (`projectId: null`) and the tax COMPARE-AND-SET have to be able to + // match ZERO rows, because that is the whole guarantee they buy. + async updateMany(args: { where: Record; data: Record }) { const current = [...rows.values()].find(row => row.id === args.where.id); if (!current) return { count: 0 }; - if ((current.projectId ?? null) !== null) return { count: 0 }; + const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); + for (const key of ["projectId", "taxAmount", "taxDeductibleBase"]) { + if (key in args.where && !eq((current as any)[key], args.where[key])) { + return { count: 0 }; + } + } rows.set(current.qbPurchaseId, { ...current, ...args.data }); return { count: 1 }; }, @@ -614,6 +619,9 @@ function createFakePrisma(initial: StoredExpense[] = []) { return { rows, + // Exposed so a test can model a concurrent writer landing between + // the read and the write. + expense, client: { async $transaction(callback: (tx: { expense: typeof expense; @@ -1639,3 +1647,46 @@ test("a second deactivation of an already-retired row is unchanged", async () => "unchanged", ); }); + +test("a tax PATCH landing mid-sync is NOT clobbered — the sync re-plans", async () => { + // DETERMINISTIC CONCURRENCY, on the interleaving that actually loses data. + // + // The sync reads a row whose recorded tax ($500) is larger than the gross + // it is about to write ($300), so its plan says "retire the whole + // classification and flag it for review". Meanwhile a bookkeeper's PATCH + // corrects the tax to $16.55 — perfectly valid against $300. + // + // Without the compare-and-set the sync writes a plan built from a figure + // that no longer exists and wipes the correction. With it, the write misses + // and the plan is recomputed against what is really there. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + taxAmount: 500, taxDeductibleBase: null, installedAtCustomer: true, + } as any, + ]); + + const stored = fake.rows.get("purchase-1") as any; + const prePatch = { ...stored }; // what the sync read + fake.rows.set("purchase-1", { ...stored, taxAmount: 16.55 }); // what the PATCH left + + let firstRead = true; + const realFindUnique = fake.expense.findUnique; + fake.expense.findUnique = async (args: any) => { + if (firstRead) { + firstRead = false; + return prePatch; + } + return realFindUnique(args); + }; + + assert.equal( + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 300 }), + "updated", + ); + const after = fake.rows.get("purchase-1") as any; + assert.equal(after.amount, 300, "the sync's own facts still land"); + assert.equal(after.taxAmount, 16.55, "the bookkeeper's correction survives"); + assert.notEqual(after.needsTaxReview, true, "and it is not flagged on a stale premise"); + assert.equal(after.installedAtCustomer, true, "nor is their tax answer discarded"); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index e7068f924..4d90d4f22 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -110,6 +110,8 @@ interface Recorder { expenses: any[]; intakeUpdates: any[]; events: any[]; + expenseUpdates: any[]; + existingExpense: any; } function recorder(overrides: Partial = {}, opts: { estimates?: { id: string }[] } = {}): Recorder { @@ -117,7 +119,10 @@ function recorder(overrides: Partial = {}, opts: { estimates?: const sendMarks: string[] = []; const expenses: any[] = []; const intakeUpdates: any[] = []; + const expenseUpdates: any[] = []; const events: any[] = []; + // Set by a test to model a Purchase that is ALREADY booked. + const state: { existingExpense: any } = { existingExpense: null }; const tx = { project: { @@ -128,8 +133,9 @@ function recorder(overrides: Partial = {}, opts: { estimates?: }), }, expense: { - findUnique: async () => null, + findUnique: async () => state.existingExpense, create: async (args: any) => { expenses.push(args.data); return { id: `exp-${expenses.length}` }; }, + update: async (args: any) => { expenseUpdates.push(args); return {}; }, }, receiptIntake: { update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, @@ -155,7 +161,11 @@ function recorder(overrides: Partial = {}, opts: { estimates?: markSendAttempted: async id => { sendMarks.push(id); return true; }, ...overrides, }; - return { deps, sendMarks, purchaseCalls, expenses, intakeUpdates, events }; + return { + deps, sendMarks, purchaseCalls, expenses, intakeUpdates, events, expenseUpdates, + set existingExpense(value: any) { state.existingExpense = value; }, + get existingExpense() { return state.existingExpense; }, + }; } test("a taxed receipt splits into a pre-tax line and a sales-tax line that reconstruct the total", () => { diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index db35ef2d8..e53363ecf 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -70,7 +70,6 @@ function workerRow(overrides: Partial = {}): WorkerRow { dedupWeakKey: null, busyPasses: 0, lastError: null, - suggestedConfidence: null, sendAttempted: false, claimToken: "claim-1", fileSha256: "s".repeat(64), From 168d899cb4033af7daa05ac05e51778568e5d48a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 02:42:15 -0700 Subject: [PATCH 087/144] fix(expenses): unblock E2E, flag invalidated allocations, guard the booking fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto Phase 1 head 8297b46b (sealed uploads, SHADOW_QUARANTINE, claim fencing, project-scoped loadPhases — Phase 1's version kept for every shared hunk, and its stricter loadPhases supersedes mine). 1. E2E was RED because I made `companyTimeZone` a REQUIRED dependency, so every caller that builds its own set threw. It is optional now, defaulting to the shared resolver — a dependency exists to be overridden, not re-stated. 2. When a gross drop invalidates only the ALLOCATION, the row is flagged as well as cleared. A silent null still read as a valid deduction: installedAtCustomer was untouched and a null base means "the whole pre-tax total", so the report would have claimed MORE than the human allocated. 4. The already-booked fill is now one guarded `updateMany` per field (`costCodeId IS NULL`, `installedAtCustomer IS NULL`, source not capture/manual) instead of read-then-write. The read is inside the transaction but a PATCH can still land in the gap — and that PATCH is exactly the authority the fill must not overrun. Regression covers it. 5. /finalize on a non-STAGING retry applies late costCodeId/installedAtCustomer only where the row is still unanswered, and returns 409 late-fields-conflict when the retry carries a DIFFERENT answer to one already recorded. 6. The manager receipt queue selects the direct project and labels through resolveExpenseProjectLabel. Co-Authored-By: Claude Fable 5.1 --- .../receipts/intake/[id]/finalize/route.ts | 21 +++- .../manager/receipts/ReceiptQueueClient.tsx | 12 +- src/app/manager/receipts/page.tsx | 6 + src/lib/qbo-expense-sync.ts | 21 +++- src/lib/receipt-intake/book.ts | 68 ++++++++--- src/lib/receipt-intake/late-fields.ts | 21 +++- tests/qbo-expense-sync.test.ts | 15 +++ tests/receipt-intake-book.test.ts | 109 ++++++++++++++++++ tests/receipt-intake-late-fields.test.ts | 2 +- 9 files changed, 242 insertions(+), 33 deletions(-) diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 8423a94b5..668574be7 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -4,6 +4,7 @@ import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/rec import { userCanAccessProject } from "@/lib/mobile-auth"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; +import { optionalBool } from "@/lib/receipt-capture-validation"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { finalizeDisposition, @@ -44,7 +45,12 @@ async function applyLateFields( const denial = await reconcileLateFields(id, lateFields, { read: rowId => prisma.receiptIntake.findUnique({ where: { id: rowId }, - select: { costCodeId: true, projectId: true, state: true, claimToken: true }, + // `installedAtCustomer` rides the same null-or-equal, + // only-before-routed, unclaimed rule as the two ids. + select: { + costCodeId: true, projectId: true, installedAtCustomer: true, + state: true, claimToken: true, + }, }), applyIfNull: async (rowId, state, toApply) => { const { count } = await prisma.receiptIntake.updateMany({ @@ -135,15 +141,17 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // disagree — silently overwriting a value a human already set is the one // outcome that loses information nobody can recover. // - // NOTE: Phase 3's `installedAtCustomer` does not exist on this model; the - // same rule will apply to it when it lands. + // Phase 3's `installedAtCustomer` rides the same rule. It is a TRI-STATE, + // so "the caller did not say" (null) is excluded from the set below exactly + // like an absent id — silence is never an answer, and never overwrites one. const lateInput = { costCodeId: typeof body.costCodeId === "string" && body.costCodeId.trim() ? body.costCodeId.trim() : null, projectId: typeof body.projectId === "string" && body.projectId.trim() ? body.projectId.trim() : null, + installedAtCustomer: optionalBool(body.installedAtCustomer), }; const lateFields = Object.fromEntries( Object.entries(lateInput).filter(([, v]) => v !== null), - ) as Partial<{ costCodeId: string; projectId: string }>; + ) as LateFields; const row = await prisma.receiptIntake.findUnique({ where: { id }, @@ -401,7 +409,10 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // would be a lie the client acts on. const current = await prisma.receiptIntake.findUnique({ where: { id }, - select: { state: true, sourceRef: true, projectId: true, dryRun: true }, + select: { + state: true, sourceRef: true, projectId: true, dryRun: true, + costCodeId: true, installedAtCustomer: true, + }, }); if (!current || current.state === "STAGING") { return NextResponse.json( diff --git a/src/app/manager/receipts/ReceiptQueueClient.tsx b/src/app/manager/receipts/ReceiptQueueClient.tsx index 71bbb1cb8..ff3cec5d4 100644 --- a/src/app/manager/receipts/ReceiptQueueClient.tsx +++ b/src/app/manager/receipts/ReceiptQueueClient.tsx @@ -1,4 +1,6 @@ -"use client"; +"use client"; + +import { resolveExpenseProjectLabel } from "@/lib/expense-attribution"; import { useRef, useState } from "react"; import { toast } from "sonner"; @@ -14,7 +16,11 @@ interface Expense { date: string | null; status: string; receiptUrl: string | null; + // BOTH sides: the queue labels a receipt by the job the money is on. + projectId?: string | null; + project?: { id: string; name: string } | null; estimate: { + projectId?: string | null; project: { id: string; name: string } | null; } | null; costCode: { code: string; name: string } | null; @@ -124,7 +130,7 @@ export default function ReceiptQueueClient({ {exp.date && Date: {new Date(exp.date).toLocaleDateString()}} - {exp.estimate?.project && Project: {exp.estimate.project.name}} + {resolveExpenseProjectLabel(exp).projectName && Project: {resolveExpenseProjectLabel(exp).projectName}} {exp.costCode && Code: {exp.costCode.code} — {exp.costCode.name}} Submitted {new Date(exp.createdAt).toLocaleDateString()} @@ -191,7 +197,7 @@ export default function ReceiptQueueClient({
{formatCurrency(Number(exp.amount))} {exp.date && {new Date(exp.date).toLocaleDateString(undefined, { timeZone: "UTC" })}} - {exp.estimate?.project && {exp.estimate.project.name}} + {resolveExpenseProjectLabel(exp).projectName && {resolveExpenseProjectLabel(exp).projectName}} {exp.qbPurchaseId && QBO transaction {exp.qbPurchaseId}} {exp.qbSyncedAt && Imported {new Date(exp.qbSyncedAt).toLocaleString()}}
diff --git a/src/app/manager/receipts/page.tsx b/src/app/manager/receipts/page.tsx index 2ad7188e5..31a817cbb 100644 --- a/src/app/manager/receipts/page.tsx +++ b/src/app/manager/receipts/page.tsx @@ -19,6 +19,9 @@ export default async function BookkeeperReceiptsPage() { prisma.expense.findMany({ where: { status: "Pending" }, include: { + // BOTH sides — the queue labels a receipt by the job the money + // is on, not by the estimate it happened to be booked against. + project: { select: { id: true, name: true } }, estimate: { include: { project: { select: { id: true, name: true } } }, }, @@ -30,6 +33,9 @@ export default async function BookkeeperReceiptsPage() { prisma.expense.findMany({ where: { qbPurchaseId: { not: null } }, include: { + // BOTH sides — the queue labels a receipt by the job the money + // is on, not by the estimate it happened to be booked against. + project: { select: { id: true, name: true } }, estimate: { include: { project: { select: { id: true, name: true } } }, }, diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 50300675e..c53434a4f 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -716,7 +716,14 @@ export function planQboExpenseUpdate( } else if (existingBase !== null) { const ceiling = Math.round((write.amount - (existingTax ?? 0)) * 100) / 100; if (!Number.isFinite(ceiling) || existingBase > ceiling) { + // NEVER A SILENT NULL. Clearing the allocation on its own leaves a + // row that still reads as a valid deduction — `installedAtCustomer` + // is untouched and a null base means "the whole pre-tax total", so + // the report would quietly claim MORE than the human allocated. + // Flagging it is what keeps the report's exclusion honest until a + // person re-splits the receipt. data.taxDeductibleBase = null; + data.needsTaxReview = true; } } @@ -1174,8 +1181,16 @@ export interface QboExpenseSyncDependencies { * and so existing tests that build dependencies by hand keep compiling. */ suggestCostCode?(input: QboCostCodeSuggestionInput): Promise; - /** The company's configured zone — `Expense.date` is a day in it, not an instant. */ - companyTimeZone(): Promise; + /** + * The company's configured zone — `Expense.date` is a day in it, not an + * instant. + * + * OPTIONAL, defaulting to the shared resolver. Making it required broke + * every caller that builds its own dependency set (the e2e reconcile spec + * among them) for a value none of them has an opinion about; a dependency + * exists to be overridden, not to be re-stated. + */ + companyTimeZone?(): Promise; now(): Date; } @@ -1378,7 +1393,7 @@ export async function syncQboExpenses( const mode = options.mode ?? "backfill"; // One read per sync. `Expense.date` is a company CALENDAR DAY, and every // writer has to agree on that or the tax report reads them in two zones. - const companyTimeZone = await dependencies.companyTimeZone(); + const companyTimeZone = await (dependencies.companyTimeZone?.() ?? resolveCompanyTimeZone()); const [purchaseRead, projects] = await Promise.all([ dependencies.readPurchases(tokens, options.since, mode, options.until), dependencies.listProjects(), diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 92f48d047..dd48c6be2 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -180,6 +180,8 @@ export interface BookPrismaClient { } | null>; create(args: any): Promise<{ id: string }>; update(args: any): Promise; + /** Guarded per-field fill — the predicate IS the guarantee. */ + updateMany(args: any): Promise<{ count: number }>; }; receiptIntake: { update(args: any): Promise; @@ -691,28 +693,58 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro return { conflict: true as const }; } - const humanCoded = - existing.costCodeSource === "capture" || existing.costCodeSource === "manual"; - const fill: Record = {}; - if (!existing.projectId && row.projectId) fill.projectId = row.projectId; - if (!existing.costCodeId && !humanCoded && costCodeId) { - fill.costCodeId = costCodeId; - fill.costCodeSource = costCodeSource; - fill.costCodeConfidence = costCodeConfidence; + // EACH FIELD GETS ITS OWN GUARDED WRITE. + // + // The read above happened inside this transaction, but a + // bookkeeper's PATCH can commit between it and these writes — + // and that PATCH is exactly the authority this fill must not + // overrun. Deciding from the read and then writing + // unconditionally is the same read-then-write shape the sync + // was already made to give up: the guarantee belongs in the + // predicate, so a row that gained an answer in the gap simply + // matches zero rows. + // + // Split per field because the conditions differ and a single + // predicate would make one field's contention veto another's + // legitimate fill. + if (!existing.projectId && row.projectId) { + await tx.expense.updateMany({ + where: { id: existing.id, projectId: null }, + data: { projectId: row.projectId }, + }); + } + if (costCodeId) { + await tx.expense.updateMany({ + where: { + id: existing.id, + costCodeId: null, + // A human's phase outranks anything booking knows. + // The explicit NULL branch matters: SQL `NOT IN` + // drops NULL rows, and an unset source is the + // common case. + OR: [ + { costCodeSource: null }, + { costCodeSource: { notIn: ["capture", "manual"] } }, + ], + }, + data: { costCodeId, costCodeSource, costCodeConfidence }, + }); } - // Tax is filled only when there is none recorded at all — - // a stored figure came either from an earlier booking of this + // Tax is filled only where there is none recorded at all — a + // stored figure came either from an earlier booking of this // same document or from a bookkeeper, and both outrank a // re-read. - if (existing.taxAmount === null && taxApplied > 0) { - fill.taxAmount = taxApplied / 100; - fill.taxAtSource = true; - } - if (existing.installedAtCustomer === null && row.installedAtCustomer !== null) { - fill.installedAtCustomer = row.installedAtCustomer; + if (taxApplied > 0) { + await tx.expense.updateMany({ + where: { id: existing.id, taxAmount: null }, + data: { taxAmount: taxApplied / 100, taxAtSource: true }, + }); } - if (Object.keys(fill).length > 0) { - await tx.expense.update({ where: { id: existing.id }, data: fill }); + if (row.installedAtCustomer !== null) { + await tx.expense.updateMany({ + where: { id: existing.id, installedAtCustomer: null }, + data: { installedAtCustomer: row.installedAtCustomer }, + }); } } diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index 202eed5d9..70b578776 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -15,11 +15,26 @@ export interface LateFields { costCodeId?: string; projectId?: string; + /** + * Phase 3's tax answer: was this material installed at a customer job? + * + * A BOOLEAN among two ids, which matters in one place — `undefined` is the + * only "not supplied". `false` is a real answer (shop consumables) and must + * survive the same null-or-equal rule as the others; treating it as absent + * would silently drop a bookkeeper saying "no" and leave the row + * unreviewed, which the tax report reads as "never claimed" rather than + * "answered no". + */ + installedAtCustomer?: boolean; } +export type LateFieldKey = "costCodeId" | "projectId" | "installedAtCustomer"; +export type LateFieldValue = string | boolean; + export interface LateFieldRow { costCodeId: string | null; projectId: string | null; + installedAtCustomer?: boolean | null; state: string; claimToken?: string | null; } @@ -33,7 +48,7 @@ export interface Denial { export interface LateFieldsDeps { read(id: string): Promise; /** updateMany fenced on {id, state, claimToken: null, : null}; returns the count. */ - applyIfNull(id: string, state: string, toApply: Record): Promise; + applyIfNull(id: string, state: string, toApply: Record): Promise; /** Re-runs the caller's authorization against a given project. */ authorize(projectId: string | null): Promise; } @@ -47,7 +62,7 @@ export async function reconcileLateFields( deps: LateFieldsDeps, ): Promise { const entries = Object.entries(lateFields).filter(([, value]) => value !== undefined) as Array< - ["costCodeId" | "projectId", string] + [LateFieldKey, LateFieldValue] >; if (entries.length === 0) return null; @@ -100,7 +115,7 @@ export async function reconcileLateFields( // change what it decided — it just makes the row disagree with the routing // it is about to publish (a receipt that now HAS a job, parked NEEDS_JOB). // The fence is applied by the caller's `applyIfNull`. - const count = await deps.applyIfNull(id, current.state, toApply as Record); + const count = await deps.applyIfNull(id, current.state, toApply as Record); if (count > 0) return null; // The CAS lost. That is NOT automatically "busy": the same zero comes back diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index ff780a32b..a7fb80b1a 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1690,3 +1690,18 @@ test("a tax PATCH landing mid-sync is NOT clobbered — the sync re-plans", asyn assert.notEqual(after.needsTaxReview, true, "and it is not flagged on a stale premise"); assert.equal(after.installedAtCustomer, true, "nor is their tax answer discarded"); }); + +test("invalidating an ALLOCATION also flags the row — never a silent null", async () => { + // Clearing the allocation on its own leaves a row that still reads as a + // valid deduction: installedAtCustomer is untouched and a null base means + // "the whole pre-tax total", so the report would quietly claim MORE than + // the human allocated. This is the report-level regression. + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", taxAmount: 10, taxDeductibleBase: 150 }, + { ...WRITE, amount: 100 }, + ); + assert.equal(plan.data.taxDeductibleBase, null, "100 - 10 = 90 < 150"); + assert.equal(plan.data.needsTaxReview, true, "and the report must skip it until re-checked"); + // The tax itself is still valid against the new gross, so it stays. + assert.ok(!("taxAmount" in plan.data)); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 4d90d4f22..160aa720a 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -136,6 +136,26 @@ function recorder(overrides: Partial = {}, opts: { estimates?: findUnique: async () => state.existingExpense, create: async (args: any) => { expenses.push(args.data); return { id: `exp-${expenses.length}` }; }, update: async (args: any) => { expenseUpdates.push(args); return {}; }, + // Models the PREDICATE. Each guarded fill has to be able to match + // ZERO rows, because that is the whole guarantee it buys. + updateMany: async (args: any) => { + expenseUpdates.push(args); + const cur = state.existingExpense ?? {}; + const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); + for (const key of ["projectId", "costCodeId", "taxAmount", "installedAtCustomer"]) { + if (key in args.where && !eq(cur[key], args.where[key])) return { count: 0 }; + } + if (Array.isArray(args.where.OR)) { + const src = cur.costCodeSource ?? null; + const ok = args.where.OR.some((b: any) => + b.costCodeSource === null + ? src === null + : src !== null && !(b.costCodeSource?.notIn ?? []).includes(src)); + if (!ok) return { count: 0 }; + } + if (state.existingExpense) Object.assign(state.existingExpense, args.data); + return { count: 1 }; + }, }, receiptIntake: { update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, @@ -1028,4 +1048,93 @@ test("the QBO core fires onExistingPurchase before it touches the attachment", ( ); // And the create hook is NOT fired on this path. assert.ok(!body.includes("onBeforeCreate"), "onBeforeCreate belongs to the create path only"); + +// ── the already-booked Purchase still needs its Phase 3 fields ───────────── + +test("an alreadyExists row is FILLED, not left blank", async () => { + // `alreadyExists` covers the lost-response retry AND a row v1 created + // before this pipeline existed. Returning it untouched left projectId, the + // phase, the provenance and the tax columns NULL forever on exactly the + // receipts the tax report is made of. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: null, costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, installedAtCustomer: null, + estimate: { projectId: null }, + }; + const result = await bookReceipt(row(), rec.deps); + assert.equal(result.outcome, "booked"); + const fill = Object.assign({}, ...rec.expenseUpdates.map((u: any) => u.data)); + assert.equal(fill.projectId, "proj-1"); + assert.equal(fill.installedAtCustomer, true); + assert.ok("taxAmount" in fill, "the tax the booking validated lands too"); +}); + +test("a human's decisions on an existing row are never overwritten", async () => { + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: "cc-human", + costCodeSource: "manual", taxAmount: 9.99, taxAtSource: true, + installedAtCustomer: false, estimate: { projectId: "proj-1" }, + }; + await bookReceipt(row(), rec.deps); + // Nothing a human owns may be written — the guarded predicates are what + // enforce it, so assert on what actually LANDED, not on what was attempted. + const after = rec.existingExpense; + assert.equal(after.costCodeId, "cc-human", "a manual phase is untouchable"); + assert.equal(after.taxAmount, 9.99, "a recorded tax outranks a re-read"); + assert.equal(after.installedAtCustomer, false, "an answered tax question is a human's"); + // The writes are ATTEMPTED and rejected by their predicates — that is the + // point of moving the guarantee into SQL, so there is nothing to assert + // about whether the statements were issued. + assert.ok(rec.expenseUpdates.every((u: any) => u.where.id === "expense-1")); +}); + +test("a PATCH landing between the read and the fill is not overrun", async () => { + // The read happens inside the transaction, but a bookkeeper's PATCH can + // still commit before these writes. Deciding from the read and writing + // unconditionally would overrun exactly the authority this fill must + // respect; the predicate is what makes the gap safe. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: null, costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, installedAtCustomer: null, + estimate: { projectId: null }, + }; + const seenByBooking = { ...rec.existingExpense }; + (rec.deps.db as any).expense.findUnique = async () => { + // Hand booking the PRE-patch snapshot, then let the PATCH land. + rec.existingExpense = { + ...rec.existingExpense, + costCodeId: "cc-human", + costCodeSource: "manual", + installedAtCustomer: false, + }; + return seenByBooking; + }; + + const result = await bookReceipt(row(), rec.deps); + assert.equal(result.outcome, "booked"); + assert.equal(rec.existingExpense.costCodeId, "cc-human", "the human's phase survives"); + assert.equal(rec.existingExpense.costCodeSource, "manual"); + assert.equal(rec.existingExpense.installedAtCustomer, false, "and their tax answer"); + // ...while the field nobody contended for is still filled. + assert.equal(rec.existingExpense.projectId, "proj-1"); +}); + +test("a Purchase already on ANOTHER job parks instead of booking", async () => { + // Filling would be guessing which job is right; overwriting would silently + // move real money between jobs. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "some-other-job", costCodeId: null, + costCodeSource: null, taxAmount: null, taxAtSource: false, + installedAtCustomer: null, estimate: { projectId: "some-other-job" }, + }; + const result = await bookReceipt(row(), rec.deps); + assert.equal(result.outcome, "needs-review"); + if (result.outcome === "needs-review") { + assert.equal(result.reason, "attribution-conflict"); + assert.equal(result.releaseStrongKey, false, "the Purchase exists — keep the key"); + } }); diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts index 08fcd76a6..1f30ae82d 100644 --- a/tests/receipt-intake-late-fields.test.ts +++ b/tests/receipt-intake-late-fields.test.ts @@ -29,7 +29,7 @@ function row(over: Partial = {}): LateFieldRow { interface Trace { deps: LateFieldsDeps; - applied: Record[]; + applied: Record[]; authorized: (string | null)[]; } From 52ba7d5fcff2705ac4a220e15636c73f419db57f Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 03:27:35 -0700 Subject: [PATCH 088/144] fix(expenses): one per-expense lock, attribution in every CAS, project-scoped labor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 9. Items 5 and 6 are start/finalize — Phase 1's files. 1. DELETE already authorized on the resolved job; a divergent-attribution test now pins it (the job it LEFT confers nothing, the job it is ON does). 2. The tax PATCH's CAS names projectId, estimateId and a new `Expense.updatedAt` row version. Access to the row was granted because of the project it was on, so a re-attribution landing in the gap means the permission check that let the request through was answered about a different job — 409 rather than a write. The column is added nullable, backfilled, then SET NOT NULL, so no DB-level default is left that `@updatedAt` does not declare. 3. New src/lib/expense-lock.ts: ONE `pg_advisory_xact_lock('expense:'||id)`, taken inside their transactions by the QBO sync, the tax PATCH and the booking fill. Per-column CAS stops a lost update but not a torn one, and the tax invariants span columns. The predicates stay — the lock orders writers that take it, the predicate protects against one that does not. After a CAS miss the sync re-plans and re-CASes; a still-contended row is LEFT ALONE, never unconditionally written, because the sync's facts survive to the next run and a discarded human answer does not. 4. The booking fill takes the same lock and pins `projectId` in every guarded predicate, so a re-attribution in the gap makes it match zero rows instead of writing a phase and a tax answer onto a job they were never about. 7. Labor coverage resolves its item fallback through the project-scoped map the expense side already used. Test fakes had to learn two real behaviours: the advisory lock is RE-ENTRANT within a transaction (serialising every call deadlocked the second one), and the apply-script parity test now selects the backfill by what it writes rather than by being the first UPDATE. Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 13 +++ prisma/schema.prisma | 6 ++ scripts/apply-expense-attribution.mjs | 9 ++ scripts/backfill-expense-attribution.mjs | 22 ++++- src/app/api/expenses/[id]/route.ts | 22 ++++- src/lib/expense-lock.ts | 56 +++++++++++++ src/lib/qbo-expense-sync.ts | 56 ++++++++++--- src/lib/receipt-intake/book.ts | 28 ++++++- tests/apply-expense-attribution.test.ts | 7 +- tests/backfill-expense-attribution.test.ts | 28 +++++++ tests/expense-edit-authz.test.ts | 82 +++++++++++++++++- tests/qbo-expense-sync.test.ts | 83 +++++++++++++++++++ tests/receipt-intake-book.test.ts | 14 +++- 13 files changed, 403 insertions(+), 23 deletions(-) create mode 100644 src/lib/expense-lock.ts diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 315fa103b..e402f93ae 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -28,6 +28,19 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30 -- Set when a re-sync invalidated a human tax classification (see the sync). ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false; +-- A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). +-- +-- Added in three steps on purpose: nullable, backfilled, then NOT NULL. A +-- single `ADD COLUMN ... NOT NULL DEFAULT now()` would leave a DB-level default +-- that `updatedAt DateTime @updatedAt` does not declare, and CI's +-- "migrations reproduce production" check compares the two. +-- +-- Each statement is independently re-runnable: IF NOT EXISTS, a predicate-bound +-- UPDATE, and a SET NOT NULL that is a no-op once applied. +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3); +UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL; +ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL; + CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); -- SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9df1b5670..d6aa1ad86 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -672,6 +672,12 @@ model Expense { purchaseOrder PurchaseOrder? @relation(fields: [purchaseOrderId], references: [id]) createdAt DateTime @default(now()) + /// Row version. Exists so the tax PATCH can COMPARE-AND-SET against the exact + /// row it authorized and validated against — attribution included. Without + /// it the CAS could only name the money columns, and a re-attribution + /// landing in the gap would slip past a check that had already decided who + /// was allowed to make it. + updatedAt DateTime @updatedAt /// Back-relation for ReceiptIntake.expenseId. Declared here rather than left /// SQL-only because `prisma migrate diff` WOULD see a foreign key that diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 6a5f22c31..a6e8b05e8 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -88,6 +88,14 @@ export const statements = [ // Set when a re-sync invalidated a human tax classification. `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false`, + // A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). + // Three steps on purpose — nullable, backfill, NOT NULL — because a single + // NOT NULL DEFAULT now() would leave a DB default that `@updatedAt` does + // not declare, and CI compares the two. Each step is re-runnable. + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3)`, + `UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL`, + `ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL`, + `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, // SET NULL, not Cascade: `estimateId` already owns this row's lifecycle. A @@ -187,6 +195,7 @@ export const expectedColumns = { Expense: [ "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", "costCodeSource", "costCodeConfidence", "taxDeductibleBase", "needsTaxReview", + "updatedAt", ], }; diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 05177bd25..3bb5bb6b9 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -442,12 +442,30 @@ export async function runBackfill({ // because clock-in already requires a phase. const timeEntries = await db.timeEntry.findMany({ where: { projectId: { in: scopedProjectIds } }, - select: { costCodeId: true, estimateItemId: true, laborCost: true, burdenCost: true }, + select: { + costCodeId: true, estimateItemId: true, laborCost: true, burdenCost: true, + // Needed to scope the item fallback to the entry's OWN job. + projectId: true, + }, }); + // PROJECT-SCOPED, exactly like the expense side. A time entry pointing at + // another job's estimate item is unattributed on the variance page, so + // resolving it through a global id->code map counted labor dollars as + // covered that the report itself does not — the same flattering error the + // expense metric had. + const laborItems = scopedItemCostCodes( + timeEntries.map(t => ({ + projectId: t.projectId, + estimate: null, + itemId: t.estimateItemId, + })), + items, + allowedCodesByProject, + ); const laborRows = timeEntries.map(t => ({ costCodeId: resolveExpenseCostCodeId( { costCodeId: t.costCodeId, itemId: t.estimateItemId }, - itemCostCodeById, + laborItems, ), amount: num(t.laborCost) + num(t.burdenCost), })); diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 42192a883..1e8d5fff0 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -7,7 +7,8 @@ import { QboManagedExpenseError, assertExpenseMutableOutsideQbo, } from "@/lib/qbo-expense-guard"; -import { resolveExpenseProjectId } from "@/lib/expense-attribution"; +import { resolveExpenseProjectId } from "@/lib/expense-attribution"; +import { lockExpense } from "@/lib/expense-lock"; import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; @@ -314,6 +315,7 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id taxDeductibleBase: true, estimateId: true, projectId: true, + updatedAt: true, estimate: { select: { projectId: true } }, }, }); @@ -507,6 +509,14 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id amount: expense.amount, taxAmount: expense.taxAmount, taxDeductibleBase: expense.taxDeductibleBase, + // THE ATTRIBUTION THE AUTHORIZATION RESTED ON. Access to this row + // was granted because of the project it was on; if it has since + // been re-attributed, the permission check that let this request + // through was answered about a different job. `updatedAt` catches + // everything else that moved. + projectId: expense.projectId, + estimateId: expense.estimateId, + updatedAt: expense.updatedAt, }; const data = { ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), @@ -528,7 +538,15 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id } : {}), }; - const written = await prisma.expense.updateMany({ where: casWhere, data }); + // The write runs under the shared per-expense lock, so this request is + // ordered against the QBO sync and the booking fill rather than merely + // racing them. The CAS stays inside it: the lock orders the writers + // that TAKE it, the predicate is what still protects against one that + // does not. + const written = await prisma.$transaction(async tx => { + await lockExpense(tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, id); + return tx.expense.updateMany({ where: casWhere, data }); + }); if (written.count === 0) { return NextResponse.json( { diff --git a/src/lib/expense-lock.ts b/src/lib/expense-lock.ts new file mode 100644 index 000000000..c60f428e5 --- /dev/null +++ b/src/lib/expense-lock.ts @@ -0,0 +1,56 @@ +// ONE lock, one key, for every writer that touches an Expense's attribution or +// its tax classification. +// +// Three writers can reach the same row concurrently: +// * the QBO sync's upsert, +// * the bookkeeper's tax PATCH, and +// * the receipt pipeline's fill for an already-booked Purchase. +// +// Each of them previously guarded itself with a compare-and-set, which stops a +// LOST UPDATE but not a torn one: a writer can still read a row, have another +// writer change several columns, and then apply a decision that was coherent +// only against the values it first saw. The tax invariants span columns — +// `taxDeductibleBase <= amount - taxAmount`, and the authorization that decided +// who may touch the row at all rests on its project — so "each column was +// written under a valid predicate" is not the same as "the row is valid". +// +// Serialising the three on a per-row advisory lock makes the read-decide-write +// sequence atomic with respect to each other. The CAS predicates stay: the lock +// orders writers that take it, the predicate is what still protects against one +// that does not (a migration, a script, a future path someone forgets to wire). +// +// `pg_advisory_xact_lock` releases at COMMIT or ROLLBACK, so there is no unlock +// to forget and a crashed transaction cannot strand the row. + +/** Structural subset of a Prisma transaction client this needs. */ +export interface AdvisoryLockClient { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; +} + +/** + * The namespaced key. `expense:` keeps these from colliding with the QBO + * sync's per-PURCHASE lock, which is a different scope: one Purchase can be + * looked up by id while the Expense it maps to is being edited by a human. + */ +export function expenseLockKey(expenseId: string): string { + return `expense:${expenseId}`; +} + +/** + * Take the per-expense lock for the rest of the transaction. + * + * `hashtextextended` is used rather than `hashtext` because it returns bigint + * directly — `pg_advisory_xact_lock` takes a bigint, and the 32-bit `hashtext` + * would have to be widened anyway while colliding far more often. A collision + * is harmless (two unrelated expenses serialise needlessly), but rarer is + * better when the alternative costs nothing. + */ +export async function lockExpense( + client: AdvisoryLockClient, + expenseId: string, +): Promise { + await client.$queryRawUnsafe( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))::text AS lock_result", + expenseLockKey(expenseId), + ); +} diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index c53434a4f..581b82bfd 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -14,6 +14,7 @@ import { isOverheadProject } from "./overhead-project"; import { dateOnlyInTimeZone } from "./tz-date"; import { resolveCompanyTimeZone } from "./company-timezone"; import { isCostCodeAllowedForProject } from "./project-phases"; +import { lockExpense } from "./expense-lock"; import { prismaPhaseDataSource } from "./project-phases-db"; // Shared with the register merge layer (register-merge.ts, Unified Money // Register plan §4) so the classification values this module WRITES can @@ -531,6 +532,7 @@ type ExpenseTransaction = { qbSyncToken: string | null; estimateId: string; projectId?: string | null; + updatedAt?: Date; taxAmount?: unknown; taxAtSource?: boolean; installedAtCustomer?: boolean | null; @@ -763,6 +765,26 @@ function planIsNoop(existing: ExistingQboExpense, plan: QboExpenseUpdatePlan): b return true; } +/** + * The values a plan was computed from. `updatedAt` covers everything else on + * the row — including the attribution the authorization rested on — so a + * re-attribution landing in the gap fails the CAS rather than being written + * over by a plan that never saw it. + */ +function casWhere(existing: { + id: string; + updatedAt?: Date; + taxAmount?: unknown; + taxDeductibleBase?: unknown; +}): Record { + return { + id: existing.id, + ...(existing.updatedAt ? { updatedAt: existing.updatedAt } : {}), + taxAmount: existing.taxAmount ?? null, + taxDeductibleBase: existing.taxDeductibleBase ?? null, + }; +} + async function lockQboExpense( transaction: ExpenseTransaction, qbPurchaseId: string, @@ -793,6 +815,7 @@ export async function upsertQboExpense( qbSyncToken: true, estimateId: true, projectId: true, + updatedAt: true, taxAmount: true, taxDeductibleBase: true, amount: true, @@ -813,6 +836,12 @@ export async function upsertQboExpense( return "imported"; } + // The per-EXPENSE lock, on top of the per-purchase one already held. + // They are different scopes: the purchase lock orders two syncs of the + // same Purchase, this one orders the sync against the tax PATCH and the + // booking fill, which know nothing about QBO purchase ids. + await lockExpense(transaction, existing.id); + const plan = planQboExpenseUpdate(existing, write); if (planIsNoop(existing, plan)) return "unchanged"; @@ -832,33 +861,34 @@ export async function upsertQboExpense( // re-plan once, which is enough because the second read is inside the // advisory lock this transaction already holds. const cas = await transaction.expense.updateMany({ - where: { - id: existing.id, - taxAmount: existing.taxAmount ?? null, - taxDeductibleBase: existing.taxDeductibleBase ?? null, - }, + where: casWhere(existing), data: plan.data, }); if (cas.count > 0) return "updated"; - // The tax values moved between the read and the write — a bookkeeper's - // PATCH landed. Re-read and RE-PLAN once against what is actually - // there. Once is enough: this transaction holds the per-purchase - // advisory lock, so the only writer that can beat it is that PATCH, and - // it has now committed. + // The row moved between the read and the write despite the lock — i.e. + // a writer that does NOT take it (a script, a migration, a path nobody + // wired). Re-read, RE-PLAN, and re-CAS: never an unconditional write, + // because the same thing can happen again and "give up and clobber" is + // not a resolution when the loser is a human's tax answer. const fresh = await transaction.expense.findUnique({ where: { qbPurchaseId: write.qbPurchaseId }, select: { id: true, qbSyncToken: true, estimateId: true, projectId: true, - taxAmount: true, taxDeductibleBase: true, amount: true, + updatedAt: true, taxAmount: true, taxDeductibleBase: true, amount: true, vendor: true, date: true, description: true, status: true, }, }); if (!fresh) return "unchanged"; const replanned = planQboExpenseUpdate(fresh, write); if (planIsNoop(fresh, replanned)) return "unchanged"; - await transaction.expense.update({ where: { id: fresh.id }, data: replanned.data }); - return "updated"; + const retry = await transaction.expense.updateMany({ + where: casWhere(fresh), + data: replanned.data, + }); + // Still contended. Leaving it is correct: the sync's facts are + // recoverable on the next run, a discarded tax correction is not. + return retry.count > 0 ? "updated" : "unchanged"; }); } diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index dd48c6be2..7567507bd 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -20,6 +20,7 @@ import { matchCostCode } from "@/lib/project-match"; import { receiptUrlRef } from "./receipt-url"; import { QBO_ATTACHMENT_MAX_BYTES } from "./intake-core"; +import { lockExpense } from "@/lib/expense-lock"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { QBTimeoutError, @@ -682,6 +683,12 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // already answered (true OR false) is a tax answer nobody but a // bookkeeper may change. if (existing) { + // The shared per-expense lock, so this fill is ORDERED against + // the tax PATCH and the QBO sync instead of racing them. The + // guarded predicates below stay: the lock orders the writers + // that take it, the predicate protects against one that does not. + await lockExpense(tx as any, existing.id); + const existingProjectId = existing.projectId ?? existing.estimate?.projectId ?? null; // ATTRIBUTION CONFLICT. The Purchase is already on a different // job than this intake row claims. Filling fields would be @@ -707,6 +714,14 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // Split per field because the conditions differ and a single // predicate would make one field's contention veto another's // legitimate fill. + // `expectedProjectId` is the attribution EVERY decision below + // was made under — the conflict check above passed against it. + // Pinning it in each predicate means a re-attribution landing + // in the gap makes the fill match zero rows rather than + // writing a phase and a tax answer onto a job they were never + // about. + const expectedProjectId = existing.projectId ?? null; + if (!existing.projectId && row.projectId) { await tx.expense.updateMany({ where: { id: existing.id, projectId: null }, @@ -717,6 +732,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro await tx.expense.updateMany({ where: { id: existing.id, + projectId: expectedProjectId ?? row.projectId, costCodeId: null, // A human's phase outranks anything booking knows. // The explicit NULL branch matters: SQL `NOT IN` @@ -736,13 +752,21 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // re-read. if (taxApplied > 0) { await tx.expense.updateMany({ - where: { id: existing.id, taxAmount: null }, + where: { + id: existing.id, + projectId: expectedProjectId ?? row.projectId, + taxAmount: null, + }, data: { taxAmount: taxApplied / 100, taxAtSource: true }, }); } if (row.installedAtCustomer !== null) { await tx.expense.updateMany({ - where: { id: existing.id, installedAtCustomer: null }, + where: { + id: existing.id, + projectId: expectedProjectId ?? row.projectId, + installedAtCustomer: null, + }, data: { installedAtCustomer: row.installedAtCustomer }, }); } diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 3ce19643d..83e90fddc 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -87,7 +87,12 @@ test("the ReceiptIntake columns are behind a to_regclass guard in both files", ( test("the backfill UPDATE only ever touches rows whose projectId is still NULL", () => { // This is the whole of its idempotency. A re-run must report 0 rows, and a // manual re-attribution must survive it. - const update = (statements as string[]).find(s => s.trimStart().startsWith("UPDATE")); + // Selected by what it WRITES, not by "the first UPDATE" — the script now + // carries a second one (the updatedAt backfill), and a positional match + // would have silently started asserting about the wrong statement. + const update = (statements as string[]).find( + s => s.trimStart().startsWith("UPDATE") && s.includes('SET "projectId"'), + ); assert.ok(update, "the script must carry the backfill UPDATE"); assert.match(update!, /e\."projectId" IS NULL/); assert.match(update!, /est\."projectId" IS NOT NULL/); diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 13499da20..85b879dcd 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -651,3 +651,31 @@ test("coverage ignores an item whose code is not a live phase of the job", () => assert.equal(scoped.size, 0); assert.equal(measureCoverage(rows, scoped).unattributed, 100); }); + +test("LABOR item dollars from another job stay unattributed too", () => { + // The expense side was scoped in round 7; the labor side still resolved + // through a global id->code map, so a time entry pointing at another job's + // estimate item counted as covered when the variance page says it is not. + const entries = [ + { projectId: "job-1", estimate: null, itemId: "item-elsewhere" }, + { projectId: "job-1", estimate: null, itemId: "item-own" }, + ]; + const items = new Map([ + ["item-elsewhere", { costCodeId: "cc-frame", estimateId: "est-2", projectId: "job-2" }], + ["item-own", { costCodeId: "cc-plumb", estimateId: "est-1", projectId: "job-1" }], + ]); + const scoped = scopedItemCostCodes(entries, items, ALL_PHASES); + + assert.equal(scoped.has("item-own"), true); + assert.equal(scoped.has("item-elsewhere"), false, "another job's item is not this job's coverage"); + + const labor = measureCoverage( + [ + { costCodeId: null, itemId: "item-elsewhere", amount: 900 }, + { costCodeId: null, itemId: "item-own", amount: 100 }, + ], + scoped, + ); + assert.equal(labor.attributed, 100); + assert.equal(labor.unattributed, 900); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index ba528a347..caeb8a20f 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -26,7 +26,11 @@ let storedExpense: Record | null; let updateArgs: { where: unknown; data: Record } | null; let estimateItems: { id: string; estimateId: string; projectId: string | null }[]; -const fakePrisma = { +const fakePrisma: any = { + // The tax PATCH writes inside a transaction that first takes the shared + // per-expense advisory lock. + $transaction: async (fn: any) => fn(fakePrisma), + $queryRawUnsafe: async () => [{ lock_result: null }], expense: { findUnique: async () => storedExpense, update: async (args: { where: unknown; data: Record }) => { @@ -39,7 +43,12 @@ const fakePrisma = { const row = storedExpense as Record | null; if (!row) return { count: 0 }; const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); - for (const key of ["amount", "taxAmount", "taxDeductibleBase"]) { + for (const key of [ + "amount", "taxAmount", "taxDeductibleBase", + // The attribution the authorization rested on, plus the row + // version that covers everything else. + "projectId", "estimateId", "updatedAt", + ]) { if (key in args.where && !eq(row[key], args.where[key])) return { count: 0 }; } updateArgs = args; @@ -115,6 +124,7 @@ beforeEach(() => { taxAtSource: true, taxDeductibleBase: null, needsTaxReview: false, + updatedAt: new Date("2026-09-01T00:00:00.000Z"), estimateId: "est-job-1", projectId: "job-1", estimate: { projectId: "job-1" }, @@ -449,3 +459,71 @@ test("the CAS names the values the decision rested on", async () => { assert.equal(where.taxAmount, 16.55); assert.equal(where.taxDeductibleBase, null); }); + +// ── #1 deletion is authorized on the RESOLVED job ────────────────────────── + +test("DELETE authorizes on the job the expense is actually on, not its estimate's", async () => { + // A re-attributed expense: projectId says job-1, the estimate still says + // job-2. Someone with access only to the OLD job must not be able to + // destroy it, and someone with access to the new one must be able to. + storedExpense = { + ...storedExpense, + projectId: "job-1", + estimateId: "est-job-2", + estimate: { projectId: "job-2" }, + }; + + currentUser = { id: "u-old", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-2"] }; + assert.equal((await del()).status, 403, "the job it LEFT confers nothing"); + assert.equal(deleteArgs, null); + + currentUser = { id: "u-new", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; + assert.equal((await del()).status, 200, "the job it is ON does"); +}); + +// ── #2 the CAS covers the attribution the authorization rested on ────────── + +test("a PATCH is refused when the row was RE-ATTRIBUTED under it", async () => { + // Access was granted because of the project this row was on. If it moved, + // the permission check that let the request through was answered about a + // different job — so the write must not land, even though every money + // value it validated is untouched. + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200, "control"); + + const original = fakePrisma.expense.updateMany; + fakePrisma.expense.updateMany = async (args: any) => { + // Model the re-attribution: the row's projectId no longer matches. + if (args.where.projectId === "job-1") return { count: 0 }; + return original(args); + }; + try { + const stale = await patch({ installedAtCustomer: true }); + assert.equal(stale.status, 409); + assert.equal((await stale.json()).code, "STALE_EXPENSE"); + } finally { + fakePrisma.expense.updateMany = original; + } +}); + +test("the CAS names projectId, estimateId and updatedAt", async () => { + await patch({ installedAtCustomer: true }); + const where = updateArgs?.where as Record; + assert.equal(where.projectId, "job-1"); + assert.equal(where.estimateId, "est-job-1"); + assert.ok(where.updatedAt instanceof Date, "the row version pins everything else"); +}); + +test("the tax PATCH writes under the shared per-expense lock", async () => { + const locks: unknown[][] = []; + const originalLock = fakePrisma.$queryRawUnsafe; + fakePrisma.$queryRawUnsafe = async (...args: unknown[]) => { locks.push(args); return [{}]; }; + try { + await patch({ installedAtCustomer: true }); + assert.equal(locks.length, 1, "exactly one lock, taken before the write"); + assert.match(String(locks[0][0]), /pg_advisory_xact_lock/); + assert.equal(locks[0][1], "expense:e1", "namespaced per expense"); + } finally { + fakePrisma.$queryRawUnsafe = originalLock; + } +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index a7fb80b1a..2d7a9fb24 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -628,8 +628,18 @@ function createFakePrisma(initial: StoredExpense[] = []) { $queryRawUnsafe: (query: string, qbPurchaseId: string) => Promise; }) => Promise) { let releaseLock: (() => void) | undefined; + // RE-ENTRANT, like the real thing. `pg_advisory_xact_lock` is + // held for the whole transaction and taking it again inside the + // same one returns immediately — the sync now takes two (per + // purchase, then per expense). A fake that serialised every + // call made the second wait on a lock the same transaction + // already held, which is a deadlock the database would never + // have. + let heldByThisTransaction = false; const transactionLock = { async $queryRawUnsafe() { + if (heldByThisTransaction) return [{ pg_advisory_xact_lock: null }]; + heldByThisTransaction = true; const previous = lockTail; lockTail = new Promise(resolve => { releaseLock = resolve; @@ -1705,3 +1715,76 @@ test("invalidating an ALLOCATION also flags the row — never a silent null", as // The tax itself is still valid against the new gross, so it stays. assert.ok(!("taxAmount" in plan.data)); }); + +// ── #3 interleaving: the sync never clobbers, and never gives up and writes ─ + +test("the sync takes the per-EXPENSE lock as well as the per-purchase one", async () => { + const fake = createFakePrisma([ + { ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null } as any, + ]); + const keys: unknown[] = []; + const wrapped = { + ...fake.client, + async $transaction(cb: any) { + return fake.client.$transaction(async (tx: any) => { + const inner = tx.$queryRawUnsafe; + tx.$queryRawUnsafe = async (...args: unknown[]) => { keys.push(args[1]); return inner(...args); }; + return cb(tx); + }); + }, + }; + await upsertQboExpense(wrapped as any, { ...WRITE, qbSyncToken: "1", amount: 400 }); + assert.ok(keys.includes("expense:expense-1"), `expense lock missing: ${JSON.stringify(keys)}`); +}); + +test("a RE-ATTRIBUTION mid-sync fails the CAS and is re-planned, not overwritten", async () => { + // The attribution moved under the sync. `updatedAt` is in the predicate, so + // the first write matches nothing; the re-plan reads the row as it now is. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + taxAmount: 500, taxDeductibleBase: null, installedAtCustomer: true, + updatedAt: new Date("2026-09-01T00:00:00.000Z"), + } as any, + ]); + const stored = fake.rows.get("purchase-1") as any; + const prePatch = { ...stored }; + fake.rows.set("purchase-1", { + ...stored, projectId: "moved-by-hand", taxAmount: 16.55, + updatedAt: new Date("2026-09-02T00:00:00.000Z"), + }); + + let firstRead = true; + const realFindUnique = fake.expense.findUnique; + fake.expense.findUnique = async (args: any) => { + if (firstRead) { firstRead = false; return prePatch; } + return realFindUnique(args); + }; + + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 300 }); + const after = fake.rows.get("purchase-1") as any; + assert.equal(after.projectId, "moved-by-hand", "the re-attribution stands"); + assert.equal(after.taxAmount, 16.55, "and the tax answer with it"); +}); + +test("a permanently contended row is LEFT ALONE, never unconditionally written", async () => { + // Both the CAS and its retry miss. The sync's own facts are recoverable on + // the next run; a discarded human answer is not, so the correct move is to + // do nothing rather than to clobber. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + taxAmount: 500, taxDeductibleBase: null, + updatedAt: new Date("2026-09-01T00:00:00.000Z"), + } as any, + ]); + const before = { ...(fake.rows.get("purchase-1") as any) }; + fake.expense.updateMany = async () => ({ count: 0 }); + let updateCalls = 0; + fake.expense.update = (async () => { updateCalls += 1; return {} as never; }) as typeof fake.expense.update; + + const result = await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 300 }); + assert.equal(result, "unchanged"); + assert.equal(updateCalls, 0, "no unconditional write anywhere on this path"); + assert.deepEqual(fake.rows.get("purchase-1"), before); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 160aa720a..562cbc142 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -142,9 +142,19 @@ function recorder(overrides: Partial = {}, opts: { estimates?: expenseUpdates.push(args); const cur = state.existingExpense ?? {}; const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); - for (const key of ["projectId", "costCodeId", "taxAmount", "installedAtCustomer"]) { + for (const key of ["costCodeId", "taxAmount", "installedAtCustomer"]) { if (key in args.where && !eq(cur[key], args.where[key])) return { count: 0 }; } + // `projectId` is pinned in every fill predicate to the + // attribution the decision was made under; the project fill + // itself pins NULL. + if ("projectId" in args.where) { + const want = args.where.projectId; + const have = cur.projectId ?? null; + if (want === null ? have !== null : !(have === null || have === want)) { + return { count: 0 }; + } + } if (Array.isArray(args.where.OR)) { const src = cur.costCodeSource ?? null; const ok = args.where.OR.some((b: any) => @@ -157,6 +167,8 @@ function recorder(overrides: Partial = {}, opts: { estimates?: return { count: 1 }; }, }, + // The fill takes the shared per-expense advisory lock first. + $queryRawUnsafe: async () => [{ lock_result: null }], receiptIntake: { update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, updateMany: async (args: any) => { intakeUpdates.push(args.data); return { count: 1 }; }, From 643a319f3ae0780c70fbbf2a845df92e36d1d0bb Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 03:54:42 -0700 Subject: [PATCH 089/144] fix(expenses): updatedAt survives the pre-deploy window; delete + backfill ordered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto Phase 1 head 8b332c77 first. Its shared reconcileLateFields and claim fence are kept; Phase 3's late fields fold INTO that helper (it already carried a note saying installedAtCustomer would land there) rather than being re-added alongside it, and a late costCodeId is validated against the job it is claimed for before reconciliation. 1. `Expense.updatedAt` gets `DEFAULT now()` in the migration, the apply script and Prisma (`@default(now()) @updatedAt`). The apply script runs against production BEFORE the build that knows the column, so for that window the OLD app is still inserting Expenses without it — NOT NULL with no default would have failed every receipt, manual entry and QBO-sync insert until the deploy landed. Order is asserted: nullable, backfill, DEFAULT, then NOT NULL. 3. `deleteExpense` (the single-expense server action) authorizes on the resolved job. It had its own copy of the bug the DELETE route had, and the earlier divergent test never touched this path. The new test does, and is mutation-checked: restoring the estimate read fails 2 of its 6 cases. 5. The backfill's cost-code writes run under the shared per-expense advisory lock and CAS on the row version their plan was computed from. This script's plan is the stalest of the four writers' — built for every row up front, applied over minutes — so it was the one still racing. A miss is counted and reported, never retried: the decision was about a state that no longer exists. Rows the project pass just filled are exempt from the version check, or the backfill would miss on a version it bumped itself. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- .../migration.sql | 19 ++- prisma/schema.prisma | 9 +- scripts/apply-expense-attribution.mjs | 13 +- scripts/backfill-expense-attribution.mjs | 61 +++++++- src/lib/time-expense-actions.ts | 35 +++-- tests/apply-expense-attribution.test.ts | 40 ++++++ tests/backfill-expense-attribution.test.ts | 82 +++++++++++ tests/expense-delete-scope.test.ts | 133 ++++++++++++++++++ 9 files changed, 368 insertions(+), 28 deletions(-) create mode 100644 tests/expense-delete-scope.test.ts diff --git a/package.json b/package.json index e9e5b9cfb..5b4a9a8ab 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index e402f93ae..0d01163c2 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -30,15 +30,22 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL -- A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). -- --- Added in three steps on purpose: nullable, backfilled, then NOT NULL. A --- single `ADD COLUMN ... NOT NULL DEFAULT now()` would leave a DB-level default --- that `updatedAt DateTime @updatedAt` does not declare, and CI's --- "migrations reproduce production" check compares the two. +-- Nullable, backfilled, DEFAULT, then NOT NULL — in that order, and the DEFAULT +-- is not optional. -- --- Each statement is independently re-runnable: IF NOT EXISTS, a predicate-bound --- UPDATE, and a SET NOT NULL that is a no-op once applied. +-- This script runs against production BEFORE the build that knows about the +-- column. For that window the OLD app is still inserting Expenses without it, +-- and a NOT NULL column with no default would fail every one of those inserts: +-- receipts, manual entries, the QBO sync. `now()` is what keeps the old code +-- writing while the new column exists. Prisma declares the same default +-- (`@default(now()) @updatedAt`), so CI's "migrations reproduce production" +-- check still sees the two agree. +-- +-- Every statement is independently re-runnable: IF NOT EXISTS, a +-- predicate-bound UPDATE, and two ALTERs that are no-ops once applied. ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3); UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL; +ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET DEFAULT now(); ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL; CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d6aa1ad86..c887ebbd6 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -677,7 +677,14 @@ model Expense { /// it the CAS could only name the money columns, and a re-attribution /// landing in the gap would slip past a check that had already decided who /// was allowed to make it. - updatedAt DateTime @updatedAt + /// + /// `@default(now())` is load-bearing for the DEPLOY, not for the model. The + /// apply script runs against production BEFORE the build that knows about + /// this column, so for that window the OLD app is still inserting Expenses + /// without it. A NOT NULL column with no default would make every one of + /// those inserts fail — receipts, manual entries, the QBO sync, all of it — + /// for as long as the window lasts. + updatedAt DateTime @default(now()) @updatedAt /// Back-relation for ReceiptIntake.expenseId. Declared here rather than left /// SQL-only because `prisma migrate diff` WOULD see a foreign key that diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index a6e8b05e8..596ed2b22 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -89,11 +89,18 @@ export const statements = [ `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false`, // A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). - // Three steps on purpose — nullable, backfill, NOT NULL — because a single - // NOT NULL DEFAULT now() would leave a DB default that `@updatedAt` does - // not declare, and CI compares the two. Each step is re-runnable. + // + // Nullable, backfill, DEFAULT, NOT NULL — and the DEFAULT is what makes the + // pre-deploy window survivable. This script runs BEFORE the build that + // knows about the column, so the OLD app is still inserting Expenses + // without it; NOT NULL with no default would fail every receipt, manual + // entry and QBO-sync insert until the deploy landed. Prisma declares the + // same default, so the migration check still sees them agree. + // + // Each step is independently re-runnable. `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3)`, `UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL`, + `ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET DEFAULT now()`, `ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL`, `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.mjs index 3bb5bb6b9..9b5da8ad8 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.mjs @@ -44,6 +44,7 @@ import { resolveExpenseProjectId, } from "../src/lib/expense-attribution.ts"; import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project.ts"; +import { lockExpense } from "../src/lib/expense-lock.ts"; import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases.ts"; import { csvCell, csvNumber } from "../src/lib/csv-safe.ts"; @@ -217,6 +218,7 @@ export function planBackfill({ costCodeConfidence: null, why: "item cost code (same project)", expectedProjectId: resolvedProjectId, + expectedUpdatedAt: expense.updatedAt ?? null, expense, }); continue; @@ -267,6 +269,7 @@ export function planBackfill({ // AFTER pass (a) has run — which is the state the write // actually meets. expectedProjectId: resolvedProjectId, + expectedUpdatedAt: expense.updatedAt ?? null, expense, }); } else { @@ -324,6 +327,23 @@ export function remainderCsv(remainder, projectNameById) { * tests/backfill-expense-attribution.test.ts can drive it with a prisma-shaped * stub and assert that a dry run makes ZERO write calls. */ +/** + * Run one write inside a transaction that first takes the shared per-expense + * advisory lock, so this script is ORDERED against the QBO sync, the tax PATCH + * and the booking fill rather than racing them. + * + * Falls back to a bare call when the injected client has no `$transaction` — + * the unit tests drive a plain stub, and requiring one there would test the + * stub rather than the script. + */ +export async function writeUnderExpenseLock(db, expenseId, run) { + if (typeof db.$transaction !== "function") return run(db); + return db.$transaction(async tx => { + await lockExpense(tx, expenseId); + return run(tx); + }); +} + export async function runBackfill({ db, apply = false, @@ -351,6 +371,8 @@ export async function runBackfill({ select: { id: true, estimateId: true, projectId: true, costCodeId: true, costCodeSource: true, itemId: true, amount: true, vendor: true, description: true, date: true, + // The row version each planned write is judged against. + updatedAt: true, estimate: { select: { projectId: true } }, }, }); @@ -516,9 +538,23 @@ export async function runBackfill({ if (!byProject.has(fill.projectId)) byProject.set(fill.projectId, []); byProject.get(fill.projectId).push(fill.id); } + // Pass (a) bumps `updatedAt` on the rows it fills, so their planned + // cost-code CAS would miss on a version the backfill itself changed. Those + // rows are exempted from the version check — the attribution and + // uncoded predicates still guard them. + const filledByProjectPass = new Set(plan.projectFills.map(fill => fill.id)); + for (const fill of plan.codeFills) { + fill.projectWasFilled = filledByProjectPass.has(fill.id); + } + for (const [projectId, ids] of byProject) { // `projectId: null` in the predicate, not just in the plan: between the // read above and this write, a re-sync or a bookkeeper may have set it. + // + // No CAS on `updatedAt` here on purpose. This pass only ever fills a + // NULL, so "the row changed" and "the column is still NULL" are the + // same question, and the id-set write is one statement per project + // rather than one per row. const result = await db.expense.updateMany({ where: { id: { in: ids }, projectId: null }, data: { projectId }, @@ -526,12 +562,33 @@ export async function runBackfill({ projectIdsWritten += result.count; } + // COST-CODE WRITES RUN UNDER THE SHARED PER-EXPENSE LOCK, one row at a + // time, and each is a compare-and-set on the row version its decision was + // made from. + // + // This script is the fourth writer of these columns, alongside the QBO + // sync, the tax PATCH and the booking fill — and the only one that had been + // left racing them. Its plan is the STALEST of the four: it is computed for + // every row up front and then applied in a loop that can take minutes, so + // "nothing changed since the read" is a much weaker assumption here than + // anywhere else. + // + // A miss is not an error and is not retried: the row moved, so the decision + // was made about a state that no longer exists. It is counted and reported, + // and a re-run will plan it again against the truth. let costCodesWritten = 0; let costCodesSkipped = 0; for (const fill of plan.codeFills) { - const result = await db.expense.updateMany({ + const result = await writeUnderExpenseLock(db, fill.id, tx => tx.expense.updateMany({ where: { id: fill.id, + // The row version the plan was computed from. Pass (a) above + // may legitimately have bumped it, so a fill whose project was + // just written is re-read rather than assumed — see the + // `expectedUpdatedAt === null` branch in the helper. + ...(fill.expectedUpdatedAt && !fill.projectWasFilled + ? { updatedAt: fill.expectedUpdatedAt } + : {}), // Everything the plan depended on, re-asserted at write time. // The plan is a snapshot taken before pass (a) ran and before // any concurrent sync or bookkeeper edit; the predicate is what @@ -552,7 +609,7 @@ export async function runBackfill({ costCodeSource: fill.costCodeSource, costCodeConfidence: fill.costCodeConfidence, }, - }); + })); costCodesWritten += result.count; if (result.count === 0) costCodesSkipped += 1; } diff --git a/src/lib/time-expense-actions.ts b/src/lib/time-expense-actions.ts index ce3a02d3c..c35abe3ba 100644 --- a/src/lib/time-expense-actions.ts +++ b/src/lib/time-expense-actions.ts @@ -194,21 +194,28 @@ export async function deleteExpense(id: string, projectId: string) { if (!hasPermission(user, "timeClock")) throw new Error("Forbidden"); const expense = await prisma.expense.findUnique({ where: { id }, - select: { - qbPurchaseId: true, - invoiceId: true, - invoicedAt: true, - estimate: { select: { projectId: true } }, - }, + select: { + qbPurchaseId: true, + invoiceId: true, + invoicedAt: true, + projectId: true, + estimate: { select: { projectId: true } }, + }, }); - if (!expense || expense.estimate.projectId !== projectId || !canAccessProject(user, expense.estimate.projectId)) { - throw new Error("Forbidden"); - } - assertExpenseMutableOutsideQbo(expense); - if (expense.invoiceId || expense.invoicedAt) throw new Error("Billed expenses cannot be deleted"); - - const deleted = await prisma.expense.deleteMany({ - where: { id, qbPurchaseId: null, invoiceId: null, invoicedAt: null }, + // Resolved, not read off the estimate. For a RE-ATTRIBUTED expense the + // estimate still names the job it left, so reading it here both admitted + // someone whose access is to that old job and refused the crew who now + // own the row — a deletion authorized against a project the expense is + // not on. + const resolvedProjectId = expense ? resolveExpenseProjectId(expense) : null; + if (!expense || resolvedProjectId !== projectId || !canAccessProject(user, resolvedProjectId)) { + throw new Error("Forbidden"); + } + assertExpenseMutableOutsideQbo(expense); + if (expense.invoiceId || expense.invoicedAt) throw new Error("Billed expenses cannot be deleted"); + + const deleted = await prisma.expense.deleteMany({ + where: { id, qbPurchaseId: null, invoiceId: null, invoicedAt: null }, }); if (deleted.count !== 1) throw new Error("Expense was billed while it was being deleted; refresh and try again"); diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 83e90fddc..45240190d 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -203,3 +203,43 @@ test("the tax-vs-gross CHECK is in both DDL paths and in the verifier", () => { ); assert.ok(verified, "and the post-run verification must assert it"); }); + +// ── updatedAt must survive the PRE-DEPLOY window (round 10, item 1) ──────── + +test("updatedAt is added nullable, backfilled, defaulted, THEN made NOT NULL", () => { + // Order is the whole point. The script runs against production BEFORE the + // build that knows about the column, so the OLD app is still inserting + // Expenses without it — NOT NULL with no default would fail every receipt, + // manual entry and QBO-sync insert until the deploy landed. + const sql = (statements as string[]).filter(s => s.includes('"updatedAt"')); + assert.equal(sql.length, 4, "add, backfill, default, not-null"); + assert.match(sql[0], /ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP\(3\)$/); + assert.ok(!/NOT NULL/.test(sql[0]), "it must land NULLABLE or the backfill cannot run"); + assert.match(sql[1], /^UPDATE [\s\S]*SET "updatedAt" = COALESCE\("createdAt"/); + assert.match(sql[2], /SET DEFAULT now\(\)/); + assert.match(sql[3], /SET NOT NULL/); + // The default must be in place BEFORE NOT NULL, or an insert racing the two + // statements still fails. + assert.ok( + (statements as string[]).indexOf(sql[2]) < (statements as string[]).indexOf(sql[3]), + "DEFAULT has to precede NOT NULL", + ); +}); + +test("every updatedAt statement is re-runnable, and matches the migration", () => { + const sql = (statements as string[]).filter(s => s.includes('"updatedAt"')); + // IF NOT EXISTS; a predicate-bound UPDATE; two idempotent ALTERs. + assert.match(sql[0], /IF NOT EXISTS/); + assert.match(sql[1], /WHERE "updatedAt" IS NULL/); + for (const statement of sql) { + assert.ok( + normalizedMigration.includes(normalize(statement).replace(/;$/, "")), + `migration.sql is missing:\n ${statement}`, + ); + } +}); + +test("Prisma declares the same default, so the migration check sees them agree", () => { + const schema = readFileSync(path.join(__dirname, "..", "prisma", "schema.prisma"), "utf8"); + assert.match(schema, /updatedAt DateTime @default\(now\(\)\) @updatedAt/); +}); diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 85b879dcd..b262ec8f1 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -30,6 +30,7 @@ type StubExpense = { vendor: string | null; description: string | null; date: Date | null; + updatedAt?: Date | null; estimate: { projectId: string | null }; }; @@ -48,6 +49,7 @@ function expense(overrides: Partial = {}): StubExpense { vendor: null, description: null, date: new Date("2026-08-01T00:00:00.000Z"), + updatedAt: new Date("2026-09-01T00:00:00.000Z"), estimate: { projectId: "job-1" }, ...overrides, }; @@ -93,6 +95,12 @@ function createStub( } if ("projectId" in where && (row.projectId ?? null) !== where.projectId) return false; if ("costCodeId" in where && (row.costCodeId ?? null) !== where.costCodeId) return false; + // The row-version CAS. Dates compare by value, not identity. + if ("updatedAt" in where) { + const want = where.updatedAt as Date | null; + const have = (row as any).updatedAt as Date | null | undefined; + if ((have?.getTime() ?? null) !== (want?.getTime() ?? null)) return false; + } if (Array.isArray(where.OR)) { const ok = (where.OR as Record[]).some(branch => { if (!("costCodeSource" in branch)) return false; @@ -679,3 +687,77 @@ test("LABOR item dollars from another job stay unattributed too", () => { assert.equal(labor.attributed, 100); assert.equal(labor.unattributed, 900); }); + +// ── the backfill is ordered against the other three writers (round 10, #5) ── + +test("each cost-code write takes the shared per-expense lock", async () => { + const locks: unknown[] = []; + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async (_q: string, key: unknown) => { locks.push(key); return [{}]; }; + + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.deepEqual(locks, ["expense:e1"], "one lock, namespaced per expense"); +}); + +test("a row that MOVED between the plan and the write is skipped, not coded", async () => { + // This script's plan is the stalest of the four writers': computed for + // every row up front, then applied in a loop that can run for minutes. The + // CAS on the row version is what stops it acting on a state that has since + // changed. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => [{}]; + + // Someone edits the row after findMany handed it to the planner. + const realUpdateMany = stub.db.expense.updateMany; + let moved = false; + (stub.db.expense as any).updateMany = async (args: any) => { + if (!moved && "updatedAt" in args.where) { + moved = true; + stub.rows[0].updatedAt = new Date("2026-09-02"); + } + return realUpdateMany(args); + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0, "the stale decision is not applied"); + assert.equal(stub.rows[0].costCodeId, null); +}); + +test("the CAS names the row version the plan was computed from", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => [{}]; + + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; + assert.deepEqual(codeWrite.where.updatedAt, new Date("2026-09-01")); +}); + +test("a row the PROJECT pass just filled is exempt from the version check", async () => { + // Pass (a) bumps updatedAt itself, so a naive CAS would miss on a version + // the backfill changed a moment earlier — and code nothing, which is + // exactly the ordering bug from round 6 wearing a different hat. + const stub = createStub( + [expense({ id: "e1", projectId: null, estimate: { projectId: "job-1" }, vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => [{}]; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.projectIds, 1); + assert.equal(result.written.costCodes, 1, "the code still lands"); + const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; + assert.ok(!("updatedAt" in codeWrite.where), "no version check on a row this run just touched"); +}); diff --git a/tests/expense-delete-scope.test.ts b/tests/expense-delete-scope.test.ts new file mode 100644 index 000000000..e8e3e4237 --- /dev/null +++ b/tests/expense-delete-scope.test.ts @@ -0,0 +1,133 @@ +/** + * `deleteExpense` — the SINGLE-expense server action (Codex round 10, item 3). + * + * The earlier divergent-attribution test covered the DELETE route, not this + * path, and this one had its own copy of the bug: it authorized against + * `expense.estimate.projectId`. For a re-attributed expense that names the job + * it LEFT, so the check both admitted someone whose access is to the old job + * and refused the crew who now own the row. + * + * Prisma, next-auth and the permission reader are patched at require() time — + * same shape as tests/job-variance-db.test.ts. No mock.module: CI is Node 20. + */ +import { test, before, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import Module from "node:module"; + +interface FakeUser { + id: string; + role: string; + permissions: Record; + projectIds: string[]; +} + +let currentUser: FakeUser | null; +let storedExpense: Record | null; +let deleteArgs: unknown; + +const fakePrisma = { + expense: { + findUnique: async () => storedExpense, + deleteMany: async (args: unknown) => { + deleteArgs = args; + return { count: 1 }; + }, + }, +}; + +let deleteExpense: (id: string, projectId: string) => Promise; + +before(async () => { + const originalRequire = Module.prototype.require; + let hit = false; + (Module.prototype as unknown as { require: (id: string) => unknown }).require = function ( + this: NodeModule, + id: string, + ) { + if (id === "@/lib/prisma") { hit = true; return { prisma: fakePrisma }; } + if (id === "@/lib/permissions") { + return { + getCurrentUserWithPermissions: async () => currentUser, + hasPermission: (user: FakeUser | null, key: string) => + !!user && (user.role === "ADMIN" || user.permissions?.[key] === true), + canAccessProject: (user: FakeUser, projectId: string) => + user.role === "ADMIN" || user.projectIds.includes(projectId), + }; + } + if (id === "next/cache") return { revalidatePath: () => {} }; + if (id === "next-auth/next") return { getServerSession: async () => ({ user: { email: "x@y.z" } }) }; + // eslint-disable-next-line prefer-rest-params + return originalRequire.apply(this, arguments as unknown as [string]); + } as typeof Module.prototype.require; + + let mod: any; + try { + mod = await import("../src/lib/time-expense-actions"); + } finally { + Module.prototype.require = originalRequire; + } + if (typeof mod.deleteExpense !== "function") { + throw new Error(`expense-delete-scope: mocks did not apply (require patch ${hit ? "WAS" : "was NOT"} hit)`); + } + deleteExpense = mod.deleteExpense; +}); + +beforeEach(() => { + currentUser = { id: "u1", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-1"] }; + // RE-ATTRIBUTED: it lives on job-1 now, its estimate still names job-2. + storedExpense = { + qbPurchaseId: null, + invoiceId: null, + invoicedAt: null, + projectId: "job-1", + estimate: { projectId: "job-2" }, + }; + deleteArgs = null; +}); + +async function attempt(projectId: string): Promise { + try { + await deleteExpense("e1", projectId); + return null; + } catch (error) { + return (error as Error).message; + } +} + +test("the job the expense is ON can delete it", async () => { + assert.equal(await attempt("job-1"), null); + assert.deepEqual(deleteArgs, { + where: { id: "e1", qbPurchaseId: null, invoiceId: null, invoicedAt: null }, + }); +}); + +test("the job it LEFT cannot — not even from that job's own page", async () => { + // Reading the estimate would have said "job-2" and allowed this. + currentUser = { id: "u2", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-2"] }; + assert.equal(await attempt("job-2"), "Forbidden"); + assert.equal(deleteArgs, null, "and nothing is deleted"); +}); + +test("access to the new job is still required, not just the right projectId", async () => { + currentUser = { id: "u3", role: "FIELD_CREW", permissions: { timeClock: true }, projectIds: ["somewhere-else"] }; + assert.equal(await attempt("job-1"), "Forbidden"); + assert.equal(deleteArgs, null); +}); + +test("an unattributed expense falls back to its estimate's job", async () => { + // The resolver's other branch: nothing to prefer, so the estimate decides. + storedExpense = { ...storedExpense, projectId: null }; + currentUser = { id: "u4", role: "MANAGER", permissions: { timeClock: true }, projectIds: ["job-2"] }; + assert.equal(await attempt("job-2"), null); +}); + +test("a row with no job at all cannot be deleted here", async () => { + storedExpense = { ...storedExpense, projectId: null, estimate: { projectId: null } }; + currentUser = { id: "u5", role: "ADMIN", permissions: {}, projectIds: [] }; + assert.equal(await attempt("job-1"), "Forbidden", "no scope to authorize against"); +}); + +test("the timeClock permission is still required", async () => { + currentUser = { id: "u6", role: "FIELD_CREW", permissions: {}, projectIds: ["job-1"] }; + assert.equal(await attempt("job-1"), "Forbidden"); +}); From c3850e17004faf669bc467086c0f57f4cf9201c1 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 04:25:55 -0700 Subject: [PATCH 090/144] fix(scripts): run under Node 20; backfill re-reads and re-plans under the lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto Phase 1 head 85d6af57. Its finalize authorization and exact-state late-field fencing are kept whole; Phase 3's `installedAtCustomer` folds into their reconcileLateFields and their authorizeLateFields replaces my duplicate phase check. 1/2. Confirmed on this branch and pinned by a new test: a late `projectId` at /finalize is refused when the session caller cannot see that job, the check is on the LATE project rather than the row's existing one, a late phase must belong to the EFFECTIVE project, and a secret forwarder skips the per-user check but not the phase rule. 3. The two data scripts imported TypeScript from src/, which plain `node` only resolves on 22.6+. They now run under `node --import=tsx` — and are renamed .mjs -> .ts, because tsx hands a .ts module to an .mjs file as CJS and the named imports fail outright. A `--help` path (no DB, no env) doubles as the CI smoke test that the documented command actually loads the import graph. @ts-nocheck keeps them exactly as unchecked as they were as .mjs; typing them properly belongs in its own change. 4. The backfill's cost-code pass now RE-READS each row under the per-expense lock and re-checks eligibility before writing, carrying the post-fill version into the CAS. The previous fix exempted rows the project pass had touched from the version check, which traded one hazard for another: an exempted row had no version guard at all. Tests cover a bookkeeper coding the row and a re-attribution, both mid-run. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- ...on.mjs => backfill-expense-attribution.ts} | 125 ++++++++++++---- ...odes.mjs => suggest-expense-cost-codes.ts} | 45 +++++- .../receipts/intake/[id]/finalize/route.ts | 5 +- tests/backfill-expense-attribution.test.ts | 80 ++++++++++- tests/finalize-late-field-authz.test.ts | 134 ++++++++++++++++++ tests/scripts-runtime-smoke.test.ts | 52 +++++++ 7 files changed, 399 insertions(+), 46 deletions(-) rename scripts/{backfill-expense-attribution.mjs => backfill-expense-attribution.ts} (85%) rename scripts/{suggest-expense-cost-codes.mjs => suggest-expense-cost-codes.ts} (73%) create mode 100644 tests/finalize-late-field-authz.test.ts create mode 100644 tests/scripts-runtime-smoke.test.ts diff --git a/package.json b/package.json index 5b4a9a8ab..7f0104bb0 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/scripts/backfill-expense-attribution.mjs b/scripts/backfill-expense-attribution.ts similarity index 85% rename from scripts/backfill-expense-attribution.mjs rename to scripts/backfill-expense-attribution.ts index 9b5da8ad8..401b0882d 100644 --- a/scripts/backfill-expense-attribution.mjs +++ b/scripts/backfill-expense-attribution.ts @@ -1,3 +1,14 @@ +// @ts-nocheck +// +// These one-shot operational scripts were `.mjs` until the rename that made +// `node --import=tsx` able to resolve their named imports from src/. They have +// never been typechecked, and this marker keeps that true rather than quietly +// changing what CI enforces in the same commit that changed the file +// extension. `tsconfig.json` excludes `scripts/`; the only reason tsc sees this +// file at all is that a test imports it. +// +// Worth typing properly — but as its own change, where a type error is a +// finding rather than rebase noise. /** * Backfill Expense attribution (Receipt Pipeline v2, Phase 3 — * docs/plans/PHASE-3-ATTRIBUTION-SPEC.md §6). @@ -24,10 +35,17 @@ * loop's discipline: overwrite an existing cost code, and overwrite a HUMAN's * (costCodeSource "capture" or "manual"). * + * RUNTIME: this file imports TypeScript from src/, so it needs a TS loader. + * node --import=tsx scripts/... + * Plain `node` works on this machine (Node 24 strips types) and FAILS on CI's + * Node 20 and on anything older — which is exactly where a one-shot data script + * gets run in a hurry. `--import=tsx` is the same loader the test suite uses, + * so there is one answer rather than a version-dependent one. + * * USAGE - * node scripts/backfill-expense-attribution.mjs # dry run - * node scripts/backfill-expense-attribution.mjs --csv out.csv # + remainder CSV - * node scripts/backfill-expense-attribution.mjs --apply # write + * node --import=tsx scripts/backfill-expense-attribution.ts # dry run + * node --import=tsx scripts/backfill-expense-attribution.ts --csv out.csv # + remainder CSV + * node --import=tsx scripts/backfill-expense-attribution.ts --apply # write * * A re-run after --apply must report 0 planned changes. That is the proof, and * it is the same rule scripts/backfill-estimate-item-cost-codes.mjs follows. @@ -37,16 +55,16 @@ import { config } from "dotenv"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { writeFileSync } from "node:fs"; -import { suggestCode } from "../src/lib/expense-cost-suggest.ts"; +import { suggestCode } from "../src/lib/expense-cost-suggest"; import { notHumanCodedExpenseWhere, resolveExpenseCostCodeId, resolveExpenseProjectId, -} from "../src/lib/expense-attribution.ts"; -import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project.ts"; -import { lockExpense } from "../src/lib/expense-lock.ts"; -import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases.ts"; -import { csvCell, csvNumber } from "../src/lib/csv-safe.ts"; +} from "../src/lib/expense-attribution"; +import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; +import { lockExpense } from "../src/lib/expense-lock"; +import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases"; +import { csvCell, csvNumber } from "../src/lib/csv-safe"; /** * Both current tiers clear this (0.9 vendor, 0.75 line), so it changes nothing @@ -350,9 +368,13 @@ export async function runBackfill({ // Annotated rather than left to inference: a bare `null` default infers the // parameter as `null`, and every caller that passes a real path (including // the test) then fails to typecheck. - csvPath = /** @type {string | null} */ (null), - writeFile = /** @type {(path: string, body: string) => void} */ (writeFileSync), - log = /** @type {(message: string) => void} */ (console.log), + // Real annotations now the file is TypeScript. They are not error-checked + // (see @ts-nocheck at the top) but they still type the EXPORT, which is + // what callers and tests see — a bare `= null` default would infer the + // parameter as `null` and reject every real path. + csvPath = null as string | null, + writeFile = writeFileSync as (path: string, body: string) => void, + log = console.log as (message: string) => void, overheadProjectId = OVERHEAD_PROJECT_ID, }) { const [projects, costCodes] = await Promise.all([ @@ -538,15 +560,6 @@ export async function runBackfill({ if (!byProject.has(fill.projectId)) byProject.set(fill.projectId, []); byProject.get(fill.projectId).push(fill.id); } - // Pass (a) bumps `updatedAt` on the rows it fills, so their planned - // cost-code CAS would miss on a version the backfill itself changed. Those - // rows are exempted from the version check — the attribution and - // uncoded predicates still guard them. - const filledByProjectPass = new Set(plan.projectFills.map(fill => fill.id)); - for (const fill of plan.codeFills) { - fill.projectWasFilled = filledByProjectPass.has(fill.id); - } - for (const [projectId, ids] of byProject) { // `projectId: null` in the predicate, not just in the plan: between the // read above and this write, a re-sync or a bookkeeper may have set it. @@ -579,16 +592,46 @@ export async function runBackfill({ let costCodesWritten = 0; let costCodesSkipped = 0; for (const fill of plan.codeFills) { - const result = await writeUnderExpenseLock(db, fill.id, tx => tx.expense.updateMany({ + const result = await writeUnderExpenseLock(db, fill.id, async tx => { + // RE-READ UNDER THE LOCK, then re-plan against what is really + // there. + // + // The previous version exempted rows that pass (a) had just filled + // from the version check, because the backfill bumps `updatedAt` + // itself. That traded one hazard for another: an exempted row had + // NO version guard at all, so anything a concurrent writer did to + // it between the plan and the write was invisible. And a row the + // project pass did NOT touch could still have moved. + // + // Reading inside the lock removes the guesswork. The version that + // goes into the CAS is the CURRENT one, so it names the state this + // decision is actually being made against — including the + // `projectId` pass (a) may have just written. + const current = await tx.expense.findUnique({ + where: { id: fill.id }, + select: { + id: true, projectId: true, costCodeId: true, costCodeSource: true, + itemId: true, updatedAt: true, estimate: { select: { projectId: true } }, + }, + }); + if (!current) return { count: 0 }; + + // The plan was made about a row that no longer looks like this. + // Re-deciding here would be a second planner with its own copy of + // the rules; skipping is honest and a re-run will plan it properly. + const stillEligible = + current.costCodeId === null && + current.costCodeSource !== "capture" && + current.costCodeSource !== "manual" && + resolveExpenseProjectId(current) === fill.expectedProjectId; + if (!stillEligible) return { count: 0 }; + + return tx.expense.updateMany({ where: { id: fill.id, - // The row version the plan was computed from. Pass (a) above - // may legitimately have bumped it, so a fill whose project was - // just written is re-read rather than assumed — see the - // `expectedUpdatedAt === null` branch in the helper. - ...(fill.expectedUpdatedAt && !fill.projectWasFilled - ? { updatedAt: fill.expectedUpdatedAt } - : {}), + // The version as READ UNDER THIS LOCK — not the one the plan + // was built from minutes ago. + ...(current.updatedAt ? { updatedAt: current.updatedAt } : {}), // Everything the plan depended on, re-asserted at write time. // The plan is a snapshot taken before pass (a) ran and before // any concurrent sync or bookkeeper edit; the predicate is what @@ -609,7 +652,8 @@ export async function runBackfill({ costCodeSource: fill.costCodeSource, costCodeConfidence: fill.costCodeConfidence, }, - })); + }); + }); costCodesWritten += result.count; if (result.count === 0) costCodesSkipped += 1; } @@ -631,7 +675,28 @@ export async function runBackfill({ }; } +const HELP = `Backfill Expense attribution (Receipt Pipeline v2, Phase 3). + + node --import=tsx scripts/backfill-expense-attribution.ts # dry run + node --import=tsx scripts/backfill-expense-attribution.ts --csv out.csv # + remainder CSV + node --import=tsx scripts/backfill-expense-attribution.ts --apply # write + +Dry run is the DEFAULT. --apply writes; re-run dry afterwards and it must +report zero planned changes. + +The --import=tsx loader is required: this script imports TypeScript from src/, +and plain node only strips types on Node 22.6+.`; + async function main() { + // --help must work with NO database and NO env. It is also the CI smoke + // test that this file can be LOADED under the documented runtime — an + // import error surfaces here rather than the first time someone runs the + // real thing against production. + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(HELP); + return; + } + const __dirname = dirname(fileURLToPath(import.meta.url)); config({ path: join(__dirname, "..", ".env.local") }); config({ path: join(__dirname, "..", ".env") }); diff --git a/scripts/suggest-expense-cost-codes.mjs b/scripts/suggest-expense-cost-codes.ts similarity index 73% rename from scripts/suggest-expense-cost-codes.mjs rename to scripts/suggest-expense-cost-codes.ts index a39aa6d1b..2d6bfd44e 100644 --- a/scripts/suggest-expense-cost-codes.mjs +++ b/scripts/suggest-expense-cost-codes.ts @@ -1,3 +1,14 @@ +// @ts-nocheck +// +// These one-shot operational scripts were `.mjs` until the rename that made +// `node --import=tsx` able to resolve their named imports from src/. They have +// never been typechecked, and this marker keeps that true rather than quietly +// changing what CI enforces in the same commit that changed the file +// extension. `tsconfig.json` excludes `scripts/`; the only reason tsc sees this +// file at all is that a test imports it. +// +// Worth typing properly — but as its own change, where a type error is a +// finding rather than rebase noise. /** * Suggest ProBuild cost codes (phases) for expenses that have none. * @@ -22,17 +33,24 @@ * lumber, drywall, paint and toilets alike. * * The rules themselves moved to src/lib/expense-cost-suggest.ts (Phase 3) so - * the QBO sync and scripts/backfill-expense-attribution.mjs run the SAME ones. + * the QBO sync and scripts/backfill-expense-attribution.ts run the SAME ones. * This script keeps its own scope and reporting; it no longer owns the regexes. * + * RUNTIME: this file imports TypeScript from src/, so it needs a TS loader. + * node --import=tsx scripts/... + * Plain `node` works on this machine (Node 24 strips types) and FAILS on CI's + * Node 20 and on anything older — which is exactly where a one-shot data script + * gets run in a hurry. `--import=tsx` is the same loader the test suite uses, + * so there is one answer rather than a version-dependent one. + * * USAGE - * node scripts/suggest-expense-cost-codes.mjs # dry run + report - * node scripts/suggest-expense-cost-codes.mjs --apply # write matches - * node scripts/suggest-expense-cost-codes.mjs --csv out.csv + * node --import=tsx scripts/suggest-expense-cost-codes.ts # dry run + report + * node --import=tsx scripts/suggest-expense-cost-codes.ts --apply # write matches + * node --import=tsx scripts/suggest-expense-cost-codes.ts --csv out.csv */ import { PrismaClient } from "@prisma/client"; -import { suggestCode } from "../src/lib/expense-cost-suggest.ts"; -import { notHumanCodedExpenseWhere } from "../src/lib/expense-attribution.ts"; +import { suggestCode } from "../src/lib/expense-cost-suggest"; +import { notHumanCodedExpenseWhere } from "../src/lib/expense-attribution"; import { config } from "dotenv"; import { fileURLToPath } from "url"; import { dirname, join } from "path"; @@ -54,7 +72,22 @@ const OVERHEAD_PROJECTS = ["Shop"]; const num = (v) => (v == null ? 0 : Number(v)); const money = (v) => `$${num(v).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; +const HELP = `Suggest cost codes for uncoded expenses (rule-based, not a model). + + node --import=tsx scripts/suggest-expense-cost-codes.ts # dry run + report + node --import=tsx scripts/suggest-expense-cost-codes.ts --apply # write matches + node --import=tsx scripts/suggest-expense-cost-codes.ts --csv out.csv + +The --import=tsx loader is required: this script imports TypeScript from src/.`; + async function main() { + // Works with no database and no env — and doubles as the CI check that this + // file still LOADS under the documented runtime. + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(HELP); + return; + } + const codes = await prisma.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }); const codeId = new Map(codes.map((c) => [c.code, c.id])); diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 668574be7..9c1882d45 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -80,7 +80,10 @@ async function applyLateFields( * Without the second, a cost code from another job rides into the Expense and * every variance report reads it as overspend on a line nobody budgeted. */ -async function authorizeFinalization( +/** Exported for tests/finalize-late-field-authz.test.ts — the project-access + * half of this is the only thing standing between a late `projectId` and a + * job the caller cannot see. */ +export async function authorizeFinalization( auth: Extract, rowProjectId: string | null, lateFields: LateFields, diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index b262ec8f1..34c998a8e 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -15,7 +15,7 @@ import { remainderCsv, runBackfill, scopedItemCostCodes, -} from "../scripts/backfill-expense-attribution.mjs"; +} from "../scripts/backfill-expense-attribution"; const OVERHEAD_ID = "overhead-project"; @@ -141,6 +141,12 @@ function createStub( }, expense: { async findMany() { return rows; }, + // The cost-code pass re-reads each row UNDER THE LOCK before + // deciding, so the stub has to serve the CURRENT row rather + // than the snapshot the planner saw. + async findUnique(args: { where: { id: string } }) { + return rows.find(row => row.id === args.where.id) ?? null; + }, async updateMany(args: { where: Record; data: Record }) { writes.push(args); let count = 0; @@ -744,20 +750,80 @@ test("the CAS names the row version the plan was computed from", async () => { assert.deepEqual(codeWrite.where.updatedAt, new Date("2026-09-01")); }); -test("a row the PROJECT pass just filled is exempt from the version check", async () => { - // Pass (a) bumps updatedAt itself, so a naive CAS would miss on a version - // the backfill changed a moment earlier — and code nothing, which is - // exactly the ordering bug from round 6 wearing a different hat. +test("a row the PROJECT pass just filled is re-read, not exempted", async () => { + // Pass (a) bumps `updatedAt` itself, so the plan's version is stale for + // exactly the rows this run just touched. The earlier fix EXEMPTED those + // from the version check, which traded one hazard for another: an exempted + // row had no version guard at all, so a concurrent writer was invisible to + // it. Re-reading under the lock removes the guess — the CAS names the + // version as it is NOW, including the projectId pass (a) wrote. const stub = createStub( - [expense({ id: "e1", projectId: null, estimate: { projectId: "job-1" }, vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [expense({ + id: "e1", projectId: null, estimate: { projectId: "job-1" }, + vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01"), + })], [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], ); (stub.db as any).$transaction = async (fn: any) => fn(stub.db); (stub.db as any).$queryRawUnsafe = async () => [{}]; + // Model the real column: pass (a)'s write bumps the version. + const realUpdateMany = stub.db.expense.updateMany; + (stub.db.expense as any).updateMany = async (args: any) => { + const result = await realUpdateMany(args); + if (result.count > 0 && "projectId" in args.data) { + stub.rows[0].updatedAt = new Date("2026-09-02"); + } + return result; + }; const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); assert.equal(result.written.projectIds, 1); assert.equal(result.written.costCodes, 1, "the code still lands"); + const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; - assert.ok(!("updatedAt" in codeWrite.where), "no version check on a row this run just touched"); + assert.deepEqual( + codeWrite.where.updatedAt, + new Date("2026-09-02"), + "the CAS names the POST-fill version, not the planner's snapshot", + ); +}); + +test("a row that became human-coded after the plan is skipped on the re-read", async () => { + // The re-read is not just a version fetch — it re-checks eligibility, so a + // bookkeeper who coded the row mid-run keeps their answer. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => { + // The PATCH lands while this row is being locked. + stub.rows[0].costCodeId = "cc-human"; + stub.rows[0].costCodeSource = "manual"; + return [{}]; + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0); + assert.equal(stub.rows[0].costCodeId, "cc-human", "the human's phase stands"); + assert.ok( + !stub.writes.some(w => "costCodeId" in w.data), + "and no write is even attempted once the re-read says ineligible", + ); +}); + +test("a row re-attributed after the plan is skipped on the re-read", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => { + stub.rows[0].projectId = "job-elsewhere"; + return [{}]; + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0, "the phase was chosen for a job it is no longer on"); + assert.equal(stub.rows[0].costCodeId, null); }); diff --git a/tests/finalize-late-field-authz.test.ts b/tests/finalize-late-field-authz.test.ts new file mode 100644 index 000000000..a678d1fe7 --- /dev/null +++ b/tests/finalize-late-field-authz.test.ts @@ -0,0 +1,134 @@ +/** + * A late `projectId` at /finalize cannot name a job the caller cannot see + * (Codex round 11, items 1 and 2 — confirming Phase 1's authorization holds on + * this branch, from the Phase 3 side). + * + * The two-step upload lets a client supply the job AFTER the bytes land. That + * is the point of the late-field path, and it is also the reason it needs its + * own access check: the project on the row was authorized at /start, and a + * different one arriving at /finalize was never checked by anything else. + * + * Phase 3 cares because a receipt filed against a job the caller cannot reach + * lands in that job's costs, its variance, and — once flagged — its tax + * deduction. + */ +import { test, before, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import Module from "node:module"; + +let accessibleProjects: string[]; +let projectPhases: Record; + +let authorizeFinalization: ( + auth: unknown, + rowProjectId: string | null, + lateFields: Record, +) => Promise; + +before(async () => { + const originalRequire = Module.prototype.require; + let hit = false; + (Module.prototype as unknown as { require: (id: string) => unknown }).require = function ( + this: NodeModule, + id: string, + ) { + if (id === "@/lib/prisma") { hit = true; return { prisma: {} }; } + if (id === "@/lib/mobile-auth") { + return { + userCanAccessProject: async (_user: unknown, projectId: string) => + accessibleProjects.includes(projectId), + }; + } + if (id === "@/lib/project-phases") { + return { + isCostCodeAllowedForProject: async (_ds: unknown, projectId: string, costCodeId: string) => + (projectPhases[projectId] ?? []).includes(costCodeId), + }; + } + if (id === "@/lib/project-phases-db") return { prismaPhaseDataSource: {} }; + // eslint-disable-next-line prefer-rest-params + return originalRequire.apply(this, arguments as unknown as [string]); + } as typeof Module.prototype.require; + + let mod: any; + try { + mod = await import("../src/app/api/receipts/intake/[id]/finalize/route"); + } finally { + Module.prototype.require = originalRequire; + } + if (typeof mod.authorizeFinalization !== "function") { + throw new Error(`finalize-late-field-authz: mocks did not apply (patch ${hit ? "WAS" : "was NOT"} hit)`); + } + authorizeFinalization = mod.authorizeFinalization; +}); + +beforeEach(() => { + accessibleProjects = ["job-mine"]; + projectPhases = { "job-mine": ["cc-plumb"], "job-theirs": ["cc-frame"] }; +}); + +const session = { ok: true, via: "session", user: { id: "u1", role: "MANAGER" } }; +const secret = { ok: true, via: "secret" }; + +test("a session caller cannot file a receipt against a job they cannot see", async () => { + const denied = await authorizeFinalization(session, null, { projectId: "job-theirs" }); + assert.ok(denied, "it must be refused"); + assert.equal(denied!.status, 403); + assert.equal((await denied!.json()).reason, "forbidden"); +}); + +test("a session caller CAN file against a job they can see", async () => { + assert.equal(await authorizeFinalization(session, null, { projectId: "job-mine" }), null); +}); + +test("the check is on the LATE project, not the one the row already had", async () => { + // The row was started against an accessible job; the late field tries to + // move it to one that is not. Authorizing the row's existing project would + // wave this through. + const denied = await authorizeFinalization(session, "job-mine", { projectId: "job-theirs" }); + assert.ok(denied); + assert.equal(denied!.status, 403); +}); + +test("a late phase must belong to the EFFECTIVE project", async () => { + // cc-frame is a phase of job-theirs, not of job-mine. + const denied = await authorizeFinalization(session, "job-mine", { costCodeId: "cc-frame" }); + assert.ok(denied); + assert.equal(denied!.status, 400); + assert.equal((await denied!.json()).error, "cost-code-not-a-phase"); +}); + +test("the effective project is the LATE one when both are supplied", async () => { + accessibleProjects = ["job-mine", "job-theirs"]; + // cc-frame is not on job-mine but IS on job-theirs, which is where the row + // is being moved to — so it must be accepted. + assert.equal( + await authorizeFinalization(session, "job-mine", { projectId: "job-theirs", costCodeId: "cc-frame" }), + null, + ); + // ...and the reverse is refused. + const denied = await authorizeFinalization(session, "job-theirs", { projectId: "job-mine", costCodeId: "cc-frame" }); + assert.ok(denied); + assert.equal(denied!.status, 400); +}); + +test("a phase with no job to check it against is refused", async () => { + const denied = await authorizeFinalization(session, null, { costCodeId: "cc-plumb" }); + assert.ok(denied); + assert.equal(denied!.status, 400); + assert.equal((await denied!.json()).error, "cost-code-without-project"); +}); + +test("a secret forwarder skips the per-user check but NOT the phase check", async () => { + // A shared-secret forwarder has no user to scope by — it resolves the job + // from the Drive folder — so project access does not apply to it. The phase + // still has to belong to the job. + assert.equal(await authorizeFinalization(secret, null, { projectId: "job-theirs" }), null); + const denied = await authorizeFinalization(secret, "job-mine", { costCodeId: "cc-frame" }); + assert.ok(denied, "the phase rule is not a per-user rule"); + assert.equal(denied!.status, 400); +}); + +test("no late fields is nothing to authorize", async () => { + assert.equal(await authorizeFinalization(session, "job-mine", {}), null); +}); diff --git a/tests/scripts-runtime-smoke.test.ts b/tests/scripts-runtime-smoke.test.ts new file mode 100644 index 000000000..87f467f99 --- /dev/null +++ b/tests/scripts-runtime-smoke.test.ts @@ -0,0 +1,52 @@ +/** + * The one-shot data scripts must LOAD under the runtime their own docs name + * (Codex round 11, item 3). + * + * They import TypeScript from src/, so they need a loader. Plain `node` happens + * to work on this machine (Node 24 strips types) and fails on CI's Node 20 — + * which is the worst possible split, because these are the scripts someone runs + * by hand, in a hurry, against production. The documented command is + * `node --import=tsx`, and this asserts the documented command actually works. + * + * `--help` is deliberately the probe: it exercises the whole import graph with + * no database, no env and no writes. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; + +const ROOT = path.join(__dirname, ".."); + +const SCRIPTS = [ + "scripts/backfill-expense-attribution.ts", + "scripts/suggest-expense-cost-codes.ts", +]; + +for (const script of SCRIPTS) { + test(`${script} loads and prints usage under \`node --import=tsx\``, () => { + const output = execFileSync( + process.execPath, + ["--import=tsx", script, "--help"], + { cwd: ROOT, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ); + // A failed import throws above (non-zero exit); this pins that the + // script reached its own argument handling rather than merely starting. + assert.match(output, /--import=tsx/, "usage text must name the required loader"); + assert.match(output, /--apply/, "and the write flag"); + }); + + test(`${script} documents the loader in its own usage block`, () => { + const source = execFileSync( + process.execPath, + ["-e", `process.stdout.write(require("fs").readFileSync(${JSON.stringify(script)}, "utf8"))`], + { cwd: ROOT, encoding: "utf8" }, + ); + // Nobody should be able to copy a bare `node scripts/...` line out of + // the header and have it fail on the Node the rest of the repo pins. + assert.ok( + !/^\s*\*\s+node scripts\//m.test(source), + "the usage block must not show a bare `node scripts/...` command", + ); + }); +} From 5db5f38f58bdaf3be21b822c8802e2a7d9a13c2d Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 04:53:38 -0700 Subject: [PATCH 091/144] fix(expenses): suggestions read the persisted row; one script writes cost codes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 12. Item 2 (finalize count/reauthorize) is Phase 1's. 1. The QBO suggester now reads the vendor and description off the PERSISTED row instead of taking them from the payload the sync just processed. Those differ exactly when the upsert refused the payload: an out-of-order webhook carries an older SyncToken, `isIncomingQboSyncTokenCurrent` correctly rejects it, and the suggestion then coded the row from a version of the purchase the database had just thrown away. The write is fenced on that row's `updatedAt` AND `qbSyncToken`, so a newer sync landing in the gap wins. 3. The backfill's project fill CASes on `projectId IS NULL AND estimateId = plannedEstimateId` — NULL alone did not say the derivation was still valid, and a row re-pointed at another estimate would have been stamped with the old estimate's project by the very pass that exists to get attribution right. The cost fill re-RUNS `planBackfill` over the freshly-read row under the lock and only writes if the answer is unchanged: eligibility was never the whole dependency, the vendor/description/item feed the decision too. 4. `suggest-expense-cost-codes.ts` loses `--apply` entirely. It was a second writer of `costCodeId` with none of the backfill's guarantees. Report-only now, through the canonical resolver and the project's phase-eligible codes, with csv-safe output; a test fails if a write or the flag returns. Test fakes learned one more real behaviour: `findMany` returns a SNAPSHOT, so a test can model "the row changed after the planner saw it". Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 15 +++ scripts/backfill-expense-attribution.ts | 82 +++++++++---- scripts/suggest-expense-cost-codes.ts | 104 ++++++++++------ src/lib/qbo-expense-sync.ts | 44 +++++-- tests/backfill-expense-attribution.test.ts | 99 ++++++++++++++- tests/qbo-expense-sync.test.ts | 133 ++++++++++++++++----- tests/scripts-runtime-smoke.test.ts | 34 ++++++ 7 files changed, 407 insertions(+), 104 deletions(-) diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index a00ba6cfc..34b988f3b 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -364,6 +364,21 @@ loading as `suggest-expense-cost-codes.mjs`. Steps: Marge. Re-run after `--apply` must report 0 changes (backfill-estimate-item-cost-codes proof rule). +**AS BUILT — there is exactly ONE writer of `costCodeId` among the scripts.** +`scripts/suggest-expense-cost-codes.ts` (the older rule-suggester) had its own +`--apply`, which made it a second writer of the same column — one with no +per-expense lock, no row-version compare-and-set, no re-plan under that lock and +no project-scoped phase check. Its `--apply` is REMOVED (Codex round 12): it is +report-only now, reads attribution through `resolveExpenseProjectId`, excludes +the overhead bucket by `OVERHEAD_PROJECT_ID` rather than by the name "Shop", +reports a match only when the writer would accept it, and emits its CSV through +`csv-safe`. `tests/scripts-runtime-smoke.test.ts` fails if a write or an +`--apply` flag ever comes back. + +Both scripts are `.ts` and run under `node --import=tsx` — they import +TypeScript from `src/`, and tsx hands a `.ts` module to an `.mjs` file as CJS, +so named imports fail outright from a `.mjs` wrapper. + ## 7. Tax paid at source report — `/reports/tax-paid-at-source` - `src/app/reports/tax-paid-at-source/page.tsx` (server component, List layout per diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 401b0882d..68622e6bc 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -174,7 +174,16 @@ export function planBackfill({ for (const expense of expenses) { const resolvedProjectId = resolveExpenseProjectId(expense); if (expense.projectId === null && resolvedProjectId) { - projectFills.push({ id: expense.id, projectId: resolvedProjectId }); + projectFills.push({ + id: expense.id, + projectId: resolvedProjectId, + // The estimate this project was DERIVED from. `projectId IS + // NULL` alone does not say the derivation is still valid: an + // expense re-pointed at a different estimate has a different + // project, and filling it from the old one would attribute the + // money to a job it was moved off. + expectedEstimateId: expense.estimateId, + }); } // NEVER re-code a row that already has a code, and never touch a @@ -555,21 +564,28 @@ export async function runBackfill({ // ── the writes ────────────────────────────────────────────────────────── let projectIdsWritten = 0; + // Grouped by (project, estimate) rather than by project alone: the estimate + // is half the predicate now, so rows derived from different estimates + // cannot share one statement. const byProject = new Map(); for (const fill of plan.projectFills) { - if (!byProject.has(fill.projectId)) byProject.set(fill.projectId, []); - byProject.get(fill.projectId).push(fill.id); + const key = `${fill.projectId}\u0000${fill.expectedEstimateId}`; + if (!byProject.has(key)) { + byProject.set(key, { projectId: fill.projectId, estimateId: fill.expectedEstimateId, ids: [] }); + } + byProject.get(key).ids.push(fill.id); } - for (const [projectId, ids] of byProject) { - // `projectId: null` in the predicate, not just in the plan: between the - // read above and this write, a re-sync or a bookkeeper may have set it. + for (const { projectId, estimateId, ids } of byProject.values()) { + // BOTH halves of the derivation, re-asserted at write time. // - // No CAS on `updatedAt` here on purpose. This pass only ever fills a - // NULL, so "the row changed" and "the column is still NULL" are the - // same question, and the id-set write is one statement per project - // rather than one per row. + // `projectId: null` says nobody has attributed the row yet; + // `estimateId` says it is still hanging off the estimate this project + // was READ from. Without the second one, an expense re-pointed at a + // different estimate between the plan and the write would be stamped + // with the old estimate's project — a silent cross-job attribution + // performed by the very pass that exists to get attribution right. const result = await db.expense.updateMany({ - where: { id: { in: ids }, projectId: null }, + where: { id: { in: ids }, projectId: null, estimateId }, data: { projectId }, }); projectIdsWritten += result.count; @@ -610,21 +626,35 @@ export async function runBackfill({ const current = await tx.expense.findUnique({ where: { id: fill.id }, select: { - id: true, projectId: true, costCodeId: true, costCodeSource: true, - itemId: true, updatedAt: true, estimate: { select: { projectId: true } }, + id: true, estimateId: true, projectId: true, costCodeId: true, + costCodeSource: true, itemId: true, vendor: true, description: true, + updatedAt: true, estimate: { select: { projectId: true } }, }, }); if (!current) return { count: 0 }; - // The plan was made about a row that no longer looks like this. - // Re-deciding here would be a second planner with its own copy of - // the rules; skipping is honest and a re-run will plan it properly. - const stillEligible = - current.costCodeId === null && - current.costCodeSource !== "capture" && - current.costCodeSource !== "manual" && - resolveExpenseProjectId(current) === fill.expectedProjectId; - if (!stillEligible) return { count: 0 }; + // RE-PLAN, don't re-use. + // + // The plan was computed minutes ago from a snapshot. Checking that + // the row is still ELIGIBLE is not the same as checking that the + // same answer still follows from it: the vendor, the description + // and the item link all feed the decision, and an edit to any of + // them makes the planned code an answer to a question nobody asked + // any more. + // + // `planBackfill` is the single copy of the rules, so it is run + // again over this one row rather than re-implemented here. + const replanned = planBackfill({ + expenses: [current], + items, + costCodeIdByCode, + scopedProjectIds, + allowedCodesByProject, + }); + const fresh = replanned.codeFills[0]; + // No longer codeable at all, or codeable as something else — either + // way the planned write is void. A re-run will plan it properly. + if (!fresh || fresh.costCodeId !== fill.costCodeId) return { count: 0 }; return tx.expense.updateMany({ where: { @@ -648,9 +678,11 @@ export async function runBackfill({ ...notHumanCodedExpenseWhere(), }, data: { - costCodeId: fill.costCodeId, - costCodeSource: fill.costCodeSource, - costCodeConfidence: fill.costCodeConfidence, + // From the RE-PLAN, so the provenance and confidence describe + // the decision that was actually just made. + costCodeId: fresh.costCodeId, + costCodeSource: fresh.costCodeSource, + costCodeConfidence: fresh.costCodeConfidence, }, }); }); diff --git a/scripts/suggest-expense-cost-codes.ts b/scripts/suggest-expense-cost-codes.ts index 2d6bfd44e..7d01b1abf 100644 --- a/scripts/suggest-expense-cost-codes.ts +++ b/scripts/suggest-expense-cost-codes.ts @@ -44,13 +44,22 @@ * so there is one answer rather than a version-dependent one. * * USAGE - * node --import=tsx scripts/suggest-expense-cost-codes.ts # dry run + report - * node --import=tsx scripts/suggest-expense-cost-codes.ts --apply # write matches + * node --import=tsx scripts/suggest-expense-cost-codes.ts # report * node --import=tsx scripts/suggest-expense-cost-codes.ts --csv out.csv + * + * READ-ONLY. There is no --apply and there must not be one: writing cost codes + * belongs to scripts/backfill-expense-attribution.ts, which is the only path + * with the per-expense lock, the compare-and-set on the row version, the + * re-plan under that lock, and the project-scoped phase check. This script + * kept a second, weaker writer alive for the same columns — two ways to code an + * expense, one of them unaware of every guarantee the other was given. */ import { PrismaClient } from "@prisma/client"; import { suggestCode } from "../src/lib/expense-cost-suggest"; -import { notHumanCodedExpenseWhere } from "../src/lib/expense-attribution"; +import { expenseNotOnProjectWhere, resolveExpenseProjectId } from "../src/lib/expense-attribution"; +import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; +import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases"; +import { csvCell, csvNumber } from "../src/lib/csv-safe"; import { config } from "dotenv"; import { fileURLToPath } from "url"; import { dirname, join } from "path"; @@ -62,22 +71,26 @@ config({ path: join(__dirname, "..", ".env") }); const prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL } } }); -const APPLY = process.argv.includes("--apply"); const csvIdx = process.argv.indexOf("--csv"); const CSV = csvIdx > -1 ? process.argv[csvIdx + 1] : null; -/** Company overhead bucket — never gets a job phase. */ -const OVERHEAD_PROJECTS = ["Shop"]; +// The overhead bucket is excluded BY ID via the canonical constant. Matching +// on the name "Shop" stopped being true the day the project could be renamed, +// and this report is read as evidence. const num = (v) => (v == null ? 0 : Number(v)); const money = (v) => `$${num(v).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; -const HELP = `Suggest cost codes for uncoded expenses (rule-based, not a model). +const HELP = `Report cost-code suggestions for uncoded expenses (rule-based, not a model). - node --import=tsx scripts/suggest-expense-cost-codes.ts # dry run + report - node --import=tsx scripts/suggest-expense-cost-codes.ts --apply # write matches + node --import=tsx scripts/suggest-expense-cost-codes.ts # report node --import=tsx scripts/suggest-expense-cost-codes.ts --csv out.csv +READ-ONLY: this script never writes. Use +scripts/backfill-expense-attribution.ts --apply to actually code expenses — it +is the only writer with the per-expense lock, the row-version CAS and the +project-scoped phase check. + The --import=tsx loader is required: this script imports TypeScript from src/.`; async function main() { @@ -91,28 +104,63 @@ async function main() { const codes = await prisma.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }); const codeId = new Map(codes.map((c) => [c.code, c.id])); + // Attribution resolved the ONE way, and the overhead bucket excluded by id. + // A row re-attributed to a live job used to be invisible here (its estimate + // still named the old one), so the report understated the work to be done. const rows = await prisma.expense.findMany({ where: { costCodeId: null, - estimate: { project: { status: "In Progress", name: { notIn: OVERHEAD_PROJECTS } } }, + ...expenseNotOnProjectWhere(OVERHEAD_PROJECT_ID), }, select: { - id: true, amount: true, vendor: true, description: true, - estimate: { select: { project: { select: { name: true } } } }, + id: true, amount: true, vendor: true, description: true, projectId: true, + project: { select: { id: true, name: true, status: true } }, + estimate: { select: { projectId: true, project: { select: { id: true, name: true, status: true } } } }, }, orderBy: { amount: "desc" }, }); + // The PROJECT'S OWN PHASES, same rule the writer applies. A suggestion the + // job could never accept is not a suggestion, and listing it as one sends a + // bookkeeper to check something that was never on the table. + const phaseRows = await prisma.estimateItem.findMany({ + where: { + costCodeId: { not: null }, + costCode: { isActive: true }, + estimate: { ...PHASE_ELIGIBLE_ESTIMATE_WHERE }, + }, + select: { costCodeId: true, estimate: { select: { projectId: true } } }, + }); + const allowedCodesByProject = new Map(); + for (const row of phaseRows) { + const projectId = row.estimate?.projectId ?? null; + if (!projectId || !row.costCodeId) continue; + if (!allowedCodesByProject.has(projectId)) allowedCodesByProject.set(projectId, new Set()); + allowedCodesByProject.get(projectId).add(row.costCodeId); + } + + const inProgress = (row) => + (row.projectId ? row.project?.status : row.estimate?.project?.status) === "In Progress"; + const matched = []; const unmatched = []; for (const r of rows) { + if (!inProgress(r)) continue; + const projectId = resolveExpenseProjectId(r); const s = suggestCode(r); - if (s && codeId.has(s.code)) matched.push({ ...r, ...s }); + const suggestedId = s ? codeId.get(s.code) : undefined; + const allowed = projectId ? allowedCodesByProject.get(projectId) : undefined; + // Reported as a match only if the WRITER would accept it. + if (s && suggestedId && allowed && allowed.has(suggestedId)) matched.push({ ...r, ...s }); else unmatched.push(r); } + const projectName = (row) => + (row.projectId ? row.project?.name : row.estimate?.project?.name) ?? ""; + const sum = (a) => a.reduce((t, r) => t + num(r.amount), 0); - console.log(`scope: active customer jobs (overhead bucket "${OVERHEAD_PROJECTS.join(", ")}" excluded)`); + console.log(`scope: In Progress customer jobs (overhead project ${OVERHEAD_PROJECT_ID} excluded)`); + console.log("READ-ONLY report. Use backfill-expense-attribution.ts --apply to write."); console.log(`uncoded rows: ${rows.length} ${money(sum(rows))}`); console.log(` confident match: ${matched.length} ${money(sum(matched))}`); console.log(` NEEDS HUMAN: ${unmatched.length} ${money(sum(unmatched))}\n`); @@ -129,35 +177,23 @@ async function main() { } if (CSV) { - const esc = (s) => `"${String(s ?? "").replace(/"/g, '""').replace(/\s+/g, " ").slice(0, 160)}"`; + // csv-safe: vendor and description are OCR output, so a leading `=` + // reaches the reader's spreadsheet as a formula. + const cell = (v) => csvCell(String(v ?? "").replace(/\s+/g, " ").slice(0, 160)); const out = [["expense_id", "project", "amount", "vendor", "suggested_code", "why", "description"].join(",")]; for (const m of matched) { - out.push([m.id, esc(m.estimate?.project?.name), num(m.amount), esc(m.vendor), m.code, esc(m.why), esc(m.description)].join(",")); + out.push([cell(m.id), cell(projectName(m)), csvNumber(num(m.amount)), cell(m.vendor), cell(m.code), cell(m.why), cell(m.description)].join(",")); } for (const u of unmatched) { - out.push([u.id, esc(u.estimate?.project?.name), num(u.amount), esc(u.vendor), "", "NEEDS_HUMAN", esc(u.description)].join(",")); + out.push([cell(u.id), cell(projectName(u)), csvNumber(num(u.amount)), cell(u.vendor), cell(""), cell("NEEDS_HUMAN"), cell(u.description)].join(",")); } writeFileSync(CSV, out.join("\n")); console.log(`\nwrote ${CSV} (${out.length - 1} rows)`); } - if (!APPLY) { - console.log("\nDRY RUN — nothing written. Re-run with --apply to save the confident matches."); - return; - } - - let n = 0; - for (const m of matched) { - // Phase 3: stamp provenance alongside the code. A row with a code and - // no source would be indistinguishable from a human's choice, and the - // capture/manual guard everywhere else keys on exactly that. - const written = await prisma.expense.updateMany({ - where: { id: m.id, costCodeId: null, ...notHumanCodedExpenseWhere() }, - data: { costCodeId: codeId.get(m.code), costCodeSource: "ai", costCodeConfidence: m.confidence }, - }); - n += written.count; - } - console.log(`\napplied ${n} cost code(s). ${unmatched.length} rows left NULL for human review.`); + console.log("\nREPORT ONLY — nothing was written. To apply:"); + console.log(" node --import=tsx scripts/backfill-expense-attribution.ts # dry run"); + console.log(" node --import=tsx scripts/backfill-expense-attribution.ts --apply # write"); } main() diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 581b82bfd..a2224db49 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -998,10 +998,12 @@ export async function deactivateQboExpense( // It also never runs on the deactivate path: a Purchase deleted in QBO is not // an occasion to guess its phase. +/** + * Only the identity. The vendor and description the suggestion reads come from + * the PERSISTED row, never from the caller — see below. + */ export interface QboCostCodeSuggestionInput { qbPurchaseId: string; - vendor: string | null; - description: string; } export interface QboCostCodeSuggestionClient { @@ -1013,6 +1015,10 @@ export interface QboCostCodeSuggestionClient { projectId: string | null; costCodeId: string | null; costCodeSource: string | null; + vendor: string | null; + description: string | null; + qbSyncToken: string | null; + updatedAt?: Date; estimate?: { projectId: string | null } | null; } | null>; updateMany(args: { @@ -1058,12 +1064,28 @@ export async function applyQboExpenseCostCodeSuggestion( */ isAllowedForProject?: (projectId: string, costCodeId: string) => Promise, ): Promise { + // THE PERSISTED ROW IS THE ONLY INPUT. + // + // The sync used to hand this function the vendor and description off the + // purchase it had just processed. That is wrong whenever the upsert did not + // actually accept them: an out-of-order QBO webhook carries an OLDER + // SyncToken, `isIncomingQboSyncTokenCurrent` correctly refuses it and + // returns "unchanged" — and the suggestion then ran on the rejected + // payload's text and coded the row from a version of the purchase the + // database deliberately threw away. + // + // Reading the row back means the suggestion can only ever describe what is + // actually stored. const stored = await client.expense.findUnique({ where: { qbPurchaseId: input.qbPurchaseId }, select: { projectId: true, costCodeId: true, costCodeSource: true, + vendor: true, + description: true, + qbSyncToken: true, + updatedAt: true, estimate: { select: { projectId: true } }, }, }); @@ -1085,7 +1107,7 @@ export async function applyQboExpenseCostCodeSuggestion( // and a row that has since been attributed must be re-scoped, not written. const expectedProjectId = stored.projectId ?? null; - const suggestion = suggestCode({ vendor: input.vendor, description: input.description }); + const suggestion = suggestCode({ vendor: stored.vendor, description: stored.description }); if (!suggestion) return "no-match"; const costCodeId = costCodeIdByCode.get(suggestion.code); @@ -1107,6 +1129,13 @@ export async function applyQboExpenseCostCodeSuggestion( // skipped rather than written on stale reasoning. projectId: expectedProjectId, costCodeId: null, + // The exact row version the suggestion was computed from. Without + // `qbSyncToken` a NEWER sync could commit between the read and this + // write, and the row would be coded from the text of a purchase it + // no longer holds — the same staleness the read above fixed, just + // one statement later. + ...(stored.updatedAt ? { updatedAt: stored.updatedAt } : {}), + qbSyncToken: stored.qbSyncToken, ...notHumanCodedExpenseWhere(), }, data: { @@ -1615,11 +1644,10 @@ export async function syncQboExpenses( // Same resilience posture as persistClassification/attachReceipt. if (dependencies.suggestCostCode) { try { - await dependencies.suggestCostCode({ - qbPurchaseId: purchase.qbPurchaseId, - vendor: purchase.vendor, - description, - }); + // Identity only. Whether this purchase's payload was ACCEPTED + // is the upsert's business, and the suggestion reads whatever + // the upsert left behind rather than what it was offered. + await dependencies.suggestCostCode({ qbPurchaseId: purchase.qbPurchaseId }); } catch (error) { console.error( "QBO cost-code suggestion failed", diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 34c998a8e..e9b047706 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -95,6 +95,7 @@ function createStub( } if ("projectId" in where && (row.projectId ?? null) !== where.projectId) return false; if ("costCodeId" in where && (row.costCodeId ?? null) !== where.costCodeId) return false; + if ("estimateId" in where && row.estimateId !== where.estimateId) return false; // The row-version CAS. Dates compare by value, not identity. if ("updatedAt" in where) { const want = where.updatedAt as Date | null; @@ -140,12 +141,16 @@ function createStub( async findMany() { return []; }, }, expense: { - async findMany() { return rows; }, + // A SNAPSHOT, like a real read. Handing back the live objects + // meant a test could not model "the row changed after the + // planner saw it" — the planner would see the change too. + async findMany() { return rows.map(row => ({ ...row })); }, // The cost-code pass re-reads each row UNDER THE LOCK before // deciding, so the stub has to serve the CURRENT row rather // than the snapshot the planner saw. async findUnique(args: { where: { id: string } }) { - return rows.find(row => row.id === args.where.id) ?? null; + const row = rows.find(candidate => candidate.id === args.where.id); + return row ? { ...row } : null; }, async updateMany(args: { where: Record; data: Record }) { writes.push(args); @@ -175,7 +180,11 @@ test("plans a projectId fill only for rows whose column is still NULL", () => { costCodeIdByCode: COST_CODE_IDS, scopedProjectIds: ["job-1"], }); - assert.deepEqual(plan.projectFills, [{ id: "needs-fill", projectId: "job-1" }]); + // The estimate the project was DERIVED from rides along, so the write can + // require the derivation is still valid. + assert.deepEqual(plan.projectFills, [ + { id: "needs-fill", projectId: "job-1", expectedEstimateId: "est-job-1" }, + ]); }); test("the item fallback wins over the rules, and is sourced 'backfill'", () => { @@ -445,7 +454,12 @@ test("apply writes both passes, each behind its own predicate", async () => { await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); const projectWrite = stub.writes.find(w => "projectId" in w.data)!; - assert.deepEqual(projectWrite.where, { id: { in: ["e1"] }, projectId: null }); + assert.deepEqual(projectWrite.where, { + id: { in: ["e1"] }, + projectId: null, + // Both halves of the derivation. + estimateId: "est-job-1", + }); const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; assert.equal(codeWrite.where.id, "e1"); @@ -827,3 +841,80 @@ test("a row re-attributed after the plan is skipped on the re-read", async () => assert.equal(result.written.costCodes, 0, "the phase was chosen for a job it is no longer on"); assert.equal(stub.rows[0].costCodeId, null); }); + +// ── mutations BEFORE the re-read (round 12, item 3) ──────────────────────── + +test("an expense re-pointed at another estimate is not stamped with the old job", async () => { + // `projectId IS NULL` alone does not say the derivation is still valid. + const stub = createStub([ + expense({ id: "e1", projectId: null, estimateId: "est-job-1", estimate: { projectId: "job-1" } }), + ]); + // The move happens AFTER the planner has read the row — the snapshot it + // planned from still says est-job-1. + const realFindMany = stub.db.expense.findMany; + let planned = false; + (stub.db.expense as any).findMany = async () => { + const snapshot = await realFindMany(); + if (!planned) { + planned = true; + stub.rows[0].estimateId = "est-job-2"; + } + return snapshot; + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.projectIds, 0, "the plan's derivation no longer holds"); + assert.equal(stub.rows[0].projectId, null); +}); + +test("a vendor edited before the re-read changes the answer, so nothing is written", async () => { + // Eligibility was not the only thing the plan depended on: the vendor feeds + // the rule. Re-checking eligibility alone would have written a phase chosen + // from text that is no longer on the row. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => { + // The edit lands while the row is being locked, i.e. BEFORE the re-read. + stub.rows[0].vendor = "General Hardware"; + stub.rows[0].description = "misc supplies"; + return [{}]; + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0, "the planned code answered a question nobody is asking now"); + assert.equal(stub.rows[0].costCodeId, null); +}); + +test("a vendor edit that points at a DIFFERENT phase is refused, not applied", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => { + // Now the rules would say FRAMING, not plumbing. + stub.rows[0].vendor = "Parr Lumber"; + return [{}]; + }; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 0, "the plan and the re-plan disagree, so neither is applied"); + assert.equal(stub.rows[0].costCodeId, null, "a re-run will plan it properly"); +}); + +test("an untouched row still codes normally through the re-plan", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => [{}]; + + const result = await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + assert.equal(result.written.costCodes, 1); + assert.equal(stub.rows[0].costCodeId, "cc-plumb"); + assert.equal(stub.rows[0].costCodeSource, "ai"); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 2d7a9fb24..a6d0a6c02 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1233,12 +1233,25 @@ type StoredForSuggestion = { projectId: string | null; costCodeId: string | null; costCodeSource: string | null; + // The suggestion reads the PERSISTED text, so the stub has to carry it. + vendor: string | null; + description: string | null; + qbSyncToken: string | null; + updatedAt?: Date; estimate?: { projectId: string | null } | null; } | null; -function fakeSuggestionClient( - stored: StoredForSuggestion = { projectId: "project-1", costCodeId: null, costCodeSource: null }, -) { +const STORED_DEFAULT: StoredForSuggestion = { + projectId: "project-1", + costCodeId: null, + costCodeSource: null, + vendor: "Summit Plumbing", + description: "[QuickBooks import] rough-in", + qbSyncToken: "3", + updatedAt: new Date("2026-09-01T00:00:00.000Z"), +}; + +function fakeSuggestionClient(stored: StoredForSuggestion = { ...STORED_DEFAULT! }) { const calls: { where: Record; data: Record }[] = []; let count = 1; return { @@ -1263,11 +1276,7 @@ test("a NULL cost code is filled with source ai and the rule's tier confidence", const fake = fakeSuggestionClient(); const result = await applyQboExpenseCostCodeSuggestion( fake.client, - { - qbPurchaseId: "purchase-1", - vendor: "Summit Plumbing", - description: "[QuickBooks import] rough-in", - }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ); assert.equal(result, "written"); @@ -1286,7 +1295,7 @@ test("the write is guarded on uncoded AND not-human-coded, with a NULL branch", const fake = fakeSuggestionClient(); await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ); const where = fake.calls[0].where; @@ -1308,6 +1317,7 @@ test("the write requires the SAME attribution the suggestion was scoped to", asy // decision was made in, and it is just as much a precondition as a // populated id. const fake = fakeSuggestionClient({ + ...STORED_DEFAULT, projectId: null, costCodeId: null, costCodeSource: null, @@ -1315,7 +1325,7 @@ test("the write requires the SAME attribution the suggestion was scoped to", asy }); await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ); assert.equal(fake.calls[0].where.projectId, null); @@ -1323,10 +1333,10 @@ test("the write requires the SAME attribution the suggestion was scoped to", asy test("a row a human already coded is refused on the STORED source, before any write", async () => { for (const costCodeSource of ["capture", "manual"]) { - const fake = fakeSuggestionClient({ projectId: "project-1", costCodeId: null, costCodeSource }); + const fake = fakeSuggestionClient({ ...STORED_DEFAULT, costCodeSource }); const result = await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ); assert.equal(result, "not-written", costCodeSource); @@ -1335,11 +1345,11 @@ test("a row a human already coded is refused on the STORED source, before any wr }); test("a row that is already coded is left alone", async () => { - const fake = fakeSuggestionClient({ projectId: "project-1", costCodeId: "cc-existing", costCodeSource: null }); + const fake = fakeSuggestionClient({ ...STORED_DEFAULT, costCodeId: "cc-existing" }); assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Ferguson", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ), "not-written", @@ -1352,13 +1362,14 @@ test("scope comes from the STORED project, not the incoming QBO match", async () // ref still says "Mueller Bathroom", so a suggester scoped to the match // would hand an overhead purchase a job phase. const fake = fakeSuggestionClient({ + ...STORED_DEFAULT, projectId: OVERHEAD_PROJECT_ID, costCodeId: null, costCodeSource: null, }); const result = await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ); assert.equal(result, "skipped-overhead"); @@ -1367,6 +1378,7 @@ test("scope comes from the STORED project, not the incoming QBO match", async () test("the stored project is resolved through the estimate when the column is NULL", async () => { const fake = fakeSuggestionClient({ + ...STORED_DEFAULT, projectId: null, costCodeId: null, costCodeSource: null, @@ -1375,7 +1387,7 @@ test("the stored project is resolved through the estimate when the column is NUL assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ), "skipped-overhead", @@ -1383,11 +1395,11 @@ test("the stored project is resolved through the estimate when the column is NUL }); test("a row with no job at all gets no job phase", async () => { - const fake = fakeSuggestionClient({ projectId: null, costCodeId: null, costCodeSource: null, estimate: null }); + const fake = fakeSuggestionClient({ ...STORED_DEFAULT!, projectId: null, estimate: null }); assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, ), "skipped-no-project", @@ -1400,7 +1412,7 @@ test("a vanished row is reported, not treated as a silent success", async () => assert.equal( await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "gone", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "gone" }, COST_CODE_IDS, ), "missing-row", @@ -1414,7 +1426,7 @@ test("a phase the JOB does not have is refused, however confident the rule was", const fake = fakeSuggestionClient(); const result = await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, async () => false, ); @@ -1427,7 +1439,7 @@ test("the scope check is asked about the row's OWN job and the resolved code", a const asked: { projectId: string; costCodeId: string }[] = []; await applyQboExpenseCostCodeSuggestion( fake.client, - { qbPurchaseId: "purchase-1", vendor: "Summit Plumbing", description: "x" }, + { qbPurchaseId: "purchase-1" }, COST_CODE_IDS, async (projectId, costCodeId) => { asked.push({ projectId, costCodeId }); return true; }, ); @@ -1436,24 +1448,25 @@ test("the scope check is asked about the row's OWN job and the resolved code", a }); test("no rule match and an unknown code both write nothing", async () => { - const fake = fakeSuggestionClient(); + // The text comes from the STORED row now, so a no-match case has to be a + // stored row the rules do not recognise. + const bland = fakeSuggestionClient({ + ...STORED_DEFAULT!, + vendor: "General Hardware", + description: "misc supplies", + }); assert.equal( - await applyQboExpenseCostCodeSuggestion( - fake.client, - { qbPurchaseId: "p", vendor: "General Hardware", description: "misc" }, - COST_CODE_IDS, - ), + await applyQboExpenseCostCodeSuggestion(bland.client, { qbPurchaseId: "p" }, COST_CODE_IDS), "no-match", ); + assert.equal(bland.calls.length, 0); + + const known = fakeSuggestionClient(); assert.equal( - await applyQboExpenseCostCodeSuggestion( - fake.client, - { qbPurchaseId: "p", vendor: "Summit Plumbing", description: "x" }, - new Map(), - ), + await applyQboExpenseCostCodeSuggestion(known.client, { qbPurchaseId: "p" }, new Map()), "unknown-code", ); - assert.equal(fake.calls.length, 0); + assert.equal(known.calls.length, 0); }); test("a failing suggester never fails the import it rides alongside", async () => { @@ -1788,3 +1801,57 @@ test("a permanently contended row is LEFT ALONE, never unconditionally written", assert.equal(updateCalls, 0, "no unconditional write anywhere on this path"); assert.deepEqual(fake.rows.get("purchase-1"), before); }); + +// ── a REJECTED payload must never feed a suggestion (round 12, item 1) ───── + +test("an out-of-order QBO payload cannot code the row from text that was refused", async () => { + // The webhook arrives late and carries an OLDER SyncToken, so the upsert + // correctly refuses it. Previously the sync then handed THAT payload's + // vendor to the suggester, and the row was coded from a version of the + // purchase the database had just thrown away. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + qbSyncToken: "5", vendor: "General Hardware", + description: "[QuickBooks import] misc supplies", + } as any, + ]); + + const stale = await upsertQboExpense(fake.client, { + ...WRITE, qbSyncToken: "2", vendor: "Summit Plumbing", + description: "[QuickBooks import] rough-in", + }); + assert.equal(stale, "unchanged", "the older token is refused"); + assert.equal(fake.rows.get("purchase-1")?.vendor, "General Hardware", "and its text never lands"); + + // The suggester reads what is STORED, so the refused "Summit Plumbing" + // cannot reach it. + const suggestion = fakeSuggestionClient({ + ...STORED_DEFAULT!, + vendor: "General Hardware", + description: "[QuickBooks import] misc supplies", + }); + assert.equal( + await applyQboExpenseCostCodeSuggestion(suggestion.client, { qbPurchaseId: "purchase-1" }, COST_CODE_IDS), + "no-match", + "the stored text matches nothing, so nothing is coded", + ); +}); + +test("the suggestion write is fenced on the row version AND the sync token", async () => { + const fake = fakeSuggestionClient(); + await applyQboExpenseCostCodeSuggestion(fake.client, { qbPurchaseId: "purchase-1" }, COST_CODE_IDS); + const where = fake.calls[0].where; + assert.equal(where.qbSyncToken, "3", "a newer sync committing in the gap must lose"); + assert.deepEqual(where.updatedAt, new Date("2026-09-01T00:00:00.000Z")); + assert.equal(where.costCodeId, null); +}); + +test("a newer sync landing between the read and the write wins", async () => { + const fake = fakeSuggestionClient(); + fake.setCount(0); // the CAS matches nothing — the token moved + assert.equal( + await applyQboExpenseCostCodeSuggestion(fake.client, { qbPurchaseId: "purchase-1" }, COST_CODE_IDS), + "not-written", + ); +}); diff --git a/tests/scripts-runtime-smoke.test.ts b/tests/scripts-runtime-smoke.test.ts index 87f467f99..80f7eca21 100644 --- a/tests/scripts-runtime-smoke.test.ts +++ b/tests/scripts-runtime-smoke.test.ts @@ -14,6 +14,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import path from "node:path"; const ROOT = path.join(__dirname, ".."); @@ -50,3 +51,36 @@ for (const script of SCRIPTS) { ); }); } + +// ── the report script must stay READ-ONLY (round 12, item 4) ─────────────── + +test("suggest-expense-cost-codes issues no writes and offers no --apply", () => { + // It used to have its own `--apply`, which made it a SECOND writer of + // `costCodeId` — one that knew nothing about the per-expense lock, the + // row-version CAS, the re-plan under that lock, or the project-scoped phase + // check the real backfill applies. Two ways to code an expense, one of them + // unaware of every guarantee the other was given. + const source = readFileSync(path.join(ROOT, "scripts/suggest-expense-cost-codes.ts"), "utf8"); + const code = source + .split("\n") + .filter(line => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join("\n"); + + for (const write of [".update(", ".updateMany(", ".create(", ".delete(", ".deleteMany("]) { + assert.ok(!code.includes(`prisma.expense${write}`), `write survived: ${write}`); + } + assert.ok( + !/process\.argv\.includes\("--apply"\)/.test(code), + "the --apply flag must be gone, not merely undocumented", + ); +}); + +test("the report scopes by the canonical overhead id, not the name \"Shop\"", () => { + const source = readFileSync(path.join(ROOT, "scripts/suggest-expense-cost-codes.ts"), "utf8"); + assert.ok(source.includes("OVERHEAD_PROJECT_ID"), "excluded by id"); + assert.ok(!/notIn: OVERHEAD_PROJECTS/.test(source), "the name-based scope is gone"); + // ...and it reads attribution the one way, so a re-attributed expense is + // not invisible to the report that is supposed to find it. + assert.ok(source.includes("resolveExpenseProjectId")); + assert.ok(source.includes("csvCell"), "OCR'd vendor text is formula-neutralized"); +}); From cebe23d254aa5ba822c49870b961750d3074324d Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:22:52 -0700 Subject: [PATCH 092/144] test(receipts): follow authorizeLateFields to the Denial shape Phase 1's helper now returns `{status, body}` rather than a NextResponse, so the assertions read `.body` instead of awaiting `.json()`. Behaviour asserted is unchanged: the late project is the one authorized, and a late phase is checked against the EFFECTIVE project. Co-Authored-By: Claude Fable 5.1 --- tests/finalize-late-field-authz.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/finalize-late-field-authz.test.ts b/tests/finalize-late-field-authz.test.ts index a678d1fe7..60a6252d5 100644 --- a/tests/finalize-late-field-authz.test.ts +++ b/tests/finalize-late-field-authz.test.ts @@ -23,7 +23,7 @@ let authorizeFinalization: ( auth: unknown, rowProjectId: string | null, lateFields: Record, -) => Promise; +) => Promise<{ status: number; body: Record } | null>; before(async () => { const originalRequire = Module.prototype.require; @@ -74,7 +74,7 @@ test("a session caller cannot file a receipt against a job they cannot see", asy const denied = await authorizeFinalization(session, null, { projectId: "job-theirs" }); assert.ok(denied, "it must be refused"); assert.equal(denied!.status, 403); - assert.equal((await denied!.json()).reason, "forbidden"); + assert.equal(denied!.body.reason, "forbidden"); }); test("a session caller CAN file against a job they can see", async () => { @@ -95,7 +95,7 @@ test("a late phase must belong to the EFFECTIVE project", async () => { const denied = await authorizeFinalization(session, "job-mine", { costCodeId: "cc-frame" }); assert.ok(denied); assert.equal(denied!.status, 400); - assert.equal((await denied!.json()).error, "cost-code-not-a-phase"); + assert.equal(denied!.body.error, "cost-code-not-a-phase"); }); test("the effective project is the LATE one when both are supplied", async () => { @@ -116,7 +116,7 @@ test("a phase with no job to check it against is refused", async () => { const denied = await authorizeFinalization(session, null, { costCodeId: "cc-plumb" }); assert.ok(denied); assert.equal(denied!.status, 400); - assert.equal((await denied!.json()).error, "cost-code-without-project"); + assert.equal(denied!.body.error, "cost-code-without-project"); }); test("a secret forwarder skips the per-user check but NOT the phase check", async () => { From bef26f48f09d7a9d5de7fd5a1c131e47796bba55 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:25:43 -0700 Subject: [PATCH 093/144] fix(expenses): any amount change re-opens a tax classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A re-sync only asked for review when the new gross BROKE an invariant (tax above the amount, or an allocation that no longer fits). An ordinary change breaks nothing and is just as capable of invalidating a human's answer: a $412.10 receipt re-syncing as $498.30 leaves $34.06 of recorded tax describing a purchase that no longer exists, and an installed-at-customer "yes" describing a different basket of goods. So any cent-level movement in the gross on a CLASSIFIED row (a tax amount, an installed-at-customer answer, or a hand allocation) now sets needsTaxReview. The classification itself is kept — it may still be right — and the report already excludes flagged rows, so the filing waits for a person rather than claiming a figure nobody re-checked. Unclassified rows are untouched, or every re-synced purchase would bury the ones that matter. Co-Authored-By: Claude Fable 5.1 --- src/lib/qbo-expense-sync.ts | 37 +++++++++++++++- tests/qbo-expense-sync.test.ts | 81 +++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index a2224db49..00abada9d 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -663,7 +663,12 @@ export interface QboExpenseUpdatePlan { */ export function planQboExpenseUpdate( existing: Pick & - Partial>, + Partial< + Pick< + ExistingQboExpense, + "amount" | "taxAmount" | "taxDeductibleBase" | "installedAtCustomer" + > + >, write: QboExpenseWrite, ): QboExpenseUpdatePlan { const existingProjectId = existing.projectId ?? null; @@ -729,6 +734,31 @@ export function planQboExpenseUpdate( } } + // ANY MOVEMENT IN THE GROSS RE-OPENS A TAX CLASSIFICATION. + // + // The two branches above only catch the amounts that break an invariant. + // An ORDINARY change does not, and it is just as capable of making a + // human's answer wrong: QuickBooks re-syncing a $412.10 receipt as $498.30 + // (a line added, a return applied, a corrected entry) leaves the recorded + // $34.06 of tax describing a purchase that no longer exists, and an + // `installedAtCustomer` yes describing a different basket of goods. The + // numbers still satisfy every CHECK, so nothing else would ever ask. + // + // "Classified" means a human's tax answer is on the row in any form — + // a tax amount, an installed-at-customer decision, or a hand allocation. + // For those rows an amount change is a REVIEW, never a silent acceptance: + // the classification is kept (it may well still be right) and the row is + // flagged, which the report reads as "not until a person looks". + const classified = + existingTax !== null || existingBase !== null || existing.installedAtCustomer != null; + const existingAmount = + existing.amount === null || existing.amount === undefined ? null : Number(existing.amount); + const amountMoved = + existingAmount !== null && + write.amount !== undefined && + Math.round(existingAmount * 100) !== Math.round(write.amount * 100); + if (classified && amountMoved) data.needsTaxReview = true; + if (existingProjectId !== null) return { fill: null, data }; const wantsProjectId = incomingProjectId !== null; @@ -818,6 +848,10 @@ export async function upsertQboExpense( updatedAt: true, taxAmount: true, taxDeductibleBase: true, + // Read for the classification test in planQboExpenseUpdate: a + // human's installed-at-customer answer counts as a tax + // classification even when no tax amount was recorded. + installedAtCustomer: true, amount: true, vendor: true, date: true, @@ -876,6 +910,7 @@ export async function upsertQboExpense( select: { id: true, qbSyncToken: true, estimateId: true, projectId: true, updatedAt: true, taxAmount: true, taxDeductibleBase: true, amount: true, + installedAtCustomer: true, vendor: true, date: true, description: true, status: true, }, }); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index a6d0a6c02..ffee9a15e 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1597,8 +1597,12 @@ test("a gross below the recorded tax CLEARS the classification and flags review" }); test("a gross that still covers the tax leaves the classification alone", () => { + // Same gross as before, so nothing about the receipt moved. const plan = planQboExpenseUpdate( - { projectId: "project-1", estimateId: "estimate-1", taxAmount: 10, taxDeductibleBase: 50 }, + { + projectId: "project-1", estimateId: "estimate-1", + amount: 100, taxAmount: 10, taxDeductibleBase: 50, + }, { ...WRITE, amount: 100 }, ); assert.ok(!("taxAmount" in plan.data)); @@ -1606,6 +1610,75 @@ test("a gross that still covers the tax leaves the classification alone", () => assert.ok(!("installedAtCustomer" in plan.data)); }); +// ── ANY amount change re-opens a classification (Codex round 13, item 1) ─── + +test("an ORDINARY increase on a classified row asks for review", () => { + // Nothing here breaks an invariant: $498.30 still covers $34.06 of tax and + // a $380 allocation. But the human classified a $412.10 receipt, and this + // is no longer that receipt. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: 412.1, taxAmount: 34.06, taxDeductibleBase: 380, + }, + { ...WRITE, amount: 498.3 }, + ); + assert.equal(plan.data.needsTaxReview, true); + // The classification is KEPT — it may still be right, and throwing away a + // human's numbers is not this function's call. It is only re-opened. + assert.ok(!("taxAmount" in plan.data), "not cleared, just flagged"); + assert.ok(!("installedAtCustomer" in plan.data)); + assert.ok(!("taxDeductibleBase" in plan.data)); +}); + +test("an ORDINARY decrease that still satisfies every check asks for review", () => { + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: 412.1, taxAmount: 34.06, taxDeductibleBase: 100, + }, + { ...WRITE, amount: 300 }, + ); + assert.equal(plan.data.needsTaxReview, true); + assert.ok(!("taxDeductibleBase" in plan.data), "the allocation still fits, so it stands"); +}); + +test("an installed-at-customer answer alone is a classification", () => { + // The row a bookkeeper answered "yes" on but never split: no tax amount, + // no allocation, and it is exactly the row the excise report reads. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: 412.1, installedAtCustomer: true, + }, + { ...WRITE, amount: 498.3 }, + ); + assert.equal(plan.data.needsTaxReview, true); +}); + +test("an UNclassified row is not flagged by an amount change", () => { + // No human answer to invalidate — flagging every re-synced purchase would + // bury the ones that matter. + const plan = planQboExpenseUpdate( + { projectId: "project-1", estimateId: "estimate-1", amount: 412.1 }, + { ...WRITE, amount: 498.3 }, + ); + assert.ok(!("needsTaxReview" in plan.data)); +}); + +test("a classified row whose gross did NOT move is left alone", () => { + // Cent-level equality, not object identity: the same money arriving as a + // Decimal string must not read as a change on every single sync. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: "412.10" as unknown as number, taxAmount: 34.06, + }, + { ...WRITE, amount: 412.1 }, + ); + assert.ok(!("needsTaxReview" in plan.data)); +}); + test("clearing a tax classification is never reported as 'unchanged'", () => { const plan = planQboExpenseUpdate( { projectId: "project-1", estimateId: "estimate-1", taxAmount: 30, taxDeductibleBase: null }, @@ -1710,8 +1783,12 @@ test("a tax PATCH landing mid-sync is NOT clobbered — the sync re-plans", asyn const after = fake.rows.get("purchase-1") as any; assert.equal(after.amount, 300, "the sync's own facts still land"); assert.equal(after.taxAmount, 16.55, "the bookkeeper's correction survives"); - assert.notEqual(after.needsTaxReview, true, "and it is not flagged on a stale premise"); assert.equal(after.installedAtCustomer, true, "nor is their tax answer discarded"); + // The gross DID move ($125.50 -> $300) on a row carrying a human's tax + // answer, so the re-plan flags it for review — a different outcome from the + // stale plan, which would have retired the classification outright. What + // the CAS protects is the correction itself, not the flag. + assert.equal(after.needsTaxReview, true, "re-opened by the real amount change"); }); test("invalidating an ALLOCATION also flags the row — never a silent null", async () => { From 3a830b6e4b8fb3ade0cb16aa8ab46648fe3d9e72 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:27:59 -0700 Subject: [PATCH 094/144] fix(expenses): PUT parses the amount once, and zero is an amount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The route validated `Number(body.amount)` and persisted `parseFloat(body.amount)`. Those disagree: "10junk" validates as NaN, which passes every check that is not a comparison, and then persists as 10 — a $207.74 receipt quietly becoming a $10 one, with the deduction-base ceiling computed from a number nobody ever stored. `body.amount ? ...` also dropped a legitimate 0, so a receipt could not be zeroed. Now: one parse, rejected unless finite and >= 0, and that same value is both what the ceiling check uses and what is written. An absent key still means "leave it alone". Co-Authored-By: Claude Fable 5.1 --- src/app/api/expenses/[id]/route.ts | 29 ++++++++++++++++---- tests/expense-edit-authz.test.ts | 44 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 1e8d5fff0..953c3d215 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -207,10 +207,29 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: // names. A PUT that merely LOWERS the amount can strand an existing // base above the new pre-tax total — the same impossible state reached // through the other door. - const resultingAmount = - body.amount !== undefined && body.amount !== null - ? Number(body.amount) - : Number(expense.amount); + // ONE PARSE, one value, used by BOTH the check below and the write. + // + // This used to validate `Number(body.amount)` and then persist + // `parseFloat(body.amount)`, which are different functions: "10junk" + // validates as NaN (silently passing every check that is not a + // comparison) and PERSISTS as 10. And `body.amount ? ...` dropped a + // legitimate 0 on the floor, so a receipt could never be zeroed. + const hasAmount = Object.prototype.hasOwnProperty.call(body, "amount"); + let nextAmount: number | undefined; + if (hasAmount && body.amount !== null && body.amount !== undefined && body.amount !== "") { + const raw = + typeof body.amount === "number" + ? body.amount + : Number(String(body.amount).trim()); + if (!Number.isFinite(raw) || raw < 0) { + return NextResponse.json( + { error: "Amount must be a number of dollars, zero or more.", field: "amount" }, + { status: 400 }, + ); + } + nextAmount = raw; + } + const resultingAmount = nextAmount ?? Number(expense.amount); const existingBase = expense.taxDeductibleBase === null ? null : Number(expense.taxDeductibleBase); if (existingBase !== null) { @@ -236,7 +255,7 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: const updatedExpense = await prisma.expense.update({ where: { id }, data: { - amount: body.amount ? parseFloat(body.amount) : undefined, + amount: nextAmount, vendor: has("vendor") ? (body.vendor || null) : undefined, // Same company-calendar-day rule as the POST — see there. date: has("date") ? (body.date ? await expenseDate(body.date) : null) : undefined, diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index caeb8a20f..df57f7a35 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -527,3 +527,47 @@ test("the tax PATCH writes under the shared per-expense lock", async () => { fakePrisma.$queryRawUnsafe = originalLock; } }); + +// ── PUT amount: one parse, one value (Codex round 13, item 8) ────────────── + +test("a junk amount is REFUSED, not silently truncated", async () => { + // "10junk" used to validate as NaN (passing every check that is not a + // comparison) and then persist as 10 via parseFloat — a $207.74 receipt + // quietly becoming a $10 one. + const res = await call({ amount: "10junk" }); + assert.equal(res.status, 400); + assert.equal((await res.json()).field, "amount"); + assert.equal(updateArgs, null, "nothing is written"); +}); + +test("a negative amount is refused", async () => { + const res = await call({ amount: -5 }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); + +test("ZERO is a real amount, not an omission", async () => { + // `body.amount ? ...` dropped it, so a receipt could never be zeroed. + const res = await call({ amount: 0 }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.amount, 0); +}); + +test("the value validated is the value persisted", async () => { + storedExpense = { ...(storedExpense as object), taxDeductibleBase: 100 } as Record; + // 100 of base + 16.55 of tax needs at least 116.55 of gross. 116.55 passes... + const ok = await call({ amount: "116.55" }); + assert.equal(ok.status, 200); + assert.equal(updateArgs?.data.amount, 116.55, "the parsed number, not a re-parse"); + // ...and a cent less is refused by the same number that would be written. + updateArgs = null; + const denied = await call({ amount: "116.54" }); + assert.equal(denied.status, 400); + assert.equal(updateArgs, null); +}); + +test("an omitted amount leaves the stored one alone", async () => { + const res = await call({ vendor: "Lowe's" }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.amount, undefined); +}); From db56fa53dea1c951503df94e9ba7dc3f34637991 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:37:39 -0700 Subject: [PATCH 095/144] feat(expenses): taxSource provenance, lock-then-read fill, one-transaction DDL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three round-13 items on the booking and rollout paths. taxSource ("ocr" | "manual"): a bookkeeper who decides a receipt has NO sales tax leaves a null taxAmount, which is indistinguishable from "nobody has looked" — so the next booking wrote an OCR figure straight over their answer. Booking stamps "ocr", the tax PATCH stamps "manual", and both fills now refuse to touch a manual row (with the explicit NULL branch SQL requires, or every legacy row would be excluded instead). PUT refuses the column by name. The existing-Expense fill takes the per-expense lock BEFORE the read it decides from, and re-reads the attribution after its guarded writes: an Expense that was re-pointed at another job while the fill ran now throws, which rolls the fills back with it, and the row parks as attribution-conflict rather than being marked BOOKED against somebody else's job. The apply script adds updatedAt WITH its default in one statement (a bare column left a window in which the OLD build's inserts landed NULL after the backfill had already passed, so SET NOT NULL could lose that race), keeps the old-shape repair as no-ops, and runs the whole DDL in a single transaction. Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 8 +- prisma/schema.prisma | 6 + scripts/apply-expense-attribution.mjs | 64 +++++--- src/app/api/expenses/[id]/route.ts | 16 +- src/lib/receipt-intake/book.ts | 121 ++++++++++++--- tests/apply-expense-attribution.test.ts | 68 +++++--- tests/expense-edit-authz.test.ts | 13 +- tests/receipt-intake-book.test.ts | 146 +++++++++++++++++- 8 files changed, 371 insertions(+), 71 deletions(-) diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 0d01163c2..a72336877 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -27,6 +27,10 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "costCodeConfidence" DECIMAL(65,3 ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30); -- Set when a re-sync invalidated a human tax classification (see the sync). ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false; +-- WHO decided the tax fields: "ocr" or "manual". A manual decision includes +-- "there is no tax here", which is a NULL taxAmount and so cannot be told from +-- "nobody has looked" without this column. Booking never overwrites "manual". +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxSource" TEXT; -- A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). -- @@ -43,9 +47,9 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL -- -- Every statement is independently re-runnable: IF NOT EXISTS, a -- predicate-bound UPDATE, and two ALTERs that are no-ops once applied. -ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3); -UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL; +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3) DEFAULT now(); ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET DEFAULT now(); +UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", now()) WHERE "updatedAt" IS NULL; ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL; CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c887ebbd6..1acf48d78 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -657,6 +657,12 @@ model Expense { /// total", which is only reached once installedAtCustomer is an explicit /// true. Set by a bookkeeper on the expense edit route. taxDeductibleBase Decimal? + /// Who decided the tax fields on this row: "ocr" (read off the receipt by + /// the intake pipeline) or "manual" (a bookkeeper, through PATCH on the + /// expense route). Booking never overwrites a "manual" decision - including + /// the decision that there is NO tax, which is a null taxAmount and so + /// cannot be told from "nobody has looked" without this column. + taxSource String? /// A re-sync changed the gross out from under a tax classification a human /// had made, so the tax fields were cleared and this row needs a person to /// look at it again. Additive and default-false; the tax report ignores it diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 596ed2b22..8f9f1c773 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -87,20 +87,33 @@ export const statements = [ `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxDeductibleBase" DECIMAL(65,30)`, // Set when a re-sync invalidated a human tax classification. `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL DEFAULT false`, + // WHO decided the tax fields: "ocr" or "manual". A manual decision + // includes "there is no tax here", which is a NULL taxAmount and so cannot + // be told from "nobody has looked" without this column. Booking never + // overwrites "manual". + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxSource" TEXT`, - // A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). + // A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2; + // ordering fixed in round 13, item 6). // - // Nullable, backfill, DEFAULT, NOT NULL — and the DEFAULT is what makes the - // pre-deploy window survivable. This script runs BEFORE the build that - // knows about the column, so the OLD app is still inserting Expenses - // without it; NOT NULL with no default would fail every receipt, manual - // entry and QBO-sync insert until the deploy landed. Prisma declares the - // same default, so the migration check still sees them agree. + // THE DEFAULT ARRIVES WITH THE COLUMN, IN THE SAME STATEMENT. // - // Each step is independently re-runnable. - `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3)`, - `UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", CURRENT_TIMESTAMP) WHERE "updatedAt" IS NULL`, + // The previous order added the column bare, backfilled, and only THEN set + // the default. That left a window in which the column existed, was + // nullable, and had no default — and this script runs against production + // BEFORE the build that knows about the column, so the OLD app is inserting + // Expenses through that window with no value for it. Every such row lands + // NULL after the backfill has already passed, and `SET NOT NULL` then + // aborts. Adding the column WITH the default means no insert can ever + // produce a NULL, so the last step cannot lose that race. + // + // The three statements after it are REPAIR for a database left in the old + // half-applied shape (column present, no default, NULLs from that window), + // and no-ops on a clean run. The whole array runs in one transaction, so a + // failure anywhere leaves the table exactly as it was. + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP(3) DEFAULT now()`, `ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET DEFAULT now()`, + `UPDATE "Expense" SET "updatedAt" = COALESCE("createdAt", now()) WHERE "updatedAt" IS NULL`, `ALTER TABLE "Expense" ALTER COLUMN "updatedAt" SET NOT NULL`, `CREATE INDEX IF NOT EXISTS "Expense_projectId_idx" ON "Expense"("projectId")`, @@ -202,7 +215,7 @@ export const expectedColumns = { Expense: [ "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", "costCodeSource", "costCodeConfidence", "taxDeductibleBase", "needsTaxReview", - "updatedAt", + "taxSource", "updatedAt", ], }; @@ -286,14 +299,27 @@ async function main() { const companyTimeZone = settings?.timeZone || DEFAULT_COMPANY_TIME_ZONE; console.log(`company time zone for the date re-anchor: ${companyTimeZone}`); - for (const sql of [...statements, reanchorSql(companyTimeZone)]) { - const label = sql.replace(/\s+/g, " ").slice(0, 84); - process.stdout.write(` ${label} ... `); - const affected = await prisma.$executeRawUnsafe(sql); - // Print the row count for the backfill: a SECOND run reporting 0 is - // the whole idempotency proof, and a silent "ok" would hide it. - console.log(sql.trimStart().startsWith("UPDATE") ? `ok (${affected} rows)` : "ok"); - } + // ONE TRANSACTION FOR THE WHOLE THING (Codex round 13, item 6). + // + // Postgres is transactional for DDL, so there is no reason for this to + // be able to stop half-way: a network blip between the backfill and + // `SET NOT NULL` used to leave production with a column the next run + // had to repair, and one wrong statement used to leave every statement + // before it applied. Either the whole shape lands or none of it does. + // + // The timeout is generous because a backfill over the Expense table on + // a cold connection is not a five-second operation, and the default + // would roll the whole thing back for being slow. + await prisma.$transaction(async tx => { + for (const sql of [...statements, reanchorSql(companyTimeZone)]) { + const label = sql.replace(/\s+/g, " ").slice(0, 84); + process.stdout.write(` ${label} ... `); + const affected = await tx.$executeRawUnsafe(sql); + // Print the row count for the backfill: a SECOND run reporting + // 0 is the whole idempotency proof, and a silent "ok" hides it. + console.log(sql.trimStart().startsWith("UPDATE") ? `ok (${affected} rows)` : "ok"); + } + }, { timeout: 300_000, maxWait: 60_000 }); // Verify shape rather than trusting the run. for (const [table, columns] of Object.entries(expectedColumns)) { diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 953c3d215..32848f4e0 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -189,6 +189,10 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: "needsTaxReview", "installedAtCustomer", "taxDeductibleBase", + // Provenance for the four above. Accepting it here would let a + // caller stamp "manual" on a row nobody answered, which is the one + // value booking treats as untouchable. + "taxSource", ]; for (const field of TAX_FIELDS_OWNED_BY_PATCH) { if (Object.prototype.hasOwnProperty.call(body, field)) { @@ -547,7 +551,17 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // leave a window where the report sees an answered row it still // refuses to count. ...(editsInstalled || editsBase || editsTaxAmount || editsTaxAtSource - ? { needsTaxReview: false } + ? { + needsTaxReview: false, + // WHO decided. Everything the intake pipeline writes is + // "ocr" and re-readable; this is a person, and booking + // must not write over it. It matters most in the case + // that leaves no other trace: a bookkeeper deciding + // there is NO tax on a receipt leaves a null taxAmount, + // which without this column cannot be told from + // "nobody has looked yet". + taxSource: "manual", + } : {}), ...(editsCostCode ? { diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 7567507bd..a713697a9 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -655,19 +655,36 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // A retry after a crash between the Purchase and this commit finds // its own Expense here (qbPurchaseId is @unique) — create it twice // and the insert would fail on that constraint anyway. - const existing = await tx.expense.findUnique({ + // LOCK FIRST, THEN READ (Codex round 13, item 4). + // + // The id is all that is needed to take the lock, and taking it + // before the read that every decision below is made from is the + // difference between "the row cannot move while I decide" and "the + // row could have moved between my read and my lock". The second + // read happens inside the lock, so it sees the winner of any race + // against the tax PATCH or the QBO sync rather than a value from + // before it. + const found = await tx.expense.findUnique({ where: { qbPurchaseId: result.qbPurchaseId }, - select: { - id: true, - projectId: true, - costCodeId: true, - costCodeSource: true, - taxAmount: true, - taxAtSource: true, - installedAtCustomer: true, - estimate: { select: { projectId: true } }, - }, + select: { id: true }, }); + if (found) await lockExpense(tx as any, found.id); + const existing = found + ? await tx.expense.findUnique({ + where: { id: found.id }, + select: { + id: true, + projectId: true, + costCodeId: true, + costCodeSource: true, + taxAmount: true, + taxAtSource: true, + taxSource: true, + installedAtCustomer: true, + estimate: { select: { projectId: true } }, + }, + }) + : null; // AN ALREADY-BOOKED PURCHASE STILL NEEDS ITS PHASE 3 FIELDS. // @@ -683,12 +700,10 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // already answered (true OR false) is a tax answer nobody but a // bookkeeper may change. if (existing) { - // The shared per-expense lock, so this fill is ORDERED against - // the tax PATCH and the QBO sync instead of racing them. The - // guarded predicates below stay: the lock orders the writers - // that take it, the predicate protects against one that does not. - await lockExpense(tx as any, existing.id); - + // The lock taken above orders this fill against the tax PATCH + // and the QBO sync. The guarded predicates below stay anyway: + // the lock orders the writers that take it, the predicate + // protects against one that does not. const existingProjectId = existing.projectId ?? existing.estimate?.projectId ?? null; // ATTRIBUTION CONFLICT. The Purchase is already on a different // job than this intake row claims. Filling fields would be @@ -756,8 +771,22 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro id: existing.id, projectId: expectedProjectId ?? row.projectId, taxAmount: null, + // A NULL taxAmount is NOT proof that nobody has + // decided. A bookkeeper who looked at the receipt + // and concluded there is no tax on it leaves + // exactly that shape, and an OCR re-read would then + // overwrite their answer with a number they had + // already rejected. `taxSource` is what tells the + // two apart. The explicit NULL branch is required: + // SQL `<> 'manual'` is NULL for a NULL column, so a + // bare not-equals would drop every legacy row. + OR: [{ taxSource: null }, { taxSource: { not: "manual" } }], + }, + data: { + taxAmount: taxApplied / 100, + taxAtSource: true, + taxSource: "ocr", }, - data: { taxAmount: taxApplied / 100, taxAtSource: true }, }); } if (row.installedAtCustomer !== null) { @@ -766,10 +795,37 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro id: existing.id, projectId: expectedProjectId ?? row.projectId, installedAtCustomer: null, + // Same reason as the tax fill: a bookkeeper's + // classification is not something a capture may + // revisit, and a NULL column needs the explicit + // branch. + OR: [{ taxSource: null }, { taxSource: { not: "manual" } }], }, data: { installedAtCustomer: row.installedAtCustomer }, }); } + + // VERIFY THE ATTRIBUTION THAT WAS ACTUALLY WRITTEN. + // + // Every predicate above is guarded, so any one of them can + // legitimately match zero rows (a human answered first). What + // must NOT happen is marking this intake row BOOKED against an + // Expense that ended up on a DIFFERENT job than the row claims: + // that is the same money-moved-between-jobs failure the + // pre-fill conflict check exists to prevent, reached instead + // through a re-attribution that commits while this runs. + // + // So the final state is re-read and compared with the intent. A + // mismatch throws, which rolls the fills back with it, and the + // row is parked for a person. + if (row.projectId) { + const after = await tx.expense.findUnique({ + where: { id: existing.id }, + select: { projectId: true, estimate: { select: { projectId: true } } }, + }); + const finalProjectId = after?.projectId ?? after?.estimate?.projectId ?? null; + if (finalProjectId !== row.projectId) throw new AttributionConflictError(); + } } const expense = existing ?? await tx.expense.create({ @@ -807,6 +863,12 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // carries a qbPurchaseId. taxAmount: taxApplied > 0 ? taxApplied / 100 : null, taxAtSource: taxApplied > 0, + // Provenance for the tax columns. "ocr" is a re-readable + // guess; "manual" (written by the tax PATCH) is a person's + // answer, and this pipeline never writes over one. Null + // when there was no tax to record, which leaves a later + // bookkeeper free to answer without arguing with a machine. + taxSource: taxApplied > 0 ? "ocr" : null, installedAtCustomer: row.installedAtCustomer, amount: amountCents / 100, vendor: row.vendor || "Unknown", @@ -918,6 +980,16 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // A lost CAS is not a fault: the successor owns this row and will book // it. Say so rather than spending an attempt on it. if (error instanceof StaleClaimError) return { outcome: "stale" }; + // The Expense is on a different job than this row claims. A send WAS + // attempted, so the strong key stays claimed and the Purchase is not + // re-sent; a person decides which job is right. + if (error instanceof AttributionConflictError) { + return { + outcome: "needs-review", + reason: "attribution-conflict", + releaseStrongKey: false, + }; + } // The Purchase EXISTS at this point. Retrying is correct and safe: the // DocNumber lookup will find it and return alreadyExists:true — and the // key must be RETAINED, which is why this attempt's send flag is passed @@ -943,6 +1015,19 @@ function describe(error: unknown): string { return "UnknownError"; } +/** + * Thrown inside the commit transaction when the Expense ended up attributed to + * a different job than the intake row claims. Throwing rather than returning is + * deliberate: it rolls the guarded fills back with it, so the row is never left + * half-filled against a job it does not belong to. + */ +class AttributionConflictError extends Error { + constructor() { + super("the expense moved to another job while booking"); + this.name = "AttributionConflictError"; + } +} + /** Thrown inside the commit transaction when the claim token no longer matches. */ class StaleClaimError extends Error { constructor() { diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 45240190d..1e2b2f1ef 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -206,39 +206,69 @@ test("the tax-vs-gross CHECK is in both DDL paths and in the verifier", () => { // ── updatedAt must survive the PRE-DEPLOY window (round 10, item 1) ──────── -test("updatedAt is added nullable, backfilled, defaulted, THEN made NOT NULL", () => { - // Order is the whole point. The script runs against production BEFORE the - // build that knows about the column, so the OLD app is still inserting - // Expenses without it — NOT NULL with no default would fail every receipt, - // manual entry and QBO-sync insert until the deploy landed. +test("updatedAt is added WITH its default, then repaired, THEN made NOT NULL", () => { + // Order is the whole point, and round 13 moved it. The script runs against + // production BEFORE the build that knows about the column, so the OLD app + // is still inserting Expenses without it. If the column arrives bare, every + // one of those inserts lands NULL — including ones that land AFTER the + // backfill has run — and `SET NOT NULL` then aborts. Arriving with the + // default means no insert can produce a NULL at all. const sql = (statements as string[]).filter(s => s.includes('"updatedAt"')); - assert.equal(sql.length, 4, "add, backfill, default, not-null"); - assert.match(sql[0], /ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP\(3\)$/); - assert.ok(!/NOT NULL/.test(sql[0]), "it must land NULLABLE or the backfill cannot run"); - assert.match(sql[1], /^UPDATE [\s\S]*SET "updatedAt" = COALESCE\("createdAt"/); - assert.match(sql[2], /SET DEFAULT now\(\)/); + assert.equal(sql.length, 4, "add-with-default, repair default, repair nulls, not-null"); + assert.match(sql[0], /ADD COLUMN IF NOT EXISTS "updatedAt" TIMESTAMP\(3\) DEFAULT now\(\)$/); + assert.ok(!/NOT NULL/.test(sql[0]), "still nullable, so a repair backfill can run"); + // Repair for a database left in the OLD half-applied shape: column present + // with no default, and NULLs from the window that shape allowed. + assert.match(sql[1], /ALTER COLUMN "updatedAt" SET DEFAULT now\(\)/); + assert.match(sql[2], /^UPDATE [\s\S]*SET "updatedAt" = COALESCE\("createdAt"/); assert.match(sql[3], /SET NOT NULL/); - // The default must be in place BEFORE NOT NULL, or an insert racing the two - // statements still fails. - assert.ok( - (statements as string[]).indexOf(sql[2]) < (statements as string[]).indexOf(sql[3]), - "DEFAULT has to precede NOT NULL", - ); + // Both repairs must precede NOT NULL, or the old-shape database still fails. + const idx = (needle: string) => (statements as string[]).indexOf(needle); + assert.ok(idx(sql[1]) < idx(sql[3]) && idx(sql[2]) < idx(sql[3]), "repair before NOT NULL"); }); test("every updatedAt statement is re-runnable, and matches the migration", () => { const sql = (statements as string[]).filter(s => s.includes('"updatedAt"')); - // IF NOT EXISTS; a predicate-bound UPDATE; two idempotent ALTERs. + // IF NOT EXISTS; two idempotent ALTERs; a predicate-bound UPDATE. assert.match(sql[0], /IF NOT EXISTS/); - assert.match(sql[1], /WHERE "updatedAt" IS NULL/); + assert.match(sql[2], /WHERE "updatedAt" IS NULL/); for (const statement of sql) { assert.ok( normalizedMigration.includes(normalize(statement).replace(/;$/, "")), - `migration.sql is missing:\n ${statement}`, + `migration.sql is missing: + ${statement}`, ); } }); +test("the DDL runs inside ONE transaction", () => { + // A blip between the backfill and SET NOT NULL used to leave production + // half-migrated. Postgres is transactional for DDL; there is no reason to + // allow a partial apply. + const script = readFileSync( + path.join(__dirname, "..", "scripts", "apply-expense-attribution.mjs"), + "utf8", + ); + const run = script.slice(script.indexOf("for (const sql of [...statements")); + assert.match(script, /await prisma\.\$transaction\(async tx =>/); + assert.match(run, /await tx\.\$executeRawUnsafe\(sql\)/, "inside the transaction, not outside it"); + assert.ok( + script.indexOf("$transaction(async tx =>") < script.indexOf("for (const sql of [...statements"), + "the loop is INSIDE the transaction", + ); +}); + +test("taxSource is declared everywhere the other tax columns are", () => { + // Codex round 13, item 5. A column that exists only in the migration is a + // P2022 on production; one that exists only in the script is a fresh CI + // database that cannot reproduce prod. + assert.ok((statements as string[]).some(s => /ADD COLUMN IF NOT EXISTS "taxSource" TEXT/.test(s))); + assert.match(migrationSql, /ADD COLUMN IF NOT EXISTS "taxSource" TEXT/); + assert.ok(expectedColumns.Expense.includes("taxSource"), "and it is verified after the run"); + const schema = readFileSync(path.join(__dirname, "..", "prisma", "schema.prisma"), "utf8"); + assert.match(schema, /taxSource\s+String\?/); +}); + test("Prisma declares the same default, so the migration check sees them agree", () => { const schema = readFileSync(path.join(__dirname, "..", "prisma", "schema.prisma"), "utf8"); assert.match(schema, /updatedAt DateTime @default\(now\(\)\) @updatedAt/); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index df57f7a35..ed9fa33be 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -213,9 +213,14 @@ test("PATCH reaches a QBO-managed row — the population the report is made of", test("PATCH touches NOTHING but the three ProBuild-only columns", async () => { await patch({ installedAtCustomer: true }); - // `needsTaxReview` rides along because answering IS clearing the flag — - // still nothing outside the ProBuild-only set. - assert.deepEqual(Object.keys(updateArgs?.data ?? {}), ["installedAtCustomer", "needsTaxReview"]); + // `needsTaxReview` and `taxSource` ride along because answering IS + // clearing the flag and recording who answered — still nothing outside the + // ProBuild-only set. + assert.deepEqual( + Object.keys(updateArgs?.data ?? {}), + ["installedAtCustomer", "needsTaxReview", "taxSource"], + ); + assert.equal(updateArgs?.data.taxSource, "manual", "a person answered, and booking must not undo it"); // A caller sending a QBO-synced field is told, not silently ignored. const res = await patch({ amount: "1.00" }); assert.equal(res.status, 400); @@ -415,7 +420,7 @@ test("PUT rejects EVERY tax-return field, by name", async () => { // believe a deduction was recorded that never was. for (const field of [ "taxAmount", "taxAtSource", "needsTaxReview", - "installedAtCustomer", "taxDeductibleBase", + "installedAtCustomer", "taxDeductibleBase", "taxSource", ]) { const res = await call({ [field]: field === "taxAtSource" ? true : 1 }); assert.equal(res.status, 400, field); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 562cbc142..ccfeb8c22 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -155,13 +155,24 @@ function recorder(overrides: Partial = {}, opts: { estimates?: return { count: 0 }; } } + // OR branches, evaluated with SQL's NULL rules: `NOT IN` and + // `<> x` are both NULL (i.e. NOT a match) for a NULL column, + // which is exactly why every guard here carries an explicit + // `{ column: null }` branch. Modelling that is the only way a + // test can catch a guard that silently drops legacy rows. if (Array.isArray(args.where.OR)) { - const src = cur.costCodeSource ?? null; - const ok = args.where.OR.some((b: any) => - b.costCodeSource === null - ? src === null - : src !== null && !(b.costCodeSource?.notIn ?? []).includes(src)); - if (!ok) return { count: 0 }; + const branchMatches = (branch: any) => + Object.entries(branch).every(([key, want]: [string, any]) => { + const have = cur[key] ?? null; + if (want && typeof want === "object" && "notIn" in want) { + return have !== null && !want.notIn.includes(have); + } + if (want && typeof want === "object" && "not" in want) { + return have !== null && have !== want.not; + } + return (want ?? null) === have; + }); + if (!args.where.OR.some(branchMatches)) return { count: 0 }; } if (state.existingExpense) Object.assign(state.existingExpense, args.data); return { count: 1 }; @@ -579,7 +590,11 @@ test("alreadyExists books identically — the lost-response retry", async () => test("an existing Expense for the same Purchase is reused, never duplicated", async () => { const r = recorder(); - (r.deps.db as any).expense.findUnique = async () => ({ id: "exp-existing" }); + // The row resolves to the job this receipt claims (through its estimate), + // which is what the post-fill attribution check re-reads. + (r.deps.db as any).expense.findUnique = async () => ({ + id: "exp-existing", estimate: { projectId: "proj-1" }, + }); const result = await bookReceipt(row(), r.deps); assert.equal((result as any).expenseId, "exp-existing"); assert.equal(r.expenses.length, 0, "no second Expense row"); @@ -1114,8 +1129,15 @@ test("a PATCH landing between the read and the fill is not overrun", async () => estimate: { projectId: null }, }; const seenByBooking = { ...rec.existingExpense }; + let reads = 0; (rec.deps.db as any).expense.findUnique = async () => { - // Hand booking the PRE-patch snapshot, then let the PATCH land. + reads += 1; + // Read 1 is the id lookup that the lock is taken on. Read 2 is the one + // every decision is made from — hand it the PRE-patch snapshot, and let + // the PATCH land in the same breath. Later reads (the post-fill + // attribution check) see the row as it really is. + if (reads === 1) return { id: "expense-1" }; + if (reads > 2) return rec.existingExpense; rec.existingExpense = { ...rec.existingExpense, costCodeId: "cc-human", @@ -1150,3 +1172,111 @@ test("a Purchase already on ANOTHER job parks instead of booking", async () => { assert.equal(result.releaseStrongKey, false, "the Purchase exists — keep the key"); } }); + +// ── tax provenance and the post-fill attribution check (round 13, 4 and 5) ── + +test("a bookkeeper's NO-TAX decision is not overwritten by an OCR re-read", async () => { + // The case a null taxAmount cannot express on its own: a person looked at + // this receipt, concluded there is no sales tax on it, and left the column + // NULL. Without `taxSource` that is indistinguishable from "nobody has + // looked", and the next booking writes an OCR figure over their answer. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: "manual", + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + const result = await bookReceipt(row(), rec.deps); + assert.equal(result.outcome, "booked"); + assert.equal(rec.existingExpense.taxAmount, null, "their answer stands"); + assert.equal(rec.existingExpense.taxSource, "manual"); + assert.equal( + rec.existingExpense.installedAtCustomer, null, + "and the capture does not answer the excise question for them either", + ); +}); + +test("a legacy row with no provenance IS filled, and stamped ocr", async () => { + // The control for the test above: `taxSource` NULL is "nobody has looked", + // and SQL's `<> 'manual'` would drop exactly these rows without the + // explicit NULL branch in the guard. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + await bookReceipt(row(), rec.deps); + assert.ok(Number(rec.existingExpense.taxAmount) > 0, "the validated tax lands"); + assert.equal(rec.existingExpense.taxSource, "ocr"); + assert.equal(rec.existingExpense.taxAtSource, true); +}); + +test("a newly created Expense records where its tax came from", async () => { + const rec = recorder(); + await bookReceipt(row(), rec.deps); + assert.equal(rec.expenses[0].taxSource, "ocr"); + assert.ok(rec.expenses[0].taxAmount > 0); +}); + +test("no tax means no provenance, so a bookkeeper can still answer", async () => { + const rec = recorder(); + await bookReceipt(row({ taxCents: 0 }), rec.deps); + assert.equal(rec.expenses[0].taxAmount, null); + assert.equal(rec.expenses[0].taxSource, null, "nobody has decided anything yet"); +}); + +test("an Expense re-attributed DURING the fill parks instead of booking", async () => { + // The pre-fill conflict check passed, the guarded fills ran, and a + // re-attribution committed underneath. Marking the intake row BOOKED here + // would tie this receipt to an Expense on somebody else's job. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: null, costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + installedAtCustomer: null, estimate: { projectId: null }, + }; + let reads = 0; + (rec.deps.db as any).expense.findUnique = async () => { + reads += 1; + if (reads === 1) return { id: "expense-1" }; + if (reads === 2) return rec.existingExpense; + // The post-fill re-read: somebody moved it. + return { projectId: "another-job", estimate: { projectId: "another-job" } }; + }; + const result = await bookReceipt(row(), rec.deps); + assert.equal(result.outcome, "needs-review"); + if (result.outcome === "needs-review") { + assert.equal(result.reason, "attribution-conflict"); + assert.equal(result.releaseStrongKey, false, "the Purchase exists — keep the key"); + } + // Nothing was marked BOOKED: the throw rolled the whole transaction back. + assert.equal( + rec.intakeUpdates.filter((u: any) => u.state === "BOOKED").length, 0, + "no booking on a row that ended up on another job", + ); +}); + +test("the per-expense lock is taken BEFORE the read the fill decides from", async () => { + // Reading first and locking second leaves the decision resting on a value + // from before the lock, which is the race the lock exists to close. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + const trace: string[] = []; + const realFind = (rec.deps.db as any).expense.findUnique; + (rec.deps.db as any).expense.findUnique = async (args: any) => { + trace.push("read"); + return realFind(args); + }; + (rec.deps.db as any).$queryRawUnsafe = async () => { + trace.push("lock"); + return [{ lock_result: null }]; + }; + await bookReceipt(row(), rec.deps); + // read (the id lookup) -> lock -> read (everything the fill decides from) + assert.deepEqual(trace.slice(0, 3), ["read", "lock", "read"]); +}); From 460ca4b4a36ab2e81e9336d359c95b3cd77109cd Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 05:41:22 -0700 Subject: [PATCH 096/144] fix(backfill): re-read the facts each write rests on, at write time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two stale-input paths the guards did not cover. The project fill pinned `estimateId`, which proves the row never left its estimate and says nothing about where that estimate now lives: an estimate moved to another job between the plan and the write stamped every one of its expenses with the OLD project. The predicate now joins the estimate in the same statement, so the derivation has to still hold at write time. The cost-code fill re-planned under the lock, but against the minutes-old snapshot of item links and job phases — so an item re-coded, or a phase removed from the job, after the snapshot was invisible, and the pass wrote a code that was only correct in the past. It now re-reads that one item and that one job's phase list inside the same transaction, applying the same eligibility rules the snapshot query does. Tests cover all three: estimate moved, item re-coded, phase removed. Each was verified to fail with the corresponding guard removed. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.ts | 73 +++++++++++++- tests/backfill-expense-attribution.test.ts | 106 ++++++++++++++++++++- 2 files changed, 174 insertions(+), 5 deletions(-) diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 68622e6bc..8a03b3bb3 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -459,6 +459,54 @@ export async function runBackfill({ allowedCodesByProject.get(projectId).add(row.costCodeId); } + // ROW-SCOPED RE-READS, for the write phase only. + // + // The two maps above are a SNAPSHOT taken before a loop that can run for + // minutes. Re-planning under the lock (below) proved the row itself had not + // moved, but it was still re-planned against those stale maps — so a phase + // deleted from the job, an estimate moved to another project, or an item + // re-coded after the snapshot would all be invisible, and the pass would + // write a code that was correct only in the past. + // + // These read the same things the snapshot did, `PHASE_ELIGIBLE_ESTIMATE_WHERE` + // and all, but for ONE project and ONE item, inside the transaction that is + // about to write. + const readAllowedCodes = async (tx, projectId) => { + const fresh = new Map(); + if (!projectId) return fresh; + const rows = await tx.estimateItem.findMany({ + where: { + costCodeId: { not: null }, + costCode: { isActive: true }, + estimate: { ...PHASE_ELIGIBLE_ESTIMATE_WHERE, projectId }, + }, + select: { costCodeId: true }, + }); + fresh.set(projectId, new Set(rows.map(r => r.costCodeId).filter(Boolean))); + return fresh; + }; + const readItem = async (tx, itemId) => { + const fresh = new Map(); + if (!itemId) return fresh; + const row = await tx.estimateItem.findUnique({ + where: { id: itemId }, + select: { + id: true, costCodeId: true, estimateId: true, + costCode: { select: { isActive: true } }, + estimate: { select: { projectId: true } }, + }, + }); + // An item whose code was cleared, or whose code was deactivated, is no + // longer a source of one — the same rule the snapshot query applies. + if (!row?.costCodeId || row.costCode?.isActive === false) return fresh; + fresh.set(row.id, { + costCodeId: row.costCodeId, + estimateId: row.estimateId, + projectId: row.estimate?.projectId ?? null, + }); + return fresh; + }; + const plan = planBackfill({ expenses, items, costCodeIdByCode, scopedProjectIds, allowedCodesByProject, }); @@ -585,7 +633,18 @@ export async function runBackfill({ // with the old estimate's project — a silent cross-job attribution // performed by the very pass that exists to get attribution right. const result = await db.expense.updateMany({ - where: { id: { in: ids }, projectId: null, estimateId }, + where: { + id: { in: ids }, projectId: null, estimateId, + // AND THE ESTIMATE STILL POINTS THERE. `estimateId` proves the + // row is on the same estimate; it says nothing about where that + // estimate now lives. An estimate moved to another project + // between the plan and this write would otherwise stamp every + // one of its expenses with the OLD project — a cross-job + // attribution performed by the pass whose whole job is getting + // attribution right. A relation filter makes it one statement, + // so there is no window between checking and writing. + estimate: { is: { projectId } }, + }, data: { projectId }, }); projectIdsWritten += result.count; @@ -644,12 +703,20 @@ export async function runBackfill({ // // `planBackfill` is the single copy of the rules, so it is run // again over this one row rather than re-implemented here. + // The item link and the phase list, RE-READ for this row inside + // the lock — not the minutes-old snapshot. See readItem / + // readAllowedCodes above. + const resolvedProjectId = current.projectId ?? current.estimate?.projectId ?? null; + const [freshItems, freshAllowed] = await Promise.all([ + readItem(tx, current.itemId), + readAllowedCodes(tx, resolvedProjectId), + ]); const replanned = planBackfill({ expenses: [current], - items, + items: freshItems, costCodeIdByCode, scopedProjectIds, - allowedCodesByProject, + allowedCodesByProject: freshAllowed, }); const fresh = replanned.codeFills[0]; // No longer codeable at all, or codeable as something else — either diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index e9b047706..78634438f 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -96,6 +96,13 @@ function createStub( if ("projectId" in where && (row.projectId ?? null) !== where.projectId) return false; if ("costCodeId" in where && (row.costCodeId ?? null) !== where.costCodeId) return false; if ("estimateId" in where && row.estimateId !== where.estimateId) return false; + // The write-time JOIN. `estimateId` proves the row is on the same + // estimate; this proves the estimate still points at the project the + // plan read off it. + if (where.estimate) { + const want = (where.estimate as any).is?.projectId ?? null; + if (((row as any).estimate?.projectId ?? null) !== want) return false; + } // The row-version CAS. Dates compare by value, not identity. if ("updatedAt" in where) { const want = where.updatedAt as Date | null; @@ -119,6 +126,9 @@ function createStub( return { writes, rows, + // Exposed so a test can change the item universe AFTER the plan has + // read it — the whole point of re-reading under the lock. + items, db: { project: { async findMany() { @@ -135,7 +145,18 @@ function createStub( }, }, estimateItem: { - async findMany() { return items; }, + // The write phase re-reads the phase list for ONE project, + // inside the lock, so the stub has to honour that filter — and + // serve whatever `items` says NOW, not a snapshot. + async findMany(args?: { where?: Record }) { + const wantProject = args?.where?.estimate?.projectId ?? null; + if (!wantProject) return items; + return items.filter(item => (item.estimate?.projectId ?? null) === wantProject); + }, + async findUnique(args: { where: { id: string } }) { + const item = items.find(candidate => candidate.id === args.where.id); + return item ? { costCode: { isActive: true }, ...item } : null; + }, }, timeEntry: { async findMany() { return []; }, @@ -457,8 +478,10 @@ test("apply writes both passes, each behind its own predicate", async () => { assert.deepEqual(projectWrite.where, { id: { in: ["e1"] }, projectId: null, - // Both halves of the derivation. + // Both halves of the derivation, plus the join that proves the second + // half is still true at write time. estimateId: "est-job-1", + estimate: { is: { projectId: "job-1" } }, }); const codeWrite = stub.writes.find(w => "costCodeId" in w.data)!; @@ -918,3 +941,82 @@ test("an untouched row still codes normally through the re-plan", async () => { assert.equal(stub.rows[0].costCodeId, "cc-plumb"); assert.equal(stub.rows[0].costCodeSource, "ai"); }); + +// ── the write re-reads what the plan assumed (Codex round 13, item 7) ─────── + +test("an ESTIMATE moved to another job after the plan is not stamped with the old one", async () => { + // `estimateId` alone cannot catch this: the row never left its estimate, + // the estimate left the job. Without the write-time join every expense on + // that estimate would be attributed to the project it used to be on. + const stub = createStub( + [expense({ id: "e1", projectId: null, estimate: { projectId: "job-1" } })], + [], + ); + const snapshot = stub.db.expense.findMany; + stub.db.expense.findMany = async () => { + const rows = await snapshot(); + // ...and now somebody re-points the estimate. + stub.rows[0].estimate = { projectId: "job-2" }; + return rows; + }; + + const result = await runBackfill({ + db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID, + }); + assert.equal(result.written.projectIds, 0, "nothing written on a stale derivation"); + assert.equal(stub.rows[0].projectId ?? null, null); +}); + +test("an ITEM re-coded after the plan does not get the planned code written", async () => { + // The plan says cc-frame because that is what the linked item said when it + // was read. Under the lock the item says something else, so the planned + // write is an answer to a question nobody is asking any more. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", itemId: "i1", vendor: "Unknown Vendor" })], + [{ id: "i1", costCodeId: "cc-frame", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + // The plan reads the item universe TWICE (the link map, then the phase + // list). Both must see the old code; the re-code lands after that, before + // the write phase re-reads this one item under the lock. + const readItems = stub.db.estimateItem.findMany; + let itemReads = 0; + stub.db.estimateItem.findMany = async (args?: any) => { + const rows = (await readItems(args)).map((item: any) => ({ ...item })); + itemReads += 1; + if (itemReads === 2) stub.items[0].costCodeId = "cc-plumb"; + return rows; + }; + + const result = await runBackfill({ + db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID, + }); + assert.equal(result.written.costCodes, 0, "the planned code is void"); + assert.equal(stub.rows[0].costCodeId ?? null, null, "and nothing else was guessed in its place"); +}); + +test("a phase REMOVED from the job after the plan blocks the write", async () => { + // The vendor rule still says cc-plumb. The job no longer has that phase, so + // writing it would put money on a phase the job does not have — the exact + // check the plan made, made again against the truth. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing" })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + // Removed AFTER the plan has read the phase list, so the plan genuinely + // decides cc-plumb and only the write-phase re-read can catch it. (Removing + // it earlier would make this pass for the wrong reason: nothing planned.) + const readItems = stub.db.estimateItem.findMany; + let itemReads = 0; + stub.db.estimateItem.findMany = async (args?: any) => { + const rows = (await readItems(args)).map((item: any) => ({ ...item })); + itemReads += 1; + if (itemReads === 2) stub.items.length = 0; + return rows; + }; + + const result = await runBackfill({ + db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID, + }); + assert.equal(result.written.costCodes, 0); + assert.equal(stub.rows[0].costCodeId ?? null, null); +}); From bc6fca3105ebbc65c72a8784dec3171af3b15d75 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:11:49 -0700 Subject: [PATCH 097/144] fix(expenses): a review flag clears only on an explicit acknowledgement needsTaxReview means a re-sync moved the gross out from under a whole tax classification, so any tax edit clearing it let a bookkeeper answering "yes, installed at customer" silently certify a tax amount and a deduction split they never looked at, and the row went straight back into the excise report. Clearing it now takes `taxReviewAck: true` carrying both taxAmount and taxDeductibleBase (installedAtCustomer stays optional: a null reads as unanswered and cannot overstate a deduction). A partial correction is still accepted and simply leaves the flag up. The modal shows a confirm checkbox on a flagged row and sends both figures with the ack, changed or not. Deletion also retires `taxSource` and counts it in the already-retired check, so a deleted purchase that a person had classified stops reporting a change forever while still claiming their provenance. Spec and PR body corrected: the backfill is `node --import=tsx scripts/backfill-expense-attribution.ts`, and the correction path is PATCH. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-2-QUEUE-AND-MEMOS-SPEC.md | 351 ++++++++++++++++++ docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 23 +- src/app/api/expenses/[id]/route.ts | 56 ++- .../[id]/time-expenses/TaxPhaseModal.tsx | 38 +- src/lib/qbo-expense-sync.ts | 11 +- tests/expense-edit-authz.test.ts | 81 +++- tests/qbo-expense-sync.test.ts | 34 +- 7 files changed, 576 insertions(+), 18 deletions(-) create mode 100644 docs/plans/PHASE-2-QUEUE-AND-MEMOS-SPEC.md diff --git a/docs/plans/PHASE-2-QUEUE-AND-MEMOS-SPEC.md b/docs/plans/PHASE-2-QUEUE-AND-MEMOS-SPEC.md new file mode 100644 index 000000000..16021253a --- /dev/null +++ b/docs/plans/PHASE-2-QUEUE-AND-MEMOS-SPEC.md @@ -0,0 +1,351 @@ +# Phase 2: Receipts tab + missing-receipt requests + Chat cards + auto-close + +Date: 2026-09-01. Parent plan: `docs/plans/RECEIPT-PIPELINE-V2-PLAN.md` (Phase 2 row). +Builds against the Phase 1 spec's `ReceiptIntake` schema (`docs/plans/PHASE-1-INTAKE-CORE-SPEC.md` +§2-3 — Phase 1 is in flight on another branch; import from it, never fork its shapes). +Planner output for the executor: build exactly this; do not guess. + +## Verified code facts (cite these, don't re-derive) + +- `/automation` is ONE server-component page (`src/app/automation/page.tsx`), gated by + `getCurrentUserWithPermissions()` + `hasPermission(user,"financialReports")` (:149-151). + Filters are URL searchParams rendered as `FilterChip` anchors (:64-85, :596-608); data + loaders live in sibling modules (`register-data.ts`), pure filter predicates in their own + testable module (`register-filters.ts`). QBO purchase deep-link precedent: + `https://qbo.intuit.com/app/expense?txnId=` (:688). +- There is NO shared `TabButton`/`EmptyState` export anywhere in `src/` despite + DESIGN_SYSTEM.md naming them — the page's own `FilterChip` + `StatCard` + (`automation/components/shared/stat-card`) are the real conventions. Use those. +- `ReviewIssue` (`prisma/schema.prisma:2624`): `@@unique([targetType,targetKey])`, `version` + OCC, `reasonCodes`/`reasonHash`/`acknowledgedCodes` (JSON strings), `displayDetails` + (never hashed), `clearedAt`, `absentSince`. NO CHECK constraint exists on `targetType` or + `reasonCodes` (`scripts/apply-review-alerts-schema.mjs` CHECKs only the status columns) — + a new targetType and reason code need ZERO DDL. +- Reason codes are a CLOSED set: `decodeReasonCodes` (`src/lib/review-alert-reasons.ts:58`) + runs `parsed.filter(isReasonCode)`, so a code missing from `KNOWN_CODES` decodes to `[]` + — which the lifecycle reads as "cleared". A new code MUST be added to the `ReasonCode` + union and `KNOWN_CODES` or every issue carrying it self-destructs on read. +- Lifecycle: `evaluateReviewIssue(targetType, targetKey, codes, displayDetails, opts)` + (`src/lib/review-alert-lifecycle.ts:231`) — ordered decision tree, OCC-retried; empty + codes → clear (step 1), new target → create gen 1 (2), cleared+non-empty → reopen gen+1 + (3), acked ⊇ current → suppress (4), same hash → touch (5), changed hash → supersede (6). + `options.episodeStatus` accepts `"PENDING" | "SUPPRESSED"` (:201). +- Delivery: `drainReviewAlerts` (`src/lib/review-alert-outbox.ts:341`) sends ONE card PER + EPISODE (per issue), ceiling `EPISODE_RATE_CEILING = 10`/run (:90), and is hard-blocked + until the `RolloutGate` baseline is complete (:354) and a real `ReviewAlertSender` exists + (none does — `unconfiguredSender`, :77). Absence/grace: `ABSENCE_GRACE_MS = 6h`, + coverage-gate floor 5 (`review-alert-evaluator.ts:228,242`) — but `reconcileIssueAbsences` + filters `targetType === "qbo-purchase"` (:284), so it never touches other target types. +- Bank data: `BankLine` (schema:2743, signed `amountCents`, `postedDate`, `rawDescriptor`, + immutable amounts), `BankLineObservation` (schema:2792; `source` STATEMENT|QBO_REGISTER, + QBO rows carry `sourceLineId` = qbTxnId). Canonical `BankLine` rows are minted ONLY from + STATEMENT observations (schema comment :2686-2699); QBO register rows arrive as + observations via `POST /api/integrations/bank-ledger/ingest` (route :504-553). No + bank-register cron exists in `vercel.json` — QBO observations are pushed by the external + `post-qbo-register.mjs` runner today; the nightly server-side pull is Phase 6. +- Pure receipt libs ALREADY EXIST — reuse, do not duplicate: + - `src/lib/receipt-policy.ts`: `classifyReceiptRequirement` (loan/fee/insurance/tax/ + owner-transfer exemptions, verbatim from the prod survey), `resolveReceiptOwner` + + `CARD_OWNERS = {8516:"CJ", 6098:"Richard", 4297:"Justin"}` (:183 — this IS the config + map; reference it, never re-declare the tails in logic), `classifyPersonToPersonPayment`. + - `src/lib/receipt-match.ts`: `extractCardRail`, exact-id BankLine→Expense receipt matcher. + Its house rule ("exact-match joins only, never amount/date/rails heuristics") guards + BankLine STATE advancement — Phase 2's fuzzy match only opens/closes a human chase + request and must never advance `BankLine.state` or write links. + - `src/lib/bank-ledger.ts:27`: `normalizePayee(raw)` (rail markers, card refs, phones, + dates, 6+-digit refs stripped; "" = no identity, never match on it). +- `Expense` (schema:580): `amount Decimal`, `date DateTime?`, `vendor String?`, + `receiptUrl`, `qbPurchaseId @unique`. Parse Decimal → cents from the STRING form + (UNIFIED-REGISTER-PLAN.md §2 cent-exactness rule), never `Number(d)*100`. +- Cron auth: `isCronAuthorized(request)` in `src/lib/cron-auth.ts` (Phase 0, fail-closed, + constant-time). `/api/cron/*` is excluded from the proxy matcher (Phase 1 spec facts). + Non-overlap pattern: `pg_try_advisory_xact_lock` short claim tx (Phase 1 spec §5). +- Chat side (verified in the two source files): + - Space: `spaces/AAQAKhvMYtg` (`qbo-clasp/sweepChatReceipts.js:69`). NOTE: + UNIFIED-REGISTER-PLAN.md §4 writes `AAQAKhvMYTg` (capital T) — that is a typo; the + running sweep's constant is authoritative. + - Bridge files in Drive folder `1WwPPvveXlweQ3LI4J-EhfdD5f7x1zzko` (Claude/Bookeeping/ + reports): `affidavit-threads.json` — Beverly WRITES, sweep READS; shape + `{threads: {"": {owner, owner_user, message_name, items:[{n, fingerprint, + date, vendor, cents, amount}]}}}` (sweepChatReceipts.js:95,108-110). And + `chat-job-answers.json` — sweep/affidavit-app APPEND, Beverly READS; records carry + `{fingerprint, job, by, by_user, signer_email, at, message, thread, text, purpose, + signed, pdf_id, pdf_url}`, deduped on (fingerprint, message) + (beverly-chat-app/chatAffidavitApp.js:601-629). + - Sign flow: interactive `cardsV2` cards with `cardId: "affidavit-"` are + posted by Beverly's bot; clicks are handled by the SEPARATE "Beverly Chat App" Apps + Script (`chatAffidavitApp.js` — signAffidavit/haveReceipt), which generates + `MissingReceiptAffidavit____.pdf` into the Drive + "Missing Receipt Memos" folder (:524-573). (The name "SIGNED-chat.pdf" from earlier + notes appears in NEITHER file — the convention above is what actually runs.) + Only the card's owner or Justin (`users/104715603105220955392`) may sign (:211). + - ProBuild's own Chat app can POST (service-account key env `GOOGLE_CHAT_SA_KEY`, + UNIFIED-REGISTER-PLAN §4) but has NO interactive endpoint (`/api/chat/events` does not + exist) — buttons on ProBuild-posted cards cannot work. Interactive signing stays with + Beverly's app. + +## 1. Goals and acceptance criteria + +1. **Receipts tab live on `/automation`** (`?tab=receipts`), grouped queue over + `ReceiptIntake` + missing-receipt issues, same `financialReports` gate as the register. + Technical: `npm run build` 0 errors; tab loader unit-tested; e2e: ADMIN sees the tab, + a FIELD_CREW session is redirected off `/automation` entirely (existing gate). +2. **Missing-receipt matcher** opens exactly one `ReviewIssue` per unmatched bank debit, + auto-closes on match, reopens on unmatch. Technical: full node:test table (§7) green; + two consecutive runs over identical input produce zero new issues/episodes. +3. **Per-owner weekday Chat card** listing that owner's open items with the three reply + options; sign handoff via the Beverly thread contract (§4). Technical: card builder + unit-tested (grouping, ack suppression, ≤10 cards, deterministic requestId); cron + returns `{skipped:"disabled"}` until `RECEIPT_REQUEST_CARDS_ENABLED=true`. +4. **No emailed PDFs anywhere** — grep gate: the new modules import no mail helper. +5. **VISUAL (gauntlet-verify consumes these verbatim; deployed preview, ADMIN session):** + - `/automation?tab=receipts` renders a "Receipts" tab control next to the existing + register view, with a count badge per group: "Needs job", "Needs review", "Booking", + "Booked today", "Missing receipts", "Duplicates". + - With no rows in a selected group, the group shows a centered muted empty message + ("Nothing here" wording, matching the register's empty-state style), not a blank + panel and not an error boundary. + - A row in "Needs job" shows vendor, date, amount, source, and a "Set job" control; + choosing a project moves the row out of the group without a full-page error. + - A row in "Booking" shows its stateReason/lastError text and a "Retry now" button. + - A row in "Booked today" shows an "Open receipt ↗" link and a "QuickBooks ↗" link + whose href starts with `https://qbo.intuit.com/app/expense?txnId=`. + - The "Missing receipts" group is sub-grouped by owner (e.g. a "CJ" heading and a + "Richard" heading), each row showing posted date, payee, amount, and card tail. + - A "Duplicates" row shows a "duplicate of" reference and a "Not a duplicate" action. + +## 2. Receipts tab + +- **Routing**: extend `parseFilters` in `page.tsx` with `tab: "register" | "receipts"` + (default `"register"`); `filterHref` preserves it. Register JSX untouched when + `tab=register`. The Receipts branch renders `` from + `src/app/automation/components/receipts/receipts-tab.tsx` (server component). +- **Loader** `src/app/automation/receipts-data.ts` (mirrors `register-data.ts` glue style): + - `fetchReceiptQueue(filters)` → one object with the six groups. Reuse the Phase 1 + exported list select/serializer from the intake GET route module (Phase 1 spec §3 + says it is exported for exactly this reuse) — never a second field list. + - Groups (states per Phase 1 §2): Needs job = `NEEDS_JOB`; Needs review = + `NEEDS_REVIEW` + `NON_RECEIPT`; Booking = `BOOKING` (surface `stateReason`, + `lastError`, `attempts`, `nextRetryAt`); Booked today = `BOOKED`/`ARCHIVED` with + `bookedAt` >= today 00:00 America/Los_Angeles (compute the boundary with the + en-CA / America/Los_Angeles idiom already in page.tsx:207); Duplicates = + `DUPLICATE` (+ `duplicateOfId`); Missing receipts = open `ReviewIssue` rows + `{targetType:"bank-line", clearedAt:null}` decoded via `decodeReasonCodes`, grouped by + `displayDetails.owner`, sorted CJ, Richard, office, Justin. + - Caps: `take` 100 per group, newest first; count badges from count queries, not from + capped lists. Whole loader wrapped in the page's degrade-honestly convention (its own + try/catch + "unavailable" card), never taking the register down. +- **Server-side filters**: searchParams `group` (one of the six, default all), + `projectId`, `owner`. Pure parser/predicate in + `src/app/automation/receipts-filters.ts` (unit-testable, mirroring + `register-filters.ts`'s reason for existing). +- **Row actions** — server actions in `src/lib/actions.ts` (repo default; the + mark-reviewed API route pattern stays for the register). Every action: session via + `getCurrentUserWithPermissions()`, `hasPermission(user,"financialReports")`, else throw; + then a guarded compare-and-swap on `state` so a racing intake worker can't be + overwritten (conditional `updateMany({where:{id, state: expectedState}})`; treat count 0 + as a stale-view error surfaced as a toast, never a silent no-op): + - `setReceiptIntakeJob(id, projectId, costCodeId?)` — allowed from + `NEEDS_JOB`/`NEEDS_REVIEW`; also `userCanAccessProject`; writes projectId/costCodeId + and state → `READ` (the worker re-routes/books from there; never jump straight to + `BOOKING` — routing owns dedup). + - `markReceiptIntakeDuplicate(id, duplicateOfId)` → state `DUPLICATE`; + `unmarkReceiptIntakeDuplicate(id)` → back to `READ` (re-runs dedup/route). + - `voidReceiptIntake(id)` → `VOID`, allowed from any non-BOOKED state (a BOOKED row is + money history — refuse). + - `retryReceiptIntake(id)` — from `BOOKING` (or `NEEDS_REVIEW` with a retryable + stateReason): `nextRetryAt = now`, `lastError = null`; the 5-min worker picks it up. + This is the "resend/retry" action; it never calls QBO inline. + - "Open receipt" is a render-time short-lived signed URL from + `resolveDocUrl(storagePath)` (`src/lib/secure-storage.ts`) — no action needed. + - "QuickBooks ↗" uses the page.tsx:688 deep-link pattern with `qbPurchaseId`. + - Missing-receipt rows: "Acknowledge" reuses the existing mark-reviewed contract + (`{id, version, reasonHash}` → `markReviewed`) — do NOT hand-roll ack writes. +- Buttons that appear on hover MUST carry the `[@media(hover:none)]` visibility classes + (CLAUDE.md hover rule). + +## 3. Missing-receipt matcher — `src/lib/receipt-requests.ts` (pure, no I/O) + +- **Reason code**: add `"MISSING_RECEIPT"` to the `ReasonCode` union AND `KNOWN_CODES` in + `src/lib/review-alert-reasons.ts` (closed-set fact above). `deriveReasonCodes` is + qbo-purchase-only — do not touch it. New constant + `RECEIPT_REQUEST_TARGET_TYPE = "bank-line"`; targetKey = `BankLine.id`. +- **Inputs** (plain rows, caller-queried): bank lines `{id, postedDate:"YYYY-MM-DD", + amountCents, rawDescriptor, checkNumber}`; expenses `{amountCents, date, vendor}`; + receipt intakes `{totalCents, txnDate, vendor, state}` (live states only — exclude + DUPLICATE/VOID/NON_RECEIPT); open issues `{targetKey}`; `now`. +- **Candidate rule**: `amountCents < 0` AND `postedDate` at least 3 calendar days before + `now` AND `classifyReceiptRequirement(line).requirement === "receipt_expected"` + (receipt-policy exemptions and money-in drop out — and an exempt or credit line with an + open issue emits a close). +- **Match rule** (suppresses creation AND drives auto-close). A bank line is "satisfied" + when some Expense or live ReceiptIntake has ALL of: + 1. amount exact: `expenseCents === -line.amountCents` (Expense.amount Decimal parsed + from its string form to integer cents; ReceiptIntake.totalCents used directly); + 2. date within ±2 calendar days of `postedDate` (null date = no match); + 3. payee agreement: `payeeMatches(normalizePayee(line.rawDescriptor), vendor)` where + `payeeMatches(a, b)` tokenizes both as + `s.toUpperCase().replace(/[^A-Z0-9 ]/g," ").split(/\s+/)` filtered to tokens of + length >= 3 that are not pure digits; match iff the token lists share >= 1 token, + OR one side's first token is a prefix (>= 4 chars) of the other's first token + (covers "LOWES #02516" vs "Lowe's Home Improvement", "HOMEDEPOT.COM" vs "Home + Depot"). Empty normalized payee or null/empty vendor NEVER matches (bank-ledger's + empty-string-is-not-an-identity rule). Amount+date alone is deliberately + insufficient — the Chevron/Cash App lesson (schema:2698); this fuzzy match only + opens/closes a chase request, never links records or advances `BankLine.state`. +- **Owner**: `resolveReceiptOwner(rawDescriptor)` (receipt-policy.ts:189). The tail→name + map stays that module's single exported `CARD_OWNERS` const — the matcher takes the + resolved owner as data and contains no tail literals. +- **Output**: `{ open: [{targetKey, displayDetails}], close: [targetKey] }` where + `displayDetails = {owner, cardTail, postedDate, amountCents, payee, rawDescriptor, + fingerprint: "pb-" + bankLineId}` (display-only; amounts/dates here never churn the + reason hash since the code set is always exactly `["MISSING_RECEIPT"]`). +- **Persistence (cron, §5)**: each `open` → `evaluateReviewIssue("bank-line", key, + ["MISSING_RECEIPT"], details, { episodeStatus: "SUPPRESSED" })`; each `close` → same + call with `[]`. Idempotency and never-duplicate-an-open-issue come from + `@@unique([targetType,targetKey])` + lifecycle steps 5/1; reopen-after-expense-deleted + is lifecycle step 3 (gen+1). `episodeStatus:"SUPPRESSED"` keeps these issues OUT of the + future per-issue card drain (one card per bank line is exactly what we don't want, and + the qbo-purchase RolloutGate must not acquire a second meaning); delivery is §4's + per-owner digest instead. + +## 4. Chat cards + Beverly sign handoff + +- **Cadence**: one card per owner (CJ, Richard only — `office` and `Justin` items are + page-only per receipt-policy.ts:170-175) per weekday morning, listing their open, + UNACKNOWLEDGED `bank-line` issues numbered 1..n. Reply options in the card text: + "reply here with a photo", "reply 'N '", "reply 'sign N' to sign a memo + instead". Never email a PDF. +- **Reused batching/grace rules**: at most `EPISODE_RATE_CEILING` (10) cards per run + (owners are 2 — assert anyway); acknowledged issues suppressed (lifecycle step 4 + semantics, computed exactly as `reviewIssueByPurchaseId` does — register-data.ts:224); + the 3-day matcher age is the grace window; a cleared issue never appears (loader filters + `clearedAt:null`). Deliberate divergence, called out: cards do NOT ride + `drainReviewAlerts` — that drainer is one-card-per-issue and blocked by the qbo-purchase + RolloutGate; the digest posts directly with Chat's own idempotency: + requestId/messageId = `receipt-req--` (a repeated id returns + the existing message), so a retried cron can never double-post. +- **Posting**: new `src/lib/receipt-request-cards.ts` — pure exported builder + a poster + using the ProBuild Chat app service account (env `GOOGLE_CHAT_SA_KEY`, space env + `GOOGLE_CHAT_REVIEW_SPACE` = `spaces/AAQAKhvMYtg`, the sweep's constant, NOT the plan + doc's `...YTg` typo). Each owner's card starts a NEW thread (threadKey + `receipt-req--`); capture the returned `message.name` + `thread.name` and + store `{threadName, messageName, n}` on each listed issue's `displayDetails`. +- **Bridge contract (what keeps ProBuild and the sweep/Beverly in sync)** — the sweep only + understands threads present in `affidavit-threads.json`, and ProBuild has no Drive + writer, so a qbo-clasp mirror closes the gap (same pattern as Phase 1 §6): + 1. ProBuild exposes `GET /api/automation/receipt-requests/threads` (auth: + `x-receipt-intake-secret`, Phase 1's machine secret; proxy public-bypass exact-match + like `api/office-tasks/ingest`) returning EXACTLY the threads-map shape from + sweepChatReceipts.js:108-110: `{threads: {"": {owner, owner_user, + message_name, items: [{n, fingerprint, date, vendor, cents, amount}]}}}` for cards + posted in the last 14 days. `fingerprint = "pb-" + bankLineId` (stable across + generations; safe inside `cardId` and PDF filenames). `owner_user` = the owner's + Chat `users/` from a new env-backed map `RECEIPT_OWNER_CHAT_USERS` (JSON, e.g. + `{"CJ":"users/...","Richard":"users/..."}`) — config, not code. ASSUMPTION: CJ's and + Richard's user ids must be collected once (Justin's is users/104715603105220955392). + 2. qbo-clasp gains `mirrorReceiptRequestThreads()` (separate Apps Script PR): poll the + endpoint after 7:45 AM, MERGE by thread key into `affidavit-threads.json` (never + clobber Beverly's own entries), write atomically like `chatSweepAppendAnswer_`. + 3. From there the EXISTING machinery runs unchanged: photo replies in the thread are + filed by the sweep (and, under Phase 1 §7, forwarded to intake with `source:"chat"` + + threadName), job-name replies resolve items, and every resolution is appended to + `chat-job-answers.json`. +- **Sign step**: signing stays with Beverly's interactive cards (ProBuild cards cannot + carry working buttons — no `/api/chat/events`). Contract: a "sign N" text reply is + recorded by the sweep into `chat-job-answers.json`; Beverly's runner, which already + reads that file, posts her `cardId: "affidavit-"` card into the SAME + thread for that item; `chatAffidavitApp.js` handles the click, enforces owner-or-Justin, + and writes the signed record + `MissingReceiptAffidavit_*.pdf` to "Missing Receipt + Memos". ASSUMPTION / companion change: Beverly's Python runner (not in any repo here — + it runs on Justin's PC) needs the small "post sign card on request" trigger; flag it in + the PR description as a required companion, never silently assume it. +- **Close the loop**: `POST /api/automation/receipt-requests/answers` (same machine + secret) accepting `{fingerprint, signed?, pdf_url?, job?, at, message, thread}` — a + qbo-clasp forwarder posts each NEW `chat-job-answers.json` record. For + `fingerprint = "pb-"` with `signed:true`: record + `{resolution:"memo-signed", pdfUrl}` into the issue's `displayDetails`, then + `evaluateReviewIssue("bank-line", key, [], details)` to clear it. Photo answers need no + handling here — the resulting Expense/ReceiptIntake closes the issue via the nightly + matcher. Unknown fingerprints (Beverly's own) → `{ok:true, ignored:true}`. + +## 5. Crons — `vercel.json` additions + +- `/api/cron/receipt-requests`, `"0 13 * * *"` (6 AM Pacific, after the overnight QBO + register push lands; NOTE: no in-repo register cron exists yet — ordering vs the + external `post-qbo-register.mjs` run is operational; restate it in the route comment). + Auth `isCronAuthorized` (cron-auth.ts, Phase 0); `maxDuration 60`; + `pg_try_advisory_xact_lock(hashtextextended('receipt-requests', 0))` claim (Phase 1 §5 + pattern — pgbouncer forbids session locks). Loads: BankLine debits from the last 60 + days + their receipt-policy inputs; Expenses and live ReceiptIntakes over the same + window ±2 days; open bank-line issues. Runs the pure matcher; applies opens/closes via + `evaluateReviewIssue`; returns `{opened, closed, touched, skipped}` counts. +- `/api/cron/receipt-request-cards`, `"30 14 * * 1-5"` (7:30 AM Pacific in PDT; drifts to + 6:30 in PST — accepted, same as every other cron here). Gated on + `RECEIPT_REQUEST_CARDS_ENABLED === "true"` (ships unset → `{skipped:"disabled"}`), so + the matcher and page run silently for a shakedown week before any Chat noise. + +## 6. Migration / env + +- **No schema change.** Reason code + `displayDetails` JSON ride existing `ReviewIssue` + columns (no CHECK blocks them — verified above); the queue reads Phase 1's + `ReceiptIntake` as-is. No `scripts/apply-*.mjs`, no `prisma/migrations/` entry. +- New env (Vercel prod): `RECEIPT_OWNER_CHAT_USERS`, `RECEIPT_REQUEST_CARDS_ENABLED` + (unset initially), and `GOOGLE_CHAT_SA_KEY` + `GOOGLE_CHAT_REVIEW_SPACE` if not already + set (Justin handles the service-account key — Claude never touches the credential; key + generation was already flagged as Justin's in UNIFIED-REGISTER-PLAN §4). + +## 7. Tests (node:test, `test/receipt-requests/*.test.mjs`; no `mock.module` — CI is Node 20) + +- `matcher.test.mjs` table: + | case | expect | + |---|---| + | debit, 4 days old, no expense | open MISSING_RECEIPT, owner from tail | + | same, matching expense (exact cents, same date, "LOWES #02516" vs "Lowe's Home Improvement") | no open; close if issue exists | + | expense date +2 / -2 days | match (close) | + | expense date +3 days | no match (open) | + | amount off by 1 cent | open | + | payee tokens disjoint ("CHEVRON" vs "CASH APP KANDI") with equal amount+date | open — amount+date alone never matches | + | credit line (amountCents > 0) | ignored; close emitted if an issue exists | + | debit 2 days old | ignored (grace) | + | loan payment / insurance / DOR descriptor | exempt via receipt-policy; close emitted if an issue exists | + | live ReceiptIntake match (totalCents/txnDate/vendor) | close | + | DUPLICATE / VOID intake rows | never satisfy a line | + | expense with null vendor or null date | never matches | + | expense deleted since last run (was matched, now absent) | line re-opens | +- `idempotency.test.mjs`: matcher + a fake lifecycle run twice on identical input — + second pass yields zero opens/closes (all same-hash touches); the open-issue input list + is respected (no duplicate open for an already-open targetKey). +- `owner.test.mjs`: `C#8516`→CJ, `C# 6098`→Richard, `C#4297`→Justin (excluded from + cards), no tail→office (excluded from cards), double-tail descriptor→no single owner. +- `cards.test.mjs`: builder groups by owner, numbers items, skips acked issues, caps at + 10 cards, requestId deterministic per owner+Pacific-date; threads-endpoint serializer + emits the exact affidavit-threads.json shape (snapshot against a literal copied from + sweepChatReceipts.js:109). +- `receipts-filters.test.mjs`: group/owner/project predicates. +- e2e (CI postgres, teardown per docs/TESTING.md): `/automation?tab=receipts` renders the + six group headings for ADMIN; the threads/answers endpoints 401 with no auth AND with a + bogus session cookie (getclients-auth-gate lesson). + +## 8. Risks / open questions (max 5) + +1. **Data freshness**: matcher truth is `BankLine`, which today fills only from monthly + statement imports; QBO register rows are observations without canonical lines. The + 3-day chase is therefore late until Phase 6's nightly pull (or until unlinked + QBO_REGISTER observations also feed the matcher — deliberately NOT done here, to avoid + dual-identity issues when a statement later mints the canonical line). Confirm Phase 6 + ordering is acceptable, or ask for the observation-feed variant as a follow-up. +2. **Beverly companion change** (§4 sign step) lives outside every repo here (Hermes + runner on Justin's PC). Until it ships, "sign N" replies are recorded but no sign card + appears — photo/job replies work day one. HUMAN DECISION on sequencing. +3. **Chat user ids for CJ/Richard** must be collected once for `RECEIPT_OWNER_CHAT_USERS` + (owner_user gates who may sign in chatAffidavitApp.js:211 — a wrong id locks the owner + out of signing their own memos). +4. **Fuzzy-match false closes**: exact-cents + ±2-day + token-overlap payee can still + close a request against a same-vendor same-amount different-purchase expense. + Accepted: a close only silences a chase; the register/variance edges still surface + unmatched purchases independently. +5. **Two chase surfaces during transition**: Beverly's own missing-receipt asks and + ProBuild's cards could both fire while Phase 2 shakes down. Mitigation: cards ship + behind `RECEIPT_REQUEST_CARDS_ENABLED`; Justin turns Beverly's ask generation off in + the same step he enables the flag (one-line runbook item in the PR). diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index 34b988f3b..0bac8d0c2 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -314,6 +314,17 @@ return. As built: * `/reports/tax-paid-at-source` counts ONLY an explicit `true`; * `Expense.taxDeductibleBase` (added in this PR) holds the resold portion of a MIXED receipt, and the report uses it in place of the pre-tax total when it is set; +* **`Expense.taxSource`** records WHO decided the tax columns: `ocr` (the intake pipeline + read it off the receipt) or `manual` (a bookkeeper, through the PATCH). Booking never + overwrites a `manual` decision — including the decision that a receipt has NO tax, which + is a null `taxAmount` and cannot be told from "nobody has looked" without this column. +* **`Expense.needsTaxReview` is cleared only by an explicit acknowledgement.** A re-sync + that moves the gross on a classified row raises it, and the report skips flagged rows. + Clearing it requires `taxReviewAck: true` in the PATCH body AND both `taxAmount` and + `taxDeductibleBase` in the same request (`installedAtCustomer` is optional — a null + reads as "unanswered" and cannot overstate a deduction). A partial correction is still + accepted; it just leaves the flag standing, because the flag means the WHOLE + classification is in doubt rather than whichever field the request happens to touch. * the correction path is **`PATCH /api/expenses/[id]`**, NOT the PUT on that route. PUT is guarded by `assertExpenseMutableOutsideQbo`, and every expense the pipeline books carries a `qbPurchaseId` — so PUT cannot reach a single row the tax report is made of, and it now @@ -343,10 +354,18 @@ Remaining mobile-repo diff (a separate PR in `gtr-probuild-mobile`): - The `/api/expenses` no-photo path keeps working unchanged for older builds; `costCodeId` is optional there on purpose, so a legacy app is never broken. -## 6. Backfill — `scripts/backfill-expense-attribution.mjs` +## 6. Backfill — `scripts/backfill-expense-attribution.ts` One-shot, dry-run DEFAULT (`--apply` to write, `--csv `), same shape and .env -loading as `suggest-expense-cost-codes.mjs`. Steps: +loading as `suggest-expense-cost-codes.ts`. It is TypeScript and runs under the tsx +loader: + +``` +node --import=tsx scripts/backfill-expense-attribution.ts # dry run +node --import=tsx scripts/backfill-expense-attribution.ts --apply # write +``` + +Steps: (a) `projectId` from `estimate.projectId` where NULL (same UPDATE as the apply script — belt and braces; report rows touched). (b) Item fallback: expenses with `costCodeId` NULL and a coded `itemId` → copy the item's diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 32848f4e0..e95895fe1 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -336,6 +336,9 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id taxAmount: true, taxAtSource: true, taxDeductibleBase: true, + // Whether this row is WAITING for a person. It decides what it + // takes to clear the flag below. + needsTaxReview: true, estimateId: true, projectId: true, updatedAt: true, @@ -357,6 +360,9 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // told, not silently ignored. const allowed = new Set([ "installedAtCustomer", "taxDeductibleBase", "taxAmount", "taxAtSource", "costCodeId", + // Not a column: the explicit "I have re-checked this flagged row" + // acknowledgement. See the needsTaxReview rule below. + "taxReviewAck", ]); const rejected = Object.keys(body).filter(key => !allowed.has(key)); if (rejected.length) { @@ -375,6 +381,49 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id const editsTaxAtSource = has("taxAtSource"); const editsCostCode = has("costCodeId"); + // CLEARING A REVIEW FLAG IS ITS OWN DECISION. + // + // `needsTaxReview` means a re-sync moved the gross out from under a + // classification a human made, so the whole classification is in doubt + // — not just whichever field the next request happens to touch. Letting + // any tax edit clear it meant a bookkeeper answering "yes, installed at + // customer" silently certified a tax amount and a deduction split they + // never looked at, and the row went straight back into the excise + // report. + // + // So clearing it takes an explicit acknowledgement AND the two figures + // the report actually reads. `installedAtCustomer` is optional: it is + // the one field whose absence cannot overstate a deduction (a null + // reads as "unanswered" and the report skips the row). + // + // A tax edit WITHOUT the ack is still accepted — a partial correction + // is normal work — it simply leaves the flag standing. + if (has("taxReviewAck") && body.taxReviewAck !== true && body.taxReviewAck !== false) { + return NextResponse.json( + { error: "taxReviewAck must be true or false." }, + { status: 400 }, + ); + } + const acknowledgesReview = body.taxReviewAck === true; + if (acknowledgesReview && !(editsTaxAmount && editsBase)) { + return NextResponse.json( + { + error: "Acknowledging a tax review needs both taxAmount and taxDeductibleBase in the same request.", + code: "TAX_REVIEW_INCOMPLETE", + }, + { status: 400 }, + ); + } + // An unflagged row has nothing to clear, so the ack is not required of + // ordinary edits; a flagged one keeps its flag until it is given. + const clearsReview = !expense.needsTaxReview || acknowledgesReview; + + // `taxReviewAck` is not a column, so a request carrying nothing else + // has no field to write. Told, not silently no-opped. + if (!editsInstalled && !editsBase && !editsTaxAmount && !editsTaxAtSource && !editsCostCode) { + return NextResponse.json({ error: "Nothing to update." }, { status: 400 }); + } + // The money permission governs anything that lands on a tax return. if ((editsInstalled || editsBase || editsTaxAmount || editsTaxAtSource) && !hasPermission(user, "financialReports")) { @@ -552,7 +601,12 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // refuses to count. ...(editsInstalled || editsBase || editsTaxAmount || editsTaxAtSource ? { - needsTaxReview: false, + // Only when the answer that justifies clearing it came + // with the request. Written in the SAME statement as + // that answer: two statements would leave a window + // where the report sees a cleared row it has not been + // given the figures for. + ...(clearsReview ? { needsTaxReview: false } : {}), // WHO decided. Everything the intake pipeline writes is // "ocr" and re-readable; this is a person, and booking // must not write over it. It matters most in the case diff --git a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx index 73fae993a..1009492fe 100644 --- a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx +++ b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx @@ -64,6 +64,9 @@ export default function TaxPhaseModal({ expense.taxDeductibleBase === null ? "" : String(expense.taxDeductibleBase), ); const [costCodeId, setCostCodeId] = useState(expense.costCodeId ?? ""); + // Only meaningful on a flagged row: the explicit "I have re-checked these + // figures" the server requires before it will clear the flag. + const [reviewAck, setReviewAck] = useState(false); const [saving, setSaving] = useState(false); const parsedTax = taxAmount.trim() === "" ? null : Number(taxAmount); @@ -89,6 +92,19 @@ export default function TaxPhaseModal({ const nextCode = costCodeId || null; if (nextCode !== expense.costCodeId) body.costCodeId = nextCode; + // ACKNOWLEDGING A REVIEW SENDS THE FIGURES, CHANGED OR NOT. + // + // The flag says the whole classification is in doubt because the gross + // moved underneath it, so "I did not edit that field" is not the same + // as "I checked it". The server refuses an ack that does not carry both + // numbers, which is what makes the confirmation mean something. + if (expense.needsTaxReview && reviewAck) { + body.taxReviewAck = true; + body.taxAmount = parsedTax; + body.taxAtSource = (parsedTax ?? 0) > 0; + body.taxDeductibleBase = nextBase; + } + if (Object.keys(body).length === 0) { onClose(); return; @@ -126,10 +142,24 @@ export default function TaxPhaseModal({ {expense.vendor || "Expense"} · {money(expense.amount)}

{expense.needsTaxReview && ( -

- QuickBooks changed this purchase's total after someone recorded its tax, so the - tax details were cleared. Please re-check them. -

+
+

+ QuickBooks changed this purchase's total after someone recorded its tax, so + these figures are in doubt. Please re-check them. +

+ +
)} diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 00abada9d..94ec2e91b 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -538,6 +538,7 @@ type ExpenseTransaction = { installedAtCustomer?: boolean | null; taxDeductibleBase?: unknown; needsTaxReview?: boolean; + taxSource?: string | null; amount: unknown; vendor: string | null; date: Date | null; @@ -611,6 +612,7 @@ export interface QboExpenseRetirementData { qbSyncedAt: Date; taxAmount: null; taxAtSource: false; + taxSource: null; installedAtCustomer: null; taxDeductibleBase: null; needsTaxReview: false; @@ -958,6 +960,11 @@ export async function deactivateQboExpense( installedAtCustomer: true, taxDeductibleBase: true, needsTaxReview: true, + // Retired with the rest of the classification, and therefore + // part of "is it already retired?" — otherwise a deleted + // purchase that a bookkeeper had classified reports "unchanged" + // forever while still carrying their provenance. + taxSource: true, }, }); if (!existing) return "unchanged"; @@ -987,7 +994,8 @@ export async function deactivateQboExpense( existing.taxAtSource === false && existing.installedAtCustomer === null && existing.taxDeductibleBase === null && - existing.needsTaxReview === false; + existing.needsTaxReview === false && + existing.taxSource === null; if ( Number(existing.amount) === 0 && existing.description === description && @@ -1007,6 +1015,7 @@ export async function deactivateQboExpense( qbSyncedAt: removal.qbSyncedAt, taxAmount: null, taxAtSource: false, + taxSource: null, installedAtCustomer: null, taxDeductibleBase: null, needsTaxReview: false, diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index ed9fa33be..456e3a0e5 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -357,17 +357,22 @@ test("a line item on this job's estimate is accepted", async () => { // ── the needsTaxReview lifecycle (Codex round 7, item 3) ─────────────────── -test("a human answer CLEARS needsTaxReview in the same write", async () => { +test("a full answer CLEARS needsTaxReview in the same write", async () => { // Two statements would leave a window where the report sees an answered - // row it still refuses to count. + // row it still refuses to count. Round 14 made the answer that justifies + // clearing it an explicit one: the ack plus both figures. storedExpense = { ...storedExpense, needsTaxReview: true }; - const res = await patch({ installedAtCustomer: true }); + const res = await patch({ + taxReviewAck: true, taxAmount: 16.55, taxDeductibleBase: 50, installedAtCustomer: true, + }); assert.equal(res.status, 200); assert.equal(updateArgs?.data.installedAtCustomer, true); assert.equal(updateArgs?.data.needsTaxReview, false, "answered, so no longer awaiting one"); }); -test("every tax field clears the flag, and a phase-only edit does not", async () => { +test("no single tax field clears the flag on its own", async () => { + // Round 14: each of these is a partial answer, and the flag says the WHOLE + // classification is in doubt. They are accepted and the flag stands. storedExpense = { ...storedExpense, needsTaxReview: true }; for (const body of [ { taxAmount: 10 }, @@ -375,14 +380,19 @@ test("every tax field clears the flag, and a phase-only edit does not", async () { taxDeductibleBase: 50 }, { installedAtCustomer: false }, ]) { - await patch(body); - assert.equal(updateArgs?.data.needsTaxReview, false, JSON.stringify(body)); + const res = await patch(body); + assert.equal(res.status, 200, JSON.stringify(body)); + assert.equal(updateArgs?.data.needsTaxReview, undefined, JSON.stringify(body)); } - // A cost-code edit is not an answer to the tax question, so the row stays - // flagged — otherwise re-phasing a receipt would quietly re-admit it to the - // filing. + // A cost-code edit is not an answer to the tax question either, and it does + // not even carry the provenance stamp. await patch({ costCodeId: null }); assert.equal(updateArgs?.data.needsTaxReview, undefined); + // On an UNflagged row the same edits clear nothing because there is + // nothing to clear, but the column is still written false as before. + storedExpense = { ...storedExpense, needsTaxReview: false }; + await patch({ taxAmount: 10 }); + assert.equal(updateArgs?.data.needsTaxReview, false); }); // ── the item link is judged on the RESOLVED job (item 4) ─────────────────── @@ -576,3 +586,56 @@ test("an omitted amount leaves the stored one alone", async () => { assert.equal(res.status, 200); assert.equal(updateArgs?.data.amount, undefined); }); + +// ── clearing a review flag is its own decision (Codex round 14, item 1) ───── + +test("an installedAtCustomer-only PATCH on a FLAGGED row leaves the flag up", async () => { + // The flag means the gross moved under the whole classification, not just + // under the field this request happens to touch. Clearing it here would + // certify a tax amount and a split nobody re-checked, and put the row + // straight back into the excise report. + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, true, "the edit still lands"); + assert.equal(updateArgs?.data.needsTaxReview, undefined, "but the flag is untouched"); +}); + +test("a full acknowledgement clears the flag", async () => { + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ + taxReviewAck: true, taxAmount: 16.55, taxDeductibleBase: 100, installedAtCustomer: true, + }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.needsTaxReview, false); + assert.equal(updateArgs?.data.taxSource, "manual"); +}); + +test("an acknowledgement without both figures is refused, not half-applied", async () => { + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: 16.55 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, "TAX_REVIEW_INCOMPLETE"); + assert.equal(updateArgs, null, "nothing is written"); +}); + +test("an UNflagged row does not need an acknowledgement", async () => { + // The ack exists to make clearing a flag deliberate. Requiring it of + // ordinary edits would just teach people to send it always. + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.needsTaxReview, false); +}); + +test("taxReviewAck on its own has nothing to write", async () => { + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ taxReviewAck: false }); + assert.equal(res.status, 400); + assert.match((await res.json()).error, /Nothing to update/); +}); + +test("a non-boolean taxReviewAck is refused", async () => { + const res = await patch({ taxReviewAck: "yes", taxAmount: 1, taxDeductibleBase: 1 }); + assert.equal(res.status, 400); + assert.match((await res.json()).error, /taxReviewAck/); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index ffee9a15e..50ecd2ee9 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1731,7 +1731,7 @@ test("a second deactivation of an already-retired row is unchanged", async () => amount: 0, status: "Reviewed" as const, description: "[QuickBooks import] Removed in QBO (deleted)", qbSyncToken: "1", - taxAmount: null, taxAtSource: false, installedAtCustomer: null, + taxAmount: null, taxAtSource: false, taxSource: null, installedAtCustomer: null, taxDeductibleBase: null, needsTaxReview: false, }; const fake = createFakePrisma([retired as any]); @@ -1932,3 +1932,35 @@ test("a newer sync landing between the read and the write wins", async () => { "not-written", ); }); + +test("deleting a MANUALLY classified purchase retires its provenance too", () => { + // Codex round 14, item 3. Leaving `taxSource` behind on a zeroed row means + // the idempotency check never sees the classification as retired, so every + // subsequent sync re-writes the same row and reports it as a change — and + // the row still claims a person stands behind figures that are now null. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + taxAmount: 34.06, taxAtSource: true, taxSource: "manual", + installedAtCustomer: true, taxDeductibleBase: 100, needsTaxReview: false, + } as any, + ]); + + return (async () => { + const removal = { + qbPurchaseId: "purchase-1", qbSyncToken: "1", + reason: "deleted", qbSyncedAt: new Date("2026-09-02T00:00:00.000Z"), + }; + assert.equal(await deactivateQboExpense(fake.client, removal), "removed"); + const row = fake.rows.get("purchase-1") as any; + assert.equal(row.amount, 0); + assert.equal(row.taxAmount, null); + assert.equal(row.taxSource, null, "nobody stands behind a purchase QBO says never happened"); + assert.equal(row.installedAtCustomer, null); + assert.equal(row.taxDeductibleBase, null); + assert.equal(row.needsTaxReview, false, "a gone purchase is not something to re-check"); + + // ...and it is now genuinely idempotent. + assert.equal(await deactivateQboExpense(fake.client, removal), "unchanged"); + })(); +}); From 4162d88ebd171886a4fe23ba3fc703d011b6cd52 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:11:55 -0700 Subject: [PATCH 098/144] fix(scripts): build the suggester's Prisma client lazily MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--help` is the CI smoke test that this file still loads, and it ran a PrismaClient constructor at module scope — which throws without DATABASE_URL, so the check failed on CI while passing on any machine with a .env. The client is now built inside main, after the help branch returns. Co-Authored-By: Claude Fable 5.1 --- scripts/suggest-expense-cost-codes.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/suggest-expense-cost-codes.ts b/scripts/suggest-expense-cost-codes.ts index 7d01b1abf..3c1f779dc 100644 --- a/scripts/suggest-expense-cost-codes.ts +++ b/scripts/suggest-expense-cost-codes.ts @@ -69,7 +69,10 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); config({ path: join(__dirname, "..", ".env.local") }); config({ path: join(__dirname, "..", ".env") }); -const prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL } } }); +// CONSTRUCTED LAZILY, inside main and after the --help branch. Building it at +// module scope threw on any machine without DATABASE_URL — including CI, where +// --help is the smoke test that this file still loads at all. +let prisma: PrismaClient | null = null; const csvIdx = process.argv.indexOf("--csv"); const CSV = csvIdx > -1 ? process.argv[csvIdx + 1] : null; @@ -101,13 +104,15 @@ async function main() { return; } - const codes = await prisma.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }); + prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL } } }); + + const codes = await prisma!.costCode.findMany({ where: { isActive: true }, select: { id: true, code: true } }); const codeId = new Map(codes.map((c) => [c.code, c.id])); // Attribution resolved the ONE way, and the overhead bucket excluded by id. // A row re-attributed to a live job used to be invisible here (its estimate // still named the old one), so the report understated the work to be done. - const rows = await prisma.expense.findMany({ + const rows = await prisma!.expense.findMany({ where: { costCodeId: null, ...expenseNotOnProjectWhere(OVERHEAD_PROJECT_ID), @@ -123,7 +128,7 @@ async function main() { // The PROJECT'S OWN PHASES, same rule the writer applies. A suggestion the // job could never accept is not a suggestion, and listing it as one sends a // bookkeeper to check something that was never on the table. - const phaseRows = await prisma.estimateItem.findMany({ + const phaseRows = await prisma!.estimateItem.findMany({ where: { costCodeId: { not: null }, costCode: { isActive: true }, @@ -198,4 +203,5 @@ async function main() { main() .catch((e) => { console.error("FAILED:", e); process.exit(1); }) - .finally(() => prisma.$disconnect()); + // Nothing to disconnect when --help returned before the client was built. + .finally(() => prisma?.$disconnect()); From b648b6fd059097d7a3379e563910862c47fb1dfe Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 06:46:45 -0700 Subject: [PATCH 099/144] fix(expenses): one tax bound for both writers, and lock what a backfill derives from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three round-15 items. An OCR tax read only had to clear "less than the total" to be booked as tax paid at source, so $90 of tax on a $100 receipt — a decimal point in the wrong place — went onto a state excise return as a $90 deduction nobody looked at. The bound the bookkeeper's PATCH already enforced now lives in expense-attribution.ts and both writers use it. The remedies differ because the situations do: PATCH refuses the request, booking cannot (the Purchase is already in QuickBooks), so it stores NULL, flags needsTaxReview, and keeps taxSource "ocr" — a machine looked and got an answer a person must replace. The sync's classification test now counts taxSource "manual" as evidence, and reads the column. A bookkeeper who decides a receipt carries NO tax leaves every other signal null, so that row — a human's explicit answer, now describing a different gross — was the one row a re-sync said nothing about. The backfill share-locks the rows its answers are DERIVED from (the estimate, the linked item, the job's phase rows) before taking the per-expense lock and reading. A predicate can catch a row that moved before the write; it cannot stop one moving during the read sequence that decides what to write. The project fill is now one expense per transaction so it can hold that lock, and a row re-pointed at an estimate or item the locks do not cover is skipped rather than written. Co-Authored-By: Claude Fable 5.1 --- scripts/backfill-expense-attribution.ts | 147 +++++++++++++++++---- src/app/api/expenses/[id]/route.ts | 16 +-- src/lib/expense-attribution.ts | 36 +++++ src/lib/qbo-expense-sync.ts | 25 +++- src/lib/receipt-intake/book.ts | 48 ++++++- tests/backfill-expense-attribution.test.ts | 118 ++++++++++++++++- tests/expense-attribution.test.ts | 24 ++++ tests/qbo-expense-sync.test.ts | 55 ++++++++ tests/receipt-intake-book.test.ts | 47 +++++++ 9 files changed, 469 insertions(+), 47 deletions(-) diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 8a03b3bb3..7b0299671 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -364,8 +364,71 @@ export function remainderCsv(remainder, projectNameById) { * stub rather than the script. */ export async function writeUnderExpenseLock(db, expenseId, run) { + return writeUnderAttributionLocks(db, { expenseId }, run); +} + +/** + * SHARE-LOCK THE ROWS A DECISION IS DERIVED FROM, in one fixed order. + * + * `FOR SHARE` blocks anyone trying to UPDATE or DELETE these rows until this + * transaction commits, while letting other readers through. That is exactly the + * shape of the hazard: the expense's own row is protected by the per-expense + * advisory lock and by compare-and-set predicates, but the FACTS the write is + * derived from live on OTHER rows — the estimate whose `projectId` supplies the + * attribution, the estimate item whose `costCodeId` is copied, and the job's + * phase rows that decide whether a code is even allowed. A predicate can catch + * a row that moved BEFORE the write; it cannot stop it moving DURING the read + * sequence that decides what to write. + * + * Order is fixed and ids are sorted, so two runs of this script can never take + * the same locks in opposite orders. They are SHARED locks, so concurrent + * readers (including another backfill) do not block each other at all — only a + * writer of those exact rows waits, briefly. + */ +async function lockRowsForShare(tx, table, ids) { + const unique = [...new Set(ids.filter(Boolean))].sort(); + if (!unique.length) return; + const params = unique.map((_, index) => `$${index + 1}`).join(", "); + await tx.$queryRawUnsafe( + `SELECT id FROM ${table} WHERE id IN (${params}) ORDER BY id FOR SHARE`, + ...unique, + ); +} + +/** + * The job's phase rows — the same universe `readAllowedCodes` reads, held still + * while it reads them. Without this, a phase deleted between the phase read and + * the write puts a code on a job that no longer has it. + */ +async function lockProjectPhaseRows(tx, projectId) { + if (!projectId) return; + await tx.$queryRawUnsafe( + `SELECT ei.id FROM "EstimateItem" ei + JOIN "Estimate" e ON e.id = ei."estimateId" + WHERE e."projectId" = $1 AND ei."costCodeId" IS NOT NULL + ORDER BY ei.id + FOR SHARE OF ei`, + projectId, + ); +} + +/** + * One transaction: the derived-from rows are share-locked FIRST, then the + * per-expense advisory lock, then the caller reads and writes. Locks before + * reads, always — a read taken before its lock describes a moment that the lock + * then fails to preserve. + * + * Falls back to a bare call when the injected client has no `$transaction`: + * some unit tests drive a plain stub, and requiring one there would test the + * stub rather than the script. + */ +export async function writeUnderAttributionLocks(db, locks, run) { + const { expenseId, estimateIds = [], estimateItemIds = [], phaseProjectId = null } = locks; if (typeof db.$transaction !== "function") return run(db); return db.$transaction(async tx => { + await lockRowsForShare(tx, '"Estimate"', estimateIds); + await lockRowsForShare(tx, '"EstimateItem"', estimateItemIds); + await lockProjectPhaseRows(tx, phaseProjectId); await lockExpense(tx, expenseId); return run(tx); }); @@ -624,30 +687,41 @@ export async function runBackfill({ byProject.get(key).ids.push(fill.id); } for (const { projectId, estimateId, ids } of byProject.values()) { - // BOTH halves of the derivation, re-asserted at write time. + // ONE EXPENSE PER TRANSACTION, under the same locks the cost pass uses. // - // `projectId: null` says nobody has attributed the row yet; - // `estimateId` says it is still hanging off the estimate this project - // was READ from. Without the second one, an expense re-pointed at a - // different estimate between the plan and the write would be stamped - // with the old estimate's project — a silent cross-job attribution - // performed by the very pass that exists to get attribution right. - const result = await db.expense.updateMany({ - where: { - id: { in: ids }, projectId: null, estimateId, - // AND THE ESTIMATE STILL POINTS THERE. `estimateId` proves the - // row is on the same estimate; it says nothing about where that - // estimate now lives. An estimate moved to another project - // between the plan and this write would otherwise stamp every - // one of its expenses with the OLD project — a cross-job - // attribution performed by the pass whose whole job is getting - // attribution right. A relation filter makes it one statement, - // so there is no window between checking and writing. - estimate: { is: { projectId } }, - }, - data: { projectId }, - }); - projectIdsWritten += result.count; + // Batching the group into a single UPDATE was cheaper, but it could not + // take a per-expense lock (the ids differ) and it read the estimate only + // through the predicate. Now the estimate is share-locked for the whole + // decision, so it cannot move while the row is being written. + for (const id of ids) { + const result = await writeUnderAttributionLocks( + db, + { expenseId: id, estimateIds: [estimateId] }, + // BOTH halves of the derivation, re-asserted at write time even + // though the locks are held: the lock stops the rows moving + // from here on, the predicate covers everything that happened + // between the plan and now, and a writer that takes no lock at + // all is still bound by it. + // + // `projectId: null` says nobody has attributed the row yet; + // `estimateId` says it is still hanging off the estimate this + // project was READ from; and the relation filter says that + // estimate still points at the project we are about to stamp. + // Without the last one, an estimate moved to another job + // between the plan and the write would stamp every one of its + // expenses with the OLD project — a cross-job attribution + // performed by the pass whose whole job is getting attribution + // right. + async tx => tx.expense.updateMany({ + where: { + id, projectId: null, estimateId, + estimate: { is: { projectId } }, + }, + data: { projectId }, + }), + ); + projectIdsWritten += result.count; + } } // COST-CODE WRITES RUN UNDER THE SHARED PER-EXPENSE LOCK, one row at a @@ -667,7 +741,18 @@ export async function runBackfill({ let costCodesWritten = 0; let costCodesSkipped = 0; for (const fill of plan.codeFills) { - const result = await writeUnderExpenseLock(db, fill.id, async tx => { + // The rows this decision is derived from, named from the PLAN — which + // is also what the re-read below is checked against, so a row that has + // since moved off them is skipped rather than written under locks that + // do not cover it. + const plannedEstimateId = fill.expense?.estimateId ?? null; + const plannedItemId = fill.expense?.itemId ?? null; + const result = await writeUnderAttributionLocks(db, { + expenseId: fill.id, + estimateIds: [plannedEstimateId], + estimateItemIds: [plannedItemId], + phaseProjectId: fill.expectedProjectId ?? null, + }, async tx => { // RE-READ UNDER THE LOCK, then re-plan against what is really // there. // @@ -691,6 +776,20 @@ export async function runBackfill({ }, }); if (!current) return { count: 0 }; + // THE LOCKS HAVE TO COVER THE ROWS THE ANSWER COMES FROM. + // + // They were taken from the plan's view of this expense. If the row + // has since been re-pointed at a different estimate or a different + // line item, the facts about to be re-read are ones nothing is + // holding still — so this is not the moment to write. Skipped and + // counted; a re-run plans it against the truth and locks the rows + // that truth actually rests on. + if ( + (current.estimateId ?? null) !== plannedEstimateId || + (current.itemId ?? null) !== plannedItemId + ) { + return { count: 0 }; + } // RE-PLAN, don't re-use. // diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index e95895fe1..b6e157ff9 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -7,7 +7,7 @@ import { QboManagedExpenseError, assertExpenseMutableOutsideQbo, } from "@/lib/qbo-expense-guard"; -import { resolveExpenseProjectId } from "@/lib/expense-attribution"; +import { isPlausibleReceiptTax, maxPlausibleTaxAmount, resolveExpenseProjectId } from "@/lib/expense-attribution"; import { lockExpense } from "@/lib/expense-lock"; import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; @@ -448,12 +448,10 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id nextInstalled = raw; } - // A CORRECTED TAX FIGURE, bounded by plausibility. WA's combined rate - // tops out around 10.6%; 12% is deliberately loose so a legitimate - // receipt is never refused, while a transposed OCR read (a $207 receipt - // "with" $207 of tax) still cannot reach a filing. Zero is allowed — - // "this receipt had no tax" is an answer a human is entitled to give. - const MAX_TAX_RATE = 0.12; + // A CORRECTED TAX FIGURE, bounded by plausibility — the SHARED bound + // (src/lib/expense-attribution.ts), because the booking pipeline judges + // an OCR read against the same number. Zero is allowed: "this receipt + // had no tax" is an answer a human is entitled to give. let nextTaxAmount: number | null = null; if (editsTaxAmount && body.taxAmount !== null) { const parsed = Number(body.taxAmount); @@ -463,8 +461,8 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id { status: 400 }, ); } - const ceiling = Math.round(Number(expense.amount) * MAX_TAX_RATE * 100) / 100; - if (parsed > ceiling) { + const ceiling = maxPlausibleTaxAmount(Number(expense.amount)); + if (!isPlausibleReceiptTax(parsed, Number(expense.amount))) { return NextResponse.json( { error: `That tax is implausible for a ${Number(expense.amount).toFixed(2)} receipt (max ${ceiling.toFixed(2)}, 12%).` }, { status: 400 }, diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index d4cfb31dc..955b69238 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -242,3 +242,39 @@ export function resolveExpenseProjectLabel( projectName: expense.estimate?.project?.name ?? null, }; } + + +/** + * THE ONE PLAUSIBILITY BOUND FOR A SALES-TAX FIGURE ON A RECEIPT. + * + * WA's combined rate tops out around 10.6%; 12% is deliberately loose so a + * legitimate receipt is never refused, while a transposed or misread figure + * (a $100 receipt "with" $90 of tax) cannot reach an excise return. + * + * It lives here because there are TWO writers of `Expense.taxAmount` and they + * must not be able to disagree: the bookkeeper's PATCH, which refuses an + * implausible figure outright, and the booking pipeline, which cannot refuse + * anything (the Purchase is already in QuickBooks) and instead stores NULL and + * flags the row for review. Same bound, different remedies. + * + * The rate is measured against the GROSS `Expense.amount`, which is what both + * writers hold; on any receipt this side of the bound the difference from a + * pre-tax basis is far smaller than the slack in the 12%. + */ +export const MAX_PLAUSIBLE_TAX_RATE = 0.12; + +/** The largest tax figure this receipt could plausibly carry, in dollars. */ +export function maxPlausibleTaxAmount(grossAmount: number): number { + if (!Number.isFinite(grossAmount) || grossAmount <= 0) return 0; + return Math.round(grossAmount * MAX_PLAUSIBLE_TAX_RATE * 100) / 100; +} + +/** + * True when `taxAmount` is a believable amount of sales tax on `grossAmount`. + * Zero is plausible ("this receipt had no tax"); a negative one is not. + */ +export function isPlausibleReceiptTax(taxAmount: number, grossAmount: number): boolean { + if (!Number.isFinite(taxAmount) || taxAmount < 0) return false; + if (taxAmount === 0) return true; + return taxAmount <= maxPlausibleTaxAmount(grossAmount); +} diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 94ec2e91b..941a7cc91 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -668,7 +668,11 @@ export function planQboExpenseUpdate( Partial< Pick< ExistingQboExpense, - "amount" | "taxAmount" | "taxDeductibleBase" | "installedAtCustomer" + | "amount" + | "taxAmount" + | "taxDeductibleBase" + | "installedAtCustomer" + | "taxSource" > >, write: QboExpenseWrite, @@ -747,12 +751,20 @@ export function planQboExpenseUpdate( // numbers still satisfy every CHECK, so nothing else would ever ask. // // "Classified" means a human's tax answer is on the row in any form — - // a tax amount, an installed-at-customer decision, or a hand allocation. + // a tax amount, an installed-at-customer decision, a hand allocation, or + // `taxSource: "manual"`. That last one is not redundant: a bookkeeper who + // decides a receipt carries NO tax leaves every one of the other three + // NULL, so without it the single most reviewable row — a human's explicit + // "no tax", now describing a different gross — is the one row a re-sync + // would say nothing about. // For those rows an amount change is a REVIEW, never a silent acceptance: // the classification is kept (it may well still be right) and the row is // flagged, which the report reads as "not until a person looks". const classified = - existingTax !== null || existingBase !== null || existing.installedAtCustomer != null; + existingTax !== null || + existingBase !== null || + existing.installedAtCustomer != null || + existing.taxSource === "manual"; const existingAmount = existing.amount === null || existing.amount === undefined ? null : Number(existing.amount); const amountMoved = @@ -852,8 +864,11 @@ export async function upsertQboExpense( taxDeductibleBase: true, // Read for the classification test in planQboExpenseUpdate: a // human's installed-at-customer answer counts as a tax - // classification even when no tax amount was recorded. + // classification even when no tax amount was recorded, and + // `taxSource: "manual"` counts even when NOTHING else is set + // (their answer was "this receipt has no tax"). installedAtCustomer: true, + taxSource: true, amount: true, vendor: true, date: true, @@ -912,7 +927,7 @@ export async function upsertQboExpense( select: { id: true, qbSyncToken: true, estimateId: true, projectId: true, updatedAt: true, taxAmount: true, taxDeductibleBase: true, amount: true, - installedAtCustomer: true, + installedAtCustomer: true, taxSource: true, vendor: true, date: true, description: true, status: true, }, }); diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index a713697a9..63d72338e 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -20,6 +20,7 @@ import { matchCostCode } from "@/lib/project-match"; import { receiptUrlRef } from "./receipt-url"; import { QBO_ATTACHMENT_MAX_BYTES } from "./intake-core"; +import { isPlausibleReceiptTax } from "@/lib/expense-attribution"; import { lockExpense } from "@/lib/expense-lock"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { @@ -593,6 +594,27 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // there is exactly one Purchase. const amountCents = expenseAmountCents(groups, row.totalCents); const taxApplied = appliedTaxCents(groups); + + // AN IMPLAUSIBLE OCR TAX IS NOT A TAX FIGURE. + // + // `buildGroups` rejects a tax read on a check and one that is >= the total, + // which leaves a wide band of nonsense it accepts: $90 of tax on a $100 + // receipt is a decimal-point or column misread, and it satisfies every + // check the pipeline had. Booked as `taxAtSource`, it goes on a state + // excise return as a $90 deduction nobody ever looked at. + // + // The bound is the SAME one the bookkeeper's PATCH enforces + // (isPlausibleReceiptTax) — the two writers of this column must not be able + // to disagree about what a believable figure is. The remedies differ + // because the situations do: PATCH refuses the request, while booking + // cannot refuse anything (the Purchase is already in QuickBooks). So the + // figure is stored as NULL, the row is flagged `needsTaxReview`, and the + // provenance still says "ocr" — a machine looked, and got an answer a + // person now has to replace. The raw read stays on `ReceiptIntake.taxCents` + // for audit, exactly as a rejected read does. + const taxIsPlausible = isPlausibleReceiptTax(taxApplied / 100, amountCents / 100); + const taxToStore = taxApplied > 0 && taxIsPlausible ? taxApplied / 100 : null; + const taxNeedsReview = taxApplied > 0 && !taxIsPlausible; // RE-VALIDATE THE PHASE AGAINST THE FINAL PROJECT. // // Both the captured code and the model's suggestion were resolved while the @@ -783,9 +805,15 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro OR: [{ taxSource: null }, { taxSource: { not: "manual" } }], }, data: { - taxAmount: taxApplied / 100, - taxAtSource: true, + // Same bound as the create path above: an + // implausible read fills NOTHING and asks for a + // person instead. Writing it here would be worse + // than on a new row — this row may already be in a + // filing period somebody has reconciled. + taxAmount: taxToStore, + taxAtSource: taxToStore !== null, taxSource: "ocr", + ...(taxNeedsReview ? { needsTaxReview: true } : {}), }, }); } @@ -861,13 +889,18 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // NOT the PUT on that route — PUT is guarded by // assertExpenseMutableOutsideQbo and every row booked here // carries a qbPurchaseId. - taxAmount: taxApplied > 0 ? taxApplied / 100 : null, - taxAtSource: taxApplied > 0, + taxAmount: taxToStore, + taxAtSource: taxToStore !== null, + // An implausible read is a question, not an answer: the row + // waits for a person and the report skips it meanwhile. + needsTaxReview: taxNeedsReview, // Provenance for the tax columns. "ocr" is a re-readable // guess; "manual" (written by the tax PATCH) is a person's // answer, and this pipeline never writes over one. Null - // when there was no tax to record, which leaves a later - // bookkeeper free to answer without arguing with a machine. + // only when there was no tax read at all, which leaves a + // later bookkeeper free to answer without arguing with a + // machine — an implausible read still counts as "a machine + // looked", which is why it keeps "ocr". taxSource: taxApplied > 0 ? "ocr" : null, installedAtCustomer: row.installedAtCustomer, amount: amountCents / 100, @@ -886,7 +919,8 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro description: `[Receipt intake] ${docRef}` + phaseCheck.note + - (taxApplied > 0 ? ` · incl. $${(taxApplied / 100).toFixed(2)} sales tax` : "") + + (taxToStore !== null ? ` · incl. $${taxToStore.toFixed(2)} sales tax` : "") + + (taxNeedsReview ? " · tax read looks wrong, needs review" : "") + ` · pending bookkeeper review`, }, select: { id: true }, diff --git a/tests/backfill-expense-attribution.test.ts b/tests/backfill-expense-attribution.test.ts index 78634438f..3c75476ce 100644 --- a/tests/backfill-expense-attribution.test.ts +++ b/tests/backfill-expense-attribution.test.ts @@ -476,7 +476,9 @@ test("apply writes both passes, each behind its own predicate", async () => { const projectWrite = stub.writes.find(w => "projectId" in w.data)!; assert.deepEqual(projectWrite.where, { - id: { in: ["e1"] }, + // One expense per statement now: the group cannot share a per-expense + // lock, so it is written a row at a time under one. + id: "e1", projectId: null, // Both halves of the derivation, plus the join that proves the second // half is still true at write time. @@ -740,12 +742,124 @@ test("each cost-code write takes the shared per-expense lock", async () => { [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], ); (stub.db as any).$transaction = async (fn: any) => fn(stub.db); - (stub.db as any).$queryRawUnsafe = async (_q: string, key: unknown) => { locks.push(key); return [{}]; }; + (stub.db as any).$queryRawUnsafe = async (query: string, ...args: unknown[]) => { + if (query.includes("pg_advisory_xact_lock")) locks.push(args[0]); + return [{}]; + }; await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); assert.deepEqual(locks, ["expense:e1"], "one lock, namespaced per expense"); }); +// ── the rows a decision is DERIVED from are held too (round 15, item 6) ──── + +/** Records every lock statement in order, with the ids it named. */ +function lockTrace(stub: ReturnType) { + const trace: { kind: string; args: unknown[] }[] = []; + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async (query: string, ...args: unknown[]) => { + const kind = query.includes("pg_advisory_xact_lock") ? "expense-lock" + : query.includes('FROM "Estimate"') ? "estimate-share" + : query.includes('JOIN "Estimate"') ? "phase-share" + : query.includes('FROM "EstimateItem"') ? "item-share" + : "other"; + trace.push({ kind, args }); + return [{}]; + }; + return trace; +} + +test("the cost fill share-locks the estimate, the item and the phase rows BEFORE the expense lock", async () => { + // A read taken before its lock describes a moment the lock then fails to + // preserve. The expense's own row is protected by the advisory lock and the + // CAS; the FACTS the answer comes from live on other rows. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", itemId: "i1", vendor: "Unknown Vendor", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + const trace = lockTrace(stub); + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + + const kinds = trace.map(entry => entry.kind); + const expenseLockAt = kinds.lastIndexOf("expense-lock"); + assert.ok(expenseLockAt >= 0, "the per-expense lock is still taken"); + const before = kinds.slice(0, expenseLockAt); + assert.ok(before.includes("estimate-share"), "the estimate is held"); + assert.ok(before.includes("item-share"), "so is the item the code is copied from"); + assert.ok(before.includes("phase-share"), "and the job's phase rows"); + // FOR SHARE, not FOR UPDATE: two readers must not block each other. + assert.ok(trace.every(entry => !String(entry.kind).includes("update"))); +}); + +test("the project fill share-locks the estimate its answer comes from", async () => { + const stub = createStub( + [expense({ id: "e1", projectId: null, estimate: { projectId: "job-1" } })], + [], + ); + const trace = lockTrace(stub); + await runBackfill({ db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID }); + + const kinds = trace.map(entry => entry.kind); + assert.deepEqual( + kinds, + ["estimate-share", "expense-lock"], + "the estimate first, then the row, then the write", + ); + assert.deepEqual(trace[0].args, ["est-job-1"], "the estimate the project was read off"); +}); + +test("an INTERLEAVED estimate move is refused: the write no-ops", async () => { + // Deterministic interleaving: the mover runs between the snapshot and the + // write, which is precisely the window the share lock closes in production + // and the predicate closes here. Either way the write must not land. + const stub = createStub( + [expense({ id: "e1", projectId: null, estimate: { projectId: "job-1" } })], + [], + ); + const sequence: string[] = []; + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async (query: string) => { + if (query.includes("FOR SHARE")) { + sequence.push("share-lock"); + // The move lands JUST BEFORE the lock takes hold — the worst case, + // and the one the predicate has to catch on its own. + stub.rows[0].estimate = { projectId: "job-2" }; + } + return [{}]; + }; + + const result = await runBackfill({ + db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID, + }); + assert.deepEqual(sequence, ["share-lock"], "the lock was attempted"); + assert.equal(result.written.projectIds, 0, "and the stale answer was not written"); + assert.equal(stub.rows[0].projectId ?? null, null); +}); + +test("an expense re-pointed at a DIFFERENT estimate is skipped by the cost fill", async () => { + // The locks were taken from the plan's view of this row. If it has since + // moved to another estimate, the facts about to be re-read are ones nothing + // is holding still, so this is not the moment to write. + const stub = createStub( + [expense({ id: "e1", projectId: "job-1", vendor: "Summit Plumbing", updatedAt: new Date("2026-09-01") })], + [{ id: "i1", costCodeId: "cc-plumb", estimateId: "est-job-1", estimate: { projectId: "job-1" } }], + ); + (stub.db as any).$transaction = async (fn: any) => fn(stub.db); + (stub.db as any).$queryRawUnsafe = async () => [{}]; + const snapshot = stub.db.expense.findMany; + stub.db.expense.findMany = async () => { + const rows = await snapshot(); + stub.rows[0].estimateId = "est-somewhere-else"; + return rows; + }; + + const result = await runBackfill({ + db: stub.db, apply: true, log: () => {}, overheadProjectId: OVERHEAD_ID, + }); + assert.equal(result.written.costCodes, 0); + assert.equal(stub.rows[0].costCodeId ?? null, null); +}); + test("a row that MOVED between the plan and the write is skipped, not coded", async () => { // This script's plan is the stalest of the four writers': computed for // every row up front, then applied in a loop that can run for minutes. The diff --git a/tests/expense-attribution.test.ts b/tests/expense-attribution.test.ts index e7f56f241..2bc346c24 100644 --- a/tests/expense-attribution.test.ts +++ b/tests/expense-attribution.test.ts @@ -14,6 +14,8 @@ import { expenseForProjectWhere, expenseForProjectsWhere, expenseHasAnyProjectWhere, + isPlausibleReceiptTax, + maxPlausibleTaxAmount, notHumanCodedExpenseWhere, resolveExpenseCostCodeId, resolveExpenseProjectId, @@ -208,3 +210,25 @@ test("an unattributed row labels as nothing at all", () => { { projectId: null, projectName: null }, ); }); + +// ── the one tax plausibility bound (Codex round 15, item 1) ──────────────── + +test("the tax bound is 12% of the gross, rounded to cents", () => { + assert.equal(maxPlausibleTaxAmount(100), 12); + assert.equal(maxPlausibleTaxAmount(207.74), 24.93); + assert.equal(maxPlausibleTaxAmount(0), 0, "no receipt, no allowance"); + assert.equal(maxPlausibleTaxAmount(-5), 0); +}); + +test("zero is plausible, a transposed read is not", () => { + // "This receipt had no tax" is a real answer. $90 on a $100 receipt is a + // decimal point in the wrong place, and it is the case that reaches an + // excise return as a $90 deduction if nothing stops it. + assert.equal(isPlausibleReceiptTax(0, 100), true); + assert.equal(isPlausibleReceiptTax(9.5, 100), true); + assert.equal(isPlausibleReceiptTax(12, 100), true, "the bound itself is allowed"); + assert.equal(isPlausibleReceiptTax(12.01, 100), false); + assert.equal(isPlausibleReceiptTax(90, 100), false); + assert.equal(isPlausibleReceiptTax(-1, 100), false); + assert.equal(isPlausibleReceiptTax(Number.NaN, 100), false); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 50ecd2ee9..044e9d1dc 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -1964,3 +1964,58 @@ test("deleting a MANUALLY classified purchase retires its provenance too", () => assert.equal(await deactivateQboExpense(fake.client, removal), "unchanged"); })(); }); + +test("a manual NO-TAX decision is a classification, and a gross change re-opens it", () => { + // Codex round 15, item 5. The bookkeeper looked at this receipt and decided + // it carries no sales tax: taxAmount null, no allocation, no + // installed-at-customer answer. Every other classification signal is + // absent, so without `taxSource` this row — a human's explicit answer, now + // describing a different gross — is the ONE row a re-sync says nothing + // about. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: 412.1, + taxAmount: null, taxDeductibleBase: null, installedAtCustomer: null, + taxSource: "manual", + }, + { ...WRITE, amount: 498.3 }, + ); + assert.equal(plan.data.needsTaxReview, true); + assert.ok(!("taxAmount" in plan.data), "their answer is kept, only re-opened"); +}); + +test("an OCR no-tax row is NOT re-opened by a gross change", () => { + // The control. Nothing here is a human answer, so flagging it would bury + // the rows that are. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: 412.1, taxAmount: null, taxDeductibleBase: null, + installedAtCustomer: null, taxSource: "ocr", + }, + { ...WRITE, amount: 498.3 }, + ); + assert.ok(!("needsTaxReview" in plan.data)); +}); + +test("the sync READS taxSource, or the rule above can never fire", () => { + // A rule that depends on a column nobody selected is a rule that does not + // exist. This is the wiring check the pure-function tests cannot make. + const fake = createFakePrisma([ + { + ...WRITE, id: "expense-1", projectId: "project-1", receiptUrl: null, + amount: 125.5, taxAmount: null, taxDeductibleBase: null, + installedAtCustomer: null, taxSource: "manual", needsTaxReview: false, + } as any, + ]); + return (async () => { + assert.equal( + await upsertQboExpense(fake.client, { ...WRITE, qbSyncToken: "1", amount: 300 }), + "updated", + ); + const row = fake.rows.get("purchase-1") as any; + assert.equal(row.needsTaxReview, true, "the manual no-tax answer was re-opened"); + assert.equal(row.taxSource, "manual", "and left standing"); + })(); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index ccfeb8c22..3235fa4e7 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -1280,3 +1280,50 @@ test("the per-expense lock is taken BEFORE the read the fill decides from", asyn // read (the id lookup) -> lock -> read (everything the fill decides from) assert.deepEqual(trace.slice(0, 3), ["read", "lock", "read"]); }); + +// ── an implausible OCR tax is flagged, never booked (round 15, item 1) ───── + +test("$90 of tax on a $100 receipt is NOT booked as tax paid at source", () => { + // buildGroups only rejects tax >= total, which leaves a wide band of + // nonsense: this satisfies every check the pipeline had and would land on a + // state excise return as a $90 deduction nobody looked at. + const rec = recorder(); + return bookReceipt(row({ totalCents: 10_000, taxCents: 9_000 }), rec.deps).then(result => { + assert.equal(result.outcome, "booked", "the Purchase is real; only the tax read is wrong"); + const created = rec.expenses[0]; + assert.equal(created.taxAmount, null, "nothing implausible reaches the report"); + assert.equal(created.taxAtSource, false); + assert.equal(created.needsTaxReview, true, "a person is asked instead"); + assert.equal(created.taxSource, "ocr", "a machine DID look — that is what needs replacing"); + assert.match(created.description, /needs review/); + assert.doesNotMatch(created.description, /incl\. \$90/); + }); +}); + +test("a believable tax on the same receipt is booked normally", () => { + // The control: same shape, a figure inside the bound. + const rec = recorder(); + return bookReceipt(row({ totalCents: 10_000, taxCents: 900 }), rec.deps).then(() => { + const created = rec.expenses[0]; + assert.equal(created.taxAmount, 9); + assert.equal(created.taxAtSource, true); + assert.equal(created.needsTaxReview, false); + assert.equal(created.taxSource, "ocr"); + }); +}); + +test("an implausible read FILLS nothing on an already-booked Purchase", () => { + // Worse here than on a new row: this one may already sit in a filing period + // somebody has reconciled. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + return bookReceipt(row({ totalCents: 10_000, taxCents: 9_000 }), rec.deps).then(() => { + assert.equal(rec.existingExpense.taxAmount, null); + assert.equal(rec.existingExpense.taxAtSource, false); + assert.equal(rec.existingExpense.needsTaxReview, true); + }); +}); From e99f9f602f6addfdc5f3294e9ff7f3e6a91ee499 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 07:27:35 -0700 Subject: [PATCH 100/144] fix(expenses): signed tax, per-decision provenance, locked phase revalidation Four round-16 items. Provenance is per decision. `taxSource` governs the two tax FIGURES and is stamped "manual" only when the PATCH actually carries one, so answering the installed-at-customer question no longer claims a person supplied tax numbers, and clearing the tax back to blank leaves the column alone (a blank is an absence, not a decision, and locking on it would freeze the row out of the pipeline forever). `installedAtCustomer` is its own evidence: non-null means answered, booking fills only a null, and a manual tax figure no longer blocks a capture from answering a question nobody touched. The phase is re-asked a third time INSIDE the booking transaction, after share-locking the job's phase rows. The two earlier checks hold nothing still, so a phase deleted mid-write still reached job cost. A code that is no longer a phase parks the row: booking it posts money to a line the job does not have, booking it uncoded silently discards a captured phase, and the Purchase already exists. The lock helper is shared with the backfill so the two writers of a cost code take it in one order. Amounts are SIGNED. A refund is a negative expense and its tax comes back with it, so the rule is direction and magnitude: the tax matches the sign of the amount (zero always allowed) and never exceeds it. Encoded in the shared bound, in the PATCH (a sign mismatch is a 400 naming the reason, never a constraint violation surfacing as a 500), and in the CHECK, which is now dropped and re-added by name so a database carrying the old refund-refusing definition is corrected. `taxAtSource` tests for zero rather than "not positive", or every credit was refused. The company-financials all-time ranking is two grouped sums again - direct rows, and legacy rows resolved through their estimate - merged in memory. Correct but unbounded is still unbounded: it was fetching every expense ever, for five numbers. The two predicates are disjoint by `projectId: null`, which is the same precedence the row-by-row resolver applies. Co-Authored-By: Claude Fable 5.1 --- .../migration.sql | 27 ++-- prisma/prisma-blind-spots.json | 2 +- scripts/apply-expense-attribution.mjs | 39 ++++-- scripts/backfill-expense-attribution.ts | 23 +--- src/app/api/expenses/[id]/route.ts | 66 ++++++++-- src/lib/company-financials-charts.ts | 81 ++++++++---- src/lib/expense-attribution.ts | 31 ++++- src/lib/expense-lock.ts | 30 +++++ src/lib/receipt-intake/book.ts | 53 +++++++- tests/apply-expense-attribution.test.ts | 30 ++++- ...mpany-financials-spend-attribution.test.ts | 115 ++++++++++++++--- tests/expense-attribution.test.ts | 19 ++- tests/expense-edit-authz.test.ts | 92 ++++++++++++- tests/receipt-intake-book.test.ts | 121 ++++++++++++++++-- 14 files changed, 602 insertions(+), 127 deletions(-) diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index a72336877..908bbf170 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -82,22 +82,29 @@ BEGIN END IF; END $$; --- TAX CANNOT EXCEED THE GROSS (Codex round 6, item 2). +-- TAX POINTS THE SAME WAY AS THE MONEY, AND IS NEVER BIGGER THAN IT +-- (Codex round 6 item 2; signed amounts, round 16 item 3). -- -- The deduction base is `amount - taxAmount`, so a tax larger than the gross -- makes it NEGATIVE and the report subtracts money from the filing. The -- taxDeductibleBase CHECK below does not cover it: a row whose allocation is -- NULL has no allocation to violate, and the negative base is computed at read -- time. This closes that hole at the only place both values are always visible. -DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_taxAmount_check' - AND conrelid = '"Expense"'::regclass) THEN - ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" - CHECK ("taxAmount" IS NULL - OR ("taxAmount" >= 0 AND "taxAmount" <= "amount")); - END IF; -END $$; +-- +-- `Expense.amount` is SIGNED: a refund, return or vendor credit is a negative +-- expense and its tax comes back with it (-$50 with -$4 of tax). The first +-- version of this constraint said `taxAmount >= 0 AND taxAmount <= amount`, +-- which refused every one of those rows and would have pushed a bookkeeper into +-- recording a credit as a positive — a filing that ADDS what it should subtract. +-- +-- Dropped and re-added rather than guarded on existence, so a database that +-- already carries the old definition is corrected rather than skipped. Both +-- statements are re-runnable. +ALTER TABLE "Expense" DROP CONSTRAINT IF EXISTS "Expense_taxAmount_check"; +ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" + CHECK ("taxAmount" IS NULL + OR "taxAmount" = 0 + OR (sign("taxAmount") = sign("amount") AND abs("taxAmount") <= abs("amount"))); -- THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). -- diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index fad7e5ade..8810bcd3d 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -92,7 +92,7 @@ { "name": "Expense_taxAmount_check", "table": "\"Expense\"", - "def": "CHECK (((\"taxAmount\" IS NULL) OR ((\"taxAmount\" >= (0)::numeric) AND (\"taxAmount\" <= amount))))" + "def": "CHECK (((\"taxAmount\" IS NULL) OR (\"taxAmount\" = (0)::numeric) OR ((sign(\"taxAmount\") = sign(amount)) AND (abs(\"taxAmount\") <= abs(amount)))))" }, { "name": "Expense_taxDeductibleBase_check", diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 8f9f1c773..8b8927d4d 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -144,19 +144,23 @@ BEGIN END IF; END $$`, - // TAX CANNOT EXCEED THE GROSS (Codex round 6, item 2). The deduction base - // is `amount - taxAmount`, so a tax above the gross makes it NEGATIVE and - // the report subtracts money from the filing. The taxDeductibleBase CHECK - // does not cover it — a NULL allocation has nothing to violate. - `DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_taxAmount_check' - AND conrelid = '"Expense"'::regclass) THEN - ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" - CHECK ("taxAmount" IS NULL - OR ("taxAmount" >= 0 AND "taxAmount" <= "amount")); - END IF; -END $$`, + // TAX POINTS THE SAME WAY AS THE MONEY, AND IS NEVER BIGGER THAN IT + // (round 6 item 2; signed amounts, round 16 item 3). The deduction base is + // `amount - taxAmount`, so a tax above the gross makes it NEGATIVE and the + // report subtracts money from the filing; the taxDeductibleBase CHECK does + // not cover that, since a NULL allocation has nothing to violate. + // + // `amount` is SIGNED — a refund or vendor credit is a negative expense and + // its tax comes back with it. The first version refused every such row. + // + // DROPPED AND RE-ADDED BY NAME rather than skipped when present, so a + // database already carrying the old definition is corrected. Both + // statements are re-runnable. + `ALTER TABLE "Expense" DROP CONSTRAINT IF EXISTS "Expense_taxAmount_check"`, + `ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" + CHECK ("taxAmount" IS NULL + OR "taxAmount" = 0 + OR (sign("taxAmount") = sign("amount") AND abs("taxAmount") <= abs("amount")))`, // THE DEDUCTION INVARIANT, IN THE DATABASE (Codex round 5, item 4). // Enforced only by the API handler before this: read the amount, validate, @@ -246,7 +250,14 @@ export const expectedCheckConstraints = [ { name: "Expense_taxAmount_check", table: "Expense", - mustMatch: [/"taxAmount" IS NULL/, /"taxAmount" >= \(?0/, /"taxAmount" <= amount/], + mustMatch: [ + /"taxAmount" IS NULL/, + /"taxAmount" = \(?0/, + // pg_get_constraintdef renders `amount` unquoted (all-lowercase, + // not a keyword) while the mixed-case column keeps its quotes. + /sign\("taxAmount"\) = sign\(amount\)/, + /abs\("taxAmount"\) <= abs\(amount\)/, + ], }, { name: "Expense_taxDeductibleBase_check", diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 7b0299671..5c147b67a 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -62,7 +62,7 @@ import { resolveExpenseProjectId, } from "../src/lib/expense-attribution"; import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; -import { lockExpense } from "../src/lib/expense-lock"; +import { lockExpense, lockProjectPhaseRowsForShare } from "../src/lib/expense-lock"; import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases"; import { csvCell, csvNumber } from "../src/lib/csv-safe"; @@ -395,22 +395,9 @@ async function lockRowsForShare(tx, table, ids) { ); } -/** - * The job's phase rows — the same universe `readAllowedCodes` reads, held still - * while it reads them. Without this, a phase deleted between the phase read and - * the write puts a code on a job that no longer has it. - */ -async function lockProjectPhaseRows(tx, projectId) { - if (!projectId) return; - await tx.$queryRawUnsafe( - `SELECT ei.id FROM "EstimateItem" ei - JOIN "Estimate" e ON e.id = ei."estimateId" - WHERE e."projectId" = $1 AND ei."costCodeId" IS NOT NULL - ORDER BY ei.id - FOR SHARE OF ei`, - projectId, - ); -} +// The job's phase rows are held by the SHARED helper in expense-lock.ts, which +// the receipt booking takes too — one statement, one lock order, so the two +// writers of a cost code cannot deadlock against each other. /** * One transaction: the derived-from rows are share-locked FIRST, then the @@ -428,7 +415,7 @@ export async function writeUnderAttributionLocks(db, locks, run) { return db.$transaction(async tx => { await lockRowsForShare(tx, '"Estimate"', estimateIds); await lockRowsForShare(tx, '"EstimateItem"', estimateItemIds); - await lockProjectPhaseRows(tx, phaseProjectId); + await lockProjectPhaseRowsForShare(tx, phaseProjectId); await lockExpense(tx, expenseId); return run(tx); }); diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index b6e157ff9..a7abb3d63 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -418,6 +418,16 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // ordinary edits; a flagged one keeps its flag until it is given. const clearsReview = !expense.needsTaxReview || acknowledgesReview; + // A tax FIGURE, not merely a tax-shaped request: `taxSource` governs + // `taxAmount` and `taxDeductibleBase`, so only a non-null value for one + // of those is a person deciding what this column describes. + // `installedAtCustomer` is NOT one of them — its own value is its + // evidence (non-null means answered) and booking already refuses to + // touch it once it is set. + const stampsTaxProvenance = + (editsTaxAmount && body.taxAmount !== null) || + (editsBase && body.taxDeductibleBase !== null); + // `taxReviewAck` is not a column, so a request carrying nothing else // has no field to write. Told, not silently no-opped. if (!editsInstalled && !editsBase && !editsTaxAmount && !editsTaxAtSource && !editsCostCode) { @@ -455,16 +465,33 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id let nextTaxAmount: number | null = null; if (editsTaxAmount && body.taxAmount !== null) { const parsed = Number(body.taxAmount); - if (!Number.isFinite(parsed) || parsed < 0) { + if (!Number.isFinite(parsed)) { return NextResponse.json( - { error: "taxAmount must be a number >= 0, or null." }, + { error: "taxAmount must be a number, or null." }, { status: 400 }, ); } - const ceiling = maxPlausibleTaxAmount(Number(expense.amount)); - if (!isPlausibleReceiptTax(parsed, Number(expense.amount))) { + const gross = Number(expense.amount); + const ceiling = maxPlausibleTaxAmount(gross); + // A REFUND'S TAX IS NEGATIVE. The rule is direction and magnitude, + // not positivity: a positive tax on a negative expense is a dropped + // minus sign, and it would ADD to a filing that should be reduced. + // Answered as a 400 with the reason, never as a constraint + // violation surfacing as a 500. + if (parsed !== 0 && Math.sign(parsed) !== Math.sign(gross)) { return NextResponse.json( - { error: `That tax is implausible for a ${Number(expense.amount).toFixed(2)} receipt (max ${ceiling.toFixed(2)}, 12%).` }, + { + error: gross < 0 + ? "This is a refund, so its tax must be negative too (or zero)." + : "Tax must be positive on a purchase (or zero).", + code: "TAX_SIGN_MISMATCH", + }, + { status: 400 }, + ); + } + if (!isPlausibleReceiptTax(parsed, gross)) { + return NextResponse.json( + { error: `That tax is implausible for a ${gross.toFixed(2)} receipt (max ${ceiling.toFixed(2)} in magnitude, 12%).` }, { status: 400 }, ); } @@ -517,10 +544,15 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // `taxAtSource` asserts "tax was charged on this receipt"; with no tax // figure behind it, it is a claim about nothing and the report filters // it out anyway. Refuse the incoherent pair rather than storing it. + // + // ZERO, not "not positive": on a refund the tax is NEGATIVE and the + // claim is still true — tax was charged on the original purchase and is + // coming back. Testing `<= 0` here refused every credit whose tax a + // bookkeeper tried to record. const resultingAtSource = editsTaxAtSource ? (nextTaxAtSource as boolean) : Boolean(expense.taxAtSource); - if (resultingAtSource && resultingTax <= 0) { + if (resultingAtSource && resultingTax === 0) { return NextResponse.json( { error: "taxAtSource can't be true with no tax amount on the receipt." }, { status: 400 }, @@ -605,14 +637,20 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // where the report sees a cleared row it has not been // given the figures for. ...(clearsReview ? { needsTaxReview: false } : {}), - // WHO decided. Everything the intake pipeline writes is - // "ocr" and re-readable; this is a person, and booking - // must not write over it. It matters most in the case - // that leaves no other trace: a bookkeeper deciding - // there is NO tax on a receipt leaves a null taxAmount, - // which without this column cannot be told from - // "nobody has looked yet". - taxSource: "manual", + // PROVENANCE IS PER DECISION, AND `taxSource` COVERS + // EXACTLY TWO COLUMNS: taxAmount and taxDeductibleBase. + // + // It is stamped only when this request actually carries + // one of those FIGURES. Two consequences, both + // deliberate: + // * answering only the installed-at-customer question + // does not claim a person supplied tax numbers, and + // * clearing the tax back to blank leaves the + // provenance alone, so a later OCR read may fill + // it — a blank is an absence, not a decision, and + // locking the column on an absence would freeze the + // row out of the pipeline forever. + ...(stampsTaxProvenance ? { taxSource: "manual" } : {}), } : {}), ...(editsCostCode diff --git a/src/lib/company-financials-charts.ts b/src/lib/company-financials-charts.ts index 15f8bc66b..ffa7f664a 100644 --- a/src/lib/company-financials-charts.ts +++ b/src/lib/company-financials-charts.ts @@ -269,7 +269,8 @@ export async function getCompanyFinancialsChartData( overheadTimeEntries, unpaidSchedules, openRetainers, - allTimeJobExpenses, + allTimeDirectTotals, + allTimeLegacyTotals, timeZone, ] = await Promise.all([ // Collected: paid schedules whose PARENT invoice isn't Draft (item 2), @@ -344,24 +345,34 @@ export async function getCompanyFinancialsChartData( where: { projectId: { in: projectIds }, status: { in: RETAINER_STATUSES }, balanceDue: { gt: 0 } }, select: { balanceDue: true, dueDate: true }, }), - // All-time top-5 ranking universe. + // All-time top-5 ranking universe, AS TWO AGGREGATES. // - // This used to be a SQL `groupBy(["estimateId"])` plus an estimate -> - // project lookup, to avoid materializing every expense row. Phase 3 - // made that WRONG rather than merely dated: grouping by estimateId - // resolves a row's job through its estimate, while the monthly spend - // series below resolves it through `resolveExpenseProjectId`. A - // re-attributed expense therefore ranked under its OLD job and was - // plotted under its new one — the same money in two different places in - // two charts on the same page. + // It was a SQL `groupBy(["estimateId"])`, which Phase 3 made wrong + // rather than merely dated: grouping by estimateId resolves a row's job + // through its estimate, while the monthly series below resolves it + // through `resolveExpenseProjectId`, so a re-attributed expense ranked + // under its OLD job and was plotted under its new one — the same money + // in two places on one page. // - // Neither `groupBy` nor a relation filter can express "projectId, or - // the estimate's projectId when it is null", so the rows are fetched - // and bucketed with the shared resolver. Scope is unchanged (all - // selectable jobs, all-time) and the select is three columns. - prisma.expense.findMany({ - where: expenseForProjectsWhere(allJobIds), - select: { amount: true, projectId: true, estimate: { select: { projectId: true } } }, + // The first fix fetched every expense row and bucketed them with the + // resolver. That was correct and unbounded: an all-time, all-jobs read + // that grows forever, to produce five numbers. + // + // The resolver's rule is a UNION of two disjoint sets — rows that carry + // a `projectId`, and legacy rows that do not and answer through their + // estimate — so it is expressible as two grouped sums with no overlap + // between them, merged in memory. `projectId: null` in the second + // predicate is what makes them disjoint, and it is the same precedence + // `resolveExpenseProjectId` applies row by row. + prisma.expense.groupBy({ + by: ["projectId"], + where: { projectId: { in: allJobIds } }, + _sum: { amount: true }, + }), + prisma.expense.groupBy({ + by: ["estimateId"], + where: { projectId: null, estimate: { projectId: { in: allJobIds } } }, + _sum: { amount: true }, }), resolveCompanyTimeZone(), ]); @@ -444,13 +455,37 @@ export async function getCompanyFinancialsChartData( // Ranking universe is ALL selectable jobs (jobProjects), all-time, ignoring // the current date-range/project filters — this is what keeps a project's // color stable no matter how the filters change. - // Same resolver as the monthly series below — that agreement is the whole - // point, and it is what the regression test pins. + // Same precedence as the monthly series below — that agreement is the + // whole point, and it is what the regression test pins. Here it is spelled + // as the union of the two disjoint groups read above rather than as a + // per-row call, because five numbers do not justify loading the ledger. const allTimeTotals = new Map(); - for (const e of allTimeJobExpenses) { - const pid = resolveExpenseProjectId(e); - if (!pid) continue; // both sides can be null on this schema - allTimeTotals.set(pid, (allTimeTotals.get(pid) ?? 0) + Number(e.amount ?? 0)); + const addAllTime = (projectId: string | null, amount: unknown) => { + if (!projectId) return; // both sides can be null on this schema + allTimeTotals.set(projectId, (allTimeTotals.get(projectId) ?? 0) + Number(amount ?? 0)); + }; + for (const group of allTimeDirectTotals) { + addAllTime(group.projectId ?? null, group._sum?.amount); + } + // The legacy half answers through its estimate, so the estimate ids it + // grouped by are resolved in ONE lookup. An estimate that has since lost + // its project contributes nothing, exactly as the row-by-row resolver + // returned null for it. + const legacyEstimateIds = allTimeLegacyTotals + .map((group) => group.estimateId) + .filter((id): id is string => Boolean(id)); + const legacyProjectByEstimate = legacyEstimateIds.length + ? new Map( + ( + await prisma.estimate.findMany({ + where: { id: { in: legacyEstimateIds } }, + select: { id: true, projectId: true }, + }) + ).map((estimate) => [estimate.id, estimate.projectId]), + ) + : new Map(); + for (const group of allTimeLegacyTotals) { + addAllTime(legacyProjectByEstimate.get(group.estimateId ?? "") ?? null, group._sum?.amount); } const topProjectIds = [...allTimeTotals.entries()] .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)) // tie-break by id: stable across reloads diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index 955b69238..3db0628c1 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -260,21 +260,40 @@ export function resolveExpenseProjectLabel( * The rate is measured against the GROSS `Expense.amount`, which is what both * writers hold; on any receipt this side of the bound the difference from a * pre-tax basis is far smaller than the slack in the 12%. + * + * AMOUNTS ARE SIGNED. A refund, a return or a vendor credit is a NEGATIVE + * expense, and the tax on it comes back too: -$50 with -$4 of tax is an + * ordinary Lowe's return. Treating "negative" as "invalid" would have made + * every credit unclassifiable and pushed a bookkeeper into recording it as a + * positive, which the excise report would then ADD to the deduction instead of + * subtracting it. So the rule is about DIRECTION and MAGNITUDE, not about + * positivity: the tax must point the same way as the money, and it can never be + * larger than the money. */ export const MAX_PLAUSIBLE_TAX_RATE = 0.12; -/** The largest tax figure this receipt could plausibly carry, in dollars. */ +/** + * The largest tax figure this receipt could plausibly carry, as a MAGNITUDE. + * Compare it against `Math.abs(taxAmount)`; the sign is a separate rule. + */ export function maxPlausibleTaxAmount(grossAmount: number): number { - if (!Number.isFinite(grossAmount) || grossAmount <= 0) return 0; - return Math.round(grossAmount * MAX_PLAUSIBLE_TAX_RATE * 100) / 100; + if (!Number.isFinite(grossAmount)) return 0; + return Math.round(Math.abs(grossAmount) * MAX_PLAUSIBLE_TAX_RATE * 100) / 100; } /** * True when `taxAmount` is a believable amount of sales tax on `grossAmount`. - * Zero is plausible ("this receipt had no tax"); a negative one is not. + * + * Zero is always plausible ("this receipt had no tax"). Otherwise the tax must + * carry the SAME SIGN as the amount — a positive tax on a refund is either a + * dropped minus sign or a filing that claims a deduction for money that came + * back — and its magnitude must be within the 12% band. The `<= |amount|` + * half of the database CHECK is implied by that band and stated there too, + * where nothing enforces the band itself. */ export function isPlausibleReceiptTax(taxAmount: number, grossAmount: number): boolean { - if (!Number.isFinite(taxAmount) || taxAmount < 0) return false; + if (!Number.isFinite(taxAmount) || !Number.isFinite(grossAmount)) return false; if (taxAmount === 0) return true; - return taxAmount <= maxPlausibleTaxAmount(grossAmount); + if (Math.sign(taxAmount) !== Math.sign(grossAmount)) return false; + return Math.abs(taxAmount) <= maxPlausibleTaxAmount(grossAmount); } diff --git a/src/lib/expense-lock.ts b/src/lib/expense-lock.ts index c60f428e5..037c96812 100644 --- a/src/lib/expense-lock.ts +++ b/src/lib/expense-lock.ts @@ -54,3 +54,33 @@ export async function lockExpense( expenseLockKey(expenseId), ); } + + +/** + * SHARE-LOCK A JOB'S PHASE ROWS for the rest of the transaction. + * + * "Is this cost code a phase of this job?" is answered from `EstimateItem` + * rows, which anyone with the estimate open can delete. Two writers depend on + * that answer being still true when they act on it — the receipt booking, which + * posts real money against the phase, and the attribution backfill, which + * writes a code onto historical rows — and neither can express the question as + * a predicate on the row it is writing. + * + * `FOR SHARE OF ei` blocks an UPDATE or DELETE of those rows until this + * transaction commits, while leaving other readers (including the other writer) + * free. Ordered by id so two holders can never take them in opposite orders. + */ +export async function lockProjectPhaseRowsForShare( + client: AdvisoryLockClient, + projectId: string | null, +): Promise { + if (!projectId) return; + await client.$queryRawUnsafe( + `SELECT ei.id FROM "EstimateItem" ei + JOIN "Estimate" e ON e.id = ei."estimateId" + WHERE e."projectId" = $1 AND ei."costCodeId" IS NOT NULL + ORDER BY ei.id + FOR SHARE OF ei`, + projectId, + ); +} diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 63d72338e..20ff831d1 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -21,7 +21,7 @@ import { matchCostCode } from "@/lib/project-match"; import { receiptUrlRef } from "./receipt-url"; import { QBO_ATTACHMENT_MAX_BYTES } from "./intake-core"; import { isPlausibleReceiptTax } from "@/lib/expense-attribution"; -import { lockExpense } from "@/lib/expense-lock"; +import { lockExpense, lockProjectPhaseRowsForShare } from "@/lib/expense-lock"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { QBTimeoutError, @@ -674,6 +674,24 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro : (row.refNumber && row.refNumber !== "NoInv" ? `Invoice ${row.refNumber}` : "Receipt"); const booked = await deps.db.$transaction(async tx => { + // THE PHASE IS RE-ASKED HERE, UNDER A LOCK (Codex round 16, item 2). + // + // The two checks before this one both happened outside any + // transaction: one before the QBO create, one after it. Neither + // holds anything still, so a phase deleted from the job while this + // transaction runs would still be written into job cost. + // + // The phase rows are share-locked first, so from this point the + // answer cannot change under us; then the same question is asked + // again. A code that is no longer a phase of this job PARKS the + // row: booking it would post money to a line the job does not have, + // and booking it UNCODED would silently discard a phase a person + // captured. Neither is ours to decide, and the Purchase already + // exists, so a human is asked instead. + await lockProjectPhaseRowsForShare(tx as any, project.id); + if (costCodeId && !(await deps.isCostCodeAllowed(project.id, costCodeId))) { + throw new PhaseRemovedError(); + } // A retry after a crash between the Purchase and this commit finds // its own Expense here (qbPurchaseId is @unique) — create it twice // and the insert would fail on that constraint anyway. @@ -822,12 +840,18 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro where: { id: existing.id, projectId: expectedProjectId ?? row.projectId, + // ITS OWN VALUE IS THE EVIDENCE. + // + // `installedAtCustomer` is a tri-state: non-null + // MEANS a person answered, so `null` is both the + // "unanswered" state and the entire guard. It is + // deliberately not gated on `taxSource`, which + // governs the two tax FIGURES: a receipt whose tax + // a bookkeeper has not touched must still be able + // to receive the capturer's installed-at-customer + // answer, and a receipt whose tax they HAVE set + // must not thereby block one. installedAtCustomer: null, - // Same reason as the tax fill: a bookkeeper's - // classification is not something a capture may - // revisit, and a NULL column needs the explicit - // branch. - OR: [{ taxSource: null }, { taxSource: { not: "manual" } }], }, data: { installedAtCustomer: row.installedAtCustomer }, }); @@ -1024,6 +1048,16 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro releaseStrongKey: false, }; } + // The phase went away underneath the write. A send WAS attempted, so + // the strong key stays claimed and the Purchase is not re-sent; a + // person re-phases the row and it books on the next pass. + if (error instanceof PhaseRemovedError) { + return { + outcome: "needs-review", + reason: "phase-changed", + releaseStrongKey: false, + }; + } // The Purchase EXISTS at this point. Retrying is correct and safe: the // DocNumber lookup will find it and return alreadyExists:true — and the // key must be RETAINED, which is why this attempt's send flag is passed @@ -1055,6 +1089,13 @@ function describe(error: unknown): string { * deliberate: it rolls the guarded fills back with it, so the row is never left * half-filled against a job it does not belong to. */ +class PhaseRemovedError extends Error { + constructor() { + super("the cost code stopped being a phase of this job while booking"); + this.name = "PhaseRemovedError"; + } +} + class AttributionConflictError extends Error { constructor() { super("the expense moved to another job while booking"); diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 1e2b2f1ef..2f933d367 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -151,7 +151,17 @@ test("the FK is SET NULL, named the way Prisma would name it, and guarded on its test("every statement is additive — nothing drops, renames, or rewrites data", () => { for (const statement of statements as string[]) { - assert.ok(!/\bDROP\b/i.test(statement), `destructive statement: ${statement}`); + // ONE exception, and it is not data: the tax CHECK is dropped and + // re-added by name so a database carrying the OLD definition (which + // refused every refund) is corrected rather than skipped. Nothing else + // may drop anything, and nothing may drop a table, column or index. + const isConstraintReplace = + /DROP CONSTRAINT IF EXISTS "Expense_taxAmount_check"/.test(statement); + assert.ok( + isConstraintReplace || !/\bDROP\b/i.test(statement), + `destructive statement: ${statement}`, + ); + assert.ok(!/DROP (TABLE|COLUMN|INDEX)/i.test(statement), `destructive: ${statement}`); assert.ok(!/\bDELETE FROM\b/i.test(statement), `destructive statement: ${statement}`); assert.ok(!/\bTRUNCATE\b/i.test(statement), `destructive statement: ${statement}`); } @@ -191,9 +201,23 @@ test("the re-anchor refuses a time zone it cannot safely interpolate", () => { }); test("the tax-vs-gross CHECK is in both DDL paths and in the verifier", () => { - const guard = (statements as string[]).find(s => s.includes("Expense_taxAmount_check")); + const guard = (statements as string[]).find( + s => s.includes("Expense_taxAmount_check") && s.includes("ADD CONSTRAINT"), + ); assert.ok(guard, "the script must carry it"); - assert.match(guard!, /"taxAmount" <= "amount"/); + // Signed: the tax points the same way as the money and is never bigger. + assert.match(guard!, /sign\("taxAmount"\) = sign\("amount"\)/); + assert.match(guard!, /abs\("taxAmount"\) <= abs\("amount"\)/); + // ...and the old definition, which refused every refund, is REPLACED rather + // than left in place on a database that already has it. + const drop = (statements as string[]).find( + s => /DROP CONSTRAINT IF EXISTS "Expense_taxAmount_check"/.test(s), + ); + assert.ok(drop, "the old definition is dropped first"); + assert.ok( + (statements as string[]).indexOf(drop!) < (statements as string[]).indexOf(guard!), + "drop before add", + ); assert.ok( normalizedMigration.includes(normalize(guard!).replace(/;$/, "")), "and the migration must carry the same statement", diff --git a/tests/company-financials-spend-attribution.test.ts b/tests/company-financials-spend-attribution.test.ts index 3cbeb0a98..9a2ec56de 100644 --- a/tests/company-financials-spend-attribution.test.ts +++ b/tests/company-financials-spend-attribution.test.ts @@ -38,7 +38,15 @@ const SETTLED = { }; const expenseCalls: any[] = []; +const groupByCalls: any[] = []; +const estimateCalls: any[] = []; +/** + * The all-time ranking is read as TWO DISJOINT GROUPED SUMS — rows that carry a + * projectId, and legacy rows that answer through their estimate. The fixture's + * re-attributed row is in the first group under its REAL job (job-b), which is + * the whole point: an estimate-keyed group would have put it under job-a. + */ const fakePrisma = { companySettings: { findUnique: async () => ({ timeZone: "America/Los_Angeles" }), @@ -46,19 +54,33 @@ const fakePrisma = { paymentSchedule: { findMany: async () => [] }, retainer: { findMany: async () => [] }, timeEntry: { findMany: async () => [] }, - estimate: { findMany: async () => [] }, + estimate: { + findMany: async (args: any) => { + estimateCalls.push(args); + // The legacy group's estimate -> project lookup. + return [{ id: "est-legacy", projectId: "job-a" }]; + }, + }, expense: { findMany: async (args: any) => { expenseCalls.push(args); - // Three expense reads, told apart by their select — the ranking one - // needs no date, the overhead one no project. + // Two expense row reads now, both date-ranged: in-range job spend + // and the overhead bucket, told apart by their select. const select = args.select ?? {}; - if (!select.date) return [REATTRIBUTED, SETTLED]; // all-time ranking universe if (!select.projectId) return []; // overhead bucket return [REATTRIBUTED, SETTLED]; // in-range job spend }, - groupBy: async () => { - throw new Error("groupBy must no longer be used: it cannot express the resolver's fallback"); + groupBy: async (args: any) => { + groupByCalls.push(args); + if (args.by?.[0] === "projectId") { + return [ + { projectId: "job-b", _sum: { amount: 5000 } }, + { projectId: "job-a", _sum: { amount: 100 } }, + ]; + } + // The legacy half: no projectId of its own, answered through the + // estimate lookup above. + return [{ estimateId: "est-legacy", _sum: { amount: 25 } }]; }, }, }; @@ -136,16 +158,77 @@ test("...and the monthly series puts the same dollars on the same job", async () ); }); -test("the ranking query is the shared both-ways predicate, not a relation-only filter", async () => { +test("the ranking covers BOTH ways a row reaches a job, not just the relation", async () => { + // The predicate that matters is unchanged in meaning — a row counts if it + // carries the project OR if its estimate does — it is now expressed as two + // disjoint aggregates instead of one OR over materialized rows (round 16, + // item 4). A relation-only filter would silently drop every re-attributed + // row, which is exactly the bug this file exists for. + groupByCalls.length = 0; + await load(); + const wheres = groupByCalls.map(call => JSON.stringify(call.where)); + assert.deepEqual(wheres, [ + JSON.stringify({ projectId: { in: ["job-a", "job-b"] } }), + JSON.stringify({ projectId: null, estimate: { projectId: { in: ["job-a", "job-b"] } } }), + ]); + // And the IN-RANGE series still reads the columns the resolver needs, so + // the two charts cannot drift apart again. + const series = expenseCalls.find(call => call.select?.projectId); + assert.equal(series.select.projectId, true); + assert.deepEqual(series.select.estimate, { select: { projectId: true } }); +}); + +// ── the ranking is an AGGREGATE, not a row scan (Codex round 16, item 4) ─── + +test("the all-time ranking materializes NO expense rows", async () => { + // Correct but unbounded is still unbounded: an all-time, all-jobs row fetch + // grows forever to produce five numbers. Every remaining `findMany` on + // Expense must therefore be date-bounded — the ranking has none, so if one + // shows up without a date filter the row scan is back. + expenseCalls.length = 0; + await load(); + for (const call of expenseCalls) { + const where = JSON.stringify(call.where ?? {}); + assert.match( + where + JSON.stringify(call ?? {}), + /date/, + `an unbounded expense row read came back: ${JSON.stringify(call.select)}`, + ); + } + assert.ok( + expenseCalls.every(call => call.select?.date === true), + "every remaining row read is the in-range series or the overhead bucket", + ); +}); + +test("the two grouped sums are DISJOINT, and in the resolver's precedence", async () => { + // Overlapping predicates would double-count a row; `projectId: null` on the + // second is what makes them a partition rather than two overlapping sets, + // and it is the same precedence resolveExpenseProjectId applies row by row. + groupByCalls.length = 0; await load(); - const ranking = expenseCalls.find(call => call.select && !call.select.date); - assert.ok(ranking, "the all-time ranking read must happen"); - assert.deepEqual(ranking.where, { - OR: [ - { projectId: { in: ["job-a", "job-b"] } }, - { projectId: null, estimate: { projectId: { in: ["job-a", "job-b"] } } }, - ], + const direct = groupByCalls.find(call => call.by?.[0] === "projectId"); + const legacy = groupByCalls.find(call => call.by?.[0] === "estimateId"); + assert.ok(direct && legacy, "both halves are read"); + assert.deepEqual(direct.where, { projectId: { in: ["job-a", "job-b"] } }); + assert.deepEqual(legacy.where, { + projectId: null, + estimate: { projectId: { in: ["job-a", "job-b"] } }, }); - assert.equal(ranking.select.projectId, true); - assert.deepEqual(ranking.select.estimate, { select: { projectId: true } }); + assert.deepEqual(direct._sum, { amount: true }); + assert.deepEqual(legacy._sum, { amount: true }); +}); + +test("the legacy half is folded in under its estimate's job", async () => { + // $25 of legacy spend on est-legacy (job-a) has to land on job-a, not be + // dropped and not be ranked under an estimate id. + estimateCalls.length = 0; + const data = await load(); + assert.deepEqual(estimateCalls[0]?.where, { id: { in: ["est-legacy"] } }); + // job-b: 5000, job-a: 100 + 25 — order unchanged, but the legacy dollars + // are counted. + assert.deepEqual( + data.spendByProject.series.map((entry: any) => entry.id), + ["job-b", "job-a", "other"], + ); }); diff --git a/tests/expense-attribution.test.ts b/tests/expense-attribution.test.ts index 2bc346c24..2df6a74e1 100644 --- a/tests/expense-attribution.test.ts +++ b/tests/expense-attribution.test.ts @@ -213,11 +213,12 @@ test("an unattributed row labels as nothing at all", () => { // ── the one tax plausibility bound (Codex round 15, item 1) ──────────────── -test("the tax bound is 12% of the gross, rounded to cents", () => { +test("the tax bound is 12% of the gross MAGNITUDE, rounded to cents", () => { assert.equal(maxPlausibleTaxAmount(100), 12); assert.equal(maxPlausibleTaxAmount(207.74), 24.93); assert.equal(maxPlausibleTaxAmount(0), 0, "no receipt, no allowance"); - assert.equal(maxPlausibleTaxAmount(-5), 0); + // A refund is a negative expense; its allowance is the same size. + assert.equal(maxPlausibleTaxAmount(-100), 12); }); test("zero is plausible, a transposed read is not", () => { @@ -229,6 +230,18 @@ test("zero is plausible, a transposed read is not", () => { assert.equal(isPlausibleReceiptTax(12, 100), true, "the bound itself is allowed"); assert.equal(isPlausibleReceiptTax(12.01, 100), false); assert.equal(isPlausibleReceiptTax(90, 100), false); - assert.equal(isPlausibleReceiptTax(-1, 100), false); assert.equal(isPlausibleReceiptTax(Number.NaN, 100), false); }); + +test("a REFUND's tax is negative, and a positive one on it is refused", () => { + // A return or vendor credit is a negative expense and the tax comes back + // with it. Refusing that shape would push a bookkeeper into recording the + // credit as a positive, which the excise report then ADDS to a deduction it + // should be reducing. + assert.equal(isPlausibleReceiptTax(-4, -50), true, "-$4 of tax on a -$50 return"); + assert.equal(isPlausibleReceiptTax(-6, -50), true, "12% of the magnitude"); + assert.equal(isPlausibleReceiptTax(-6.01, -50), false, "and no further"); + assert.equal(isPlausibleReceiptTax(4, -50), false, "a dropped minus sign"); + assert.equal(isPlausibleReceiptTax(-4, 50), false, "and the same the other way"); + assert.equal(isPlausibleReceiptTax(0, -50), true, "a credit can carry no tax"); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 456e3a0e5..68ffb748a 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -213,14 +213,13 @@ test("PATCH reaches a QBO-managed row — the population the report is made of", test("PATCH touches NOTHING but the three ProBuild-only columns", async () => { await patch({ installedAtCustomer: true }); - // `needsTaxReview` and `taxSource` ride along because answering IS - // clearing the flag and recording who answered — still nothing outside the - // ProBuild-only set. + // `needsTaxReview` rides along because answering IS clearing the flag. + // `taxSource` does NOT: it governs the two tax FIGURES, and this request + // carries neither (round 16, item 1). assert.deepEqual( Object.keys(updateArgs?.data ?? {}), - ["installedAtCustomer", "needsTaxReview", "taxSource"], + ["installedAtCustomer", "needsTaxReview"], ); - assert.equal(updateArgs?.data.taxSource, "manual", "a person answered, and booking must not undo it"); // A caller sending a QBO-synced field is told, not silently ignored. const res = await patch({ amount: "1.00" }); assert.equal(res.status, 400); @@ -639,3 +638,86 @@ test("a non-boolean taxReviewAck is refused", async () => { assert.equal(res.status, 400); assert.match((await res.json()).error, /taxReviewAck/); }); + +// ── signed expenses: refunds carry negative tax (Codex round 16, item 3) ─── + +test("a -$50 refund accepts -$4 of tax", async () => { + storedExpense = { + ...(storedExpense as object), + amount: -50, taxAmount: null, taxDeductibleBase: null, + } as Record; + const res = await patch({ taxAmount: -4, taxAtSource: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxAmount, -4); +}); + +test("...and refuses +$4 with a 400, not a constraint violation", async () => { + // A positive tax on a negative expense is a dropped minus sign. Caught in + // the handler, so the caller is told why — the database CHECK behind it + // would surface as a 500 with nothing to act on. + storedExpense = { + ...(storedExpense as object), + amount: -50, taxAmount: null, taxDeductibleBase: null, + } as Record; + const res = await patch({ taxAmount: 4 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, "TAX_SIGN_MISMATCH"); + assert.equal(updateArgs, null, "nothing is written"); +}); + +test("a refund's tax is still bounded by magnitude", async () => { + storedExpense = { + ...(storedExpense as object), + amount: -50, taxAmount: null, taxDeductibleBase: null, + } as Record; + assert.equal((await patch({ taxAmount: -6 })).status, 200, "12% of $50"); + updateArgs = null; + const tooBig = await patch({ taxAmount: -45 }); + assert.equal(tooBig.status, 400); + assert.match((await tooBig.json()).error, /implausible/); + assert.equal(updateArgs, null); +}); + +test("a purchase still refuses a negative tax", async () => { + storedExpense = { + ...(storedExpense as object), + amount: 207.74, taxAmount: null, taxDeductibleBase: null, + } as Record; + const res = await patch({ taxAmount: -4 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, "TAX_SIGN_MISMATCH"); +}); + +// ── provenance is per decision (Codex round 16, item 1) ──────────────────── + +test("answering ONLY installedAtCustomer does not claim the tax figures", async () => { + // Stamping "manual" here would freeze the tax columns out of the pipeline + // on the strength of an answer to a different question. + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxSource, undefined); +}); + +test("CLEARING the tax back to blank leaves the provenance alone", async () => { + // A blank is an absence, not a decision. Locking the column on it would + // stop OCR ever filling the figure again. + const res = await patch({ taxAmount: null, taxAtSource: false }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxAmount, null, "the clear still lands"); + assert.equal(updateArgs?.data.taxSource, undefined, "but nobody claimed a figure"); +}); + +test("supplying either FIGURE stamps manual", async () => { + await patch({ taxAmount: 16.55 }); + assert.equal(updateArgs?.data.taxSource, "manual"); + const afterAmount = updateArgs; + await patch({ taxDeductibleBase: 50 }); + assert.notEqual(updateArgs, afterAmount, "a second write happened"); + assert.equal(updateArgs?.data.taxSource, "manual"); +}); + +test("a phase-only edit touches neither the flag nor the provenance", async () => { + await patch({ costCodeId: null }); + assert.equal(updateArgs?.data.taxSource, undefined); + assert.equal(updateArgs?.data.needsTaxReview, undefined); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 3235fa4e7..f6b952620 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -1075,6 +1075,7 @@ test("the QBO core fires onExistingPurchase before it touches the attachment", ( ); // And the create hook is NOT fired on this path. assert.ok(!body.includes("onBeforeCreate"), "onBeforeCreate belongs to the create path only"); +}); // ── the already-booked Purchase still needs its Phase 3 fields ───────────── @@ -1190,10 +1191,10 @@ test("a bookkeeper's NO-TAX decision is not overwritten by an OCR re-read", asyn assert.equal(result.outcome, "booked"); assert.equal(rec.existingExpense.taxAmount, null, "their answer stands"); assert.equal(rec.existingExpense.taxSource, "manual"); - assert.equal( - rec.existingExpense.installedAtCustomer, null, - "and the capture does not answer the excise question for them either", - ); + // ...while the excise question, which they did NOT answer, is still filled + // from the capture. `taxSource` governs the tax figures only; the + // installed-at-customer answer is its own evidence (round 16, item 1). + assert.equal(rec.existingExpense.installedAtCustomer, true); }); test("a legacy row with no provenance IS filled, and stamped ocr", async () => { @@ -1272,13 +1273,15 @@ test("the per-expense lock is taken BEFORE the read the fill decides from", asyn trace.push("read"); return realFind(args); }; - (rec.deps.db as any).$queryRawUnsafe = async () => { - trace.push("lock"); + (rec.deps.db as any).$queryRawUnsafe = async (query: string) => { + trace.push(query.includes("FOR SHARE") ? "phase-share" : "lock"); return [{ lock_result: null }]; }; await bookReceipt(row(), rec.deps); - // read (the id lookup) -> lock -> read (everything the fill decides from) - assert.deepEqual(trace.slice(0, 3), ["read", "lock", "read"]); + // The job's phase rows are share-locked first (round 16, item 2), then the + // id lookup, then the per-expense lock, then the read every decision is + // made from. + assert.deepEqual(trace.slice(0, 4), ["phase-share", "read", "lock", "read"]); }); // ── an implausible OCR tax is flagged, never booked (round 15, item 1) ───── @@ -1327,3 +1330,105 @@ test("an implausible read FILLS nothing on an already-booked Purchase", () => { assert.equal(rec.existingExpense.needsTaxReview, true); }); }); + +// ── the two provenances do not gate each other (round 16, item 1) ────────── + +test("an ANSWERED installedAtCustomer is never overwritten, whatever taxSource says", () => { + // Its own value is the evidence: non-null means a person answered. This is + // the "no" case, which is the one that costs money if it is flipped — a + // false reads as "not resold" and keeps the receipt off the excise return. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + installedAtCustomer: false, estimate: { projectId: "proj-1" }, + }; + return bookReceipt(row({ installedAtCustomer: true }), rec.deps).then(() => { + assert.equal(rec.existingExpense.installedAtCustomer, false, "their answer stands"); + }); +}); + +test("a manual TAX figure does not block the capture's excise answer", () => { + // The cross-field regression: `taxSource: "manual"` guards taxAmount and + // taxDeductibleBase. Letting it also guard installedAtCustomer meant a + // bookkeeper correcting a tax figure silently stopped every later capture + // from answering a question they never touched. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: 16.55, taxAtSource: true, taxSource: "manual", + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + return bookReceipt(row(), rec.deps).then(() => { + assert.equal(rec.existingExpense.installedAtCustomer, true, "the capture answers it"); + assert.equal(rec.existingExpense.taxAmount, 16.55, "and their figure is untouched"); + assert.equal(rec.existingExpense.taxSource, "manual"); + }); +}); + +// ── the phase is held still across the money write (round 16, item 2) ────── + +test("a phase REMOVED between the read and the write parks, it does not book", async () => { + // Deterministic interleaving on the window that matters. The code was a + // phase of this job at both earlier checks; a person deletes it from the + // estimate while the booking transaction runs. + // + // Booking it would post money to a line the job no longer has. Booking it + // UNCODED would silently discard a phase a person captured. Neither is this + // pipeline's call, and the Purchase already exists — so the row parks. + let asked = 0; + const rec = recorder({ + isCostCodeAllowed: async () => { + asked += 1; + return asked < 3; // valid before the send and after it, gone by the write + }, + }); + const result = await bookReceipt(row({ costCodeId: "cc-demo" }), rec.deps); + + assert.equal(asked, 3, "asked again inside the transaction"); + assert.equal(result.outcome, "needs-review"); + if (result.outcome === "needs-review") { + assert.equal(result.reason, "phase-changed"); + assert.equal(result.releaseStrongKey, false, "the Purchase exists — keep the key"); + } + assert.equal(rec.expenses.length, 0, "no Expense was written"); + assert.equal( + rec.intakeUpdates.filter((update: any) => update.state === "BOOKED").length, 0, + "and nothing was marked BOOKED", + ); +}); + +test("the in-transaction check happens AFTER the phase rows are locked", async () => { + // Asking before the lock answers about a moment the lock then fails to + // preserve — the whole point of taking it. + const order: string[] = []; + const rec = recorder({ + isCostCodeAllowed: async () => { + order.push("ask"); + return true; + }, + }); + (rec.deps.db as any).$queryRawUnsafe = async (query: string) => { + if (query.includes("FOR SHARE")) order.push("phase-share"); + return [{ lock_result: null }]; + }; + await bookReceipt(row({ costCodeId: "cc-demo" }), rec.deps); + // ask (pre-send), ask (post-create), share-lock, ask (inside the tx) + assert.deepEqual(order, ["ask", "ask", "phase-share", "ask"]); +}); + +test("a row with NO phase is not parked by this check", async () => { + // Nothing to revalidate, so the extra question is not even asked — and an + // uncoded receipt still books, exactly as before. + let asked = 0; + const rec = recorder({ + isCostCodeAllowed: async () => { + asked += 1; + return false; // the captured code was never a phase of this job + }, + }); + const result = await bookReceipt(row({ costCodeId: "cc-from-another-job" }), rec.deps); + assert.equal(result.outcome, "booked", "it books UNCODED, as it always did"); + assert.equal(asked, 2, "no third question once there is no code left to check"); + assert.equal(rec.expenses[0].costCodeId, null); +}); From 31fe0ef193e5427a077492f13e7309e6ca227b0c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 08:13:19 -0700 Subject: [PATCH 101/144] fix(expenses): one signed-credit model, and phase validity as a transactional invariant Four round-17 items. ONE SIGNED MODEL FOR CREDITS. A return or vendor credit is a negative expense whose tax comes back with it, and half the pipeline still assumed money only goes out. The tax report summed `taxAmount > 0`, so every credit was dropped and the deduction went on claiming tax that had been refunded; the sync's invalidation compared `existingTax > amount`, and -4 > -50 is true, so it retired the classification on every credit it saw; the deductible-base CHECK demanded `base >= 0`, which made a credit unallocatable. All three are now sign-and-magnitude: the tax and the allocation point the way the money does and never exceed it. A reduced refund that can no longer carry its recorded tax is flagged for review with the tax nulled, never an aborted import. The base CHECK is dropped and re-added by name, in the migration, the apply script and the blind-spots snapshot. PROVENANCE. A PATCH carrying `taxAmount: null` is a bookkeeper saying there is no sales tax on this receipt, so it stamps `taxSource: "manual"` and booking will not write an OCR guess over it; an OMITTED key still leaves the column alone. `taxReviewAck` is now accepted only with both figures present, non-null and coherent with the amount, so an acknowledgement cannot certify an empty row back into the excise report. FINALIZE. The row read selects `installedAtCustomer`, the merge treats it as a captured field, and the publish CAS fences on it. It was the one path that could silently replace a tax answer: the merge saw no stored value, so a late `false` overwrote a captured `true`. PHASE VALIDITY IS NOW A TRANSACTIONAL INVARIANT. `assertPhaseOfProjectTx` locks Project, Estimate, EstimateItem and CostCode FOR SHARE in one fixed order and then answers on the caller's own transaction. Booking, the manual PATCH, finalize and the QBO suggester all use it, so an estimate archived or reassigned, or a cost code deactivated, between the check and the write can no longer be written into job cost. The phase data source also stops handing back deactivated codes, which the validation path had been trusting. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- .../migration.sql | 23 +- prisma/prisma-blind-spots.json | 2 +- scripts/apply-expense-attribution.mjs | 24 +- scripts/backfill-expense-attribution.ts | 5 +- src/app/api/expenses/[id]/route.ts | 118 +++++++-- .../receipts/intake/[id]/finalize/route.ts | 54 ++++- .../[id]/time-expenses/TaxPhaseModal.tsx | 26 +- src/lib/expense-lock.ts | 30 --- src/lib/phase-invariant.ts | 161 +++++++++++++ src/lib/project-phases-db.ts | 8 + src/lib/qbo-expense-sync.ts | 105 +++++--- src/lib/receipt-intake/book.ts | 46 ++-- src/lib/receipt-intake/late-fields.ts | 4 +- src/lib/tax-at-source-report.ts | 9 +- tests/apply-expense-attribution.test.ts | 2 +- tests/expense-edit-authz.test.ts | 85 ++++++- tests/phase-invariant.test.ts | 225 ++++++++++++++++++ tests/qbo-expense-sync.test.ts | 75 ++++++ tests/receipt-intake-book.test.ts | 86 +++++-- tests/receipt-intake-late-fields.test.ts | 51 ++++ tests/tax-at-source-query.test.ts | 5 +- tests/tax-at-source-report.test.ts | 24 ++ 23 files changed, 1011 insertions(+), 161 deletions(-) create mode 100644 src/lib/phase-invariant.ts create mode 100644 tests/phase-invariant.test.ts diff --git a/package.json b/package.json index 7f0104bb0..bcd9a114f 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 908bbf170..08bcdd767 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -117,16 +117,19 @@ ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxAmount_check" -- Prisma cannot express a CHECK, so this lives here by hand and is recorded in -- prisma/prisma-blind-spots.json; scripts/check-migrations-match.mjs asserts it. -- Safe to add: `taxDeductibleBase` is new and every existing row is NULL. -DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_taxDeductibleBase_check' - AND conrelid = '"Expense"'::regclass) THEN - ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" - CHECK ("taxDeductibleBase" IS NULL - OR ("taxDeductibleBase" >= 0 - AND "taxDeductibleBase" <= "amount" - COALESCE("taxAmount", 0))); - END IF; -END $$; +-- SIGNED, for the same reason the tax check is: a return or vendor credit is a +-- negative expense, and the resold portion of it is negative too. `base >= 0` +-- made every credit unallocatable, which is the shape that pushes a bookkeeper +-- into recording it as a positive and ADDING to a filing that should shrink. +-- +-- Dropped and re-added by name, so a database carrying the unsigned definition +-- is corrected rather than skipped. Both statements are re-runnable. +ALTER TABLE "Expense" DROP CONSTRAINT IF EXISTS "Expense_taxDeductibleBase_check"; +ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" + CHECK ("taxDeductibleBase" IS NULL + OR "taxDeductibleBase" = 0 + OR (sign("taxDeductibleBase") = sign("amount") + AND abs("taxDeductibleBase") <= abs("amount" - COALESCE("taxAmount", 0)))); UPDATE "Expense" e SET "projectId" = est."projectId" FROM "Estimate" est diff --git a/prisma/prisma-blind-spots.json b/prisma/prisma-blind-spots.json index 8810bcd3d..b699f5675 100644 --- a/prisma/prisma-blind-spots.json +++ b/prisma/prisma-blind-spots.json @@ -97,7 +97,7 @@ { "name": "Expense_taxDeductibleBase_check", "table": "\"Expense\"", - "def": "CHECK (((\"taxDeductibleBase\" IS NULL) OR ((\"taxDeductibleBase\" >= (0)::numeric) AND (\"taxDeductibleBase\" <= (amount - COALESCE(\"taxAmount\", (0)::numeric))))))" + "def": "CHECK (((\"taxDeductibleBase\" IS NULL) OR (\"taxDeductibleBase\" = (0)::numeric) OR ((sign(\"taxDeductibleBase\") = sign(amount)) AND (abs(\"taxDeductibleBase\") <= abs((amount - COALESCE(\"taxAmount\", (0)::numeric)))))))" }, { "name": "Inspection_required_date_check", diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 8b8927d4d..4dacf3362 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -167,16 +167,15 @@ END $$`, // then UPDATE. A QBO re-sync changing `amount` between those two statements // leaves a row the tax report TRUSTS verbatim. Prisma cannot express a // CHECK, so it is hand-written here and in prisma-blind-spots.json. - `DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint - WHERE conname = 'Expense_taxDeductibleBase_check' - AND conrelid = '"Expense"'::regclass) THEN - ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" - CHECK ("taxDeductibleBase" IS NULL - OR ("taxDeductibleBase" >= 0 - AND "taxDeductibleBase" <= "amount" - COALESCE("taxAmount", 0))); - END IF; -END $$`, + // SIGNED, for the same reason the tax check is: the resold portion of a + // return is negative. `base >= 0` made every credit unallocatable. + // Dropped and re-added by name so an old definition is corrected. + `ALTER TABLE "Expense" DROP CONSTRAINT IF EXISTS "Expense_taxDeductibleBase_check"`, + `ALTER TABLE "Expense" ADD CONSTRAINT "Expense_taxDeductibleBase_check" + CHECK ("taxDeductibleBase" IS NULL + OR "taxDeductibleBase" = 0 + OR (sign("taxDeductibleBase") = sign("amount") + AND abs("taxDeductibleBase") <= abs("amount" - COALESCE("taxAmount", 0))))`, // RE-ANCHOR THE LEGACY UTC-MIDNIGHT ROWS (Codex round 6, item 1). // @@ -264,10 +263,11 @@ export const expectedCheckConstraints = [ table: "Expense", mustMatch: [ /"taxDeductibleBase" IS NULL/, - /"taxDeductibleBase" >= \(?0/, + /"taxDeductibleBase" = \(?0/, + /sign\("taxDeductibleBase"\) = sign\(amount\)/, // pg_get_constraintdef renders `amount` UNQUOTED (all-lowercase, // not a keyword) while the mixed-case columns keep their quotes. - /amount - COALESCE\("taxAmount"/, + /abs\("taxDeductibleBase"\) <= abs\(\(?amount - COALESCE\("taxAmount"/, ], }, ]; diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 5c147b67a..2f50e07ea 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -62,7 +62,8 @@ import { resolveExpenseProjectId, } from "../src/lib/expense-attribution"; import { OVERHEAD_PROJECT_ID } from "../src/lib/overhead-project"; -import { lockExpense, lockProjectPhaseRowsForShare } from "../src/lib/expense-lock"; +import { lockExpense } from "../src/lib/expense-lock"; +import { lockPhaseRowsForShare } from "../src/lib/phase-invariant"; import { PHASE_ELIGIBLE_ESTIMATE_WHERE } from "../src/lib/project-phases"; import { csvCell, csvNumber } from "../src/lib/csv-safe"; @@ -415,7 +416,7 @@ export async function writeUnderAttributionLocks(db, locks, run) { return db.$transaction(async tx => { await lockRowsForShare(tx, '"Estimate"', estimateIds); await lockRowsForShare(tx, '"EstimateItem"', estimateItemIds); - await lockProjectPhaseRowsForShare(tx, phaseProjectId); + await lockPhaseRowsForShare(tx, phaseProjectId); await lockExpense(tx, expenseId); return run(tx); }); diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index a7abb3d63..fbe3511b2 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -12,6 +12,7 @@ import { lockExpense } from "@/lib/expense-lock"; import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; @@ -404,29 +405,54 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id { status: 400 }, ); } + // AN ACKNOWLEDGEMENT MUST CARRY REAL FIGURES. + // + // Clearing the flag says "I have re-checked these numbers", so the + // request has to contain numbers to have checked: both keys present, + // both non-null, both finite, and both coherent with the amount — the + // same sign and magnitude rules the writes themselves enforce. An ack + // carrying `taxAmount: null` would otherwise certify an empty row back + // into the excise report. const acknowledgesReview = body.taxReviewAck === true; - if (acknowledgesReview && !(editsTaxAmount && editsBase)) { - return NextResponse.json( - { - error: "Acknowledging a tax review needs both taxAmount and taxDeductibleBase in the same request.", - code: "TAX_REVIEW_INCOMPLETE", - }, - { status: 400 }, - ); + if (acknowledgesReview) { + const gross = Number(expense.amount); + const ackTax = editsTaxAmount ? Number(body.taxAmount) : Number.NaN; + const ackBase = editsBase ? Number(body.taxDeductibleBase) : Number.NaN; + const coherent = (value: number) => + Number.isFinite(value) && + (value === 0 || Math.sign(value) === Math.sign(gross)) && + Math.abs(value) <= Math.abs(gross); + const complete = + editsTaxAmount && body.taxAmount !== null && + editsBase && body.taxDeductibleBase !== null && + coherent(ackTax) && coherent(ackBase); + if (!complete) { + return NextResponse.json( + { + error: "Acknowledging a tax review needs a real taxAmount and taxDeductibleBase in the same request.", + code: "TAX_REVIEW_INCOMPLETE", + }, + { status: 400 }, + ); + } } // An unflagged row has nothing to clear, so the ack is not required of // ordinary edits; a flagged one keeps its flag until it is given. const clearsReview = !expense.needsTaxReview || acknowledgesReview; - // A tax FIGURE, not merely a tax-shaped request: `taxSource` governs - // `taxAmount` and `taxDeductibleBase`, so only a non-null value for one - // of those is a person deciding what this column describes. - // `installedAtCustomer` is NOT one of them — its own value is its + // A DECISION ABOUT THE TAX FIGURES, which an explicit null IS. + // + // `taxSource` governs `taxAmount` and `taxDeductibleBase`. Sending + // `taxAmount: null` is not an absence — it is a bookkeeper looking at + // the receipt and saying there is no sales tax on it, and booking must + // not then write an OCR guess over that. What leaves the column alone + // is OMITTING the key: the request said nothing about tax, so nobody + // decided anything and a later read may still fill it. + // + // `installedAtCustomer` is NOT one of these — its own value is its // evidence (non-null means answered) and booking already refuses to // touch it once it is set. - const stampsTaxProvenance = - (editsTaxAmount && body.taxAmount !== null) || - (editsBase && body.taxDeductibleBase !== null); + const stampsTaxProvenance = editsTaxAmount || editsBase; // `taxReviewAck` is not a column, so a request carrying nothing else // has no field to write. Told, not silently no-opped. @@ -512,9 +538,24 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id let nextBase: number | null = null; if (editsBase && body.taxDeductibleBase !== null) { const parsed = Number(body.taxDeductibleBase); - if (!Number.isFinite(parsed) || parsed < 0) { + if (!Number.isFinite(parsed)) { return NextResponse.json( - { error: "taxDeductibleBase must be a number ≥ 0, or null." }, + { error: "taxDeductibleBase must be a number, or null." }, + { status: 400 }, + ); + } + // Signed, like the amount it is a portion of: the resold part of a + // -$50 return is negative too. A base pointing the other way is a + // dropped minus sign, and it would ADD to a filing that should be + // reduced. + if (parsed !== 0 && Math.sign(parsed) !== Math.sign(Number(expense.amount))) { + return NextResponse.json( + { + error: Number(expense.amount) < 0 + ? "This is a refund, so its deductible amount must be negative too (or zero)." + : "The deductible amount must be positive on a purchase (or zero).", + code: "BASE_SIGN_MISMATCH", + }, { status: 400 }, ); } @@ -531,9 +572,15 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id const resultingTax = editsTaxAmount ? (nextTaxAmount ?? 0) : Number(expense.taxAmount ?? 0); - if (resultingBase !== null) { + if (resultingBase !== null && resultingBase !== 0) { const ceiling = Math.round((Number(expense.amount) - resultingTax) * 100) / 100; - if (!Number.isFinite(ceiling) || resultingBase > ceiling) { + // MAGNITUDE, because both sides are signed. On a refund the ceiling + // is negative and `base > ceiling` would pass anything. + if ( + !Number.isFinite(ceiling) || + Math.sign(resultingBase) !== Math.sign(ceiling) || + Math.abs(resultingBase) > Math.abs(ceiling) + ) { return NextResponse.json( { error: `The deduction base can't exceed the pre-tax receipt total (${ceiling.toFixed(2)}).` }, { status: 400 }, @@ -575,6 +622,12 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id { status: resolved.status }, ); } + // A FIRST pass, outside the transaction, purely so the + // caller gets a clean 400 instead of a rolled-back write. The + // ANSWER THAT COUNTS is re-taken inside the transaction below, + // where the rows it depends on are locked — this one can go + // stale between here and the write and that is fine, because + // nothing acts on it. const onProject = await isCostCodeAllowedForProject( prismaPhaseDataSource, projectId, @@ -667,9 +720,32 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // that TAKE it, the predicate is what still protects against one that // does not. const written = await prisma.$transaction(async tx => { - await lockExpense(tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, id); - return tx.expense.updateMany({ where: casWhere, data }); + const raw = tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }; + await lockExpense(raw, id); + // THE PHASE ANSWER THAT COUNTS, taken here (round 17, item 5). + // + // The check above ran on the global client and held nothing: an + // estimate archived or reassigned, or the code deactivated, between + // it and this write would still be stamped onto the row. This one + // locks the four tables the answer rests on and reads them on this + // transaction's snapshot, so it cannot go stale before the update. + if (editsCostCode && nextCostCodeId) { + const verdict = await assertPhaseOfProjectTx(raw, projectId, nextCostCodeId); + if (!verdict.ok) return { count: 0, phaseRejected: verdict.reason } as const; + } + const result = await tx.expense.updateMany({ where: casWhere, data }); + return { count: result.count, phaseRejected: null } as const; }); + if (written.phaseRejected) { + return NextResponse.json( + { + error: "That cost code stopped being one of this project's phases while you were editing.", + code: "PHASE_NOT_ON_PROJECT", + reason: written.phaseRejected, + }, + { status: 400 }, + ); + } if (written.count === 0) { return NextResponse.json( { diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index 9c1882d45..d7e00aff0 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -3,6 +3,7 @@ import { prisma } from "@/lib/prisma"; import { authenticateIntake, STAFF_READ_ROLES, type IntakeAuth } from "@/lib/receipt-intake/intake-auth"; import { userCanAccessProject } from "@/lib/mobile-auth"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { optionalBool } from "@/lib/receipt-capture-validation"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; @@ -162,6 +163,13 @@ export async function POST(req: Request, context: { params: Promise<{ id: string id: true, source: true, state: true, stateReason: true, sourceRef: true, storagePath: true, mimeType: true, projectId: true, costCodeId: true, dryRun: true, createdById: true, fileSha256: true, expectedSha256: true, uploadLeaseVersion: true, + // A CAPTURED TAX ANSWER, and therefore part of the merge and the + // publish CAS. Leaving it out of this read made the publish the one + // path that could overwrite it: `mergeCapturedFields` saw no stored + // value, so a late `false` landed on a row that already said `true` + // and the excise report changed answer with nothing recording that + // the first one existed. + installedAtCustomer: true, }, }); if (!row) return NextResponse.json({ ok: false, reason: "not-found" }, { status: 404 }); @@ -334,7 +342,11 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // the job at /start and sent a different one at finalize simply won, and // nothing recorded that the first answer had ever existed. const merged = mergeCapturedFields( - { projectId: row.projectId, costCodeId: row.costCodeId }, + { + projectId: row.projectId, + costCodeId: row.costCodeId, + installedAtCustomer: row.installedAtCustomer, + }, lateFields, ); if ("status" in merged) return NextResponse.json(merged.body, { status: merged.status }); @@ -372,10 +384,32 @@ export async function POST(req: Request, context: { params: Promise<{ id: string // ONE shared seal-and-publish, also used by the worker's stale-STAGING // sweep, so the two publishers cannot diverge on ordering or fencing. + // Set by the commit below when the phase stopped being one while we were + // publishing. Distinct from a lost CAS, which means somebody else moved the + // row — this means the row is fine and the world changed around it. + let phaseRejectedAtPublish: string | null = null; const outcome = await sealAndPublish(row.storagePath, id, check, { seal: sealObject, - commit: async (canonicalPath, values) => { - const { count } = await prisma.receiptIntake.updateMany({ + commit: async (canonicalPath, values) => prisma.$transaction(async tx => { + // THE PHASE ANSWER THAT COUNTS (round 17, item 5). + // + // `authorizePhase` above ran on the global client and held nothing. + // This one locks the four tables the answer rests on and reads them + // on the transaction that is about to publish, so an estimate + // archived or reassigned, or a code deactivated, in that window + // cannot be published onto the row. + if (merged.resulting.costCodeId) { + const verdict = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + merged.resulting.projectId, + merged.resulting.costCodeId, + ); + if (!verdict.ok) { + phaseRejectedAtPublish = verdict.reason; + return 0; + } + } + const { count } = await tx.receiptIntake.updateMany({ // Fenced on the EXACT state and reason observed, on the row // being unclaimed, and on every captured value this publish was // validated against. Anything that moved between the read and @@ -397,13 +431,25 @@ export async function POST(req: Request, context: { params: Promise<{ id: string }, }); return count; - }, + }), dropUpload: uploadPath => deleteObjectOrRecord(uploadPath, "sealed").then(() => undefined), }); if (!outcome) { return NextResponse.json({ ok: false, reason: "storage-unavailable" }, { status: 503 }); } + // Checked BEFORE the lost-CAS branch: nothing was published, and answering + // "already finalized" would be a lie the client acts on. + if (phaseRejectedAtPublish) { + return NextResponse.json( + { + ok: false, + error: "phase-not-on-project", + reason: `the phase stopped being one of this job's phases while publishing (${phaseRejectedAtPublish})`, + }, + { status: 409 }, + ); + } if (!outcome.published) { // The CAS lost — which is now TWO different things. Either another // publisher moved the row (the caller's answer is "already finalized"), diff --git a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx index 1009492fe..137c707d4 100644 --- a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx +++ b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx @@ -71,8 +71,13 @@ export default function TaxPhaseModal({ const parsedTax = taxAmount.trim() === "" ? null : Number(taxAmount); const effectiveTax = parsedTax ?? 0; - const taxCeiling = Math.round(expense.amount * MAX_TAX_RATE * 100) / 100; - const baseCeiling = Math.round((expense.amount - effectiveTax) * 100) / 100; + // SIGNED. A return or vendor credit is a negative expense: its tax and its + // deductible portion are negative too, and the server refuses a figure + // pointing the other way. The inputs follow the receipt rather than + // assuming every expense is money going out. + const isCredit = expense.amount < 0; + const taxCeiling = Math.round(Math.abs(expense.amount) * MAX_TAX_RATE * 100) / 100; + const baseCeiling = Math.round(Math.abs(expense.amount - effectiveTax) * 100) / 100; async function save() { // Only what actually changed. The endpoint refuses unknown keys, and @@ -196,15 +201,19 @@ export default function TaxPhaseModal({ setTaxAmount(event.target.value)} placeholder="0.00" /> - Up to {money(taxCeiling)} (12% of the receipt). Leave blank if the read was wrong and - you don't know the figure. + {isCredit + ? `This is a refund, so enter the tax as a negative, down to -${money(taxCeiling)} (12% of the receipt).` + : `Up to ${money(taxCeiling)} (12% of the receipt).`}{" "} + Leave blank if the read was wrong and you don't know the figure. @@ -215,15 +224,16 @@ export default function TaxPhaseModal({ setBase(event.target.value)} - placeholder={`whole pre-tax total — ${money(Math.max(baseCeiling, 0))}`} + placeholder={`whole pre-tax total — ${isCredit ? "-" : ""}${money(baseCeiling)}`} /> Leave blank to claim the whole pre-tax total. Set it when only part of the receipt was - resold to the client; must be between 0 and {money(Math.max(baseCeiling, 0))}. + resold to the client; it must point the same way as the receipt and cannot exceed{" "} + {isCredit ? `-${money(baseCeiling)}` : money(baseCeiling)}. diff --git a/src/lib/expense-lock.ts b/src/lib/expense-lock.ts index 037c96812..c60f428e5 100644 --- a/src/lib/expense-lock.ts +++ b/src/lib/expense-lock.ts @@ -54,33 +54,3 @@ export async function lockExpense( expenseLockKey(expenseId), ); } - - -/** - * SHARE-LOCK A JOB'S PHASE ROWS for the rest of the transaction. - * - * "Is this cost code a phase of this job?" is answered from `EstimateItem` - * rows, which anyone with the estimate open can delete. Two writers depend on - * that answer being still true when they act on it — the receipt booking, which - * posts real money against the phase, and the attribution backfill, which - * writes a code onto historical rows — and neither can express the question as - * a predicate on the row it is writing. - * - * `FOR SHARE OF ei` blocks an UPDATE or DELETE of those rows until this - * transaction commits, while leaving other readers (including the other writer) - * free. Ordered by id so two holders can never take them in opposite orders. - */ -export async function lockProjectPhaseRowsForShare( - client: AdvisoryLockClient, - projectId: string | null, -): Promise { - if (!projectId) return; - await client.$queryRawUnsafe( - `SELECT ei.id FROM "EstimateItem" ei - JOIN "Estimate" e ON e.id = ei."estimateId" - WHERE e."projectId" = $1 AND ei."costCodeId" IS NOT NULL - ORDER BY ei.id - FOR SHARE OF ei`, - projectId, - ); -} diff --git a/src/lib/phase-invariant.ts b/src/lib/phase-invariant.ts new file mode 100644 index 000000000..e7c1e8741 --- /dev/null +++ b/src/lib/phase-invariant.ts @@ -0,0 +1,161 @@ +// "IS THIS COST CODE A PHASE OF THIS JOB?" AS A TRANSACTIONAL INVARIANT. +// +// Five writers ask that question and then act on the answer: the receipt +// booking, the manual expense PATCH, the two-step finalize, the QBO cost-code +// suggester, and the attribution backfill. Every one of them used to ask it +// through the global Prisma client, outside whatever transaction it was about +// to write in — so the answer was true when it was given and nothing kept it +// true until the write landed. +// +// The facts it depends on live on four tables and any of them can move: +// +// * `Project` — the job itself can be deleted. +// * `Estimate` — can be ARCHIVED, moved to another project, or dropped out +// of the eligible statuses (a "Rejected" revision is not +// committed work). +// * `EstimateItem`— the line that carries the code can be deleted or recoded. +// * `CostCode` — can be DEACTIVATED company-wide. +// +// So the check locks all four FOR SHARE, in one fixed order, and only then +// answers. FOR SHARE blocks an UPDATE or DELETE of those rows until the +// caller's transaction commits, while leaving other readers (including another +// caller of this helper) free — the answer cannot go stale between here and the +// write, and two callers cannot deadlock against each other because the order +// never varies. +// +// It is deliberately NOT a Prisma query. The whole point is that it runs on the +// CALLER'S transaction client, so it sees, and holds, the same snapshot the +// write will use. A helper that quietly reached for the global client would +// reintroduce the bug it exists to close. +import { + PHASE_ELIGIBLE_ESTIMATE_STATUSES, + SAFETY_COST_CODE, + shouldIncludeSafetyPhase, +} from "@/lib/project-phases"; + +/** The structural subset of a Prisma transaction client this needs. */ +export interface PhaseTxClient { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; +} + +/** Why a code is not a phase of a job — the reason, not just "no". */ +export type PhaseRejection = + | "no-project" + | "project-missing" + | "not-a-phase" + | "code-inactive"; + +export type PhaseVerdict = + | { ok: true } + | { ok: false; reason: PhaseRejection }; + +/** + * Read LAZILY, not at module load. Several unit tests patch + * `@/lib/project-phases` at require time and hand back only the export they + * care about; a module-level `.map` over a constant that is absent from the + * stub crashes the whole import for tests that never call this function. + */ +function eligibleEstimateStatuses(): string[] { + return (PHASE_ELIGIBLE_ESTIMATE_STATUSES ?? []).map((status) => status); +} + +/** + * Take the four share locks, in the ONE order every caller uses. + * + * Exported so a caller that needs the job's whole phase list held still (the + * attribution backfill re-reads it under the lock) can take the same locks in + * the same order rather than inventing a second ordering to deadlock against. + */ +export async function lockPhaseRowsForShare( + tx: PhaseTxClient, + projectId: string | null, + costCodeId?: string | null, +): Promise { + if (!projectId) return; + // 1. The job. + await tx.$queryRawUnsafe(`SELECT id FROM "Project" WHERE id = $1 FOR SHARE`, projectId); + // 2. Its estimates. Ordered, so two holders acquire them the same way. + await tx.$queryRawUnsafe( + `SELECT id FROM "Estimate" WHERE "projectId" = $1 ORDER BY id FOR SHARE`, + projectId, + ); + // 3. The line items that carry the codes. `FOR SHARE OF ei` keeps the lock + // off the joined estimate rows, which step 2 already holds. + await tx.$queryRawUnsafe( + `SELECT ei.id FROM "EstimateItem" ei + JOIN "Estimate" e ON e.id = ei."estimateId" + WHERE e."projectId" = $1 + ORDER BY ei.id + FOR SHARE OF ei`, + projectId, + ); + // 4. The cost code itself, when the caller named one: `isActive` is a + // company-wide switch that has nothing to do with this job. + if (costCodeId) { + await tx.$queryRawUnsafe(`SELECT id FROM "CostCode" WHERE id = $1 FOR SHARE`, costCodeId); + } +} + +/** + * Lock, then answer: is `costCodeId` a phase of `projectId` RIGHT NOW, and will + * it still be when this transaction commits? + * + * Mirrors `resolveProjectPhaseCodes` exactly, in SQL: + * * the estimate must belong to this project, be un-archived, and be in one + * of the eligible statuses (committed work, not a draft or a rejection); + * * the cost code must be active; + * * the company Safety phase is allowed on an In Progress project without an + * estimate item, because it is appended to the list rather than discovered + * from one. + * + * A null `costCodeId` is vacuously fine — there is nothing to check — and no + * locks are taken for it. + */ +export async function assertPhaseOfProjectTx( + tx: PhaseTxClient, + projectId: string | null, + costCodeId: string | null, +): Promise { + if (!costCodeId) return { ok: true }; + if (!projectId) return { ok: false, reason: "no-project" }; + + await lockPhaseRowsForShare(tx, projectId, costCodeId); + + const project = (await tx.$queryRawUnsafe( + `SELECT id, status FROM "Project" WHERE id = $1`, + projectId, + )) as { id: string; status: string | null }[]; + if (!project?.length) return { ok: false, reason: "project-missing" }; + + const code = (await tx.$queryRawUnsafe( + `SELECT id, code, "isActive" FROM "CostCode" WHERE id = $1`, + costCodeId, + )) as { id: string; code: string; isActive: boolean }[]; + // An unknown code and a deactivated one are the same answer to the caller: + // it is not a phase anybody may post money to. + if (!code?.length || !code[0].isActive) return { ok: false, reason: "code-inactive" }; + + // The Safety phase is company-wide and never appears on an estimate, so it + // is checked before the estimate join rather than through it. + if (code[0].code === SAFETY_COST_CODE && shouldIncludeSafetyPhase(project[0].status)) { + return { ok: true }; + } + + const statuses = eligibleEstimateStatuses(); + const statusParams = statuses.map((_, index) => `$${index + 3}`).join(", "); + const onProject = (await tx.$queryRawUnsafe( + `SELECT 1 AS ok + FROM "EstimateItem" ei + JOIN "Estimate" e ON e.id = ei."estimateId" + WHERE e."projectId" = $1 + AND ei."costCodeId" = $2 + AND e."archivedAt" IS NULL + AND e.status IN (${statusParams}) + LIMIT 1`, + projectId, + costCodeId, + ...statuses, + )) as unknown[]; + + return onProject?.length ? { ok: true } : { ok: false, reason: "not-a-phase" }; +} diff --git a/src/lib/project-phases-db.ts b/src/lib/project-phases-db.ts index b6b3d4b37..41b2300ec 100644 --- a/src/lib/project-phases-db.ts +++ b/src/lib/project-phases-db.ts @@ -25,6 +25,14 @@ export const prismaPhaseDataSource: PhaseDataSource = { where: { estimate: { projectId, ...PHASE_ELIGIBLE_ESTIMATE_WHERE }, costCodeId: { not: null }, + // A DEACTIVATED code is not a phase anybody may post to. It was + // read through and handed back with `isActive: false` attached, + // which every caller then had to remember to check — and the + // validation path did not, so a code retired company-wide still + // passed "is this a phase of this job?". The Safety phase has + // always been filtered this way (getSafetyCostCode returns null + // for an inactive row); this makes the estimate half agree. + costCode: { isActive: true }, }, select: { costCode: { select: { id: true, code: true, name: true, description: true, isActive: true } } }, distinct: ["costCodeId"], diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 941a7cc91..3c6a103b3 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -15,6 +15,7 @@ import { dateOnlyInTimeZone } from "./tz-date"; import { resolveCompanyTimeZone } from "./company-timezone"; import { isCostCodeAllowedForProject } from "./project-phases"; import { lockExpense } from "./expense-lock"; +import { assertPhaseOfProjectTx, type PhaseTxClient as PhaseTxLike } from "./phase-invariant"; import { prismaPhaseDataSource } from "./project-phases-db"; // Shared with the register merge layer (register-merge.ts, Unified Money // Register plan §4) so the classification values this module WRITES can @@ -707,7 +708,7 @@ export function planQboExpenseUpdate( ? null : Number(existing.taxDeductibleBase); - // A LOWERED GROSS INVALIDATES THE WHOLE TAX CLASSIFICATION. + // A GROSS THAT CANNOT CARRY THE RECORDED TAX INVALIDATES IT. // // If QuickBooks now says the purchase was smaller than the tax a human // recorded, that tax is about a receipt this row no longer describes. @@ -715,20 +716,40 @@ export function planQboExpenseUpdate( // money from the filing; the database CHECK would also refuse the write and // take the entire QBO import down with it. // + // MAGNITUDES AND SIGNS, not `>`. Amounts are signed: a refund is a negative + // expense carrying negative tax, and `-4 > -50` is true, so the old + // comparison retired the classification on every credit it ever saw. The + // test is the database CHECK's own: the tax must point the same way as the + // money and be no larger than it. A REDUCED refund (-$50 becoming -$3 with + // -$4 of tax still on the row) fails that, and the answer is the same as + // for a purchase — clear the classification and ask a person — never an + // aborted sync. + // // So the classification is CLEARED, not clamped — a guessed-down tax is // still a guess on a tax return — and `needsTaxReview` marks the row so a // person is asked rather than the silence being mistaken for "no tax". // `costCodeSource` is deliberately untouched: which PHASE the money is on // is a separate question the gross does not bear on. - if (existingTax !== null && existingTax > write.amount) { + const taxCannotFitGross = + existingTax !== null && + existingTax !== 0 && + (Math.sign(existingTax) !== Math.sign(write.amount) || + Math.abs(existingTax) > Math.abs(write.amount)); + if (taxCannotFitGross) { data.taxAmount = null; data.taxAtSource = false; data.installedAtCustomer = null; data.taxDeductibleBase = null; data.needsTaxReview = true; - } else if (existingBase !== null) { + } else if (existingBase !== null && existingBase !== 0) { + // Same rule for the allocation: it points the way the money does and + // never exceeds the pre-tax remainder in magnitude. const ceiling = Math.round((write.amount - (existingTax ?? 0)) * 100) / 100; - if (!Number.isFinite(ceiling) || existingBase > ceiling) { + const baseCannotFit = + !Number.isFinite(ceiling) || + Math.sign(existingBase) !== Math.sign(ceiling) || + Math.abs(existingBase) > Math.abs(ceiling); + if (baseCannotFit) { // NEVER A SILENT NULL. Clearing the allocation on its own leaves a // row that still reads as a valid deduction — `installedAtCustomer` // is untouched and a null base means "the whole pre-tax total", so @@ -1066,6 +1087,13 @@ export interface QboCostCodeSuggestionInput { } export interface QboCostCodeSuggestionClient { + /** + * Present on a real Prisma client, absent on the pure unit-test stubs. + * When it is here the phase check and the write run in ONE transaction, + * under the shared locks; when it is not, the injected + * `isAllowedForProject` is the check. Neither path skips the question. + */ + $transaction?(callback: (tx: QboCostCodeSuggestionClient & PhaseTxLike) => Promise): Promise; expense: { findUnique(args: { where: { qbPurchaseId: string }; @@ -1180,30 +1208,53 @@ export async function applyQboExpenseCostCodeSuggestion( return "phase-not-on-project"; } + // ONE definition of the write, used by both paths below. + const suggestionWhere = { + qbPurchaseId: input.qbPurchaseId, + // Everything the decision depended on, re-asserted at write time. + // A row re-attributed or coded between the read above and here is + // skipped rather than written on stale reasoning. + projectId: expectedProjectId, + costCodeId: null, + // The exact row version the suggestion was computed from. Without + // `qbSyncToken` a NEWER sync could commit between the read and this + // write, and the row would be coded from the text of a purchase it no + // longer holds — the same staleness the read above fixed, one statement + // later. + ...(stored.updatedAt ? { updatedAt: stored.updatedAt } : {}), + qbSyncToken: stored.qbSyncToken, + ...notHumanCodedExpenseWhere(), + }; + const suggestionData = { + costCodeId, + // "ai" is the spec's value for "a machine chose this". The rules are + // regexes; the label is about provenance, not about technique. + costCodeSource: "ai", + costCodeConfidence: suggestion.confidence, + }; + + // THE TRANSACTIONAL FORM OF THE SAME QUESTION (round 17, item 5). + // + // The check above answers on the global client and holds nothing: an + // estimate archived, or the code deactivated, between it and the write + // would still be stamped onto the row by an automated pass. Where the + // client can open a transaction, the check is re-taken inside it under the + // shared locks and the write happens on that same snapshot. + if (typeof client.$transaction === "function") { + return client.$transaction(async tx => { + const verdict = await assertPhaseOfProjectTx(tx, projectId, costCodeId); + if (!verdict.ok) return "phase-not-on-project"; + const inTx = await tx.expense.updateMany({ + where: suggestionWhere, + data: suggestionData, + }); + return inTx.count > 0 ? "written" : "not-written"; + }); + } + const written = await client.expense.updateMany({ - where: { - qbPurchaseId: input.qbPurchaseId, - // Everything the decision depended on, re-asserted at write time. - // A row re-attributed or coded between the read above and here is - // skipped rather than written on stale reasoning. - projectId: expectedProjectId, - costCodeId: null, - // The exact row version the suggestion was computed from. Without - // `qbSyncToken` a NEWER sync could commit between the read and this - // write, and the row would be coded from the text of a purchase it - // no longer holds — the same staleness the read above fixed, just - // one statement later. - ...(stored.updatedAt ? { updatedAt: stored.updatedAt } : {}), - qbSyncToken: stored.qbSyncToken, - ...notHumanCodedExpenseWhere(), - }, - data: { - costCodeId, - // "ai" is the spec's value for "a machine chose this". The rules are - // regexes; the label is about provenance, not about technique. - costCodeSource: "ai", - costCodeConfidence: suggestion.confidence, - }, + where: suggestionWhere, + data: suggestionData, }); return written.count > 0 ? "written" : "not-written"; } diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 20ff831d1..261b41cfd 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -21,7 +21,8 @@ import { matchCostCode } from "@/lib/project-match"; import { receiptUrlRef } from "./receipt-url"; import { QBO_ATTACHMENT_MAX_BYTES } from "./intake-core"; import { isPlausibleReceiptTax } from "@/lib/expense-attribution"; -import { lockExpense, lockProjectPhaseRowsForShare } from "@/lib/expense-lock"; +import { lockExpense } from "@/lib/expense-lock"; +import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { startOfDateInTimeZone } from "@/lib/tz-date"; import { QBTimeoutError, @@ -674,24 +675,28 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro : (row.refNumber && row.refNumber !== "NoInv" ? `Invoice ${row.refNumber}` : "Receipt"); const booked = await deps.db.$transaction(async tx => { - // THE PHASE IS RE-ASKED HERE, UNDER A LOCK (Codex round 16, item 2). + // THE PHASE IS RE-ASKED HERE, THROUGH THIS TRANSACTION + // (round 16 item 2; round 17 item 5). // - // The two checks before this one both happened outside any - // transaction: one before the QBO create, one after it. Neither - // holds anything still, so a phase deleted from the job while this + // The two checks before this one both went through the global + // datasource, outside any transaction: one before the QBO create, + // one after it. Neither holds anything still, so an estimate + // archived, reassigned, or a cost code deactivated while this // transaction runs would still be written into job cost. // - // The phase rows are share-locked first, so from this point the - // answer cannot change under us; then the same question is asked - // again. A code that is no longer a phase of this job PARKS the - // row: booking it would post money to a line the job does not have, - // and booking it UNCODED would silently discard a phase a person - // captured. Neither is ours to decide, and the Purchase already - // exists, so a human is asked instead. - await lockProjectPhaseRowsForShare(tx as any, project.id); - if (costCodeId && !(await deps.isCostCodeAllowed(project.id, costCodeId))) { - throw new PhaseRemovedError(); - } + // `assertPhaseOfProjectTx` locks the four tables the answer depends + // on and then answers on THIS transaction's snapshot, so from here + // the answer cannot change before the write. A code that is no + // longer a phase PARKS the row: booking it would post money to a + // line the job does not have, and booking it UNCODED would silently + // discard a phase a person captured. Neither is ours to decide, and + // the Purchase already exists, so a human is asked instead. + const phaseStillValid = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + project.id, + costCodeId, + ); + if (!phaseStillValid.ok) throw new PhaseRemovedError(phaseStillValid.reason); // A retry after a crash between the Purchase and this commit finds // its own Expense here (qbPurchaseId is @unique) — create it twice // and the insert would fail on that constraint anyway. @@ -1054,7 +1059,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro if (error instanceof PhaseRemovedError) { return { outcome: "needs-review", - reason: "phase-changed", + reason: `phase-changed:${error.reason}`, releaseStrongKey: false, }; } @@ -1090,9 +1095,12 @@ function describe(error: unknown): string { * half-filled against a job it does not belong to. */ class PhaseRemovedError extends Error { - constructor() { - super("the cost code stopped being a phase of this job while booking"); + /** Carries WHY, so the parked row names the thing a person has to fix. */ + readonly reason: string; + constructor(reason: string) { + super(`the cost code stopped being a phase of this job while booking (${reason})`); this.name = "PhaseRemovedError"; + this.reason = reason; } } diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index 70b578776..845f2263b 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -202,7 +202,7 @@ export type CapturedValues = Record export interface CapturedMerge { /** Only the fields that are actually changing. */ - apply: Record; + apply: Record; /** * The ORIGINAL captured values, for a CAS on the publishing update: if any * of them moved between the read and the write, the publish must lose. @@ -227,7 +227,7 @@ export function mergeCapturedFields( captured: CapturedValues, lateFields: LateFields, ): Denial | CapturedMerge { - const apply: Record = {}; + const apply: Record = {}; const guard: Record = {}; const conflicts: Record = {}; diff --git a/src/lib/tax-at-source-report.ts b/src/lib/tax-at-source-report.ts index 9119f5d13..0531a9f86 100644 --- a/src/lib/tax-at-source-report.ts +++ b/src/lib/tax-at-source-report.ts @@ -15,7 +15,10 @@ // must never be spent as a deduction. Nothing defaults this: it is set at // capture by the person holding the material, or corrected afterwards by a // bookkeeper on the expense edit route, and -// * `taxAmount > 0` — a zero is an answer (no tax), not an absence. +// * `taxAmount != 0` — a zero is an answer (no tax), not an absence. NOT +// `> 0`: a return or vendor credit is a NEGATIVE expense carrying negative +// tax, and it belongs on the filing as a SUBTRACTION. Excluding it would +// leave the deduction claiming tax on material that went back to the store. // // ...and one NEGATIVE condition, which is about the row's LIFECYCLE rather // than its content: `needsTaxReview` must be false. A QBO re-sync that moves @@ -366,7 +369,9 @@ export async function queryTaxAtSourceRows(filters: TaxAtSourceFilters): Promise where: { taxAtSource: true, installedAtCustomer: true, - taxAmount: { gt: 0 }, + // Signed: credits subtract. Excluding them would claim a + // deduction for tax that was refunded. + taxAmount: { not: 0 }, // A row whose gross moved under a human's tax answer is NOT a // deduction until a person looks again. Without this the "null // taxDeductibleBase means the whole pre-tax total" rule would claim diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 2f933d367..c1fed20aa 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -156,7 +156,7 @@ test("every statement is additive — nothing drops, renames, or rewrites data", // refused every refund) is corrected rather than skipped. Nothing else // may drop anything, and nothing may drop a table, column or index. const isConstraintReplace = - /DROP CONSTRAINT IF EXISTS "Expense_taxAmount_check"/.test(statement); + /DROP CONSTRAINT IF EXISTS "Expense_(taxAmount|taxDeductibleBase)_check"/.test(statement); assert.ok( isConstraintReplace || !/\bDROP\b/i.test(statement), `destructive statement: ${statement}`, diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 68ffb748a..5978aa35e 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -698,13 +698,22 @@ test("answering ONLY installedAtCustomer does not claim the tax figures", async assert.equal(updateArgs?.data.taxSource, undefined); }); -test("CLEARING the tax back to blank leaves the provenance alone", async () => { - // A blank is an absence, not a decision. Locking the column on it would - // stop OCR ever filling the figure again. +test("an EXPLICIT null tax is a manual 'no tax' decision", async () => { + // Round 17, item 2. Sending the key with null is a bookkeeper looking at + // the receipt and saying there is no sales tax on it. Booking must not then + // write an OCR guess over that answer, which is what `taxSource` prevents. const res = await patch({ taxAmount: null, taxAtSource: false }); assert.equal(res.status, 200); - assert.equal(updateArgs?.data.taxAmount, null, "the clear still lands"); - assert.equal(updateArgs?.data.taxSource, undefined, "but nobody claimed a figure"); + assert.equal(updateArgs?.data.taxAmount, null, "the clear lands"); + assert.equal(updateArgs?.data.taxSource, "manual", "and it is recorded as theirs"); +}); + +test("an OMITTED tax key leaves the provenance alone", async () => { + // The request said nothing about tax, so nobody decided anything and a + // later OCR read may still fill it. + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxSource, undefined); }); test("supplying either FIGURE stamps manual", async () => { @@ -721,3 +730,69 @@ test("a phase-only edit touches neither the flag nor the provenance", async () = assert.equal(updateArgs?.data.taxSource, undefined); assert.equal(updateArgs?.data.needsTaxReview, undefined); }); + +// ── an acknowledgement must carry real figures (round 17, item 2) ────────── + +test("an ack whose taxAmount is null is refused, and the flag stays", async () => { + // "I re-checked these numbers" has to contain numbers. A null would + // certify an empty row straight back into the excise report. + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: null, taxDeductibleBase: 50 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, "TAX_REVIEW_INCOMPLETE"); + assert.equal(updateArgs, null, "nothing is written, so the flag stands"); +}); + +test("an ack whose figures point the wrong way is refused", async () => { + // Coherent means sign and magnitude, the same rules the writes enforce. + storedExpense = { + ...(storedExpense as object), needsTaxReview: true, amount: 207.74, + } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: -16.55, taxDeductibleBase: 50 }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); + +test("an ack whose base exceeds the receipt is refused", async () => { + storedExpense = { + ...(storedExpense as object), needsTaxReview: true, amount: 207.74, + } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: 16.55, taxDeductibleBase: 500 }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); + +test("a coherent ack still clears the flag", async () => { + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: 16.55, taxDeductibleBase: 50 }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.needsTaxReview, false); +}); + +// ── signed credits, end to end (round 17, item 1) ────────────────────────── + +test("a refund accepts a NEGATIVE deduction base, and refuses a positive one", async () => { + storedExpense = { + ...(storedExpense as object), amount: -50, taxAmount: -4, taxDeductibleBase: null, + } as Record; + const ok = await patch({ taxDeductibleBase: -40 }); + assert.equal(ok.status, 200); + assert.equal(updateArgs?.data.taxDeductibleBase, -40); + + updateArgs = null; + const wrongWay = await patch({ taxDeductibleBase: 40 }); + assert.equal(wrongWay.status, 400); + assert.equal((await wrongWay.json()).code, "BASE_SIGN_MISMATCH"); + assert.equal(updateArgs, null); +}); + +test("a refund's base is bounded by the pre-tax MAGNITUDE", async () => { + storedExpense = { + ...(storedExpense as object), amount: -50, taxAmount: -4, taxDeductibleBase: null, + } as Record; + assert.equal((await patch({ taxDeductibleBase: -46 })).status, 200, "-50 + 4 of tax"); + updateArgs = null; + const tooBig = await patch({ taxDeductibleBase: -47 }); + assert.equal(tooBig.status, 400); + assert.equal(updateArgs, null); +}); diff --git a/tests/phase-invariant.test.ts b/tests/phase-invariant.test.ts new file mode 100644 index 000000000..b8057b18e --- /dev/null +++ b/tests/phase-invariant.test.ts @@ -0,0 +1,225 @@ +/** + * "Is this cost code a phase of this job?" as a TRANSACTIONAL invariant + * (Codex round 17, item 5). + * + * Five writers asked that question through the global Prisma client and then + * wrote on the answer in a transaction that never held it. The facts live on + * four other tables — Project, Estimate, EstimateItem, CostCode — and every one + * of them can move in that window: an estimate archived or reassigned, a line + * item deleted, a cost code retired company-wide. + * + * These tests drive the helper against a scripted database so each of those + * interleavings is deterministic, and pin the LOCK ORDER, which is the only + * thing standing between two callers and a deadlock. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertPhaseOfProjectTx, lockPhaseRowsForShare } from "../src/lib/phase-invariant"; + +interface World { + project: { id: string; status: string } | null; + costCode: { id: string; code: string; isActive: boolean } | null; + /** One row per estimate item, with the estimate facts it hangs off. */ + items: { + costCodeId: string; + projectId: string; + estimateStatus: string; + archivedAt: Date | null; + }[]; +} + +function db(world: World) { + const queries: string[] = []; + const tx = { + async $queryRawUnsafe(query: string, ...args: unknown[]) { + queries.push(query.replace(/\s+/g, " ").trim()); + if (/FOR SHARE/.test(query)) return []; + if (/FROM "Project" WHERE id/.test(query)) { + return world.project ? [world.project] : []; + } + if (/FROM "CostCode" WHERE id/.test(query)) { + return world.costCode ? [world.costCode] : []; + } + if (/FROM "EstimateItem"/.test(query)) { + // The predicate, modelled: project, code, un-archived, and an + // ELIGIBLE status (the statuses arrive as parameters). + const [projectId, costCodeId, ...statuses] = args as string[]; + const hit = world.items.some( + item => + item.projectId === projectId && + item.costCodeId === costCodeId && + item.archivedAt === null && + statuses.includes(item.estimateStatus), + ); + return hit ? [{ ok: 1 }] : []; + } + return []; + }, + }; + return { tx, queries }; +} + +const LIVE: World = { + project: { id: "job-1", status: "In Progress" }, + costCode: { id: "cc-plumb", code: "03-PLUMB", isActive: true }, + items: [ + { costCodeId: "cc-plumb", projectId: "job-1", estimateStatus: "Approved", archivedAt: null }, + ], +}; + +const clone = (world: World): World => ({ + project: world.project ? { ...world.project } : null, + costCode: world.costCode ? { ...world.costCode } : null, + items: world.items.map(item => ({ ...item })), +}); + +// ── the happy answer ─────────────────────────────────────────────────────── + +test("a real phase of the job passes", async () => { + const { tx } = db(clone(LIVE)); + assert.deepEqual(await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), { ok: true }); +}); + +test("no cost code is nothing to check, and takes no locks", async () => { + const { tx, queries } = db(clone(LIVE)); + assert.deepEqual(await assertPhaseOfProjectTx(tx, "job-1", null), { ok: true }); + assert.deepEqual(queries, [], "a vacuous question locks nothing"); +}); + +test("a phase with no job to check it against is refused", async () => { + const { tx, queries } = db(clone(LIVE)); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, null, "cc-plumb"), + { ok: false, reason: "no-project" }, + ); + assert.deepEqual(queries, []); +}); + +// ── the lock order IS the deadlock protection ────────────────────────────── + +test("four tables are share-locked, in one fixed order, BEFORE anything is read", async () => { + const { tx, queries } = db(clone(LIVE)); + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"); + + const locks = queries.filter(query => /FOR SHARE/.test(query)); + assert.equal(locks.length, 4); + assert.match(locks[0], /FROM "Project"/); + assert.match(locks[1], /FROM "Estimate"/); + assert.match(locks[2], /FROM "EstimateItem"/); + assert.match(locks[3], /FROM "CostCode"/); + // Every read happens after every lock — a read taken first describes a + // moment the lock then fails to preserve. + const lastLock = queries.map(q => /FOR SHARE/.test(q)).lastIndexOf(true); + const firstRead = queries.findIndex(q => !/FOR SHARE/.test(q)); + assert.ok(firstRead > lastLock, "locks, then reads"); + // Ordered id scans, so two holders acquire the same rows the same way. + assert.match(locks[1], /ORDER BY id/); + assert.match(locks[2], /ORDER BY ei\.id/); +}); + +test("the same lock order is available to a caller that needs the whole list", async () => { + // The backfill re-reads a job's phases under the lock; it must not invent a + // second ordering to deadlock against the booking. + const { tx, queries } = db(clone(LIVE)); + await lockPhaseRowsForShare(tx, "job-1"); + const tables = queries.map(query => query.match(/FROM "(\w+)"/)?.[1]); + assert.deepEqual(tables, ["Project", "Estimate", "EstimateItem"]); + assert.ok(queries.every(query => /FOR SHARE/.test(query))); +}); + +// ── the interleavings ────────────────────────────────────────────────────── + +test("an ESTIMATE REASSIGNED to another job stops being this job's phase", async () => { + // The item still carries the code; the estimate it hangs off now belongs to + // somebody else's job. Writing the code here would post money onto a line + // this job does not have. + const world = clone(LIVE); + world.items[0].projectId = "job-2"; + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "not-a-phase" }, + ); +}); + +test("an ARCHIVED estimate stops being committed work", async () => { + const world = clone(LIVE); + world.items[0].archivedAt = new Date("2026-09-02T00:00:00.000Z"); + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "not-a-phase" }, + ); +}); + +test("an estimate that fell out of the eligible statuses stops counting", async () => { + // A "Rejected" revision is not committed work — the same predicate the + // clock-in route applies. + const world = clone(LIVE); + world.items[0].estimateStatus = "Rejected"; + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "not-a-phase" }, + ); +}); + +test("a DEACTIVATED cost code is refused, and named as such", async () => { + // Company-wide retirement has nothing to do with this job, so it is a + // different answer from "not a phase of yours". + const world = clone(LIVE); + world.costCode!.isActive = false; + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "code-inactive" }, + ); +}); + +test("a cost code that does not exist is the same answer as a retired one", async () => { + const world = clone(LIVE); + world.costCode = null; + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "code-inactive" }, + ); +}); + +test("a DELETED job accepts nothing", async () => { + const world = clone(LIVE); + world.project = null; + const { tx } = db(world); + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "project-missing" }, + ); +}); + +// ── the Safety phase is company-wide, not an estimate line ───────────────── + +test("the Safety phase passes on an In Progress job with no estimate item", async () => { + const world = clone(LIVE); + world.costCode = { id: "cc-safety", code: "32-SAFETY", isActive: true }; + world.items = []; + const { tx } = db(world); + assert.deepEqual(await assertPhaseOfProjectTx(tx, "job-1", "cc-safety"), { ok: true }); +}); + +test("...but not on a job that is not In Progress, and not while retired", async () => { + const notRunning = clone(LIVE); + notRunning.project = { id: "job-1", status: "Completed" }; + notRunning.costCode = { id: "cc-safety", code: "32-SAFETY", isActive: true }; + notRunning.items = []; + assert.deepEqual( + await assertPhaseOfProjectTx(db(notRunning).tx, "job-1", "cc-safety"), + { ok: false, reason: "not-a-phase" }, + ); + + const retired = clone(LIVE); + retired.costCode = { id: "cc-safety", code: "32-SAFETY", isActive: false }; + assert.deepEqual( + await assertPhaseOfProjectTx(db(retired).tx, "job-1", "cc-safety"), + { ok: false, reason: "code-inactive" }, + ); +}); diff --git a/tests/qbo-expense-sync.test.ts b/tests/qbo-expense-sync.test.ts index 044e9d1dc..b97e815a1 100644 --- a/tests/qbo-expense-sync.test.ts +++ b/tests/qbo-expense-sync.test.ts @@ -2019,3 +2019,78 @@ test("the sync READS taxSource, or the rule above can never fire", () => { assert.equal(row.taxSource, "manual", "and left standing"); })(); }); + +// ── signed credits in the sync (Codex round 17, item 1) ──────────────────── + +test("a REFUND's negative tax is not retired by a re-sync", () => { + // `-4 > -50` is true, so the old comparison retired the classification on + // every credit it ever saw: a -$50 return with -$4 of tax lost its tax + // fields on the next sync, and the filing quietly stopped netting it. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: -50, taxAmount: -4, taxDeductibleBase: -40, + }, + { ...WRITE, amount: -50 }, + ); + assert.ok(!("taxAmount" in plan.data), "the classification stands"); + assert.ok(!("taxDeductibleBase" in plan.data)); + assert.ok(!("needsTaxReview" in plan.data), "and nothing to re-check"); +}); + +test("a REDUCED refund that can no longer carry its tax is flagged, not aborted", () => { + // QBO now says the credit was only $3 while the row still records $4 of tax + // coming back. That violates |tax| <= |amount| — the database CHECK would + // refuse the write and take the whole import down with it — so the + // classification is cleared and a person is asked. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: -50, taxAmount: -4, taxDeductibleBase: -40, + }, + { ...WRITE, amount: -3 }, + ); + assert.equal(plan.data.taxAmount, null); + assert.equal(plan.data.taxAtSource, false); + assert.equal(plan.data.taxDeductibleBase, null); + assert.equal(plan.data.installedAtCustomer, null); + assert.equal(plan.data.needsTaxReview, true, "asked, never silently dropped"); +}); + +test("a credit that FLIPS to a purchase invalidates the classification", () => { + // The tax now points against the money: whatever this row is, it is not the + // receipt somebody classified. + const plan = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: -50, taxAmount: -4, taxDeductibleBase: null, + }, + { ...WRITE, amount: 50 }, + ); + assert.equal(plan.data.taxAmount, null); + assert.equal(plan.data.needsTaxReview, true); +}); + +test("a refund's ALLOCATION is judged on magnitude too", () => { + // -$46 of allocation against a -$50 credit with -$4 of tax is exact; a + // shrunken credit strands it and the row is flagged rather than left + // claiming more than the receipt. + const fits = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: -50, taxAmount: -4, taxDeductibleBase: -46, + }, + { ...WRITE, amount: -50 }, + ); + assert.ok(!("needsTaxReview" in fits.data)); + + const stranded = planQboExpenseUpdate( + { + projectId: "project-1", estimateId: "estimate-1", + amount: -50, taxAmount: -4, taxDeductibleBase: -46, + }, + { ...WRITE, amount: -20 }, + ); + assert.equal(stranded.data.taxDeductibleBase, null); + assert.equal(stranded.data.needsTaxReview, true); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index f6b952620..73fb174cb 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -112,6 +112,12 @@ interface Recorder { events: any[]; expenseUpdates: any[]; existingExpense: any; + /** + * The world the IN-TRANSACTION phase invariant reads (round 17, item 5): + * the job's status and whether the cost code is still active. A test that + * models an archive, a deactivation or a deleted job moves these. + */ + state: { existingExpense: any; projectStatus: string; costCodeActive: boolean }; } function recorder(overrides: Partial = {}, opts: { estimates?: { id: string }[] } = {}): Recorder { @@ -122,7 +128,13 @@ function recorder(overrides: Partial = {}, opts: { estimates?: const expenseUpdates: any[] = []; const events: any[] = []; // Set by a test to model a Purchase that is ALREADY booked. - const state: { existingExpense: any } = { existingExpense: null }; + const state: { existingExpense: any; projectStatus: string; costCodeActive: boolean } = { + existingExpense: null, + // What the phase invariant reads back for this job and code. A test + // that models an archive, a deactivation or a deleted job moves these. + projectStatus: "In Progress", + costCodeActive: true, + }; const tx = { project: { @@ -179,7 +191,25 @@ function recorder(overrides: Partial = {}, opts: { estimates?: }, }, // The fill takes the shared per-expense advisory lock first. - $queryRawUnsafe: async () => [{ lock_result: null }], + // The phase invariant asks its questions in SQL, on THIS transaction + // (round 17, item 5). The fake answers them from the same injected + // `isCostCodeAllowed` rule every test already sets, so a test writes + // one rule and both the pre-send checks and the in-transaction one obey + // it. Locks return whatever; only the three reads matter. + $queryRawUnsafe: async (query: string, ...args: any[]) => { + if (/FROM "Project" WHERE id/.test(query) && /status/.test(query)) { + return [{ id: args[0], status: state.projectStatus }]; + } + if (/FROM "CostCode" WHERE id/.test(query)) { + return state.costCodeActive + ? [{ id: args[0], code: "03-PLUMB", isActive: true }] + : [{ id: args[0], code: "03-PLUMB", isActive: false }]; + } + if (/FROM "EstimateItem"/.test(query) && /LIMIT 1/.test(query)) { + return (await deps.isCostCodeAllowed(args[0], args[1])) ? [{ ok: 1 }] : []; + } + return [{ lock_result: null }]; + }, receiptIntake: { update: async (args: any) => { intakeUpdates.push(args.data); return {}; }, updateMany: async (args: any) => { intakeUpdates.push(args.data); return { count: 1 }; }, @@ -206,6 +236,8 @@ function recorder(overrides: Partial = {}, opts: { estimates?: }; return { deps, sendMarks, purchaseCalls, expenses, intakeUpdates, events, expenseUpdates, + /** The world the in-transaction phase invariant reads. */ + state, set existingExpense(value: any) { state.existingExpense = value; }, get existingExpense() { return state.existingExpense; }, }; @@ -1273,15 +1305,31 @@ test("the per-expense lock is taken BEFORE the read the fill decides from", asyn trace.push("read"); return realFind(args); }; - (rec.deps.db as any).$queryRawUnsafe = async (query: string) => { - trace.push(query.includes("FOR SHARE") ? "phase-share" : "lock"); - return [{ lock_result: null }]; + const realQuery = (rec.deps.db as any).$queryRawUnsafe; + (rec.deps.db as any).$queryRawUnsafe = async (query: string, ...args: any[]) => { + trace.push( + query.includes("FOR SHARE") ? "phase-share" + : query.includes("pg_advisory_xact_lock") ? "lock" + : "phase-read", + ); + return realQuery(query, ...args); }; await bookReceipt(row(), rec.deps); - // The job's phase rows are share-locked first (round 16, item 2), then the - // id lookup, then the per-expense lock, then the read every decision is - // made from. - assert.deepEqual(trace.slice(0, 4), ["phase-share", "read", "lock", "read"]); + // The phase invariant runs FIRST — four share locks then its three reads + // (round 17, item 5) — and only then the id lookup, the per-expense lock, + // and the read every fill decision is made from. + assert.equal(trace[0], "phase-share", "nothing is read before the locks"); + const advisoryAt = trace.indexOf("lock"); + const idReadAt = trace.indexOf("read"); + assert.ok(idReadAt >= 0 && advisoryAt > idReadAt, "the id lookup, then its lock"); + assert.ok( + trace.lastIndexOf("read") > advisoryAt, + "and the decisive read happens INSIDE the per-expense lock", + ); + assert.ok( + trace.slice(0, advisoryAt).filter(entry => entry === "phase-share").length >= 4, + "Project, Estimate, EstimateItem and CostCode are all held", + ); }); // ── an implausible OCR tax is flagged, never booked (round 15, item 1) ───── @@ -1388,7 +1436,8 @@ test("a phase REMOVED between the read and the write parks, it does not book", a assert.equal(asked, 3, "asked again inside the transaction"); assert.equal(result.outcome, "needs-review"); if (result.outcome === "needs-review") { - assert.equal(result.reason, "phase-changed"); + // The reason names the thing a person has to fix. + assert.equal(result.reason, "phase-changed:not-a-phase"); assert.equal(result.releaseStrongKey, false, "the Purchase exists — keep the key"); } assert.equal(rec.expenses.length, 0, "no Expense was written"); @@ -1408,13 +1457,22 @@ test("the in-transaction check happens AFTER the phase rows are locked", async ( return true; }, }); - (rec.deps.db as any).$queryRawUnsafe = async (query: string) => { + const passThrough = (rec.deps.db as any).$queryRawUnsafe; + (rec.deps.db as any).$queryRawUnsafe = async (query: string, ...args: any[]) => { if (query.includes("FOR SHARE")) order.push("phase-share"); - return [{ lock_result: null }]; + return passThrough(query, ...args); }; await bookReceipt(row({ costCodeId: "cc-demo" }), rec.deps); - // ask (pre-send), ask (post-create), share-lock, ask (inside the tx) - assert.deepEqual(order, ["ask", "ask", "phase-share", "ask"]); + // ask (pre-send), ask (post-create), then the share locks, and only then + // the question that decides the write. + assert.deepEqual(order.slice(0, 2), ["ask", "ask"]); + const lastShare = order.lastIndexOf("phase-share"); + assert.ok(lastShare > 1, "the locks come after the two stateless checks"); + assert.equal(order[order.length - 1], "ask", "and the decisive question comes last"); + assert.equal( + order.filter(entry => entry === "ask").length, 3, + "asked once more, inside the transaction", + ); }); test("a row with NO phase is not parked by this check", async () => { diff --git a/tests/receipt-intake-late-fields.test.ts b/tests/receipt-intake-late-fields.test.ts index 1f30ae82d..27ba8b03a 100644 --- a/tests/receipt-intake-late-fields.test.ts +++ b/tests/receipt-intake-late-fields.test.ts @@ -381,3 +381,54 @@ test("finalize authorizes the effective project on EVERY session call, before an assert.ok(gate < finalize.indexOf(write), `the gate precedes ${write}`); } }); + +// ── installedAtCustomer is CAPTURED at publish time (round 17, item 3) ────── + +test("the finalize route reads installedAtCustomer and merges it as a captured field", () => { + // The merge rules below are only worth anything if the route actually hands + // the stored value in. It did not: the row read selected projectId and + // costCodeId only, so `mergeCapturedFields` saw no stored answer and a late + // `false` landed on a row that already said `true` — the one path that + // could silently overwrite a tax answer. + const select = FINALIZE.slice(FINALIZE.indexOf("const row = await prisma.receiptIntake.findUnique")); + const head = select.slice(0, select.indexOf("});")); + assert.match(head, /installedAtCustomer: true/, "the row read must select it"); + + const merge = FINALIZE.slice(FINALIZE.indexOf("mergeCapturedFields(")); + const call = merge.slice(0, merge.indexOf("lateFields,")); + assert.match(call, /installedAtCustomer: row\.installedAtCustomer/, "and pass it to the merge"); +}); + +test("a CONFLICTING answer at finalize is a 409, and publishes nothing", () => { + // Stored true, late false. This is the shape that decides how a purchase is + // taxed, so the publish must refuse rather than pick a winner. + const merged = mergeCapturedFields( + { projectId: "proj-a", costCodeId: null, installedAtCustomer: true }, + { installedAtCustomer: false } as never, + ); + assert.ok("status" in merged); + assert.equal(merged.status, 409); + assert.equal(merged.body.error, "late-fields-conflict"); +}); + +test("the publish CAS carries installedAtCustomer, so a concurrent finalize loses", () => { + // Two finalizations of the same row: the first fills the tax answer, the + // second was validated against a row that did not have one. Without the + // captured value in the fence the second would publish over it and both + // would report success. + const merged = mergeCapturedFields( + { projectId: "proj-a", costCodeId: null, installedAtCustomer: null }, + { installedAtCustomer: true } as never, + ); + assert.ok(!("status" in merged)); + assert.equal( + merged.guard.installedAtCustomer, null, + "fenced on the value this publish was validated against", + ); + assert.deepEqual(merged.apply, { installedAtCustomer: true } as never); + // ...and the route applies that guard to the publishing update. + const commit = FINALIZE.slice(FINALIZE.indexOf("state: \"RECEIVED\"")); + const fence = FINALIZE.slice(FINALIZE.indexOf("where: { id, ...publishFence(row)")); + assert.match(fence.slice(0, 120), /\.\.\.merged\.guard/, "the CAS includes every captured value"); + assert.ok(commit.length > 0); +}); diff --git a/tests/tax-at-source-query.test.ts b/tests/tax-at-source-query.test.ts index 470434532..bf9078713 100644 --- a/tests/tax-at-source-query.test.ts +++ b/tests/tax-at-source-query.test.ts @@ -94,7 +94,10 @@ test("the where clause asks for all three conditions POSITIVELY", async () => { // `true`, not `{ not: false }`: a NULL means "nobody said", and a NULL must // never be spent as a tax deduction. assert.equal(where.installedAtCustomer, true); - assert.deepEqual(where.taxAmount, { gt: 0 }, "a $0 tax row is an ANSWER, not a candidate"); + // A figure is PRESENT, signed. `{ gt: 0 }` dropped every vendor credit, + // so the deduction went on claiming tax that had been refunded (round 17, + // item 1). A $0 tax row is still an ANSWER, not a candidate. + assert.deepEqual(where.taxAmount, { not: 0 }, "credits belong on the filing too"); }); test("the date window is the COMPANY quarter, in instants", async () => { diff --git a/tests/tax-at-source-report.test.ts b/tests/tax-at-source-report.test.ts index 73ab5915e..dfdacd16a 100644 --- a/tests/tax-at-source-report.test.ts +++ b/tests/tax-at-source-report.test.ts @@ -338,3 +338,27 @@ test("the exclusion is written POSITIVELY so unattributed rows survive it", () = ], }); }); + +// ── a credit SUBTRACTS from the filing (Codex round 17, item 1) ── + +test("a return nets against the purchases in the same month", () => { + // The excise deduction is the cost of articles actually resold. Material + // that went back to the store was not resold, and the credit carries the + // tax back with it, so it belongs in the total as a subtraction rather than + // being excluded and leaving the deduction overstated. + const purchase = row({ + id: "buy", dayKey: "2026-09-04", + receiptTotalCents: 20774, deductionBaseCents: 19119, taxCents: 1655, + }); + const credit = row({ + id: "return", dayKey: "2026-09-11", + receiptTotalCents: -5000, deductionBaseCents: -4600, taxCents: -400, + }); + const { summary, months } = groupTaxAtSource([purchase, credit]); + assert.equal(summary.count, 2); + assert.equal(summary.taxCents, 1255, "1655 - 400"); + assert.equal(summary.deductionBaseCents, 14519, "19119 - 4600"); + assert.equal(summary.receiptTotalCents, 15774); + assert.equal(months.length, 1, "same month, one group"); + assert.equal(months[0].taxCents, 1255); +}); From 6247f27bcee2a134546cd49b6fac548fd707ff16 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 09:02:02 -0700 Subject: [PATCH 102/144] fix(expenses): four tax states, capture provenance, and the phase invariant everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six round-18 items. taxSource is now four explicit states — null (unreviewed), "ocr", "manual" (a person's figure) and "manual-none" (a person saying this receipt has no sales tax) — documented in the spec with a table, and composed through taxNotHumanDecidedWhere() rather than tested for by hand. An acknowledgement needs the taxAmount key: a figure, or an explicit null. Omitting it is a 400, because a request that says nothing about tax has nothing to certify. A blank deduction base is no longer a null with a remembered meaning: the server computes and stores amount - tax, sign intact. The modal derives taxAtSource from the figure through the shared rule, so a refund's negative tax no longer stores "no tax here" and drops the credit out of the excise report. ReceiptIntake now records WHO captured a phase: "user" for a signed-in person, "machine" for a shared-secret forwarder, derived from the caller at every door and never read off a body. Booking copies the distinction, so a Drive folder name books as a correctable "machine" phase instead of borrowing the authority of a person who picked it. Every Expense writer that sets a cost code now runs assertPhaseOfProjectTx inside its write transaction — the POST route, the legacy PUT, createExpenseCore and the Drive receipt ingest join booking, the PATCH and the QBO suggester — and a tripwire test fails when a new writer appears without it. The rollout script no longer swallows the CompanySettings query: only an ABSENT row falls back to the app default, and an unreadable one aborts rather than re-anchoring a whole table into a zone nobody chose. The re-anchor is idempotent by marker (attributionAnchoredAt), because the time-of-day predicate is not one for a company configured as UTC. The already-booked recovery fills a null receiptUrl and never replaces an existing one: a receipt nobody can open is the difference between a defensible deduction and a number in a spreadsheet. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 58 ++++++-- package.json | 4 +- .../migration.sql | 8 ++ prisma/schema.prisma | 13 ++ scripts/apply-expense-attribution.mjs | 67 ++++++++- src/app/api/expenses/[id]/route.ts | 136 +++++++++++++----- src/app/api/expenses/route.ts | 36 ++++- .../api/integrations/receipt-ingest/route.ts | 34 ++++- .../receipts/intake/[id]/finalize/route.ts | 11 +- src/app/api/receipts/intake/route.ts | 5 +- src/app/api/receipts/intake/start/route.ts | 3 +- .../[id]/time-expenses/TaxPhaseModal.tsx | 15 +- src/lib/expense-attribution.ts | 54 +++++++ src/lib/qbo-expense-sync.ts | 3 +- src/lib/receipt-capture-validation.ts | 18 +++ src/lib/receipt-intake/book.ts | 45 +++++- src/lib/receipt-intake/late-fields.ts | 10 +- src/lib/time-expense-core.ts | 21 ++- tests/apply-expense-attribution.test.ts | 75 ++++++++++ tests/expense-attribution.test.ts | 34 +++++ tests/expense-edit-authz.test.ts | 82 +++++++++-- tests/expense-phase-scope.test.ts | 44 +++++- tests/expense-writer-phase-guard.test.ts | 123 ++++++++++++++++ tests/receipt-intake-book.test.ts | 71 ++++++++- tests/receipt-intake-worker.test.ts | 1 + 25 files changed, 879 insertions(+), 92 deletions(-) create mode 100644 tests/expense-writer-phase-guard.test.ts diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index 0bac8d0c2..1a5f1602d 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -314,17 +314,57 @@ return. As built: * `/reports/tax-paid-at-source` counts ONLY an explicit `true`; * `Expense.taxDeductibleBase` (added in this PR) holds the resold portion of a MIXED receipt, and the report uses it in place of the pre-tax total when it is set; -* **`Expense.taxSource`** records WHO decided the tax columns: `ocr` (the intake pipeline - read it off the receipt) or `manual` (a bookkeeper, through the PATCH). Booking never - overwrites a `manual` decision — including the decision that a receipt has NO tax, which - is a null `taxAmount` and cannot be told from "nobody has looked" without this column. +* **`Expense.taxSource` is a four-state column**, not a label. It governs the two tax + FIGURES (`taxAmount`, `taxDeductibleBase`) and nothing else — `installedAtCustomer` is + its own evidence, since a non-null value already means somebody answered. + + | state | meaning | who writes it | may an automated pass overwrite the figures? | + |---|---|---|---| + | `null` | nobody has looked, or nobody has looked SINCE the figures were invalidated | the initial insert | yes | + | `"ocr"` | the intake pipeline read the figures off the receipt | booking | yes — it is a guess, and re-readable | + | `"manual"` | a person supplied an amount | the PATCH, when the request carries a tax figure | **no** | + | `"manual-none"` | a person looked and said this receipt carries NO sales tax | the PATCH, when the request carries `taxAmount: null` | **no** | + + The last two are `HUMAN_TAX_SOURCES` (`src/lib/expense-attribution.ts`); booking and the + QBO sync both compose `taxNotHumanDecidedWhere()` rather than testing for one of them by + hand. `"manual-none"` exists because a null `taxAmount` alone cannot say whether a person + decided there is no tax or nobody has looked yet — and OCR overwriting the first of those + is a bookkeeper's answer silently replaced by a machine's guess. + + OMITTING the `taxAmount` key leaves `taxSource` untouched: that request said nothing + about tax, so nobody decided anything and a later read may still fill it. +* **A blank deduction base means "the whole pre-tax total", and the SERVER stores it.** + Not a null with a remembered meaning: when a PATCH writes the tax figures and leaves + `taxDeductibleBase` blank, the row is saved with `amount − taxAmount` (sign intact, so a + refund's base is negative). Legacy nulls stay readable and the report still treats them + as the whole pre-tax total; nothing new is added to them. * **`Expense.needsTaxReview` is cleared only by an explicit acknowledgement.** A re-sync that moves the gross on a classified row raises it, and the report skips flagged rows. - Clearing it requires `taxReviewAck: true` in the PATCH body AND both `taxAmount` and - `taxDeductibleBase` in the same request (`installedAtCustomer` is optional — a null - reads as "unanswered" and cannot overstate a deduction). A partial correction is still - accepted; it just leaves the flag standing, because the flag means the WHOLE - classification is in doubt rather than whichever field the request happens to touch. + Clearing it requires `taxReviewAck: true` AND the `taxAmount` key in the same request — + either a coherent figure (→ `"manual"`) or an explicit `null` meaning "this receipt has + no sales tax" (→ `"manual-none"`). A request that OMITS `taxAmount` is a 400: it has + nothing to certify. `taxDeductibleBase` is optional (blank is computed, as above) and + `installedAtCustomer` is optional (a null reads as "unanswered" and cannot overstate a + deduction). A partial correction without the ack is still accepted; it just leaves the + flag standing, because the flag means the WHOLE classification is in doubt rather than + whichever field the request happens to touch. +* **`ReceiptIntake.costCodeSource` records WHO captured a phase**: `"user"` (a signed-in + person, through the app or the mobile capture screen) or `"machine"` (a shared-secret + forwarder resolving it from a Drive folder or a mail rule). Booking copies the + distinction onto the Expense — a user's capture books as `capture` and is untouchable, a + machine's books as `machine` and stays correctable by the backfill and the QBO + suggester. It is derived from the CALLER at every door (`captureActorSource(auth.via)`), + never read off a request body, and a row captured before the column existed is treated + as a machine guess. +* **Phase validity is a transactional invariant.** `assertPhaseOfProjectTx` + (`src/lib/phase-invariant.ts`) share-locks `Project` → `Estimate` → `EstimateItem` → + `CostCode` in that fixed order and then answers on the CALLER'S transaction, so an + estimate archived or reassigned, or a cost code deactivated, between the check and the + write cannot be written into job cost. Every Expense writer that sets a cost code calls + it inside its write transaction: the POST route, the PATCH, the legacy PUT, + `createExpenseCore`, the Drive receipt ingest, receipt booking (which parks the row as + `phase-changed:` rather than booking an uncoded receipt) and the QBO suggester. + `tests/expense-writer-phase-guard.test.ts` fails when a new writer appears without it. * the correction path is **`PATCH /api/expenses/[id]`**, NOT the PUT on that route. PUT is guarded by `assertExpenseMutableOutsideQbo`, and every expense the pipeline books carries a `qbPurchaseId` — so PUT cannot reach a single row the tax report is made of, and it now diff --git a/package.json b/package.json index bcd9a114f..55bee153a 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 08bcdd767..e7f40351f 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -31,6 +31,10 @@ ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "needsTaxReview" BOOLEAN NOT NULL -- "there is no tax here", which is a NULL taxAmount and so cannot be told from -- "nobody has looked" without this column. Booking never overwrites "manual". ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxSource" TEXT; +-- The re-anchor marker. See the UPDATE the rollout script runs: a predicate on +-- the time-of-day cannot tell a row that has already been re-anchored from one +-- legitimately written at local midnight, so the fact is recorded. +ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "attributionAnchoredAt" TIMESTAMP(3); -- A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2). -- @@ -143,5 +147,9 @@ DO $$ BEGIN IF to_regclass('"ReceiptIntake"') IS NOT NULL THEN ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; + -- "user" | "machine": who supplied the captured phase. Booking copies the + -- distinction onto the Expense so an automated pass may correct a machine's + -- guess and may never touch a person's answer. + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT; END IF; END $$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1acf48d78..5ae764924 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -663,6 +663,11 @@ model Expense { /// the decision that there is NO tax, which is a null taxAmount and so /// cannot be told from "nobody has looked" without this column. taxSource String? + /// Set by the one-off UTC re-anchor in the Phase 3 rollout script. A row that + /// carries it has already had its `date` moved from UTC midnight to the + /// company's calendar day, so a re-run skips it — the predicate alone cannot + /// tell a re-anchored row from one legitimately written at local midnight. + attributionAnchoredAt DateTime? /// A re-sync changed the gross out from under a tax classification a human /// had made, so the tax fields were cleared and this row needs a person to /// look at it again. Additive and default-false; the tax report ignores it @@ -3103,6 +3108,14 @@ model ReceiptIntake { /// counts, so a missing answer never inflates the deduction. Set at capture /// (the mobile toggle) and copied onto the Expense at booking. installedAtCustomer Boolean? + /// WHO supplied `costCodeId` on this row: "user" (a signed-in person + /// through the app or the mobile capture screen) or "machine" (a + /// shared-secret forwarder resolving it from a Drive folder or a mail + /// rule). Booking copies the distinction onto the Expense, because a + /// machine's phase is a guess a later pass may correct and a person's + /// is not. Nullable: rows captured before this column existed say + /// nothing, and "nothing" is treated as a machine guess. + costCodeSource String? createdById String? createdBy User? @relation(fields: [createdById], references: [id], onDelete: SetNull) diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 4dacf3362..33fe12116 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -62,6 +62,36 @@ export function targetMatches(actual, expectDb, expectHost) { */ export const DEFAULT_COMPANY_TIME_ZONE = "America/Los_Angeles"; +/** + * ONLY AN ABSENT SETTINGS ROW FALLS BACK. + * + * This decides the zone every legacy `Expense.date` is re-anchored INTO, so a + * wrong answer silently moves receipts between quarters on a tax filing. There + * are three distinguishable cases and they must not be collapsed: + * + * * no row at all -> a database that has never had settings written; the + * app's own default is the honest answer. + * * a row with a blank zone -> same thing said differently. + * * a row we could not read -> NOT an answer. The previous version wrapped + * the query in `.catch(() => [undefined])`, so a + * permissions error, a typo'd column, or a dropped + * connection read as "no settings" and quietly re-anchored + * a whole table into Pacific. + * + * So the caller passes the QUERY RESULT and this returns the zone; an error is + * the caller's to throw, and main() no longer swallows it. + */ +export function pickCompanyTimeZone(rows) { + if (!Array.isArray(rows)) { + throw new Error("CompanySettings query did not return rows; refusing to guess the company time zone"); + } + const zone = rows[0]?.timeZone; + if (zone === undefined || zone === null || String(zone).trim() === "") { + return { timeZone: DEFAULT_COMPANY_TIME_ZONE, from: "default" }; + } + return { timeZone: String(zone).trim(), from: "settings" }; +} + /** Built per-run so the zone is the company's, not a hard-coded guess. */ export function reanchorSql(timeZone) { // The zone is interpolated, so it must be a real IANA name and nothing @@ -69,9 +99,23 @@ export function reanchorSql(timeZone) { if (!/^[A-Za-z][A-Za-z0-9+_-]*(\/[A-Za-z0-9+_-]+)*$/.test(timeZone)) { throw new Error(`Refusing to interpolate a suspicious time zone: ${timeZone}`); } + // IDEMPOTENT BY MARKER, not by shape. + // + // The predicate on the time-of-day was the whole guard: re-anchoring moves + // a row off 00:00 UTC, so a second run "could not match it again". That is + // true only when the company zone has an offset. For a company configured + // as UTC the rewrite is the identity, so every row stayed at midnight and + // stayed eligible forever — and a row legitimately written at local + // midnight in such a company is indistinguishable from an un-anchored one. + // + // `attributionAnchoredAt` records the fact instead of inferring it. The + // time-of-day predicate stays, because it is what selects the LEGACY rows + // (everything time-expense-core wrote has always carried a real time). return `UPDATE "Expense" - SET "date" = (("date"::date)::timestamp AT TIME ZONE '${timeZone}') AT TIME ZONE 'UTC' + SET "date" = (("date"::date)::timestamp AT TIME ZONE '${timeZone}') AT TIME ZONE 'UTC', + "attributionAnchoredAt" = now() WHERE "date" IS NOT NULL + AND "attributionAnchoredAt" IS NULL AND "date"::time = TIME '00:00:00'`; } @@ -92,6 +136,10 @@ export const statements = [ // be told from "nobody has looked" without this column. Booking never // overwrites "manual". `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "taxSource" TEXT`, + // The re-anchor marker (see reanchorSql): a predicate on the time-of-day + // cannot tell an already-re-anchored row from one legitimately written at + // local midnight, so the fact is recorded on the row. + `ALTER TABLE "Expense" ADD COLUMN IF NOT EXISTS "attributionAnchoredAt" TIMESTAMP(3)`, // A ROW VERSION FOR THE TAX CORRECTION PATH (Codex round 9, item 2; // ordering fixed in round 13, item 6). @@ -210,6 +258,7 @@ END $$`, IF to_regclass('"ReceiptIntake"') IS NOT NULL THEN ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "taxAtSource" BOOLEAN NOT NULL DEFAULT false; ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "installedAtCustomer" BOOLEAN; + ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT; END IF; END $$`, ]; @@ -218,7 +267,7 @@ export const expectedColumns = { Expense: [ "projectId", "taxAmount", "taxAtSource", "installedAtCustomer", "costCodeSource", "costCodeConfidence", "taxDeductibleBase", "needsTaxReview", - "taxSource", "updatedAt", + "taxSource", "attributionAnchoredAt", "updatedAt", ], }; @@ -303,12 +352,16 @@ async function main() { } // The company zone, for the legacy re-anchor below. Read before the DDL - // so a missing CompanySettings row fails loudly rather than half-way. - const [settings] = await prisma.$queryRawUnsafe( + // so a bad answer fails before anything is written. + // + // NOT wrapped in a catch: an unreadable settings table is not "no + // settings", and treating it as such re-anchors a whole table into a + // zone nobody chose. An absent row is the only thing that falls back. + const settingsRows = await prisma.$queryRawUnsafe( `SELECT "timeZone" FROM "CompanySettings" WHERE id = 'singleton'`, - ).catch(() => [undefined]); - const companyTimeZone = settings?.timeZone || DEFAULT_COMPANY_TIME_ZONE; - console.log(`company time zone for the date re-anchor: ${companyTimeZone}`); + ); + const { timeZone: companyTimeZone, from: zoneFrom } = pickCompanyTimeZone(settingsRows); + console.log(`company time zone for the date re-anchor: ${companyTimeZone} (from ${zoneFrom})`); // ONE TRANSACTION FOR THE WHOLE THING (Codex round 13, item 6). // diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index fbe3511b2..036cd18b8 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -7,7 +7,12 @@ import { QboManagedExpenseError, assertExpenseMutableOutsideQbo, } from "@/lib/qbo-expense-guard"; -import { isPlausibleReceiptTax, maxPlausibleTaxAmount, resolveExpenseProjectId } from "@/lib/expense-attribution"; +import { + isPlausibleReceiptTax, + maxPlausibleTaxAmount, + resolveExpenseProjectId, + taxIsAtSource, +} from "@/lib/expense-attribution"; import { lockExpense } from "@/lib/expense-lock"; import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; @@ -257,7 +262,19 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: // and the description. `undefined` tells Prisma "leave it alone"; // an explicitly-sent null still clears the field. const has = (key: string) => Object.prototype.hasOwnProperty.call(body, key); - const updatedExpense = await prisma.expense.update({ + // THE PHASE ANSWER THAT COUNTS, taken with the write (round 18, item 4). + // Same reason as the POST and the PATCH: the check above holds nothing, + // and this route stamps "manual", which no automated pass may correct. + const legacyWrite = await prisma.$transaction(async tx => { + if (editsCostCode && nextCostCodeId) { + const verdict = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + resolveExpenseProjectId(expense), + nextCostCodeId, + ); + if (!verdict.ok) return { expense: null, phaseRejected: verdict.reason } as const; + } + const updated = await tx.expense.update({ where: { id }, data: { amount: nextAmount, @@ -277,9 +294,21 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: } : {}), }, + }); + return { expense: updated, phaseRejected: null } as const; }); + if (legacyWrite.phaseRejected) { + return NextResponse.json( + { + error: "That cost code stopped being one of this project's phases.", + code: "PHASE_NOT_ON_PROJECT", + reason: legacyWrite.phaseRejected, + }, + { status: 400 }, + ); + } - return NextResponse.json(updatedExpense); + return NextResponse.json(legacyWrite.expense); } catch (error) { if (error instanceof QboManagedExpenseError) { return NextResponse.json({ error: error.message }, { status: 409 }); @@ -405,31 +434,39 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id { status: 400 }, ); } - // AN ACKNOWLEDGEMENT MUST CARRY REAL FIGURES. + // AN ACKNOWLEDGEMENT MUST SAY WHAT WAS DECIDED. + // + // Clearing the flag says "I have re-checked this receipt", and there + // are exactly two answers a person can have re-checked TO: // - // Clearing the flag says "I have re-checked these numbers", so the - // request has to contain numbers to have checked: both keys present, - // both non-null, both finite, and both coherent with the amount — the - // same sign and magnitude rules the writes themselves enforce. An ack - // carrying `taxAmount: null` would otherwise certify an empty row back - // into the excise report. + // * a figure — `taxAmount` a coherent number, or + // * "there is no sales tax on this receipt" — `taxAmount` an explicit + // null, which is a decision and is recorded as `manual-none`. + // + // What is NOT an answer is omitting the key: that request says nothing + // about tax at all, so there is nothing to certify and the flag stands. + // A blank `taxDeductibleBase` is fine either way — the server computes + // and stores the whole pre-tax total below rather than leaving a null + // whose meaning has to be remembered by every reader. const acknowledgesReview = body.taxReviewAck === true; if (acknowledgesReview) { const gross = Number(expense.amount); - const ackTax = editsTaxAmount ? Number(body.taxAmount) : Number.NaN; - const ackBase = editsBase ? Number(body.taxDeductibleBase) : Number.NaN; - const coherent = (value: number) => - Number.isFinite(value) && - (value === 0 || Math.sign(value) === Math.sign(gross)) && - Math.abs(value) <= Math.abs(gross); - const complete = - editsTaxAmount && body.taxAmount !== null && - editsBase && body.taxDeductibleBase !== null && - coherent(ackTax) && coherent(ackBase); - if (!complete) { + const coherent = (value: unknown) => { + const parsed = Number(value); + return ( + Number.isFinite(parsed) && + (parsed === 0 || Math.sign(parsed) === Math.sign(gross)) && + Math.abs(parsed) <= Math.abs(gross) + ); + }; + const answered = + editsTaxAmount && + (body.taxAmount === null || coherent(body.taxAmount)) && + (!editsBase || body.taxDeductibleBase === null || coherent(body.taxDeductibleBase)); + if (!answered) { return NextResponse.json( { - error: "Acknowledging a tax review needs a real taxAmount and taxDeductibleBase in the same request.", + error: "Acknowledging a tax review needs taxAmount in the same request — a figure, or an explicit null meaning this receipt has no sales tax.", code: "TAX_REVIEW_INCOMPLETE", }, { status: 400 }, @@ -440,19 +477,25 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // ordinary edits; a flagged one keeps its flag until it is given. const clearsReview = !expense.needsTaxReview || acknowledgesReview; - // A DECISION ABOUT THE TAX FIGURES, which an explicit null IS. + // WHICH OF THE FOUR STATES THIS REQUEST PUTS THE ROW IN. // - // `taxSource` governs `taxAmount` and `taxDeductibleBase`. Sending - // `taxAmount: null` is not an absence — it is a bookkeeper looking at - // the receipt and saying there is no sales tax on it, and booking must - // not then write an OCR guess over that. What leaves the column alone - // is OMITTING the key: the request said nothing about tax, so nobody - // decided anything and a later read may still fill it. + // `taxSource` governs `taxAmount` and `taxDeductibleBase` (see + // HUMAN_TAX_SOURCES in expense-attribution.ts): + // + // * an explicit `taxAmount: null` is a person saying this receipt has + // NO sales tax -> "manual-none". Not an absence: it is the answer a + // null figure cannot express on its own, and booking must not write + // an OCR guess over it. + // * any other tax-figure edit -> "manual". + // * OMITTING both keys leaves the column alone, so a row nobody has + // spoken about stays open to an automated read. // // `installedAtCustomer` is NOT one of these — its own value is its // evidence (non-null means answered) and booking already refuses to // touch it once it is set. const stampsTaxProvenance = editsTaxAmount || editsBase; + const nextTaxSource = + editsTaxAmount && body.taxAmount === null ? "manual-none" : "manual"; // `taxReviewAck` is not a column, so a request carrying nothing else // has no field to write. Told, not silently no-opped. @@ -596,9 +639,17 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // claim is still true — tax was charged on the original purchase and is // coming back. Testing `<= 0` here refused every credit whose tax a // bookkeeper tried to record. + // DERIVED WHEN THE REQUEST DOES NOT SAY. `taxAtSource` is the fact that + // tax was charged, and it follows the figure — so a request that + // changes the figure and stays silent about the flag gets the flag that + // matches, rather than a 400 about a pair it never asserted. A caller + // that DOES assert both still has to make them agree. + const derivesAtSource = editsTaxAmount && !editsTaxAtSource; const resultingAtSource = editsTaxAtSource ? (nextTaxAtSource as boolean) - : Boolean(expense.taxAtSource); + : derivesAtSource + ? taxIsAtSource(nextTaxAmount) + : Boolean(expense.taxAtSource); if (resultingAtSource && resultingTax === 0) { return NextResponse.json( { error: "taxAtSource can't be true with no tax amount on the receipt." }, @@ -673,11 +724,30 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id estimateId: expense.estimateId, updatedAt: expense.updatedAt, }; + // A BLANK DEDUCTION BASE IS NOT A NULL WITH A MEANING. + // + // "Null means the whole pre-tax total" is a rule every reader has to + // remember, and the report is one of several. When a person edits the + // tax figures and leaves the base blank, the server computes and stores + // what they meant — `amount - tax` — so the row says it outright. The + // legacy nulls stay readable; nothing new adds to them. + const computedBase = + stampsTaxProvenance && (editsBase ? nextBase === null : resultingBase === null) + ? Math.round((Number(expense.amount) - resultingTax) * 100) / 100 + : null; + const writesBase = editsBase || computedBase !== null; + const data = { ...(editsInstalled ? { installedAtCustomer: nextInstalled } : {}), - ...(editsBase ? { taxDeductibleBase: nextBase } : {}), + ...(writesBase + ? { taxDeductibleBase: computedBase !== null ? computedBase : nextBase } + : {}), ...(editsTaxAmount ? { taxAmount: nextTaxAmount } : {}), - ...(editsTaxAtSource ? { taxAtSource: nextTaxAtSource as boolean } : {}), + ...(editsTaxAtSource + ? { taxAtSource: nextTaxAtSource as boolean } + : derivesAtSource + ? { taxAtSource: resultingAtSource } + : {}), // A human just answered, so the row is no longer awaiting one. // Cleared in the SAME write as the answer: two statements would // leave a window where the report sees an answered row it still @@ -703,7 +773,7 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // it — a blank is an absence, not a decision, and // locking the column on an absence would freeze the // row out of the pipeline forever. - ...(stampsTaxProvenance ? { taxSource: "manual" } : {}), + ...(stampsTaxProvenance ? { taxSource: nextTaxSource } : {}), } : {}), ...(editsCostCode diff --git a/src/app/api/expenses/route.ts b/src/app/api/expenses/route.ts index 4dd340318..0cd50cae1 100644 --- a/src/app/api/expenses/route.ts +++ b/src/app/api/expenses/route.ts @@ -5,6 +5,7 @@ import { authenticateMobileOrSession, userCanAccessProject } from "@/lib/mobile- import { resolveCostCode } from "@/lib/cost-coding"; import { prismaCostCodingDataSource } from "@/lib/cost-coding-db"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; +import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; @@ -154,8 +155,23 @@ export async function POST(req: NextRequest) { costTypeId = resolved.costTypeId; } - const newExpense = await prisma.expense.create({ - data: { + // THE PHASE ANSWER THAT COUNTS, taken with the write (round 18, item 4). + // + // The check above ran on the global client and holds nothing: an + // estimate archived or reassigned, or the code deactivated, between it + // and the insert would still be stamped onto a brand new row — as + // "capture", which no automated pass may then correct. + const created = await prisma.$transaction(async tx => { + if (costCodeId) { + const verdict = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + projectId, + costCodeId, + ); + if (!verdict.ok) return { expense: null, phaseRejected: verdict.reason } as const; + } + const expense = await tx.expense.create({ + data: { estimateId, // Phase 3: born with its job. Resolved above in both branches // (derived from the estimate on the web path, checked against @@ -173,10 +189,22 @@ export async function POST(req: NextRequest) { description: description || null, receiptUrl: receiptUrl || null, status: "Pending", - }, + }, + }); + return { expense, phaseRejected: null } as const; }); + if (created.phaseRejected) { + return NextResponse.json( + { + error: "That cost code stopped being one of this project's phases.", + code: "PHASE_NOT_ON_PROJECT", + reason: created.phaseRejected, + }, + { status: 400 }, + ); + } - return NextResponse.json(newExpense); + return NextResponse.json(created.expense); } catch (error: any) { console.error("Error creating expense:", error); return NextResponse.json( diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index 500b75730..20951db70 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/prisma"; +import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { resolveProjectPhaseCodes } from "@/lib/project-phases"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "@/lib/company-timezone"; @@ -117,17 +118,35 @@ export async function POST(req: Request) { .filter(Boolean) .join("; "); - await prisma.expense.create({ + // A MATCHED PHASE IS STILL A CLAIM ABOUT THIS JOB (round 18, item 4). + // + // `matchCostCode` is a string match over the company's codes; it knows + // nothing about which phases this job carries, and nothing here held + // the answer still. The invariant locks the four tables it rests on and + // answers on the transaction that inserts the row; a code that is not + // (or no longer) a phase of this job is dropped rather than posted, + // with the same warning an unmatched category already produces. + const ingested = await prisma.$transaction(async tx => { + let phaseId = costCode?.id ?? null; + if (phaseId) { + const verdict = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + project.id, + phaseId, + ); + if (!verdict.ok) phaseId = null; + } + await tx.expense.create({ data: { estimateId, projectId: project.id, - costCodeId: costCode?.id ?? null, + costCodeId: phaseId, // The category came from the Apps Script's Gemini read, not // from a person — "ai", never "capture", so nothing downstream // treats it as a human's answer. No confidence: matchCostCode // is a string match and has no score to report, and inventing // one would be a guess presented as a measurement. - costCodeSource: costCode ? "ai" : null, + costCodeSource: phaseId ? "ai" : null, costCodeConfidence: null, amount, vendor: body.vendor || "Unknown", @@ -138,8 +157,15 @@ export async function POST(req: Request) { `[Drive import] ${docRef} · ${group.category}` + (lineSummary ? ` · ${lineSummary}` : "") + ` · pending bookkeeper review`, - }, + }, + }); + return phaseId; }); + if (costCode && ingested === null) { + warnings.push( + `"${group.category}" matched ${costCode.code}, which is not a phase of this job — expense created without one`, + ); + } created++; } diff --git a/src/app/api/receipts/intake/[id]/finalize/route.ts b/src/app/api/receipts/intake/[id]/finalize/route.ts index d7e00aff0..4355f4f7b 100644 --- a/src/app/api/receipts/intake/[id]/finalize/route.ts +++ b/src/app/api/receipts/intake/[id]/finalize/route.ts @@ -5,7 +5,7 @@ import { userCanAccessProject } from "@/lib/mobile-auth"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; -import { optionalBool } from "@/lib/receipt-capture-validation"; +import { captureActorSource, optionalBool } from "@/lib/receipt-capture-validation"; import { MAX_STORED_BYTES } from "@/lib/receipt-intake/intake-core"; import { finalizeDisposition, @@ -156,6 +156,15 @@ export async function POST(req: Request, context: { params: Promise<{ id: string const lateFields = Object.fromEntries( Object.entries(lateInput).filter(([, v]) => v !== null), ) as LateFields; + // A LATE PHASE CARRIES THE SAME PROVENANCE A CAPTURED ONE DOES. + // + // Derived from the CALLER, never read off the body: a signed-in person + // answering is "user", a shared-secret forwarder resolving the job from a + // Drive folder is "machine". Booking makes the first untouchable and leaves + // the second correctable, which is the whole point of recording it. + if (lateFields.costCodeId) { + lateFields.costCodeSource = captureActorSource(auth.via); + } const row = await prisma.receiptIntake.findUnique({ where: { id }, diff --git a/src/app/api/receipts/intake/route.ts b/src/app/api/receipts/intake/route.ts index ca8b99775..35863df8c 100644 --- a/src/app/api/receipts/intake/route.ts +++ b/src/app/api/receipts/intake/route.ts @@ -5,7 +5,7 @@ import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; import { authorizePhase } from "@/lib/receipt-intake/late-fields"; import { isCostCodeAllowedForProject } from "@/lib/project-phases"; -import { optionalBool } from "@/lib/receipt-capture-validation"; +import { captureActorSource, optionalBool } from "@/lib/receipt-capture-validation"; import { resolveInstalledAtCustomer } from "@/lib/expense-attribution"; import { prismaPhaseDataSource } from "@/lib/project-phases-db"; import { receiptObjectSize, uploadReceiptObject } from "@/lib/receipt-intake/bucket"; @@ -305,6 +305,9 @@ export async function POST(req: Request) { dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId: parsed.projectId, costCodeId: parsed.costCodeId, + // Only meaningful when a phase was actually supplied; a row + // with no captured code has no captured provenance either. + costCodeSource: parsed.costCodeId ? captureActorSource(auth.via) : null, installedAtCustomer: resolveInstalledAtCustomer(parsed.installedAtCustomer), createdById: auth.via === "session" ? auth.user.id : null, // Only a shared-secret forwarder may assert this: it is the diff --git a/src/app/api/receipts/intake/start/route.ts b/src/app/api/receipts/intake/start/route.ts index fc5f7d0f3..bf1b70806 100644 --- a/src/app/api/receipts/intake/start/route.ts +++ b/src/app/api/receipts/intake/start/route.ts @@ -3,7 +3,7 @@ import { NextResponse } from "next/server"; import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; import { userCanAccessProject } from "@/lib/mobile-auth"; -import { optionalBool } from "@/lib/receipt-capture-validation"; +import { captureActorSource, optionalBool } from "@/lib/receipt-capture-validation"; import { SECURE_BUCKET } from "@/lib/secure-storage"; import { getSupabase } from "@/lib/supabase"; import { authenticateIntake } from "@/lib/receipt-intake/intake-auth"; @@ -148,6 +148,7 @@ export async function POST(req: Request) { dryRun: process.env.RECEIPT_INTAKE_DRYRUN !== "false", projectId, costCodeId, + costCodeSource: costCodeId ? captureActorSource(auth.via) : null, // Tri-state, and nothing defaults it: silence is "nobody said", // which is never claimed on the excise return. installedAtCustomer: optionalBool(body.installedAtCustomer), diff --git a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx index 137c707d4..d1af63db5 100644 --- a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx +++ b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx @@ -2,6 +2,7 @@ import { useState } from "react"; import { toast } from "sonner"; +import { taxIsAtSource } from "@/lib/expense-attribution"; /** * The bookkeeper's correction surface for the WA "tax paid at source" report @@ -89,8 +90,9 @@ export default function TaxPhaseModal({ if (parsedTax !== expense.taxAmount) { body.taxAmount = parsedTax; // `taxAtSource` is the factual "tax was charged here"; it follows - // the figure rather than being a second thing to get wrong. - body.taxAtSource = (parsedTax ?? 0) > 0; + // the figure rather than being a second thing to get wrong. SIGNED: + // a return carries negative tax and the fact still holds. + body.taxAtSource = taxIsAtSource(parsedTax); } const nextBase = base.trim() === "" ? null : Number(base); if (nextBase !== expense.taxDeductibleBase) body.taxDeductibleBase = nextBase; @@ -106,7 +108,7 @@ export default function TaxPhaseModal({ if (expense.needsTaxReview && reviewAck) { body.taxReviewAck = true; body.taxAmount = parsedTax; - body.taxAtSource = (parsedTax ?? 0) > 0; + body.taxAtSource = taxIsAtSource(parsedTax); body.taxDeductibleBase = nextBase; } @@ -231,9 +233,10 @@ export default function TaxPhaseModal({ placeholder={`whole pre-tax total — ${isCredit ? "-" : ""}${money(baseCeiling)}`} /> - Leave blank to claim the whole pre-tax total. Set it when only part of the receipt was - resold to the client; it must point the same way as the receipt and cannot exceed{" "} - {isCredit ? `-${money(baseCeiling)}` : money(baseCeiling)}. + Leave blank and the whole pre-tax total is recorded for you ( + {isCredit ? `-${money(baseCeiling)}` : money(baseCeiling)}). Set it when only part of + the receipt was resold to the client; it must point the same way as the receipt and + cannot exceed that amount. diff --git a/src/lib/expense-attribution.ts b/src/lib/expense-attribution.ts index 3db0628c1..120ff6107 100644 --- a/src/lib/expense-attribution.ts +++ b/src/lib/expense-attribution.ts @@ -297,3 +297,57 @@ export function isPlausibleReceiptTax(taxAmount: number, grossAmount: number): b if (Math.sign(taxAmount) !== Math.sign(grossAmount)) return false; return Math.abs(taxAmount) <= maxPlausibleTaxAmount(grossAmount); } + + +/** + * `Expense.taxSource` — WHO decided the two tax FIGURES (`taxAmount` and + * `taxDeductibleBase`), as four explicit states rather than a boolean dressed + * up as a string: + * + * null nobody has looked, or nobody has looked SINCE the figures + * were invalidated. An automated read may fill them. + * "ocr" the intake pipeline read them off the receipt. A guess, and + * re-readable: a later pass or a person may replace it. + * "manual" a person supplied an amount. Untouchable by any automated + * pass. + * "manual-none" a person looked and said this receipt carries NO sales tax. + * Also untouchable — and it is the state a null `taxAmount` + * cannot express on its own, which is the entire reason this + * column exists. + * + * The last two are the human states, and both must survive a booking, a + * re-sync, and a backfill. + */ +export const HUMAN_TAX_SOURCES = ["manual", "manual-none"] as const; + +/** + * The `where` fragment for "an automated pass may write the tax figures here". + * + * The explicit NULL branch is not decoration: SQL `NOT IN (…)` is NULL for a + * NULL column, so a bare `notIn` silently excludes every legacy row — which is + * the exact opposite of the intent, since those are the rows most in need of a + * first read. + */ +export function taxNotHumanDecidedWhere(): { + OR: ({ taxSource: null } | { taxSource: { notIn: string[] } })[]; +} { + return { + OR: [ + { taxSource: null }, + { taxSource: { notIn: [...HUMAN_TAX_SOURCES] } }, + ], + }; +} + +/** + * `taxAtSource` is the FACT that sales tax was charged on this receipt, and it + * follows the figure rather than being a second thing to get wrong. + * + * Signed: a return carries NEGATIVE tax and the fact is just as true — the tax + * was charged, and is now coming back. `> 0` read every credit as "no tax + * here", which is how a refund's tax quietly left the filing. + */ +export function taxIsAtSource(taxAmount: number | null | undefined): boolean { + if (taxAmount === null || taxAmount === undefined) return false; + return Number.isFinite(taxAmount) && taxAmount !== 0; +} diff --git a/src/lib/qbo-expense-sync.ts b/src/lib/qbo-expense-sync.ts index 3c6a103b3..eefa2c89a 100644 --- a/src/lib/qbo-expense-sync.ts +++ b/src/lib/qbo-expense-sync.ts @@ -7,6 +7,7 @@ import { after } from "next/server"; import { suggestCode } from "./expense-cost-suggest"; import { HUMAN_COST_CODE_SOURCES, + HUMAN_TAX_SOURCES, notHumanCodedExpenseWhere, resolveExpenseProjectId, } from "./expense-attribution"; @@ -785,7 +786,7 @@ export function planQboExpenseUpdate( existingTax !== null || existingBase !== null || existing.installedAtCustomer != null || - existing.taxSource === "manual"; + (HUMAN_TAX_SOURCES as readonly string[]).includes(existing.taxSource ?? ""); const existingAmount = existing.amount === null || existing.amount === undefined ? null : Number(existing.amount); const amountMoved = diff --git a/src/lib/receipt-capture-validation.ts b/src/lib/receipt-capture-validation.ts index ede66e4f2..1fe775cfd 100644 --- a/src/lib/receipt-capture-validation.ts +++ b/src/lib/receipt-capture-validation.ts @@ -20,3 +20,21 @@ export function optionalBool(value: unknown): boolean | null { if (value === "false") return false; return null; } + + +/** + * WHO supplied a captured phase. + * + * A signed-in person picking a phase on their phone is an ANSWER. A + * shared-secret forwarder resolving one from a Drive folder name or a mail rule + * is a GUESS that happens to arrive at capture time, and it has no more + * standing than the suggester's. Booking copies the distinction onto the + * Expense, where "capture" is untouchable and a machine's phase stays + * correctable by the backfill and the QBO suggester. + * + * Recorded at the door, because that is the only place the caller's identity is + * still in hand. + */ +export function captureActorSource(via: "session" | "secret"): "user" | "machine" { + return via === "session" ? "user" : "machine"; +} diff --git a/src/lib/receipt-intake/book.ts b/src/lib/receipt-intake/book.ts index 261b41cfd..77dd9b8ad 100644 --- a/src/lib/receipt-intake/book.ts +++ b/src/lib/receipt-intake/book.ts @@ -20,7 +20,7 @@ import { matchCostCode } from "@/lib/project-match"; import { receiptUrlRef } from "./receipt-url"; import { QBO_ATTACHMENT_MAX_BYTES } from "./intake-core"; -import { isPlausibleReceiptTax } from "@/lib/expense-attribution"; +import { isPlausibleReceiptTax, taxNotHumanDecidedWhere } from "@/lib/expense-attribution"; import { lockExpense } from "@/lib/expense-lock"; import { assertPhaseOfProjectTx } from "@/lib/phase-invariant"; import { startOfDateInTimeZone } from "@/lib/tz-date"; @@ -50,6 +50,13 @@ export interface BookableRow { dryRun: boolean; projectId: string | null; costCodeId: string | null; + /** + * WHO supplied `costCodeId`: "user" (a signed-in person) or "machine" (a + * shared-secret forwarder). Null on rows captured before this existed, and + * treated as a machine guess — the safe direction, since it leaves the + * phase correctable rather than freezing an unattributed guess in place. + */ + costCodeSource: string | null; suggestedCostCodeId: string | null; /** The model's confidence in that phase suggestion, 0..1. */ suggestedConfidence: number | null; @@ -659,8 +666,17 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // captured code that the final project does not carry is dropped by // resolvePhase, and calling the survivor "capture" would then be a // claim about a decision that did not stick. + // A MACHINE'S CAPTURE IS NOT A HUMAN'S. + // + // Everything that arrived as `row.costCodeId` used to book as + // "capture", which `HUMAN_COST_CODE_SOURCES` makes untouchable — so a + // Drive folder name or a mail rule could pin a phase that no later pass + // was allowed to correct, with exactly the authority of a person who + // picked it. The intake row records which it was; booking carries that + // through. + const capturedByHuman = row.costCodeSource === "user"; const costCodeSource = costCodeId - ? (costCodeId === row.costCodeId ? "capture" : "ai") + ? (costCodeId === row.costCodeId ? (capturedByHuman ? "capture" : "machine") : "ai") : null; const costCodeConfidence = costCodeSource === "ai" ? row.suggestedConfidence : null; const driveFileId = driveFileIdOf(row); @@ -725,6 +741,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro taxAmount: true, taxAtSource: true, taxSource: true, + receiptUrl: true, installedAtCustomer: true, estimate: { select: { projectId: true } }, }, @@ -825,7 +842,7 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro // two apart. The explicit NULL branch is required: // SQL `<> 'manual'` is NULL for a NULL column, so a // bare not-equals would drop every legacy row. - OR: [{ taxSource: null }, { taxSource: { not: "manual" } }], + ...taxNotHumanDecidedWhere(), }, data: { // Same bound as the create path above: an @@ -862,6 +879,28 @@ export async function bookReceipt(row: BookableRow, deps: BookDependencies): Pro }); } + // A RECOVERED ROW USUALLY HAS NO LINK TO THE DOCUMENT. + // + // v1 created plenty of Expenses with a null `receiptUrl`, and a + // crash between the Purchase and this commit leaves one too. + // Booking knows exactly where the bytes are, and a receipt + // nobody can open is the difference between a defensible + // deduction and a number in a spreadsheet. + // + // Guarded on `receiptUrl: null`, so an existing link — a Drive + // URL somebody fixed by hand, or one an earlier pass wrote — is + // never replaced by this one. + if (receiptUrl) { + await tx.expense.updateMany({ + where: { + id: existing.id, + projectId: expectedProjectId ?? row.projectId, + receiptUrl: null, + }, + data: { receiptUrl }, + }); + } + // VERIFY THE ATTRIBUTION THAT WAS ACTUALLY WRITTEN. // // Every predicate above is guarded, so any one of them can diff --git a/src/lib/receipt-intake/late-fields.ts b/src/lib/receipt-intake/late-fields.ts index 845f2263b..cd00c5130 100644 --- a/src/lib/receipt-intake/late-fields.ts +++ b/src/lib/receipt-intake/late-fields.ts @@ -14,6 +14,13 @@ export interface LateFields { costCodeId?: string; + /** + * Derived, never taken from the request body: "user" when a signed-in + * person supplied the phase, "machine" when a shared-secret forwarder did. + * It rides with `costCodeId` so a late phase carries the same provenance a + * captured one does. + */ + costCodeSource?: string; projectId?: string; /** * Phase 3's tax answer: was this material installed at a customer job? @@ -28,11 +35,12 @@ export interface LateFields { installedAtCustomer?: boolean; } -export type LateFieldKey = "costCodeId" | "projectId" | "installedAtCustomer"; +export type LateFieldKey = "costCodeId" | "costCodeSource" | "projectId" | "installedAtCustomer"; export type LateFieldValue = string | boolean; export interface LateFieldRow { costCodeId: string | null; + costCodeSource?: string | null; projectId: string | null; installedAtCustomer?: boolean | null; state: string; diff --git a/src/lib/time-expense-core.ts b/src/lib/time-expense-core.ts index 9790c5530..47e49f23a 100644 --- a/src/lib/time-expense-core.ts +++ b/src/lib/time-expense-core.ts @@ -2,6 +2,7 @@ import { prisma } from "./prisma"; import { resolveCostCode } from "./cost-coding"; import { prismaCostCodingDataSource } from "./cost-coding-db"; import { isCostCodeAllowedForProject } from "./project-phases"; +import { assertPhaseOfProjectTx } from "./phase-invariant"; import { prismaPhaseDataSource } from "./project-phases-db"; import { resolveExpenseProjectId } from "./expense-attribution"; import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "./company-timezone"; @@ -211,8 +212,21 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str : null; if (expenseDate && Number.isNaN(expenseDate.getTime())) throw new Error("A valid expense date is required"); - return prisma.expense.create({ - data: { + // THE PHASE ANSWER THAT COUNTS, taken with the write (round 18, item 4). + // The check above answers on the global client and holds nothing; this one + // locks the four tables it rests on and reads them on the transaction that + // inserts the row. + return prisma.$transaction(async tx => { + if (costCodeId) { + const verdict = await assertPhaseOfProjectTx( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + estimate.projectId, + costCodeId, + ); + if (!verdict.ok) throw new Error("That cost code isn't one of this project's phases."); + } + return tx.expense.create({ + data: { estimateId, // Phase 3: the estimate's project, already resolved and validated // above (including the change-order cross-check). @@ -231,7 +245,8 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str receiptUrl, changeOrderId: changeOrder?.id ?? null, isBillable: data.isBillable ?? Boolean(changeOrder), - }, + }, + }); }); } diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index c1fed20aa..e021d6051 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -22,6 +22,7 @@ import { expectedColumns, expectedConstraints, expectedIndexes, + pickCompanyTimeZone, reanchorSql, statements, targetMatches, @@ -297,3 +298,77 @@ test("Prisma declares the same default, so the migration check sees them agree", const schema = readFileSync(path.join(__dirname, "..", "prisma", "schema.prisma"), "utf8"); assert.match(schema, /updatedAt DateTime @default\(now\(\)\) @updatedAt/); }); + +// ── the company zone is not guessed (Codex round 18, item 5) ─────────────── + +test("only an ABSENT settings row falls back to the app default", () => { + // This zone decides which quarter every legacy receipt lands in. An empty + // result is a database that has never had settings written, and the app's + // own default is the honest answer for it. + assert.deepEqual(pickCompanyTimeZone([]), { + timeZone: "America/Los_Angeles", + from: "default", + }); + assert.deepEqual(pickCompanyTimeZone([{ timeZone: null }]), { + timeZone: "America/Los_Angeles", + from: "default", + }); + assert.deepEqual(pickCompanyTimeZone([{ timeZone: " " }]), { + timeZone: "America/Los_Angeles", + from: "default", + }); +}); + +test("a configured zone is used verbatim, and reported as such", () => { + assert.deepEqual(pickCompanyTimeZone([{ timeZone: "America/New_York" }]), { + timeZone: "America/New_York", + from: "settings", + }); + assert.deepEqual(pickCompanyTimeZone([{ timeZone: " UTC " }]), { + timeZone: "UTC", + from: "settings", + }); +}); + +test("an UNREADABLE settings query is not an answer", () => { + // The old code wrapped the query in `.catch(() => [undefined])`, so a + // permissions error or a dropped connection read as "no settings" and + // quietly re-anchored a whole table into Pacific. + assert.throws(() => pickCompanyTimeZone(undefined as never), /refusing to guess/i); + assert.throws(() => pickCompanyTimeZone(null as never), /refusing to guess/i); +}); + +test("the script does not swallow the settings query error", () => { + const script = readFileSync( + path.join(__dirname, "..", "scripts", "apply-expense-attribution.mjs"), + "utf8", + ); + const read = script.slice(script.indexOf('SELECT "timeZone" FROM "CompanySettings"')); + assert.ok( + !/\.catch\(/.test(read.slice(0, 200)), + "an unreadable settings table must abort, not fall back", + ); +}); + +test("the re-anchor is idempotent by MARKER, not by shape", () => { + // The time-of-day predicate was the whole guard, and it is not one for a + // company configured as UTC: there the rewrite is the identity, so every + // row stayed at midnight and stayed eligible forever. + const sql = reanchorSql("America/Los_Angeles"); + assert.match(sql, /"attributionAnchoredAt" IS NULL/, "already-anchored rows are skipped"); + assert.match(sql, /SET[\s\S]*"attributionAnchoredAt" = now\(\)/, "and the fact is recorded"); + // The legacy selector stays: everything written by time-expense-core has + // always carried a real time-of-day. + assert.match(sql, /"date"::time = TIME '00:00:00'/); + // The marker column ships with the rest of the DDL, in both files. + assert.ok((statements as string[]).some(s => /"attributionAnchoredAt" TIMESTAMP\(3\)/.test(s))); + assert.match(migrationSql, /"attributionAnchoredAt" TIMESTAMP\(3\)/); +}); + +test("ReceiptIntake.costCodeSource ships behind the same guard as the other two", () => { + // Phase 1 owns that table; the column is additive and skipped when the + // table is not there yet (round 18, item 3). + const guarded = (statements as string[]).find(s => s.includes("ReceiptIntake")); + assert.match(guarded!, /"costCodeSource" TEXT/); + assert.match(migrationSql, /ALTER TABLE "ReceiptIntake" ADD COLUMN IF NOT EXISTS "costCodeSource" TEXT/); +}); diff --git a/tests/expense-attribution.test.ts b/tests/expense-attribution.test.ts index 2df6a74e1..515fbd971 100644 --- a/tests/expense-attribution.test.ts +++ b/tests/expense-attribution.test.ts @@ -9,6 +9,8 @@ */ import assert from "node:assert/strict"; import test from "node:test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { HUMAN_COST_CODE_SOURCES, expenseForProjectWhere, @@ -17,6 +19,7 @@ import { isPlausibleReceiptTax, maxPlausibleTaxAmount, notHumanCodedExpenseWhere, + taxIsAtSource, resolveExpenseCostCodeId, resolveExpenseProjectId, resolveExpenseProjectLabel, @@ -245,3 +248,34 @@ test("a REFUND's tax is negative, and a positive one on it is refused", () => { assert.equal(isPlausibleReceiptTax(-4, 50), false, "and the same the other way"); assert.equal(isPlausibleReceiptTax(0, -50), true, "a credit can carry no tax"); }); + +// ── taxAtSource follows the figure, signed (Codex round 18, item 1) ──────── + +test("taxIsAtSource is true for any non-zero figure, either direction", () => { + // The FACT is "tax was charged on this receipt". On a return it is just as + // true — the tax was charged, and is now coming back. `> 0` read every + // credit as "no tax here", which is how a refund's tax left the filing. + assert.equal(taxIsAtSource(16.55), true); + assert.equal(taxIsAtSource(-4), true, "a credit still carries the fact"); + assert.equal(taxIsAtSource(0), false, "zero is an answer: no tax"); + assert.equal(taxIsAtSource(null), false, "and silence is not a claim"); + assert.equal(taxIsAtSource(undefined), false); + assert.equal(taxIsAtSource(Number.NaN), false); +}); + +test("the tax & phase modal derives the flag from the figure, not from its sign", () => { + // The modal is the only writer a bookkeeper touches directly. It computed + // `(parsedTax ?? 0) > 0`, so saving a refund's -$4 of tax silently stored + // taxAtSource=false and dropped the row out of the excise report — on both + // the ordinary save and the review acknowledgement. + const modal = readFileSync( + path.join(__dirname, "..", "src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx"), + "utf8", + ); + const assignments = [...modal.matchAll(/body\.taxAtSource = ([^;]+);/g)].map(m => m[1].trim()); + assert.equal(assignments.length, 2, "the ordinary save and the ack both set it"); + for (const assignment of assignments) { + assert.equal(assignment, "taxIsAtSource(parsedTax)", "both go through the shared rule"); + } + assert.ok(!/taxAtSource[^;]*>\s*0/.test(modal), "no positive-only copy survives"); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 5978aa35e..51562935b 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -610,14 +610,28 @@ test("a full acknowledgement clears the flag", async () => { assert.equal(updateArgs?.data.taxSource, "manual"); }); -test("an acknowledgement without both figures is refused, not half-applied", async () => { +test("an acknowledgement that OMITS taxAmount is refused, not half-applied", async () => { + // Round 18: a request that says nothing about tax has nothing to certify. + // Supplying only the amount IS enough (the base is computed), so the + // refusal is specifically about the key being absent. storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; - const res = await patch({ taxReviewAck: true, taxAmount: 16.55 }); + const res = await patch({ taxReviewAck: true, taxDeductibleBase: 50 }); assert.equal(res.status, 400); assert.equal((await res.json()).code, "TAX_REVIEW_INCOMPLETE"); assert.equal(updateArgs, null, "nothing is written"); }); +test("an acknowledgement with only the tax figure is enough", async () => { + storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; + const res = await patch({ taxReviewAck: true, taxAmount: 16.55 }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.needsTaxReview, false); + assert.equal(updateArgs?.data.taxSource, "manual"); + // ...and the blank base is STORED as the whole pre-tax total rather than + // left as a null whose meaning every reader has to remember. + assert.equal(updateArgs?.data.taxDeductibleBase, 191.19, "207.74 - 16.55"); +}); + test("an UNflagged row does not need an acknowledgement", async () => { // The ack exists to make clearing a flag deliberate. Requiring it of // ordinary edits would just teach people to send it always. @@ -698,14 +712,17 @@ test("answering ONLY installedAtCustomer does not claim the tax figures", async assert.equal(updateArgs?.data.taxSource, undefined); }); -test("an EXPLICIT null tax is a manual 'no tax' decision", async () => { - // Round 17, item 2. Sending the key with null is a bookkeeper looking at - // the receipt and saying there is no sales tax on it. Booking must not then - // write an OCR guess over that answer, which is what `taxSource` prevents. +test("an EXPLICIT null tax is a manual 'no tax' decision, in its own state", async () => { + // Round 18: the four states are null (unreviewed), "ocr", "manual" (a + // person's figure) and "manual-none" (a person's "there is no tax here"). + // The last is the one a null `taxAmount` cannot express on its own, which + // is the entire reason the column exists. const res = await patch({ taxAmount: null, taxAtSource: false }); assert.equal(res.status, 200); assert.equal(updateArgs?.data.taxAmount, null, "the clear lands"); - assert.equal(updateArgs?.data.taxSource, "manual", "and it is recorded as theirs"); + assert.equal(updateArgs?.data.taxSource, "manual-none"); + // With no tax, the whole receipt is the pre-tax total, and it is stored. + assert.equal(updateArgs?.data.taxDeductibleBase, 207.74); }); test("an OMITTED tax key leaves the provenance alone", async () => { @@ -733,14 +750,17 @@ test("a phase-only edit touches neither the flag nor the provenance", async () = // ── an acknowledgement must carry real figures (round 17, item 2) ────────── -test("an ack whose taxAmount is null is refused, and the flag stays", async () => { - // "I re-checked these numbers" has to contain numbers. A null would - // certify an empty row straight back into the excise report. +test("a FLAGGED no-tax workflow: ack + null clears the flag as manual-none", async () => { + // The whole point of the flag is that a person looks again. "I looked, and + // this receipt has no sales tax" is one of the two answers they can reach, + // and refusing it would leave the row flagged forever. storedExpense = { ...(storedExpense as object), needsTaxReview: true } as Record; - const res = await patch({ taxReviewAck: true, taxAmount: null, taxDeductibleBase: 50 }); - assert.equal(res.status, 400); - assert.equal((await res.json()).code, "TAX_REVIEW_INCOMPLETE"); - assert.equal(updateArgs, null, "nothing is written, so the flag stands"); + const res = await patch({ taxReviewAck: true, taxAmount: null }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxAmount, null); + assert.equal(updateArgs?.data.taxSource, "manual-none"); + assert.equal(updateArgs?.data.needsTaxReview, false, "answered, so no longer waiting"); + assert.equal(updateArgs?.data.taxDeductibleBase, 207.74, "the whole receipt, stored"); }); test("an ack whose figures point the wrong way is refused", async () => { @@ -796,3 +816,37 @@ test("a refund's base is bounded by the pre-tax MAGNITUDE", async () => { assert.equal(tooBig.status, 400); assert.equal(updateArgs, null); }); + +// ── the four taxSource states, end to end (Codex round 18, item 2) ───────── + +test("a BLANK base on an ordinary tax edit is stored, not left null", async () => { + // "Null means the whole pre-tax total" is a rule every reader has to + // remember; the server writes what the person meant instead. + const res = await patch({ taxAmount: 16.55, taxAtSource: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxDeductibleBase, 191.19); +}); + +test("an EXPLICIT base is honoured, not overwritten by the computed one", async () => { + const res = await patch({ taxAmount: 16.55, taxDeductibleBase: 100 }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxDeductibleBase, 100, "a mixed receipt keeps its split"); +}); + +test("a REFUND's blank base is computed with the sign intact", async () => { + storedExpense = { + ...(storedExpense as object), amount: -50, taxAmount: null, taxDeductibleBase: null, + } as Record; + const res = await patch({ taxAmount: -4, taxAtSource: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxDeductibleBase, -46); +}); + +test("an installedAtCustomer-only edit computes nothing and claims nothing", async () => { + // It is not a tax figure, so it neither stamps provenance nor invents a + // deduction base for a row nobody has priced. + const res = await patch({ installedAtCustomer: true }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.taxSource, undefined); + assert.equal(updateArgs?.data.taxDeductibleBase, undefined); +}); diff --git a/tests/expense-phase-scope.test.ts b/tests/expense-phase-scope.test.ts index a601961fd..281536a9e 100644 --- a/tests/expense-phase-scope.test.ts +++ b/tests/expense-phase-scope.test.ts @@ -17,7 +17,11 @@ import { resolveCostCode, type CostCodingDataSource } from "../src/lib/cost-codi // mobile-auth, which throws at import time unless NEXTAUTH_SECRET is set — // true in CI, and a unit test has no business needing a JWT secret. import { resolveInstalledAtCustomer } from "../src/lib/expense-attribution"; -import { optionalBool } from "../src/lib/receipt-capture-validation"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { captureActorSource, optionalBool } from "../src/lib/receipt-capture-validation"; + +const ROOT = path.resolve(__dirname, ".."); import { authorizePhase } from "../src/lib/receipt-intake/late-fields"; // ── the two checks every phase writer must run ───────────────────────────── @@ -143,3 +147,41 @@ test("optionalBool is tri-state across JSON and multipart", () => { assert.equal(optionalBool(silent), null, JSON.stringify(silent) ?? "undefined"); } }); + +// ── who supplied the captured phase (Codex round 18, item 3) ─────────────── + +test("a signed-in person is 'user'; a shared-secret forwarder is 'machine'", () => { + // A person picking a phase on their phone is an ANSWER. A forwarder + // resolving one from a Drive folder name is a GUESS that happens to arrive + // at capture time, and it has no more standing than the suggester's. + assert.equal(captureActorSource("session"), "user"); + assert.equal(captureActorSource("secret"), "machine"); +}); + +test("both intake doors record the actor with the captured phase", () => { + // The value has to be written where the caller's identity is still in hand. + for (const rel of [ + "src/app/api/receipts/intake/route.ts", + "src/app/api/receipts/intake/start/route.ts", + ]) { + const source = readFileSync(path.join(ROOT, rel), "utf8"); + assert.match(source, /captureActorSource\(auth\.via\)/, `${rel} does not record the actor`); + // ...and only when a phase was actually captured: a row with no code + // has no captured provenance either. + assert.match(source, /costCodeSource: [\w.]+\s*\?\s*captureActorSource/, rel); + } +}); + +test("a late phase at finalize carries the same provenance", () => { + const source = readFileSync( + path.join(ROOT, "src/app/api/receipts/intake/[id]/finalize/route.ts"), + "utf8", + ); + assert.match(source, /lateFields\.costCodeSource = captureActorSource\(auth\.via\)/); + // Derived from the CALLER, never read off the body — otherwise a forwarder + // could label its own guess a person's answer. + assert.ok( + !/costCodeSource:\s*(?:body|json|form)/.test(source), + "provenance must not be taken from the request", + ); +}); diff --git a/tests/expense-writer-phase-guard.test.ts b/tests/expense-writer-phase-guard.test.ts new file mode 100644 index 000000000..c73ec45df --- /dev/null +++ b/tests/expense-writer-phase-guard.test.ts @@ -0,0 +1,123 @@ +/** + * EVERY Expense writer that sets a cost code answers the phase question INSIDE + * its write transaction (Codex round 18, item 4). + * + * There are six of them, written at different times by different hands, and + * five had the same shape: ask `isCostCodeAllowedForProject` on the global + * client, then write in a transaction that holds nothing. An estimate archived + * or reassigned, or a cost code deactivated, in that window still landed on the + * row — and on the routes that stamp "capture" or "manual" it landed as + * something no automated pass is allowed to correct. + * + * This is a TRIPWIRE, not a proof: it fails when a NEW writer appears without + * the invariant, which is the failure mode a behavioural test cannot see. The + * behaviour itself is covered by tests/phase-invariant.test.ts (the rules and + * the lock order) and by the concurrency test at the bottom of this file. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { assertPhaseOfProjectTx } from "../src/lib/phase-invariant"; + +const ROOT = path.resolve(__dirname, ".."); + +function walk(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir)) { + const full = path.join(dir, entry); + if (statSync(full).isDirectory()) walk(full, out); + else if (full.endsWith(".ts") || full.endsWith(".tsx")) out.push(full); + } + return out; +} + +/** + * A file that writes `costCodeId` onto an Expense. + * + * Scoped to the STATEMENT, not the file: billing-core builds ChangeOrderItems + * carrying a `costCodeId` and, separately, stamps invoice ids onto expenses. A + * file-wide search reads that pair as "an expense writer setting a phase" and + * fails on a file that does nothing of the sort. The window is generous enough + * to span a formatted `data: { ... }` block and short enough not to run into + * the next unrelated statement. + */ +function writesAnExpenseCostCode(source: string): boolean { + const writes = /(?:prisma|tx|transaction|client)\.expense\.(?:create|update|updateMany)\s*\(/g; + for (let match = writes.exec(source); match; match = writes.exec(source)) { + const window = source.slice(match.index, match.index + 2000); + // `costCodeId` as an assigned VALUE inside the payload — never a + // `select: { costCodeId: true }` and never a `where` predicate. Both + // spellings count: `costCodeId: ` and the property SHORTHAND + // `costCodeId,` that two of these routes use. Missing the shorthand is + // how a tripwire quietly stops covering the writer it was written for. + if (/data:\s*\{[\s\S]{0,1500}?costCodeId(:\s*(?!true\b)|\s*[,}])/.test(window)) return true; + } + return false; +} + +test("every Expense writer that sets a cost code calls assertPhaseOfProjectTx", () => { + const offenders: string[] = []; + for (const file of walk(path.join(ROOT, "src"))) { + const source = readFileSync(file, "utf8"); + if (!writesAnExpenseCostCode(source)) continue; + if (source.includes("assertPhaseOfProjectTx")) continue; + offenders.push(path.relative(ROOT, file).replace(/\\/g, "/")); + } + assert.deepEqual( + offenders, + [], + "these write a phase onto an Expense without the transactional invariant:\n " + + offenders.join("\n "), + ); +}); + +test("the writers we know about are actually in the scanned set", () => { + // A tripwire that scans nothing passes forever. This pins that the scan + // really does reach the six writers, so a future refactor that moves one + // out of `src/` fails here rather than silently shrinking the net. + const expected = [ + "src/app/api/expenses/route.ts", + "src/app/api/expenses/[id]/route.ts", + "src/app/api/integrations/receipt-ingest/route.ts", + "src/lib/time-expense-core.ts", + "src/lib/receipt-intake/book.ts", + "src/lib/qbo-expense-sync.ts", + ]; + const scanned = walk(path.join(ROOT, "src")) + .filter(file => writesAnExpenseCostCode(readFileSync(file, "utf8"))) + .map(file => path.relative(ROOT, file).replace(/\\/g, "/")); + for (const writer of expected) { + assert.ok(scanned.includes(writer), `${writer} is no longer detected as an Expense cost-code writer`); + } +}); + +// ── the concurrency the tripwire cannot see ──────────────────────────────── + +test("a phase deactivated between the check and the write is refused, not written", async () => { + // The interleaving, deterministically: the caller's own validation passed + // (that is the premise), and the code is deactivated before the write. The + // invariant re-asks on the writing transaction and answers no. + const world = { isActive: true }; + const tx = { + async $queryRawUnsafe(query: string, ...args: unknown[]) { + if (/FOR SHARE/.test(query)) { + // The deactivation lands as the locks are taken — the last + // moment it can, and the one a pre-transaction check misses. + world.isActive = false; + return []; + } + if (/FROM "Project" WHERE id/.test(query)) return [{ id: args[0], status: "In Progress" }]; + if (/FROM "CostCode" WHERE id/.test(query)) { + return [{ id: args[0], code: "03-PLUMB", isActive: world.isActive }]; + } + if (/FROM "EstimateItem"/.test(query)) return [{ ok: 1 }]; + return []; + }, + }; + + assert.deepEqual( + await assertPhaseOfProjectTx(tx, "job-1", "cc-plumb"), + { ok: false, reason: "code-inactive" }, + "the write must not proceed on an answer that was true a moment ago", + ); +}); diff --git a/tests/receipt-intake-book.test.ts b/tests/receipt-intake-book.test.ts index 73fb174cb..025c75adf 100644 --- a/tests/receipt-intake-book.test.ts +++ b/tests/receipt-intake-book.test.ts @@ -80,6 +80,7 @@ function row(overrides: Partial = {}): BookableRow { dryRun: false, projectId: "proj-1", costCodeId: null, + costCodeSource: null, suggestedCostCodeId: "cc-plumb", suggestedConfidence: 0.82, taxAtSource: true, @@ -154,7 +155,7 @@ function recorder(overrides: Partial = {}, opts: { estimates?: expenseUpdates.push(args); const cur = state.existingExpense ?? {}; const eq = (a: unknown, b: unknown) => (a ?? null) === (b ?? null); - for (const key of ["costCodeId", "taxAmount", "installedAtCustomer"]) { + for (const key of ["costCodeId", "taxAmount", "installedAtCustomer", "receiptUrl"]) { if (key in args.where && !eq(cur[key], args.where[key])) return { count: 0 }; } // `projectId` is pinned in every fill predicate to the @@ -1490,3 +1491,71 @@ test("a row with NO phase is not parked by this check", async () => { assert.equal(asked, 2, "no third question once there is no code left to check"); assert.equal(rec.expenses[0].costCodeId, null); }); + +// ── a machine's capture is not a human's (round 18, item 3) ──────────────── + +test("a phase captured by a PERSON books as untouchable 'capture'", () => { + const rec = recorder(); + return bookReceipt(row({ costCodeId: "cc-demo", costCodeSource: "user" }), rec.deps).then(() => { + assert.equal(rec.expenses[0].costCodeId, "cc-demo"); + assert.equal(rec.expenses[0].costCodeSource, "capture"); + }); +}); + +test("a phase captured by a FORWARDER books as correctable 'machine'", () => { + // A Drive folder name is a guess. Booking it as "capture" gave it exactly + // the authority of a person who picked it, and froze it against every later + // pass that could have corrected it. + const rec = recorder(); + return bookReceipt(row({ costCodeId: "cc-demo", costCodeSource: "machine" }), rec.deps).then(() => { + assert.equal(rec.expenses[0].costCodeId, "cc-demo"); + assert.equal(rec.expenses[0].costCodeSource, "machine"); + }); +}); + +test("a row captured before the column existed is treated as a machine guess", () => { + // The safe direction: it leaves the phase correctable rather than freezing + // an unattributed guess in place forever. + const rec = recorder(); + return bookReceipt(row({ costCodeId: "cc-demo", costCodeSource: null }), rec.deps).then(() => { + assert.equal(rec.expenses[0].costCodeSource, "machine"); + }); +}); + +// ── a recovered row gets its receipt link (round 18, item 6) ─────────────── + +test("an existing Expense with NO receiptUrl is given one", () => { + // v1 created plenty of these, and a crash between the Purchase and the + // commit leaves one too. A receipt nobody can open is the difference + // between a defensible deduction and a number in a spreadsheet. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, receiptUrl: null, + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + return bookReceipt(row(), rec.deps).then(result => { + assert.equal(result.outcome, "booked"); + assert.ok(rec.existingExpense.receiptUrl, "the link is filled"); + }); +}); + +test("an EXISTING receiptUrl is never replaced", () => { + // Somebody may have fixed it by hand, or an earlier pass wrote a better + // one. The guard is the predicate, so a value that appears between the read + // and the write survives as well. + const rec = recorder(); + rec.existingExpense = { + id: "expense-1", projectId: "proj-1", costCodeId: null, costCodeSource: null, + taxAmount: null, taxAtSource: false, taxSource: null, + receiptUrl: "https://drive.google.com/file/d/HAND-FIXED/view", + installedAtCustomer: null, estimate: { projectId: "proj-1" }, + }; + return bookReceipt(row(), rec.deps).then(() => { + assert.equal( + rec.existingExpense.receiptUrl, + "https://drive.google.com/file/d/HAND-FIXED/view", + "the existing link stands", + ); + }); +}); diff --git a/tests/receipt-intake-worker.test.ts b/tests/receipt-intake-worker.test.ts index e53363ecf..346f2faa2 100644 --- a/tests/receipt-intake-worker.test.ts +++ b/tests/receipt-intake-worker.test.ts @@ -49,6 +49,7 @@ function workerRow(overrides: Partial = {}): WorkerRow { dryRun: true, projectId: "proj-1", costCodeId: null, + costCodeSource: null, suggestedCostCodeId: null, suggestedConfidence: null, taxAtSource: false, From 0962751f1b0a6ab1cee696de3f6dbe3d5bee5129 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 09:32:48 -0700 Subject: [PATCH 103/144] fix(proxy): allowlist anonymous Server Action dispatch instead of denylisting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Next's action IDs are GLOBAL: the path a `next-action` POST is sent to only decides whose middleware runs first, not which action runs. The guard was a DENYLIST (legal pages + machine endpoints), which is the wrong shape for a global namespace — every other public-bypass path was a live anonymous dispatcher: /api/auth, /api/mobile, /api/pdf/*, /api/portal, /api/payments, /api/selections/*, /login, /share/*, and the asset patterns. Now: an action dispatch with no session cookie is 403 BEFORE the public bypass unless the path is allowlisted. The allowlist is the output of an audit of the anonymous route trees for server actions invoked from their client components: /portal/** approveEstimate, approveContract, approveChangeOrder, mark*Viewed, createDecision, submitSelectionProposal, portalCreateMoodBoard, setPortalStageOverride, ... /sub-portal/** subPortalUploadCOI and the sub sign-in flow. Audited and deliberately NOT allowlisted: /login (next-auth signIn is a plain POST to /api/auth/*, not an action), /share/** (server component reading Prisma; its one client child imports no actions), the legal pages, and every /api route (Next dispatches an action to the page URL the client is on, never to a route handler). Tests drive real requests through the proxy — 20 refused paths, 8 allowlisted ones, prefix-not-substring cases, and the same paths without the header — because the bug was an ORDERING one that a helper-level assertion cannot see. Mutation-tested: removing the guard and widening it to a substring both fail. Co-Authored-By: Claude Fable 5.1 --- src/proxy.ts | 102 +++++++++++++----------------- tests/receipt-intake-auth.test.ts | 82 +++++++++++++++++++----- 2 files changed, 110 insertions(+), 74 deletions(-) diff --git a/src/proxy.ts b/src/proxy.ts index f30d6d776..8be97ac76 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -67,58 +67,43 @@ const MOBILE_AUTHENTICATED_ROUTE_PATTERNS = [ // Play specifically requires a public account-deletion URL. const PUBLIC_PROXY_BYPASS_PATTERN = /^\/(?:api\/health$|api\/health\/pipeline\/?$|api\/(?:auth|cron|twilio|webhook|payments|portal|integrations|mcp(?:\/|$)|version|pdf\/(?:estimates|invoices|change-orders)|sub-portal|mobile|selections\/(?:item-comments|ai-sort|link-schedule))(?:\/|$)|api\/office-tasks\/ingest\/?$|api\/receipts\/intake\/?$|api\/receipts\/intake\/start\/?$|api\/receipts\/intake\/[^/]+\/(?:archived|finalize)\/?$|login(?:\/|$)|portal(?:\/|$)|sub-portal(?:\/|$)|share(?:\/|$)|privacy(?:\/|$)|terms(?:\/|$)|account-deletion(?:\/|$)|support(?:\/|$)|_next\/(?:static|image)(?:\/|$)|favicon\.ico$|.*\.(?:png|jpg|svg|webmanifest)$)/; -// The legal pages are static server components that define no Server Actions. -// Next's action IDs are global, so a bypassed path is a place an anonymous caller -// could POST a `next-action` header for someone else's action; the portal routes -// accept that tradeoff because they genuinely have anonymous actions, these don't. -const LEGAL_PAGE_PATTERN = /^\/(?:privacy|terms|account-deletion|support)(?:\/|$)/; - -// The receipt-intake machine endpoints are route handlers that define no Server -// Actions, and their ONLY gate is an in-handler `x-receipt-intake-secret` check. -// That check never runs for an action dispatch: Next's action IDs are global, so -// a `next-action` POST on a bypassed path invokes SOMEONE ELSE'S action and -// never reaches this route's code at all. The stale-cookie guard above does not -// cover it either — a machine caller carries no session cookie, so it takes the -// anonymous path straight into the bypass. +// WHICH PATHS MAY DISPATCH A SERVER ACTION WITHOUT A SESSION. +// +// Next's action IDs are GLOBAL: a `next-action` POST carries the id of the +// action to run, and the path it is sent to only decides which route's +// middleware and layout run first. So an anonymous action dispatch aimed at ANY +// path that skips the proxy invokes whatever action that id names — the route's +// own gate (a shared secret in a handler, a token check in a page) never runs, +// because the request never reaches the handler at all. +// +// This was previously a DENYLIST (legal pages + machine endpoints), which is the +// wrong shape for a global namespace: every public-bypass path nobody thought to +// list — /api/auth, /api/mobile, /api/pdf/*, /login, /share/*, the static asset +// patterns — was a live anonymous action dispatcher. An allowlist inverts the +// default, so a new public path is closed until someone opens it deliberately. // -// Same reasoning as LEGAL_PAGE_PATTERN, and the same conclusion: bypassing the -// proxy is never allowed to also mean bypassing the Server Action boundary. The -// portal/api routes above accept that tradeoff because they genuinely have -// anonymous actions; these do not. +// The list is the OUTPUT OF AN AUDIT, not a guess. Grepping the anonymous route +// trees for server-action imports invoked from their client components: // -// Exact match, mirroring the bypass entries themselves — a descendant that is -// NOT bypassed still hits withAuth and needs no special case here. -// NOTE on the matcher (bottom of this file): its FIRST entry is -// `{ source: "/:path*", has: [{ type: "header", key: "next-action" }] }`, which -// routes EVERY next-action request through this proxy regardless of path — the -// exclusion list in the second entry does not apply to them. So these paths were -// already reaching the code below; what waved them through was -// PUBLIC_PROXY_BYPASS_PATTERN returning next() before any action check ran. -// Widening this pattern is therefore the whole fix, and the matcher is left -// alone: adding /api/cron, /api/webhook and friends to it would put middleware -// (including a DB-touching staff lookup) in front of every ordinary webhook and -// cron request, which is real latency and a new failure point on a hot path, -// for no additional protection. +// /portal/** — approveEstimate, approveContract, approveChangeOrder, +// markEstimateViewed / markInvoiceViewed / markContractViewed, +// createDecision, submitSelectionProposal, portalCreateMoodBoard, +// setPortalStageOverride, addTaskCommentAsSub, ... Each +// authorizes on its own client/token check inside the action. +// /sub-portal/** — subPortalUploadCOI and the sub sign-in flow. // -// Deliberately NOT here: api/portal, api/payments, api/sub-portal and -// api/selections/*. Those genuinely serve anonymous Server Actions — that is the -// tradeoff their bypass exists for, and 403ing them would break the client -// portal. -const MACHINE_ENDPOINT_PATTERN = new RegExp( - "^\/api\/(?:" + [ - // Receipt Pipeline v2 — secret/Bearer in the handler, exact paths. - "receipts\/intake(?:\/start|\/[^/]+\/(?:archived|finalize))?", - // Cron: Bearer CRON_SECRET, checked in each route. - "cron\/[^?]*", - // Machine-to-machine ingest: each carries its own shared secret. - "integrations\/[^?]*", - // Stripe / Twilio: signature-verified, never session-authenticated. - "webhook(?:s)?\/[^?]*", - "twilio\/[^?]*", - // Ops probe: Bearer CRON_SECRET or a staff session, checked in-route. - "health\/pipeline", - ].join("|") + ")\/?$", -); +// Audited and deliberately NOT here, because they define and dispatch none: +// /login — next-auth `signIn()`, which is a plain POST to +// /api/auth/*, not an action dispatch. +// /share/** — a server component that reads Prisma directly; its one +// client child (ShareStudio) imports no actions. +// /privacy, /terms, /account-deletion, /support — static legal pages. +// /api/** — route handlers. Next dispatches an action to the URL the +// client is ON (a page), never to an API route, so no +// legitimate dispatch is aimed at one. That includes the +// machine endpoints whose only gate lives in the handler, +// and the mobile/PDF/auth routes. +const ANONYMOUS_SERVER_ACTION_PATHS = /^\/(?:portal|sub-portal)(?:\/|$)/; // Test-only action dispatchers that get the proxy bypass below. Explicit, not a // prefix match: the proxy checks only the environment gates, never the route's @@ -209,14 +194,17 @@ export default async function proxy(req: any, event: any) { return NextResponse.next(); } - // Legal pages are readable by anyone but are not an action endpoint. - if (isServerAction && typeof pathname === "string" && LEGAL_PAGE_PATTERN.test(pathname)) { - return new NextResponse("Forbidden", { status: 403 }); - } - - // Neither are the machine endpoints, whose only gate lives in the handler - // that an action dispatch never reaches. - if (isServerAction && typeof pathname === "string" && MACHINE_ENDPOINT_PATTERN.test(pathname)) { + // AN ANONYMOUS ACTION DISPATCH IS REFUSED UNLESS THE PATH IS ALLOWLISTED, + // and this runs BEFORE the public bypass — the bypass returning next() ahead + // of any action check is exactly what made every public path a dispatcher. + // + // A request carrying a session cookie has already been through the staff + // check above, so this is only about anonymous ones. + if ( + isServerAction + && !hasNextAuthSessionCookie(req) + && !(typeof pathname === "string" && ANONYMOUS_SERVER_ACTION_PATHS.test(pathname)) + ) { return new NextResponse("Forbidden", { status: 403 }); } diff --git a/tests/receipt-intake-auth.test.ts b/tests/receipt-intake-auth.test.ts index 7b70cfe33..c5ce15151 100644 --- a/tests/receipt-intake-auth.test.ts +++ b/tests/receipt-intake-auth.test.ts @@ -427,10 +427,17 @@ test("isCronAuthorized fails closed on an unset secret and is constant-time", as } }); -test("machine endpoints refuse a Server Action dispatch; portal actions still work", async () => { - // The matcher's FIRST entry already routes every next-action request here - // regardless of path, so these were reaching the proxy — the bypass was - // waving them through before any action check ran. +test("ANONYMOUS action dispatch: allowlisted paths pass, everything else is 403", async () => { + // Next's action IDs are GLOBAL — the path a `next-action` POST is sent to + // only decides whose middleware runs first, not which action runs. So every + // public-bypass path was an anonymous dispatcher for any action in the app, + // and the old denylist (legal pages + machine endpoints) closed the two + // somebody had thought of while /api/auth, /api/mobile, /api/pdf/*, /login, + // /share/* and the asset patterns stayed open. + // + // These are RUNTIME dispatches through the real proxy, not assertions about + // a helper: the bug was an ORDERING one (the bypass returned next() before + // any action check ran), and only driving the request end to end can see it. const { default: proxy } = await loadProxy(); const { NextRequest } = await import("next/server"); const event = { waitUntil() {} } as any; @@ -445,39 +452,80 @@ test("machine endpoints refuse a Server Action dispatch; portal actions still wo }), event); try { + // REFUSED — none of these define an anonymous Server Action, and each + // one bypasses the proxy for its own unrelated reason. for (const path of [ + // Machine endpoints: their only gate is a secret checked in the + // handler, which an action dispatch never reaches. "/api/cron/receipt-intake-worker", "/api/health/pipeline", "/api/integrations/qbo-receipts/create", "/api/webhook/stripe", "/api/twilio/sms", + "/api/receipts/intake", + "/api/receipts/intake/start", + "/api/receipts/intake/abc123/finalize", + // Public bypasses the old denylist never mentioned. These are the + // regression: every one of them dispatched actions anonymously. + "/api/auth/session", + "/api/mobile/projects", + "/api/pdf/estimates/abc123", + "/api/portal/verify", + "/api/payments/deposit-ingest", + "/api/selections/item-comments", + "/login", + "/share/room/sometoken", + // Legal pages, as before. + "/privacy", + "/terms", + "/account-deletion", + "/support", ]) { const res = await dispatch(path); assert.ok(res, path); - assert.equal(res.status, 403, `${path} must refuse an action dispatch`); + assert.equal(res.status, 403, `${path} must refuse an anonymous action dispatch`); + // NextResponse.next() carries x-middleware-next: 1. Anything else + // means the proxy kept control, which is the point. assert.equal(res.headers.get("x-middleware-next"), null, path); } - // Routes that GENUINELY serve anonymous Server Actions are untouched — - // 403ing them would break the client portal. - for (const path of ["/api/portal/verify", "/api/payments/deposit-ingest", "/api/selections/item-comments"]) { + // ALLOWED — the client portal and the sub portal genuinely dispatch + // actions with no session (approveEstimate, markInvoiceViewed, + // subPortalUploadCOI, the sub sign-in flow). Each authorizes on its own + // client/token check INSIDE the action; 403ing them here would break + // the portal outright. + for (const path of [ + "/portal", + "/portal/estimates/cmpd8mblp0004od6iufe0jfzc", + "/portal/invoices/abc123", + "/portal/projects/abc123/selections", + "/portal/clip", + "/sub-portal", + "/sub-portal/login", + "/sub-portal/projects/abc123", + ]) { const res = await dispatch(path); - assert.equal(res!.headers.get("x-middleware-next"), "1", `${path} must still pass`); + assert.equal(res!.headers.get("x-middleware-next"), "1", `${path} must still dispatch`); } - // And an ordinary cron call is unaffected. - const normal = await proxy( - new NextRequest("https://probuild.test/api/cron/receipt-intake-worker", { method: "GET" }), - event, - ); - assert.equal(normal!.headers.get("x-middleware-next"), "1"); + // The allowlist is a PREFIX of path segments, not a substring: a route + // that merely starts with the same letters is not the portal. + for (const path of ["/portalx", "/sub-portalx", "/api/portal-ish"]) { + const res = await dispatch(path); + assert.equal(res!.status, 403, path); + } + + // And an ordinary request — no next-action header — is untouched on + // every one of those paths. + for (const path of ["/api/cron/receipt-intake-worker", "/api/receipts/intake", "/login", "/share/room/t"]) { + const res = await proxy(new NextRequest(`https://probuild.test${path}`, { method: "GET" }), event); + assert.equal(res!.headers.get("x-middleware-next"), "1", `${path} without the header`); + } } finally { env.NODE_ENV = prod; } }); -// ── sourceRef shape, per source (round-13 item 4) ────────────────────────── - test("a sourceRef must carry a real id for its source, not just the prefix", async () => { const { validateSourceRef, decideSource, MAX_SOURCE_REF_BYTES } = await import("../src/lib/receipt-intake/intake-core"); From 51a5ef4e0944073b0de4e52a8cc27923bc3590b8 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 2 Sep 2026 09:50:04 -0700 Subject: [PATCH 104/144] fix(expenses): a blank tax says which blank it is, and a fallback job is re-resolved under lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five round-19 items. A blank tax field meant two different things and the payload could not tell them apart. It now says: `{ taxAmount: null, taxKnown: false }` is "nobody has read it yet" — it stamps no provenance, keeps a review flag up, and is refused as an acknowledgement (TAX_UNKNOWN); `{ taxAmount: null, taxKnown: true }` is "I looked, there is no tax", recorded as manual-none. The modal asks which with a pair of radios when the field is empty, and refuses NaN or Infinity before serializing (JSON turns both into null, which the server would otherwise read as a deliberate "no tax"). The server refuses them too, because the modal is one caller of many. On a FLAGGED row an acknowledgement now needs both taxAmount and taxDeductibleBase present — each a figure or an explicit null. The flag says the whole classification is in doubt, and certifying one figure while staying silent about the other is the half-answer it exists to prevent. An expense with no projectId of its own answers through its estimate, and that estimate can be moved to another job mid-request. PATCH, PUT and DELETE now share-lock the estimate inside their transaction, re-resolve the job, re-check the actor against THAT job, and carry it in the write predicate — so a row that moved is refused (403 when the actor may not touch the new job, 409 when the row moved underneath them) rather than written under a stale permission. The QBO suggester does the same, so its phase check and its write agree about which job they are for. The new-expense form offers this job's phases instead of every active cost code in the company. The server refused everything else anyway; the picker was inviting a refusal. Co-Authored-By: Claude Fable 5.1 --- package.json | 4 +- src/app/api/expenses/[id]/route.ts | 175 +++++++++++++-- .../[id]/time-expenses/ExpensesTab.tsx | 5 + .../[id]/time-expenses/TaxPhaseModal.tsx | 63 +++++- .../[id]/time-expenses/TimeExpensesClient.tsx | 13 +- src/lib/expense-attribution.ts | 56 +++++ src/lib/qbo-expense-sync.ts | 35 ++- tests/expense-edit-authz.test.ts | 205 +++++++++++++++++- tests/expense-phase-picker.test.tsx | 77 +++++++ tests/qbo-expense-sync.test.ts | 104 +++++++++ 10 files changed, 704 insertions(+), 33 deletions(-) create mode 100644 tests/expense-phase-picker.test.tsx diff --git a/package.json b/package.json index 55bee153a..eee841579 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:proxy-health": "tsx --test tests/proxy-health-pipeline.test.ts", "test:qbo-payments-outage": "tsx --test tests/qbo-payments-outage.test.ts", "test:receipt-intake": "tsx --test tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts", - "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts", + "test:expense-attribution": "tsx --test tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/tax-at-source-query.test.ts tests/company-financials-spend-attribution.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts tests/expense-phase-picker.test.tsx", "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/bank-ledger-status-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts tests/deposit-review.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", @@ -29,7 +29,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts", + "test:unit": "tsx --test tests/proxy-mobile-routes.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/wa-breaks.test.ts tests/logistics-formalize.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts tests/commercial-siding-qbo-evidence.test.ts tests/inspection-core.test.ts tests/portal-inspections.test.ts tests/percent-complete.test.ts tests/project-financials-earned.test.ts tests/percent-complete-cron.test.ts tests/percent-complete-recalc-race.test.ts tests/percent-complete-co-tasks.test.ts tests/percent-complete-backfill.test.ts tests/percent-complete-digest-guard.test.ts tests/qb-timed-fetch.test.ts tests/pipeline-health.test.ts tests/cron-auth.test.ts tests/pipeline-digest-route.test.ts tests/qbo-token-refresh-timeout.test.ts tests/proxy-health-pipeline.test.ts tests/qbo-payments-outage.test.ts tests/receipt-intake-keys.test.ts tests/receipt-intake-route-state.test.ts tests/receipt-intake-read.test.ts tests/receipt-intake-book.test.ts tests/receipt-intake-worker.test.ts tests/receipt-intake-auth.test.ts tests/apply-receipt-intake.test.ts tests/receipt-intake-archive-contract.test.ts tests/receipt-intake-cutover.test.ts tests/secure-storage-classification.test.ts tests/receipt-intake-raw-sql.test.ts tests/receipt-intake-stored-object.test.ts tests/receipt-intake-phases.test.ts tests/receipt-intake-cleanup.test.ts tests/receipt-intake-late-fields.test.ts tests/receipt-intake-reject.test.ts tests/receipt-intake-claim-release.test.ts tests/receipt-url.test.ts tests/apply-expense-attribution.test.ts tests/expense-attribution.test.ts tests/expense-cost-suggest.test.ts tests/backfill-expense-attribution.test.ts tests/tax-at-source-report.test.ts tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts tests/qbo-purchase-classification-persistence.test.ts tests/company-financials-spend-attribution.test.ts tests/tax-at-source-query.test.ts tests/expense-phase-scope.test.ts tests/expense-edit-authz.test.ts tests/expense-date-timezone.test.ts tests/expense-delete-scope.test.ts tests/scripts-runtime-smoke.test.ts tests/finalize-late-field-authz.test.ts tests/phase-invariant.test.ts tests/expense-writer-phase-guard.test.ts tests/expense-phase-picker.test.tsx", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/app/api/expenses/[id]/route.ts b/src/app/api/expenses/[id]/route.ts index 036cd18b8..fadffcab9 100644 --- a/src/app/api/expenses/[id]/route.ts +++ b/src/app/api/expenses/[id]/route.ts @@ -8,9 +8,11 @@ import { assertExpenseMutableOutsideQbo, } from "@/lib/qbo-expense-guard"; import { + expenseStillOnProjectWhere, isPlausibleReceiptTax, maxPlausibleTaxAmount, resolveExpenseProjectId, + resolveExpenseProjectUnderLock, taxIsAtSource, } from "@/lib/expense-attribution"; import { lockExpense } from "@/lib/expense-lock"; @@ -41,6 +43,7 @@ export async function DELETE(req: NextRequest, { params }: { params: Promise<{ i select: { qbPurchaseId: true, projectId: true, + estimateId: true, estimate: { select: { projectId: true } }, }, }); @@ -54,9 +57,44 @@ export async function DELETE(req: NextRequest, { params }: { params: Promise<{ i return NextResponse.json({ error: "Forbidden" }, { status: 403 }); } - await prisma.expense.deleteMany({ where: { id, qbPurchaseId: null } }); + // AND AGAIN, UNDER LOCK, FOR A FALLBACK-ATTRIBUTED ROW (round 19, item 3). + // + // A row with no `projectId` answers through its estimate, and somebody + // can move that estimate to another job between the check above and + // this delete. The row would then be destroyed under a permission that + // was granted for a job it is no longer on. + const removed = await prisma.$transaction(async tx => { + const locked = await resolveExpenseProjectUnderLock( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + expense, + ); + if (!locked || !canAccessProject(user, locked)) return { count: 0, denied: true } as const; + const result = await tx.expense.deleteMany({ + where: { + id, + qbPurchaseId: null, + // The predicate carries the answer, so a row that moved in + // the gap matches nothing rather than being deleted. + ...expenseStillOnProjectWhere(expense, locked), + }, + }); + return { count: result.count, denied: false } as const; + }); + if (removed.denied) return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + if (removed.count === 0) { + // The row moved (or a QBO id appeared) between the read and the + // delete. Reporting success would tell the caller their row is gone + // when it is not. + return NextResponse.json( + { + error: "This expense changed while you were deleting it. Reopen it and try again.", + code: "EXPENSE_REATTRIBUTED", + }, + { status: 409 }, + ); + } - return NextResponse.json({ success: true }); + return NextResponse.json({ success: true, deleted: removed.count }); } catch (error) { if (error instanceof QboManagedExpenseError) { return NextResponse.json({ error: error.message }, { status: 409 }); @@ -266,16 +304,28 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: // Same reason as the POST and the PATCH: the check above holds nothing, // and this route stamps "manual", which no automated pass may correct. const legacyWrite = await prisma.$transaction(async tx => { + // The same locked re-resolve as the DELETE: this route stamps + // "manual" and rewrites the amount, and a fallback-attributed row + // can change jobs between the authorization above and this write. + const lockedProjectId = await resolveExpenseProjectUnderLock( + tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, + expense, + ); + if (!lockedProjectId || !canAccessProject(user, lockedProjectId)) { + return { expense: null, phaseRejected: null, denied: "forbidden" } as const; + } if (editsCostCode && nextCostCodeId) { const verdict = await assertPhaseOfProjectTx( tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }, resolveExpenseProjectId(expense), nextCostCodeId, ); - if (!verdict.ok) return { expense: null, phaseRejected: verdict.reason } as const; + if (!verdict.ok) { + return { expense: null, phaseRejected: verdict.reason, denied: null } as const; + } } - const updated = await tx.expense.update({ - where: { id }, + const written = await tx.expense.updateMany({ + where: { id, ...expenseStillOnProjectWhere(expense, lockedProjectId) }, data: { amount: nextAmount, vendor: has("vendor") ? (body.vendor || null) : undefined, @@ -295,8 +345,24 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: : {}), }, }); - return { expense: updated, phaseRejected: null } as const; + if (written.count === 0) { + return { expense: null, phaseRejected: null, denied: "moved" } as const; + } + const updated = await tx.expense.findUnique({ where: { id } }); + return { expense: updated, phaseRejected: null, denied: null } as const; }); + if (legacyWrite.denied === "forbidden") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + if (legacyWrite.denied) { + return NextResponse.json( + { + error: "This expense moved to another job while you were editing it.", + code: "EXPENSE_REATTRIBUTED", + }, + { status: 409 }, + ); + } if (legacyWrite.phaseRejected) { return NextResponse.json( { @@ -390,8 +456,13 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // told, not silently ignored. const allowed = new Set([ "installedAtCustomer", "taxDeductibleBase", "taxAmount", "taxAtSource", "costCodeId", - // Not a column: the explicit "I have re-checked this flagged row" - // acknowledgement. See the needsTaxReview rule below. + // Not a column: which of the two things a NULL taxAmount means. + // `taxKnown: false` is "I do not know what the tax was", which is + // the state the pipeline starts in; `taxKnown: true` alongside a + // null amount is "I looked, and there is none". See below. + "taxKnown", + // Not a column either: the explicit "I have re-checked this flagged + // row" acknowledgement. See the needsTaxReview rule below. "taxReviewAck", ]); const rejected = Object.keys(body).filter(key => !allowed.has(key)); @@ -448,6 +519,28 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // A blank `taxDeductibleBase` is fine either way — the server computes // and stores the whole pre-tax total below rather than leaving a null // whose meaning has to be remembered by every reader. + // A NULL TAX AMOUNT IS TWO DIFFERENT ANSWERS, and the payload has to + // say which (round 19, item 2): + // + // * `{ taxAmount: null, taxKnown: false }` — "I do not know what the + // tax on this receipt was". That is where the row already is, so it + // changes no provenance, keeps any review flag up, and CANNOT + // acknowledge a review: there is nothing to certify. + // * `{ taxAmount: null, taxKnown: true }` — "I looked; there is no + // sales tax on this receipt". A decision, recorded as + // `manual-none`, and a complete answer to a review. + // + // `taxKnown` defaults to TRUE when the key is absent, because the only + // caller that sends a bare `taxAmount: null` today is a bookkeeper + // clearing a figure — and the modal now always says which it means. + if (has("taxKnown") && typeof body.taxKnown !== "boolean") { + return NextResponse.json( + { error: "taxKnown must be true or false." }, + { status: 400 }, + ); + } + const taxIsUnknown = editsTaxAmount && body.taxAmount === null && body.taxKnown === false; + const acknowledgesReview = body.taxReviewAck === true; if (acknowledgesReview) { const gross = Number(expense.amount); @@ -459,14 +552,33 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id Math.abs(parsed) <= Math.abs(gross) ); }; + // "I do not know" is not an answer to a review, and clearing the + // flag on it would put an unpriced row back into the excise report. + if (taxIsUnknown) { + return NextResponse.json( + { + error: "This receipt is flagged for review, so \"tax unknown\" cannot clear it. Enter the tax, or say the receipt has none.", + code: "TAX_UNKNOWN", + }, + { status: 400 }, + ); + } + // BOTH KEYS, on a flagged row (round 19, item 1). The flag means + // the whole classification is in doubt, and the two figures are the + // whole classification — so certifying one while staying silent + // about the other is exactly the half-answer the flag exists to + // prevent. Each may be a coherent number or an explicit null. + const bothPresent = expense.needsTaxReview ? editsTaxAmount && editsBase : editsTaxAmount; const answered = - editsTaxAmount && + bothPresent && (body.taxAmount === null || coherent(body.taxAmount)) && (!editsBase || body.taxDeductibleBase === null || coherent(body.taxDeductibleBase)); if (!answered) { return NextResponse.json( { - error: "Acknowledging a tax review needs taxAmount in the same request — a figure, or an explicit null meaning this receipt has no sales tax.", + error: expense.needsTaxReview + ? "Acknowledging a tax review needs both taxAmount and taxDeductibleBase in the same request — each a figure, or an explicit null." + : "Acknowledging a tax review needs taxAmount in the same request — a figure, or an explicit null meaning this receipt has no sales tax.", code: "TAX_REVIEW_INCOMPLETE", }, { status: 400 }, @@ -475,7 +587,8 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id } // An unflagged row has nothing to clear, so the ack is not required of // ordinary edits; a flagged one keeps its flag until it is given. - const clearsReview = !expense.needsTaxReview || acknowledgesReview; + const clearsReview = + (!expense.needsTaxReview && !taxIsUnknown) || (acknowledgesReview && !taxIsUnknown); // WHICH OF THE FOUR STATES THIS REQUEST PUTS THE ROW IN. // @@ -493,7 +606,10 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // `installedAtCustomer` is NOT one of these — its own value is its // evidence (non-null means answered) and booking already refuses to // touch it once it is set. - const stampsTaxProvenance = editsTaxAmount || editsBase; + // "I do not know" leaves the provenance where it was — null or "ocr" — + // so a later read may still fill the figure. Only the other two + // outcomes are decisions. + const stampsTaxProvenance = (editsTaxAmount || editsBase) && !taxIsUnknown; const nextTaxSource = editsTaxAmount && body.taxAmount === null ? "manual-none" : "manual"; @@ -536,7 +652,7 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id const parsed = Number(body.taxAmount); if (!Number.isFinite(parsed)) { return NextResponse.json( - { error: "taxAmount must be a number, or null." }, + { error: "taxAmount must be a finite number, or null." }, { status: 400 }, ); } @@ -581,9 +697,11 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id let nextBase: number | null = null; if (editsBase && body.taxDeductibleBase !== null) { const parsed = Number(body.taxDeductibleBase); + // NaN and Infinity included: `Number("")` is 0 and + // `Number("abc")` is NaN, and neither is a deduction. if (!Number.isFinite(parsed)) { return NextResponse.json( - { error: "taxDeductibleBase must be a number, or null." }, + { error: "taxDeductibleBase must be a finite number, or null." }, { status: 400 }, ); } @@ -732,7 +850,8 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // what they meant — `amount - tax` — so the row says it outright. The // legacy nulls stay readable; nothing new adds to them. const computedBase = - stampsTaxProvenance && (editsBase ? nextBase === null : resultingBase === null) + stampsTaxProvenance && + (editsBase ? nextBase === null : resultingBase === null) ? Math.round((Number(expense.amount) - resultingTax) * 100) / 100 : null; const writesBase = editsBase || computedBase !== null; @@ -792,6 +911,13 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id const written = await prisma.$transaction(async tx => { const raw = tx as unknown as { $queryRawUnsafe(q: string, ...v: unknown[]): Promise }; await lockExpense(raw, id); + // THE JOB, RE-RESOLVED UNDER LOCK (round 19, item 3). For a row + // with no `projectId` of its own the answer lives on the estimate, + // which somebody else can re-point while this request decides. + const lockedProjectId = await resolveExpenseProjectUnderLock(raw, expense); + if (!lockedProjectId || !canAccessProject(user, lockedProjectId)) { + return { count: 0, phaseRejected: null, denied: "forbidden" } as const; + } // THE PHASE ANSWER THAT COUNTS, taken here (round 17, item 5). // // The check above ran on the global client and held nothing: an @@ -800,12 +926,23 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id // locks the four tables the answer rests on and reads them on this // transaction's snapshot, so it cannot go stale before the update. if (editsCostCode && nextCostCodeId) { - const verdict = await assertPhaseOfProjectTx(raw, projectId, nextCostCodeId); - if (!verdict.ok) return { count: 0, phaseRejected: verdict.reason } as const; + const verdict = await assertPhaseOfProjectTx(raw, lockedProjectId, nextCostCodeId); + if (!verdict.ok) { + return { count: 0, phaseRejected: verdict.reason, denied: null } as const; + } } - const result = await tx.expense.updateMany({ where: casWhere, data }); - return { count: result.count, phaseRejected: null } as const; + const result = await tx.expense.updateMany({ + where: { ...casWhere, ...expenseStillOnProjectWhere(expense, lockedProjectId) }, + data, + }); + return { count: result.count, phaseRejected: null, denied: null } as const; }); + // TWO different answers, deliberately. "You may not touch this job" is a + // 403 about the ACTOR; a lost predicate is a 409 about the ROW, and the + // client's remedy differs (ask for access vs. reopen and retry). + if (written.denied === "forbidden") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } if (written.phaseRejected) { return NextResponse.json( { diff --git a/src/app/projects/[id]/time-expenses/ExpensesTab.tsx b/src/app/projects/[id]/time-expenses/ExpensesTab.tsx index c4ba1b21b..9309c4c20 100644 --- a/src/app/projects/[id]/time-expenses/ExpensesTab.tsx +++ b/src/app/projects/[id]/time-expenses/ExpensesTab.tsx @@ -29,6 +29,7 @@ interface Expense { installedAtCustomer?: boolean | null; taxDeductibleBase?: unknown; needsTaxReview?: boolean; + taxSource?: string | null; } interface Props { @@ -418,6 +419,10 @@ export default function ExpensesTab({ projectId, expenses: initialExpenses, onAd ? null : num(expense.taxDeductibleBase), needsTaxReview: Boolean(expense.needsTaxReview), + // Which of the four states the row is in, so + // the panel can tell "no tax here" from + // "nobody has read it yet". + taxSource: expense.taxSource ?? null, costCodeId: expense.costCode?.id ?? null, })} className={`mr-3 text-xs underline transition ${expense.needsTaxReview ? "text-amber-600 font-semibold" : "text-slate-400 hover:text-hui-primary"}`} diff --git a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx index d1af63db5..53d0607db 100644 --- a/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx +++ b/src/app/projects/[id]/time-expenses/TaxPhaseModal.tsx @@ -28,6 +28,8 @@ export interface TaxPhaseExpense { installedAtCustomer: boolean | null; taxDeductibleBase: number | null; needsTaxReview: boolean; + /** null | "ocr" | "manual" | "manual-none" — see expense-attribution.ts. */ + taxSource: string | null; costCodeId: string | null; } @@ -64,6 +66,10 @@ export default function TaxPhaseModal({ const [base, setBase] = useState( expense.taxDeductibleBase === null ? "" : String(expense.taxDeductibleBase), ); + // WHICH KIND OF BLANK the tax field is (round 19, item 2). A blank on its + // own means "I do not know", which is where the row already is; saying + // there is NO tax is a decision and has to be made deliberately. + const [taxKnown, setTaxKnown] = useState(expense.taxSource === "manual-none"); const [costCodeId, setCostCodeId] = useState(expense.costCodeId ?? ""); // Only meaningful on a flagged row: the explicit "I have re-checked these // figures" the server requires before it will clear the flag. @@ -84,11 +90,32 @@ export default function TaxPhaseModal({ // Only what actually changed. The endpoint refuses unknown keys, and // sending a field back unchanged would stamp provenance on it for no // reason. + // NOTHING NON-FINITE LEAVES THIS FORM. `Number("1e999")` is Infinity + // and `Number("1.2.3")` is NaN; JSON.stringify turns both into `null`, + // which the server would read as a deliberate "no tax". Caught here so + // the person sees the field they mistyped, and caught again on the + // server because this is not the only caller. + const badNumber = (raw: string) => raw.trim() !== "" && !Number.isFinite(Number(raw)); + if (badNumber(taxAmount)) { + toast.error("The tax amount must be a number."); + return; + } + if (badNumber(base)) { + toast.error("The deductible amount must be a number."); + return; + } + const body: Record = {}; const nextInstalled = installed === "unknown" ? null : installed === "yes"; if (nextInstalled !== expense.installedAtCustomer) body.installedAtCustomer = nextInstalled; - if (parsedTax !== expense.taxAmount) { + const taxStateChanged = + parsedTax !== expense.taxAmount || + (parsedTax === null && taxKnown !== (expense.taxSource === "manual-none")); + if (taxStateChanged) { body.taxAmount = parsedTax; + // WHICH BLANK this is. Only sent alongside the amount, because on + // its own it is not an edit. + if (parsedTax === null) body.taxKnown = taxKnown; // `taxAtSource` is the factual "tax was charged here"; it follows // the figure rather than being a second thing to get wrong. SIGNED: // a return carries negative tax and the fact still holds. @@ -107,7 +134,10 @@ export default function TaxPhaseModal({ // numbers, which is what makes the confirmation mean something. if (expense.needsTaxReview && reviewAck) { body.taxReviewAck = true; + // BOTH figures, changed or not: the server requires both keys on a + // flagged row, because the flag is about the whole classification. body.taxAmount = parsedTax; + if (parsedTax === null) body.taxKnown = taxKnown; body.taxAtSource = taxIsAtSource(parsedTax); body.taxDeductibleBase = nextBase; } @@ -214,9 +244,36 @@ export default function TaxPhaseModal({ {isCredit ? `This is a refund, so enter the tax as a negative, down to -${money(taxCeiling)} (12% of the receipt).` - : `Up to ${money(taxCeiling)} (12% of the receipt).`}{" "} - Leave blank if the read was wrong and you don't know the figure. + : `Up to ${money(taxCeiling)} (12% of the receipt).`} + {/* + * A BLANK FIELD IS AMBIGUOUS, so the form asks which blank + * it is. "I do not know" is where the row already sits and + * changes nothing; "no tax on this receipt" is a decision + * the pipeline may never overwrite. + */} + {parsedTax === null && ( +
+ + +
+ )} diff --git a/src/lib/company-timezone.ts b/src/lib/company-timezone.ts index 0db4f5ca7..f1f87fa49 100644 --- a/src/lib/company-timezone.ts +++ b/src/lib/company-timezone.ts @@ -1,5 +1,6 @@ import { prisma } from "./prisma"; import { validTimeZone, DEFAULT_COMPANY_TIME_ZONE } from "./tz-date"; +export type { CalendarDateVerdict } from "./tz-date"; // Pure date/time-zone primitives live in tz-date.ts (no prisma import, so // they stay importable from anything that must be unit-testable without a @@ -9,7 +10,10 @@ import { validTimeZone, DEFAULT_COMPANY_TIME_ZONE } from "./tz-date"; // startOfDateInTimeZone / endOfDateInTimeZone / dateInputInTimeZone don't // need to change their import path. export { + CALENDAR_DATE_BAD_SHAPE, + CALENDAR_DATE_NOT_REAL, DEFAULT_COMPANY_TIME_ZONE, + classifyCalendarDate, dateOnlyInTimeZone, startOfDateInTimeZone, endOfDateInTimeZone, diff --git a/src/lib/time-expense-core.ts b/src/lib/time-expense-core.ts index ed9e1efea..4b5eb940b 100644 --- a/src/lib/time-expense-core.ts +++ b/src/lib/time-expense-core.ts @@ -11,7 +11,7 @@ import { resolveExpenseProjectUnderLock, } from "./expense-attribution"; import { prismaPhaseDataSource } from "./project-phases-db"; -import { dateOnlyInTimeZone, resolveCompanyTimeZone } from "./company-timezone"; +import { CALENDAR_DATE_NOT_REAL, classifyCalendarDate, dateOnlyInTimeZone, resolveCompanyTimeZone } from "./company-timezone"; import { resolveScheduleTaskIdForPunch } from "./punch-task-binding"; import { toCompanyDayKey } from "./company-day"; import { assertExpenseMutableOutsideQbo } from "./qbo-expense-guard"; @@ -80,8 +80,17 @@ export async function createTimeEntryCore(data: CreateTimeEntryCoreInput, actor: if (data.burdenCost != null && (!Number.isFinite(data.burdenCost) || data.burdenCost < 0)) { throw new Error("Burden cost cannot be negative"); } - const startTime = /^\d{4}-\d{2}-\d{2}$/.test(data.date) - ? dateOnlyInTimeZone(data.date, await resolveCompanyTimeZone()) + // The shared validator, so an impossible day (`2026-02-31`) is refused by + // the same rule everywhere rather than throwing out of the parser with a + // different message (round 47, item 2). + const startVerdict = classifyCalendarDate(data.date); + // An impossible day is refused here rather than retried as an instant: + // `new Date("2026-02-31")` rolls forward to 3 March instead of failing. + if (startVerdict.kind === "invalid" && startVerdict.reason === CALENDAR_DATE_NOT_REAL) { + throw new Error("A valid time-entry date is required"); + } + const startTime = startVerdict.kind === "valid" + ? dateOnlyInTimeZone(startVerdict.date, await resolveCompanyTimeZone()) : new Date(data.date); if (Number.isNaN(startTime.getTime())) throw new Error("A valid time-entry date is required"); @@ -211,11 +220,19 @@ export async function createExpenseCore(data: CreateExpenseCoreInput, actor: str } receiptUrl = receipt.url; } - const expenseDate = data.date - ? /^\d{4}-\d{2}-\d{2}$/.test(data.date) - ? dateOnlyInTimeZone(data.date, await resolveCompanyTimeZone()) - : new Date(data.date) - : null; + // Same rule as every other intake. `omitted` keeps the existing "no date" + // behaviour (null, not today); `invalid` falls through to the instant + // parse, and an unparseable value is refused rather than stored as an + // Invalid Date. + const expenseVerdict = classifyCalendarDate(data.date); + if (expenseVerdict.kind === "invalid" && expenseVerdict.reason === CALENDAR_DATE_NOT_REAL) { + throw new Error("A valid expense date is required"); + } + const expenseDate = expenseVerdict.kind === "omitted" + ? null + : expenseVerdict.kind === "valid" + ? dateOnlyInTimeZone(expenseVerdict.date, await resolveCompanyTimeZone()) + : new Date(data.date as string); if (expenseDate && Number.isNaN(expenseDate.getTime())) throw new Error("A valid expense date is required"); // THE PHASE ANSWER THAT COUNTS, taken with the write (round 18, item 4). diff --git a/src/lib/tz-date.ts b/src/lib/tz-date.ts index 373bb0c6c..c097fe15d 100644 --- a/src/lib/tz-date.ts +++ b/src/lib/tz-date.ts @@ -158,6 +158,66 @@ function instantForWallClock( } /** Store date-only business values at local noon to preserve their calendar date. */ +/** + * THREE ANSWERS, NOT TWO (Codex round 47, item 2). + * + * Every intake that accepts a calendar day had the same two-branch shape: + * "matches YYYY-MM-DD" or "everything else", where everything else silently + * became `new Date()` — today. So a receipt whose date arrived as `07/14/2026` + * or `Jul 14 2026` was BOOKED ON THE DAY IT WAS IMPORTED, in whatever quarter + * that fell in, with nothing in the response saying so. And a value that + * passes the shape but is not a real day — `2026-02-31`, the shape a bad OCR + * read produces most often — reached the parser, which threw, and the route + * answered 500. + * + * Those are three different situations and they need three different answers: + * + * * OMITTED — no date was sent. Defaulting to today is legitimate, but the + * caller has to be TOLD, which is why this is a distinct verdict rather + * than folded in with the failures. + * * VALID — a real calendar day, ready for `dateOnlyInTimeZone`. + * * INVALID — something was sent and it is not a date. That is a caller bug + * and belongs in a 400 naming the offending value, never a guess and never + * a stack trace. + * + * The calendar check is a `Date.UTC` round trip, the same one `dateParts` uses: + * JavaScript rolls `2026-02-31` forward to 3 March rather than rejecting it, so + * "did the day I put in come back out" is the question that actually detects it. + */ +/** + * The two ways a supplied value fails, named so a caller can tell them apart + * without re-implementing the regex. The distinction matters: a value that is + * not YYYY-MM-DD at all may still be a legitimate full TIMESTAMP, and callers + * that accept both try that next. A value that IS YYYY-MM-DD but names no real + * day must never be retried as an instant — `new Date("2026-02-31")` does not + * fail, it rolls forward to 3 March, which is how an impossible date became a + * silently wrong one. + */ +export const CALENDAR_DATE_BAD_SHAPE = "must use YYYY-MM-DD"; +export const CALENDAR_DATE_NOT_REAL = "is not a real calendar date"; + +export type CalendarDateVerdict = + | { kind: "omitted" } + | { kind: "valid"; date: string } + | { kind: "invalid"; value: string; reason: string }; + +export function classifyCalendarDate(value: unknown): CalendarDateVerdict { + if (value === null || value === undefined) return { kind: "omitted" }; + if (typeof value !== "string") { + return { kind: "invalid", value: String(value), reason: "must be a string in YYYY-MM-DD form" }; + } + const trimmed = value.trim(); + if (trimmed === "") return { kind: "omitted" }; + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(trimmed); + if (!match) return { kind: "invalid", value: trimmed, reason: CALENDAR_DATE_BAD_SHAPE }; + const [year, month, day] = [Number(match[1]), Number(match[2]), Number(match[3])]; + const check = new Date(Date.UTC(year, month - 1, day)); + if (check.getUTCFullYear() !== year || check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) { + return { kind: "invalid", value: trimmed, reason: CALENDAR_DATE_NOT_REAL }; + } + return { kind: "valid", date: trimmed }; +} + export function dateOnlyInTimeZone(date: string, timeZone: string): Date { return instantForWallClock(date, timeZone, 12, 0, 0, 0, "date"); } diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index beba83da3..de4d9a51e 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -16,8 +16,15 @@ import assert from "node:assert/strict"; import test from "node:test"; import { readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; import path from "node:path"; import { + APPLY_TARGETS, + PRODUCTION_BASELINE_MIGRATION, + parseTarget, + resolveTargetDatabaseUrl, + targetBanner, + targetHostVerdict, expectedCheckConstraints, expectedColumns, expectedConstraints, @@ -1407,3 +1414,145 @@ test("the index verifier checks that an index is USABLE, not just present", () = assert.match(script, /toConcurrentIndexSql\(rebuild\)/); assert.match(script, /is STILL invalid after a rebuild/, "and gives up loudly rather than looping"); }); + + +// ── WHICH DATABASE (cross-PR rule, round 46) ─────────────────────────────── + +/** + * A developer with a local Postgres in their shell could run this script, + * watch every "verified ..." line print, and merge believing production had + * the columns. `--expect-db` / `--expect-host` did not stop it: the operator + * supplies BOTH sides of that comparison, so a local server satisfies it as + * easily as the real one. The target has to be named, and prod's URL has to + * come from the deployed env file rather than the shell. + */ +test("no --target is a refusal, not a default", () => { + const missing = parseTarget(["node", "apply.mjs", "--yes"]); + assert.match(missing.error ?? "", /--target is required/); + // A misspelling is not a silent fallback to prod either. + const wrong = parseTarget(["node", "apply.mjs", "--target", "production"]); + assert.match(wrong.error ?? "", /Unknown --target/); + const bare = parseTarget(["node", "apply.mjs", "--target"]); + assert.match(bare.error ?? "", /Unknown --target/); + assert.equal(parseTarget(["node", "apply.mjs", "--target", "prod"]).name, "prod"); + assert.equal(parseTarget(["node", "apply.mjs", "--target", "ci", "--yes"]).name, "ci"); +}); + +test("an ambient DATABASE_URL cannot impersonate production", () => { + // The failure this exists for, exactly: a local database in the shell. + const ambient = { DATABASE_URL: "postgresql://probuild:probuild@localhost:5432/probuild" } as unknown as NodeJS.ProcessEnv; + const files = { + ".env.production.local": + "NEXTAUTH_SECRET=irrelevant\n" + + 'DATABASE_URL="postgresql://postgres.ref:pw@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true"\n', + }; + const io = { + env: ambient, + exists: (file: unknown) => String(file) in files, + read: (file: unknown) => files[String(file) as keyof typeof files], + }; + + const prod = resolveTargetDatabaseUrl("prod", io); + assert.equal(prod.from, ".env.production.local", "the file, never the shell"); + assert.match(prod.url ?? "", /pooler\.supabase\.com/); + assert.doesNotMatch(prod.url ?? "", /localhost/, "the ambient value is not consulted at all"); + + // ...and with no such file, prod REFUSES rather than falling back to it. + const noFile = resolveTargetDatabaseUrl("prod", { ...io, exists: () => false }); + assert.match(noFile.error ?? "", /\.env\.production\.local, which does not exist/); + assert.equal(noFile.url, undefined, "no URL is produced, so no DDL can run"); + + // The CI target is the one that DOES read the environment. + const ci = resolveTargetDatabaseUrl("ci", io); + assert.equal(ci.url, ambient.DATABASE_URL); + assert.equal(ci.from, "process.env.DATABASE_URL"); +}); + +test("each target refuses the other one's host", () => { + assert.match( + targetHostVerdict("prod", "postgresql://u:p@localhost:5432/probuild") ?? "", + /expects the Supabase pooler, but the URL points at localhost/, + ); + assert.equal( + targetHostVerdict("prod", "postgresql://u:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres"), + null, + ); + // The reverse guard: the CI path must never be pointed at production, so + // the throwaway-container mode cannot become a way around the prod checks. + assert.match( + targetHostVerdict("ci", "postgresql://u:p@db.ghzdbzdnwjxazvmcefbh.supabase.co:5432/postgres") ?? "", + /must never point at .*supabase\.co — that is production/, + ); + assert.equal(targetHostVerdict("ci", "postgresql://u:p@localhost:5432/probuild_apply"), null); +}); + +test("only prod demands the production baseline row", () => { + assert.equal(APPLY_TARGETS.prod.requireBaseline, true); + assert.equal(APPLY_TARGETS.ci.requireBaseline, false); + // The name is the one CLAUDE.md documents as marked applied in prod by the + // deliberate one-off `migrate resolve --applied` step. + assert.equal(PRODUCTION_BASELINE_MIGRATION, "20260814000000_baseline_production"); + assert.ok( + readFileSync(path.resolve(__dirname, "..", "prisma", "migrations", PRODUCTION_BASELINE_MIGRATION, "migration.sql"), "utf8").length > 0, + "and it is a real migration in this repo", + ); +}); + +test("the banner names the database and REDACTS the credentials", () => { + const line = targetBanner("prod", { + url: "postgresql://postgres.ref:sup3rs3cret@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true", + from: ".env.production.local", + db: "postgres", + host: "10.0.0.5", + }); + assert.doesNotMatch(line, /sup3rs3cret/, "the password never reaches the terminal"); + assert.match(line, /:\*\*\*\*@/); + assert.match(line, /TARGET prod/); + assert.match(line, /db="postgres"/); + assert.match(line, /server="10\.0\.0\.5"/); + assert.match(line, /from \.env\.production\.local/); +}); + +test("the CI driver passes --target ci, so the prod guard cannot be met by accident", () => { + const driver = readFileSync( + path.resolve(__dirname, "..", "scripts", "ci-apply-expense-attribution-e2e.mjs"), + "utf8", + ); + assert.match(driver, /"--target", "ci"/); + assert.doesNotMatch(driver, /"--target", "prod"/); +}); + +test("THE ACTUAL SCRIPT refuses an ambient local URL, before any DDL", () => { + // The refusal has to happen in the real process, not just in the pure + // helpers: `main()` could call them and ignore the answer. Both attempts + // below exit before a PrismaClient is even constructed, which is why this + // needs no database. + const script = path.resolve(__dirname, "..", "scripts", "apply-expense-attribution.mjs"); + const ambient = { + ...process.env, + DATABASE_URL: "postgresql://probuild:probuild@localhost:5432/probuild", + }; + const attempt = (args: string[]) => { + try { + const stdout = execFileSync(process.execPath, [script, ...args], { + env: ambient, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + }); + return { code: 0, output: stdout }; + } catch (error) { + const failure = error as { status?: number; stdout?: string; stderr?: string }; + return { code: failure.status ?? -1, output: `${failure.stdout ?? ""}${failure.stderr ?? ""}` }; + } + }; + + const noTarget = attempt(["--yes", "--expect-db", "probuild", "--expect-host", "127.0.0.1"]); + assert.notEqual(noTarget.code, 0, "it must not run"); + assert.match(noTarget.output, /REFUSING: --target is required/); + assert.doesNotMatch(noTarget.output, /applied|verified/, "nothing was executed against the database"); + + // ...and naming prod does not rescue it: the URL would come from + // .env.production.local, which is not checked in and is not on CI. + const asProd = attempt(["--target", "prod", "--yes", "--expect-db", "probuild", "--expect-host", "127.0.0.1"]); + assert.notEqual(asProd.code, 0); + assert.match(asProd.output, /REFUSING/); + assert.doesNotMatch(asProd.output, /localhost/, "the ambient URL is not even echoed as a candidate"); +}); diff --git a/tests/apply-script-index-db.test.ts b/tests/apply-script-index-db.test.ts index 33a1bccdc..1a639956c 100644 --- a/tests/apply-script-index-db.test.ts +++ b/tests/apply-script-index-db.test.ts @@ -19,6 +19,9 @@ import assert from "node:assert/strict"; import { PrismaClient } from "@prisma/client"; import { INDEX_STATEMENTS, + expectedIndexes, + indexDrift, + readIndexCatalog, rebuildInvalidIndex, toConcurrentIndexSql, } from "../scripts/apply-expense-attribution.mjs"; @@ -168,6 +171,61 @@ test("...and a rebuild blocked by real duplicates fails LOUDLY, naming them", { } }); + +test("an index that is BOTH invalid and DRIFTED is verified from the REBUILT one", { skip }, async () => { + // ROUND 47, ITEM 3. The rebuild was correct and the verification that + // followed it was not: `indexDrift` was handed the PRE-rebuild snapshot, so + // an index that was invalid AND the wrong shape got rebuilt into the right + // shape and then reported as still drifted — with an instruction to drop + // the index the rebuild had just made correct. + await seed(); + const expected = (expectedIndexes as { + name: string; table: string; unique: boolean; keyColumns: string[]; predicate: RegExp | null; + }[]).find(index => index.name === INDEX)!; + try { + // Drift AND invalidity in one fixture: not unique, no partial + // predicate, one key column instead of two, and marked unusable the way + // an interrupted CONCURRENTLY build leaves it. + await db!.$executeRawUnsafe(`DROP INDEX CONCURRENTLY IF EXISTS "${INDEX}"`); + await db!.$executeRawUnsafe(`CREATE INDEX "${INDEX}" ON "Expense" ("sourceFileId")`); + await db!.$executeRawUnsafe( + `UPDATE pg_index SET indisvalid = false, indisready = false + WHERE indexrelid = (SELECT oid FROM pg_class WHERE relname = $1)`, + INDEX, + ); + + const stale = await readIndexCatalog(db, INDEX); + assert.equal(stale.is_valid, false, "the fixture really is unusable"); + const staleDrift = indexDrift(expected, stale); + assert.ok(staleDrift, `CONTROL: the pre-rebuild snapshot reports drift (${staleDrift})`); + + const repair = await rebuildInvalidIndex(db, INDEX, stale); + assert.equal(repair.ok, true, `the rebuild should have worked: ${repair.error}`); + + // THE CONTROL, restated as the bug: the snapshot the old code verified + // from still says "drifted" even though the database is now correct. + assert.ok( + indexDrift(expected, stale), + "the stale row is still drifted — verifying from it is the bug", + ); + + const fresh = await readIndexCatalog(db, INDEX); + assert.equal(indexDrift(expected, fresh), null, "the REBUILT index is the right shape"); + assert.deepEqual( + { valid: fresh.is_valid, ready: fresh.is_ready }, + { valid: true, ready: true }, + ); + assert.match(fresh.def, /CREATE UNIQUE INDEX/); + assert.match(fresh.def, /WHERE \("sourceFileId" IS NOT NULL\)/); + } finally { + await cleanup(); + await db!.$executeRawUnsafe(`DROP INDEX CONCURRENTLY IF EXISTS "${INDEX}"`).catch(() => {}); + await db!.$executeRawUnsafe( + toConcurrentIndexSql((INDEX_STATEMENTS as string[]).find(sql => sql.includes(INDEX))!), + ); + } +}); + after(async () => { await db?.$disconnect(); }); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index bd256fb46..6a0b35524 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -13,6 +13,13 @@ import { test, before, beforeEach } from "node:test"; import assert from "node:assert/strict"; import Module from "node:module"; +import type { ComponentType } from "react"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import TaxPhaseModal, { + buildTaxPatchBody, + type TaxPhaseExpense, +} from "../src/app/projects/[id]/time-expenses/TaxPhaseModal"; interface FakeUser { id: string; @@ -2002,3 +2009,161 @@ test("both handlers refuse the OTHER one's ack shape", async () => { assert.equal(booleanToPut.status, 400); assert.equal((await booleanToPut.json()).code, "TAX_REVIEW_ACK_MALFORMED"); }); + +// ── the MODAL's ack body, through the REAL route (round 47, item 1) ──────── + +/** + * The bug this closes could not be seen from either side alone. The server + * correctly requires all three fields to clear a flag; the modal sent the two + * numbers, and sent `installedAtCustomer` only when the person CHANGED it. So + * the most ordinary case there is — re-check the figures, leave the + * yes/no/unknown answer as it was — sent two of three keys and came back 400 + * every time, which means the flag could not be cleared through the UI at all. + * + * These drive the modal's own body builder into the real PATCH handler, so a + * change to either side has to keep them agreeing. + */ +const FLAGGED: TaxPhaseExpense = { + id: "e1", + vendor: "Lowe's", + description: null, + amount: 207.74, + taxAmount: 16.55, + taxAtSource: true, + installedAtCustomer: true, + taxDeductibleBase: null, + needsTaxReview: true, + taxSource: "manual", + costCodeId: null, +}; + +/** The form as the modal opens it: every field showing the stored value. */ +const UNTOUCHED = { + installed: "yes" as const, + taxAmount: "16.55", + taxKnown: false, + base: "", + costCodeId: "", + reviewAck: true, +}; + +test("the modal's ack carries the installation answer even when it did not change", async () => { + const body = buildTaxPatchBody(FLAGGED, UNTOUCHED); + assert.equal(body.taxReviewAck, true); + assert.equal(body.taxAmount, 16.55, "the figure, re-certified"); + assert.equal(body.taxDeductibleBase, null, "and the base, re-certified as 'the whole pre-tax total'"); + assert.ok("installedAtCustomer" in body, "the field the excise report keys on is NAMED"); + assert.equal(body.installedAtCustomer, true, "as the value on screen"); +}); + +test("...and that body clears the flag through the real handler", async () => { + storedExpense = { + ...(storedExpense as object), + installedAtCustomer: true, + needsTaxReview: true, + taxSource: "manual", + } as Record; + const res = await patch(buildTaxPatchBody(FLAGGED, UNTOUCHED)); + assert.equal(res.status, 200, `the ack must be accepted: ${JSON.stringify(await res.clone().json())}`); + assert.equal(updateArgs?.data.needsTaxReview, false, "and the flag actually comes down"); +}); + +test("CONTROL: the body the modal used to send is refused", async () => { + // Verbatim the old rule: the ack block sent the two numbers, and + // `installedAtCustomer` went only when it changed — which here it did not. + const preFix = { ...buildTaxPatchBody(FLAGGED, UNTOUCHED) }; + delete preFix.installedAtCustomer; + + storedExpense = { + ...(storedExpense as object), + installedAtCustomer: true, + needsTaxReview: true, + taxSource: "manual", + } as Record; + const res = await patch(preFix); + assert.equal(res.status, 400, "this is what the bookkeeper got, every time"); + assert.equal((await res.json()).code, "TAX_REVIEW_ACK_MALFORMED"); + assert.equal(updateArgs, null, "and the flag stayed up"); +}); + +test("a CHANGED installation answer still travels, and still only once", async () => { + // The other branch: the ordinary block already wanted to send it. The two + // blocks must not disagree about the value, and the key must not be sent + // twice with different values. + const body = buildTaxPatchBody(FLAGGED, { ...UNTOUCHED, installed: "unknown" }); + assert.equal(body.installedAtCustomer, null); + storedExpense = { + ...(storedExpense as object), + installedAtCustomer: true, + needsTaxReview: true, + taxSource: "manual", + } as Record; + const res = await patch(body); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.installedAtCustomer, null, "the person's new answer is what lands"); +}); + +test("an UNFLAGGED row still sends only what changed", async () => { + // The exception is scoped to a flag being cleared. On an ordinary edit, + // re-sending an untouched field would stamp provenance on it for no reason. + const body = buildTaxPatchBody( + { ...FLAGGED, needsTaxReview: false }, + { ...UNTOUCHED, reviewAck: false, base: "50" }, + ); + assert.deepEqual(body, { taxDeductibleBase: 50 }); +}); + +test("the checkbox says what it certifies", () => { + // The label is part of the contract: the body now re-certifies the + // installation answer, so the person ticking the box has to be told that is + // what they are doing. + const markup = renderToStaticMarkup(createElement( + TaxPhaseModal as unknown as ComponentType>, + { expense: FLAGGED, phases: [], onClose: () => {}, onSaved: () => {} }, + )); + assert.match(markup, /installed at a customer job/i); + assert.match(markup, /re-checked the tax/i); +}); + +// ── the DATE, on the PUT (Codex round 47, item 2) ────────────────────────── + +test("an IMPOSSIBLE but well-shaped date is a 400, not a 500", async () => { + // `2026-02-31` passes `/^\d{4}-\d{2}-\d{2}$/`, which was the whole test + // this route applied, and then throws inside `dateOnlyInTimeZone` — so a + // typo came back as "Failed to update expense" with a 500, and the value + // that caused it was nowhere in the response. + const res = await call({ date: "2026-02-31" }); + assert.equal(res.status, 400); + const json = await res.json(); + assert.match(json.error, /not a real calendar date/); + assert.equal(json.date, "2026-02-31"); + assert.equal(updateArgs, null, "and nothing was written"); +}); + +test("...and unparseable junk is refused rather than stored as an Invalid Date", async () => { + const res = await call({ date: "yesterday" }); + assert.equal(res.status, 400); + assert.equal(updateArgs, null); +}); + +test("a real calendar day and a full timestamp both still work", async () => { + const day = await call({ date: "2026-08-14" }); + assert.equal(day.status, 200); + assert.equal( + (updateArgs?.data.date as Date).toISOString().slice(0, 10), + "2026-08-14", + "company noon on the day sent, not UTC midnight the day before", + ); + + updateArgs = null; + const instant = await call({ date: "2026-08-14T18:30:00.000Z" }); + assert.equal(instant.status, 200); + const written = updateArgs as { data: Record } | null; + assert.equal((written?.data.date as Date).toISOString(), "2026-08-14T18:30:00.000Z"); +}); + +test("clearing the date is still possible", async () => { + const res = await call({ date: null }); + assert.equal(res.status, 200); + assert.equal(updateArgs?.data.date, null); +}); diff --git a/tests/receipt-ingest-attribution.test.ts b/tests/receipt-ingest-attribution.test.ts index c363aedd8..81a826bd5 100644 --- a/tests/receipt-ingest-attribution.test.ts +++ b/tests/receipt-ingest-attribution.test.ts @@ -250,7 +250,10 @@ test("the pair is written from the LOCKED estimate, together", async () => { const res = await post(PAYLOAD); assert.equal(res.status, 200); assert.deepEqual(await res.json(), { - ok: true, created: 1, projectId: "job-1", projectName: "Berg ADU", warnings: [], + // `dateSource` joined the response in round 47, item 2: "this row is + // dated today because nobody sent a date" is the one outcome a caller + // cannot work out from its own payload. + ok: true, created: 1, projectId: "job-1", projectName: "Berg ADU", dateSource: "supplied", warnings: [], }); assert.equal(created.length, 1); assert.equal(created[0].projectId, "job-1"); @@ -515,3 +518,80 @@ test("the id is stored EXACTLY as sent, never derived from the url", async () => assert.equal(created[0].sourceFileId, "1AbC_-dEf"); assert.equal(created[0].receiptUrl, "https://example.test/whatever", "the url is still the human link"); }); + +// ── the DATE (Codex round 47, item 2) ────────────────────────────────────── + +/** + * Three situations, and the route used to have two answers for them. + * + * * OMITTED — legitimate, books today, and NOBODY WAS TOLD. + * * MALFORMED (`07/14/2026`, `Jul 14 2026`) — fell through the same branch + * and also booked today, so a receipt from another quarter landed in this + * one with a `created: 1` that looked like success. + * * IMPOSSIBLE (`2026-02-31`, the shape a bad OCR read produces most often) + * — passed the shape test, reached `dateOnlyInTimeZone`, and threw: a 500 + * for a caller's typo. + */ +const dayOf = (value: unknown) => new Date(value as Date).toISOString().slice(0, 10); + +test("a supplied date is used, and reported as supplied", async () => { + const res = await post(PAYLOAD); + const json = await res.json(); + assert.equal(json.ok, true); + assert.equal(json.dateSource, "supplied"); + // Noon company time on the day sent, not UTC midnight (which reads as the + // day before in Pacific and files the receipt in the wrong month). + assert.equal(dayOf(created[0].date), "2026-08-14"); +}); + +test("an OMITTED date books today, and the response SAYS it did", async () => { + const { date: _omitted, ...noDate } = PAYLOAD; + const res = await post({ ...noDate, fileId: "drive-file-omitted" }); + const json = await res.json(); + assert.equal(res.status, 200, "omitting a date is allowed"); + assert.equal(json.dateSource, "defaulted-today"); + assert.ok( + json.warnings.some((warning: string) => /dated today/i.test(warning)), + `the caller is told: ${JSON.stringify(json.warnings)}`, + ); + assert.equal(dayOf(created[0].date), new Date().toISOString().slice(0, 10)); +}); + +test("a MALFORMED date is a 400 naming it, not a silent booking on today", async () => { + for (const bad of ["07/14/2026", "Jul 14 2026", "2026-8-14", "yesterday"]) { + created = []; + const res = await post({ ...PAYLOAD, date: bad, fileId: `drive-file-${bad}` }); + assert.equal(res.status, 400, `${bad} must be refused`); + const json = await res.json(); + assert.equal(json.reason, "invalid-date"); + assert.equal(json.date, bad, "the offending value is named"); + assert.deepEqual(created, [], "and nothing was written"); + } +}); + +test("an IMPOSSIBLE but well-shaped date is a 400, not a 500", async () => { + // CONTROL for the old behaviour: this value passes `/^\d{4}-\d{2}-\d{2}$/`, + // which is the whole test the route used to apply, and then throws inside + // the parser. A 500 tells the Apps Script nothing it can act on, and it + // retries the same bytes forever. + const res = await post({ ...PAYLOAD, date: "2026-02-31", fileId: "drive-file-impossible" }); + assert.equal(res.status, 400); + const json = await res.json(); + assert.equal(json.reason, "invalid-date"); + assert.equal(json.date, "2026-02-31"); + assert.match(json.detail, /not a real calendar date/); + assert.deepEqual(created, [], "nothing was written"); +}); + +test("the date is judged BEFORE any group is inserted", async () => { + // Ordering matters: a document refused for its date must not leave half its + // groups behind, the same rule the malformed-group check follows. + const res = await post({ + ...PAYLOAD, + date: "2026-02-31", + fileId: "drive-file-order", + groups: [{ category: "Plumbing", amount: 10 }, { category: "Plumbing", amount: 20 }], + }); + assert.equal(res.status, 400); + assert.deepEqual(created, []); +}); diff --git a/tests/tz-date.test.ts b/tests/tz-date.test.ts index e2fc8490a..97d171b67 100644 --- a/tests/tz-date.test.ts +++ b/tests/tz-date.test.ts @@ -8,6 +8,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { addCalendarDaysInTimeZone, + classifyCalendarDate, addDaysToKey, dayKeyInTimeZone, startOfDateInTimeZone, @@ -84,3 +85,46 @@ test("startOfDateInTimeZone and endOfDateInTimeZone bracket a full company-local assert.ok(end.getTime() > start.getTime()); assert.ok(end.getTime() - start.getTime() < 86_400_000); // 23:59:59.999, just under 24h }); + +// ── the calendar-day validator (Codex round 47, item 2) ──────────────────── + +test("classifyCalendarDate separates OMITTED from SUPPLIED-AND-WRONG", () => { + // Three answers, because the intakes need three different behaviours: an + // absent date may default to today, a malformed one may NOT (that is how a + // receipt from another quarter got booked into this one), and an + // impossible one may not reach the parser (that is how a typo became a + // 500). + for (const omitted of [null, undefined, "", " "]) { + assert.deepEqual(classifyCalendarDate(omitted), { kind: "omitted" }, String(omitted)); + } + assert.deepEqual(classifyCalendarDate("2026-08-14"), { kind: "valid", date: "2026-08-14" }); + assert.deepEqual(classifyCalendarDate(" 2026-08-14 "), { kind: "valid", date: "2026-08-14" }); + + for (const malformed of ["07/14/2026", "Jul 14 2026", "2026-8-14", "20260814", "yesterday"]) { + const verdict = classifyCalendarDate(malformed); + assert.equal(verdict.kind, "invalid", malformed); + assert.equal((verdict as { value: string }).value, malformed.trim()); + assert.match((verdict as { reason: string }).reason, /YYYY-MM-DD/); + } +}); + +test("...and it rejects a well-shaped IMPOSSIBLE day", () => { + // The shape a bad OCR read produces most often. JavaScript rolls + // 2026-02-31 forward to 3 March rather than refusing it, so only a round + // trip detects it. + for (const impossible of ["2026-02-31", "2026-13-01", "2026-00-10", "2026-04-31", "2025-02-29"]) { + const verdict = classifyCalendarDate(impossible); + assert.equal(verdict.kind, "invalid", impossible); + assert.match((verdict as { reason: string }).reason, /not a real calendar date/); + } + // ...while a real leap day passes. + assert.deepEqual(classifyCalendarDate("2024-02-29"), { kind: "valid", date: "2024-02-29" }); +}); + +test("a non-string is invalid, not omitted", () => { + // `{ date: 20260814 }` is a caller bug. Treating it as absent would book + // the row on today and say nothing. + for (const value of [20260814, {}, [], true]) { + assert.equal(classifyCalendarDate(value).kind, "invalid", JSON.stringify(value)); + } +}); From 77544007a91eaf98729c1adce9ce6f6b4658a2c4 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 3 Sep 2026 10:28:18 -0700 Subject: [PATCH 141/144] fix(expenses): pin the apply script to a PROJECT REF, and take the date tests off the real Prisma singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `--target prod` guard could not tell production from a migrated staging clone: Supabase's pooler hostnames are shared REGIONALLY and every project's database is called `postgres`, so `--expect-db`, `--expect-host` and the baseline-migration row all match either one. The identity lives in the URL USERNAME (`postgres.`). `projectRefVerdict` now reads it (from the username, or from a `db..supabase.co` host for a direct connection) and requires it to equal `APPLY_EXPECT_PROJECT_REF` — the shared name all five apply scripts use. UNSET is a refusal, not a skipped check; `--target ci` does not ask for one; the redacted identity line names the ref. PR body tells Justin to set it. CI red on 53831788: the new PUT date test's SUCCESS path fell through to the real Prisma singleton (`resolveCompanyTimeZone`), which throws in CI where DATABASE_URL is unset — a local .env masked it. The harness now stubs `@/lib/company-timezone` with the REAL parsers and a fixed zone, so no test in that file can reach a database. `npm run test:unit` verified with DATABASE_URL UNSET: 2134 pass, 0 fail. Co-Authored-By: Claude Fable 5.1 --- scripts/apply-expense-attribution.mjs | 70 ++++++++++++++++++++++++- tests/apply-expense-attribution.test.ts | 61 +++++++++++++++++++++ tests/expense-edit-authz.test.ts | 8 +++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 89d3643d0..1ff1683c9 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -83,6 +83,13 @@ export const APPLY_TARGETS = { requireBaseline: true, hostMustMatch: /(^|\.)pooler\.supabase\.com$/i, hostDescription: "the Supabase pooler", + // THE HOST IS NOT THE IDENTITY. Supabase's pooler hostnames are shared + // REGIONALLY — `aws-0-us-west-2.pooler.supabase.com` is every project + // in that region — and the database is called `postgres` in all of + // them. A migrated staging clone therefore matches the host, the + // database name AND the baseline row. What actually names the project + // is the URL's USERNAME: `postgres.`. + requireProjectRef: true, }, ci: { envFile: null, @@ -90,6 +97,9 @@ export const APPLY_TARGETS = { requireBaseline: false, hostMustNotMatch: /supabase\.(co|com)$/i, hostDescription: "a throwaway container", + // A container has no project ref, and requiring one would only mean + // inventing a fake to satisfy the check. + requireProjectRef: false, }, }; @@ -164,9 +174,60 @@ export function targetHostVerdict(name, url) { return null; } +/** + * The Supabase PROJECT REF out of a connection URL, or null. + * + * The pooler username is `postgres.`; a direct connection uses a + * bare `postgres` with the ref in the HOST (`db..supabase.co`). Both are + * read, because a future change of connection style must not silently turn the + * check off. + */ +export function projectRefFromUrl(url) { + let parsed; + try { + parsed = new URL(url); + } catch { + return null; + } + const user = decodeURIComponent(parsed.username ?? ""); + const dotted = /^postgres\.([a-z0-9]+)$/i.exec(user); + if (dotted) return dotted[1]; + const host = /^db\.([a-z0-9]+)\.supabase\.co$/i.exec(parsed.hostname ?? ""); + return host ? host[1] : null; +} + +/** + * Is this the project the operator meant? `APPLY_EXPECT_PROJECT_REF` is the + * shared name every apply script uses, so setting it once covers all of them. + * + * UNSET IS A REFUSAL, not a skip. A guard that disables itself when its input + * is missing protects nothing on the machine that matters — the one where + * somebody is running this in a hurry. + */ +export function projectRefVerdict(name, url, env = process.env) { + const target = APPLY_TARGETS[name]; + if (!target?.requireProjectRef) return null; + const expected = (env.APPLY_EXPECT_PROJECT_REF ?? "").trim(); + if (!expected) { + return `--target ${name} requires APPLY_EXPECT_PROJECT_REF (the Supabase project ref, e.g. the value in postgres.). Set it and re-run.`; + } + const actual = projectRefFromUrl(url); + if (!actual) { + return `--target ${name} could not read a project ref from the connection URL — expected a postgres. username or a db..supabase.co host.`; + } + if (actual !== expected) { + return `REFUSING: this URL is for project ${actual}, not ${expected}. The pooler host and the database name are shared across projects in a region, so they cannot tell production from a staging clone.`; + } + return null; +} + /** The one line printed before any DDL, with the credentials removed. */ export function targetBanner(name, { url, from, db, host }) { - return `TARGET ${name}: db="${db}" server="${host || "(local socket)"}" url=${maskUrl(url)} (from ${from})`; + const ref = projectRefFromUrl(url); + return ( + `TARGET ${name}: db="${db}" server="${host || "(local socket)"}" ` + + `project="${ref ?? "(none)"}" url=${maskUrl(url)} (from ${from})` + ); } export function maskUrl(url) { @@ -1689,6 +1750,13 @@ async function main() { console.error(`REFUSING: ${hostProblem}`); process.exit(1); } + // ...and WHICH project, not just which kind of host. Checked before the + // client is even constructed, so a wrong ref never opens a connection. + const refProblem = projectRefVerdict(chosen.name, url); + if (refProblem) { + console.error(refProblem.startsWith("REFUSING") ? refProblem : `REFUSING: ${refProblem}`); + process.exit(1); + } const prisma = new PrismaClient({ datasources: { db: { url } } }); try { diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index de4d9a51e..39b732bc0 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -22,6 +22,8 @@ import { APPLY_TARGETS, PRODUCTION_BASELINE_MIGRATION, parseTarget, + projectRefFromUrl, + projectRefVerdict, resolveTargetDatabaseUrl, targetBanner, targetHostVerdict, @@ -1556,3 +1558,62 @@ test("THE ACTUAL SCRIPT refuses an ambient local URL, before any DDL", () => { assert.match(asProd.output, /REFUSING/); assert.doesNotMatch(asProd.output, /localhost/, "the ambient URL is not even echoed as a candidate"); }); + +test("the HOST is not the identity — the project ref is", () => { + // Supabase pooler hostnames are shared REGIONALLY and every project's + // database is called `postgres`, so host + database name + baseline row + // all match a migrated staging clone just as well as they match + // production. The ref in the URL username is the only thing that does not. + const PROD = "postgresql://postgres.ghzdbzdnwjxazvmcefbh:pw@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true"; + const CLONE = "postgresql://postgres.stagingprojectref:pw@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true"; + + assert.equal(projectRefFromUrl(PROD), "ghzdbzdnwjxazvmcefbh"); + assert.equal(projectRefFromUrl(CLONE), "stagingprojectref"); + // The direct-connection spelling puts the ref in the HOST instead. Read + // too, so changing connection style cannot silently disable the check. + assert.equal( + projectRefFromUrl("postgresql://postgres:pw@db.ghzdbzdnwjxazvmcefbh.supabase.co:5432/postgres"), + "ghzdbzdnwjxazvmcefbh", + ); + assert.equal(projectRefFromUrl("postgresql://probuild:probuild@localhost:5432/probuild"), null); + + const env = { APPLY_EXPECT_PROJECT_REF: "ghzdbzdnwjxazvmcefbh" } as unknown as NodeJS.ProcessEnv; + assert.equal(projectRefVerdict("prod", PROD, env), null, "the real project passes"); + const refused = projectRefVerdict("prod", CLONE, env); + assert.match(refused ?? "", /this URL is for project stagingprojectref, not ghzdbzdnwjxazvmcefbh/); + assert.match(refused ?? "", /shared across projects in a region/, "and it says WHY the other checks did not catch it"); +}); + +test("an UNSET APPLY_EXPECT_PROJECT_REF is a refusal, not a skipped check", () => { + // A guard that turns itself off when its input is missing protects nothing + // on the machine that matters — the one where somebody is in a hurry. + const PROD = "postgresql://postgres.ghzdbzdnwjxazvmcefbh:pw@aws-0-us-west-2.pooler.supabase.com:6543/postgres"; + for (const env of [{}, { APPLY_EXPECT_PROJECT_REF: "" }, { APPLY_EXPECT_PROJECT_REF: " " }]) { + assert.match( + projectRefVerdict("prod", PROD, env as unknown as NodeJS.ProcessEnv) ?? "", + /requires APPLY_EXPECT_PROJECT_REF/, + ); + } + // The name is shared with the other apply scripts on purpose: one variable, + // set once, covers all of them. + assert.match( + projectRefVerdict("prod", PROD, {} as unknown as NodeJS.ProcessEnv) ?? "", + /APPLY_EXPECT_PROJECT_REF/, + ); + // ...and CI never asks for one: a throwaway container has no project ref. + assert.equal( + projectRefVerdict("ci", "postgresql://probuild:probuild@localhost:5432/probuild_apply", {} as unknown as NodeJS.ProcessEnv), + null, + ); +}); + +test("the banner names the project, still redacted", () => { + const line = targetBanner("prod", { + url: "postgresql://postgres.ghzdbzdnwjxazvmcefbh:sup3rs3cret@aws-0-us-west-2.pooler.supabase.com:6543/postgres", + from: ".env.production.local", + db: "postgres", + host: "10.0.0.5", + }); + assert.match(line, /project="ghzdbzdnwjxazvmcefbh"/); + assert.doesNotMatch(line, /sup3rs3cret/); +}); diff --git a/tests/expense-edit-authz.test.ts b/tests/expense-edit-authz.test.ts index 6a0b35524..62c1d0d76 100644 --- a/tests/expense-edit-authz.test.ts +++ b/tests/expense-edit-authz.test.ts @@ -13,6 +13,7 @@ import { test, before, beforeEach } from "node:test"; import assert from "node:assert/strict"; import Module from "node:module"; +import * as tzDate from "../src/lib/tz-date"; import type { ComponentType } from "react"; import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; @@ -161,6 +162,13 @@ before(async () => { id: string, ) { if (id === "@/lib/prisma") return { prisma: fakePrisma }; + // The company zone comes from the DATABASE. Every other test here + // stops before that read; the date tests do not, and in CI (no + // DATABASE_URL) the singleton throws on construction. The date + // PARSERS are the real ones — they are what is under test. + if (id === "@/lib/company-timezone") { + return { ...tzDate, resolveCompanyTimeZone: async () => "America/Los_Angeles" }; + } if (id === "@/lib/permissions") { return { getCurrentUserWithPermissions: async () => currentUser, From ed1f9e707492c26b480201d2cab9d85ae2d04f33 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 3 Sep 2026 11:26:44 -0700 Subject: [PATCH 142/144] =?UTF-8?q?fix(expenses):=20PR=20#442=20round-18?= =?UTF-8?q?=20gate=20=E2=80=94=20drain-window=20bridge=20for=20Drive=20rec?= =?UTF-8?q?eipts,=20variance=20pass-through,=20coverage=20parity,=20item?= =?UTF-8?q?=20ownership=20under=20lock,=20target=20guard=20on=20the=20mone?= =?UTF-8?q?y=20backfill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. [P1] A mixed-version deploy could permanently duplicate a Drive receipt. Old instances insert with sourceFileId/sourceGroupIndex NULL — their client predates both columns — so the new route's dedupe (fast path AND locked re-check) could not see those rows, a retried delivery inserted the whole receipt again, and the post-deploy backfill made the duplicate permanent. New drain-window trigger `probuild_expense_source_file_bridge` (created pre-deploy, dropped by --post-deploy AFTER the stamping backfill, in the committed migration too) derives the id from receiptUrl with the backfill's own expression and takes the SAME advisory key the route takes (`hashtextextended('receipt-ingest:' || sourceFileId, 0)`), so the two versions serialize. The ordinal counts within the TRANSACTION, not the table, so a re-delivery collides with the rows already there instead of landing on fresh ordinals. The route's dedupe also matches a legacy row by exact receiptUrl when sourceFileId is NULL. 2. [P1] "Clear phase" still did not clear the variance report: `job-variance-db.ts` selected `costCodeSource` and dropped it when building `VarianceExpense`, so `manual-none` arrived as undefined, read as "nobody has spoken", and the item fallback went on charging the phase. Passed through, and proved through the REAL loader over a seeded database. 3. [P2] The coverage headline measured a different universe than the report it describes: item metadata was loaded only for items EXPENSES link to (so labor attributed through `estimateItemId` read as unattributed), and WRITE eligibility (active code + committed estimate) was applied to a MEASUREMENT the page makes without either filter. The item universe now covers time entries too, and `scopedItemCostCodes` takes the phase gate as OPTIONAL: writes pass it, measurements pass null. `runBackfill` returns the numbers so a test can compare them with `computeProjectVariance` on one seed. 4. [P2] The locked re-read reintroduced the cross-job-item bug: `readItem` discarded an item whose code was null or retired, which `planBackfill` cannot tell from "no such item", so a corrupt link fell through to regex inference instead of `item-outside-estimate`. Ownership now survives a missing code, and the item's OWNING estimate joins the locked union in canonical order with a re-read that refuses if that ownership moved. 5. [P2] The money backfill had no target guard at all — it loaded .env.local and wrote with whatever DATABASE_URL was in the shell, while the DDL script already had one. The guard is now a shared module, `scripts/lib/apply-target.mjs` (pure, no entrypoint, proved inert on import), imported by both: `--apply` requires `--target prod|ci` plus --expect-db/--expect-host, the same project-ref/host/baseline verification, and prints the redacted identity line before any write. The CI e2e driver now runs the backfill with `--target ci`. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 17 + .../migration.sql | 84 ++++ scripts/apply-expense-attribution.mjs | 427 ++++++++---------- scripts/backfill-expense-attribution.ts | 207 +++++++-- scripts/ci-apply-expense-attribution-e2e.mjs | 12 + scripts/lib/apply-target.mjs | 277 ++++++++++++ .../api/integrations/receipt-ingest/route.ts | 26 +- src/lib/job-variance-db.ts | 10 + tests/apply-expense-attribution.test.ts | 95 +++- tests/apply-scripts-inert-on-import.test.ts | 42 +- tests/attribution-lock-order-db.test.ts | 146 +++++- tests/backfill-coverage-parity-db.test.ts | 281 ++++++++++++ tests/receipt-ingest-attribution.test.ts | 72 ++- tests/receipt-ingest-drain-window-db.test.ts | 352 +++++++++++++++ tests/scripts-runtime-smoke.test.ts | 47 ++ 15 files changed, 1815 insertions(+), 280 deletions(-) create mode 100644 scripts/lib/apply-target.mjs create mode 100644 tests/backfill-coverage-parity-db.test.ts create mode 100644 tests/receipt-ingest-drain-window-db.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae0569897..5a593a1da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,23 @@ jobs: env: PHASE_INVARIANT_DB_TEST_URL: postgresql://probuild:probuild@localhost:5432/probuild_migrations + # THE MIXED-VERSION DRAIN WINDOW. An old instance's INSERT is raw SQL that + # names neither new column -- a shape today's Prisma client cannot express -- + # and the fix is a trigger that stamps it and takes the route's advisory lock. + # Neither half exists outside a real server. + - name: Receipt-ingest drain-window bridge against real Postgres + run: npx tsx --test tests/receipt-ingest-drain-window-db.test.ts + env: + PHASE_INVARIANT_DB_TEST_URL: postgresql://probuild:probuild@localhost:5432/probuild_migrations + + # The backfill's coverage headline against the report it claims to describe, + # on one seeded database. A hand-built fixture cannot catch the two readers + # drifting apart, because the fixture is written by whoever wrote the code. + - name: Backfill coverage parity with the variance report + run: npx tsx --test tests/backfill-coverage-parity-db.test.ts + env: + PHASE_INVARIANT_DB_TEST_URL: postgresql://probuild:probuild@localhost:5432/probuild_migrations + # An index can EXIST and enforce nothing. A failed CREATE INDEX # CONCURRENTLY leaves the index behind with the right name and # `indisvalid = false`; the planner ignores it, a UNIQUE one guards diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index f5cf85ae0..33d794b6f 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -430,6 +430,84 @@ BEFORE UPDATE OF "amount" ON "Expense" FOR EACH ROW EXECUTE FUNCTION probuild_expense_amount_tax_guard(); +-- THE DRAIN-WINDOW BRIDGE FOR DRIVE RECEIPTS (Codex round 48, item 1). +-- +-- Old instances insert receipt rows with NULL sourceFileId/sourceGroupIndex, +-- because their Prisma client predates both columns. The new route dedupes on +-- sourceFileId, so it cannot see those rows and a retried delivery inserts the +-- whole receipt twice. This trigger stamps the id from receiptUrl on INSERT and +-- takes the SAME per-file advisory lock the route takes, so the two versions +-- serialize; the ordinal counts within the transaction, so a re-delivery lands +-- on the ordinals already there and the partial unique index refuses it. +-- +-- Like the two guards above it is drain-window scaffolding: created here and +-- dropped at the end of this file, so a fresh database finishes in the shape +-- production finishes in. +CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() + RETURNS trigger + LANGUAGE plpgsql + AS $bridge$ + DECLARE + derived TEXT; + next_index INT; + counter_key TEXT; + BEGIN + -- A row that already names its file speaks for itself: the new build + -- inserted it, and it has chosen its own group ordinal. + IF NEW."sourceFileId" IS NULL AND NEW."receiptUrl" IS NOT NULL THEN + derived := COALESCE( + substring(NEW."receiptUrl" from '/d/([A-Za-z0-9_-]+)'), + substring(NEW."receiptUrl" from '[?&]id=([A-Za-z0-9_-]+)') + ); + NEW."sourceFileId" := derived; + END IF; + + IF NEW."sourceFileId" IS NULL THEN + RETURN NEW; + END IF; + + -- THE SAME LOCK THE ROUTE TAKES, so an old-version insert and a + -- new-version request for one file cannot both believe they are first. + -- Transaction-scoped: released at COMMIT or ROLLBACK, nothing to leak. + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) + ); + + -- THE ORDINAL COUNTS WITHIN THIS TRANSACTION, NOT WITHIN THE TABLE. + -- + -- MAX(existing) + 1 was the obvious rule and it is the wrong one: it + -- makes a RE-DELIVERY of a document the table already holds land on + -- fresh ordinals and insert cleanly, which is the duplicate this whole + -- bridge exists to stop. Counting per transaction instead means the N + -- groups of one delivery get 0,1,2..., and a SECOND delivery of the + -- same file starts at 0 again -- colliding with the row already there + -- and aborting on the partial unique index + -- ("sourceFileId", "sourceGroupIndex"). An old instance that retries a + -- document therefore fails loudly instead of duplicating it, and its + -- Apps Script does not archive the file. + -- + -- set_config(..., true) is transaction-local, so the counter cannot + -- survive a COMMIT or leak into another session. The key is hashed + -- because a Drive file id is not a legal GUC name. + IF NEW."sourceGroupIndex" IS NULL THEN + counter_key := 'probuild.bridge_' || md5(NEW."sourceFileId"); + next_index := COALESCE(NULLIF(current_setting(counter_key, true), '')::int, -1) + 1; + PERFORM set_config(counter_key, next_index::text, true); + NEW."sourceGroupIndex" := next_index; + END IF; + + RETURN NEW; + END; + $bridge$; + +DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge ON "Expense"; + +CREATE TRIGGER probuild_expense_source_file_bridge + BEFORE INSERT ON "Expense" + FOR EACH ROW + EXECUTE FUNCTION probuild_expense_source_file_bridge(); + + -- THE BACKFILL READS THE ESTIMATE UNDER A LOCK (Codex round 32). A plain -- `UPDATE ... FROM "Estimate"` join takes no row lock, so under READ COMMITTED @@ -585,3 +663,9 @@ DROP TRIGGER IF EXISTS probuild_expense_amount_tax_guard ON "Expense"; DROP FUNCTION IF EXISTS probuild_expense_amount_tax_guard(); DROP TRIGGER IF EXISTS probuild_expense_amount_tax_ack ON "Expense"; DROP FUNCTION IF EXISTS probuild_expense_amount_tax_ack(); + +-- ...and the bridge comes out with them. It is the most expensive of the +-- three to leave standing: an advisory lock on every expense insert that +-- carries a Drive url, forever. +DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge ON "Expense"; +DROP FUNCTION IF EXISTS probuild_expense_source_file_bridge(); diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 1ff1683c9..5258b8321 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -49,208 +49,42 @@ export function resolveDatabaseUrl() { throw new Error("DATABASE_URL not found in process.env, .env.local, or .env"); } -/** - * WHICH DATABASE, SAID OUT LOUD (cross-PR rule, round 46). - * - * `resolveDatabaseUrl` above prefers an AMBIENT `DATABASE_URL`. That is the - * right default for a driver that hands the script a throwaway container, and - * the wrong one for a person: a developer with a local Postgres in their shell - * runs this, watches every "verified ..." line print, and merges believing - * production has the columns. Nothing in the output contradicts them — - * `--expect-db postgres --expect-host ...` can be satisfied by a local server - * as easily as by the real one, because the operator supplies both sides of - * that comparison. - * - * So the TARGET is now an explicit argument, and each target decides where the - * URL may come from: - * - * * `--target prod` reads `.env.production.local` and IGNORES the ambient - * `DATABASE_URL` entirely — the file Vercel writes is the only thing that - * can name production — and additionally requires the pooler host and the - * production baseline migration row. - * * `--target ci` is the throwaway container: ambient `DATABASE_URL`, no - * baseline row (a database built from `migrate deploy` in a fresh - * container has one, but a hand-rolled fixture may not), and it REFUSES a - * Supabase-looking URL so the CI path can never be pointed at prod. - * - * Both are named on the command line. There is deliberately no default: a - * missing `--target` is an error, not a guess. - */ -export const APPLY_TARGETS = { - prod: { - envFile: ".env.production.local", - allowAmbient: false, - requireBaseline: true, - hostMustMatch: /(^|\.)pooler\.supabase\.com$/i, - hostDescription: "the Supabase pooler", - // THE HOST IS NOT THE IDENTITY. Supabase's pooler hostnames are shared - // REGIONALLY — `aws-0-us-west-2.pooler.supabase.com` is every project - // in that region — and the database is called `postgres` in all of - // them. A migrated staging clone therefore matches the host, the - // database name AND the baseline row. What actually names the project - // is the URL's USERNAME: `postgres.`. - requireProjectRef: true, - }, - ci: { - envFile: null, - allowAmbient: true, - requireBaseline: false, - hostMustNotMatch: /supabase\.(co|com)$/i, - hostDescription: "a throwaway container", - // A container has no project ref, and requiring one would only mean - // inventing a fake to satisfy the check. - requireProjectRef: false, - }, +import { + APPLY_TARGETS, + PRODUCTION_BASELINE_MIGRATION, + maskUrl, + parseTarget, + projectRefFromUrl, + projectRefVerdict, + resolveTargetDatabaseUrl, + targetBanner, + targetHostVerdict, + resolveTargetOrRefuse, + targetMatches, + verifyTargetIdentity, +} from "./lib/apply-target.mjs"; + +// Re-exported so this file stays the one import site its tests and the CI +// driver already use; the DEFINITIONS live in scripts/lib/apply-target.mjs so +// the money backfill checks the same things (round 48, item 5). +export { + APPLY_TARGETS, + PRODUCTION_BASELINE_MIGRATION, + maskUrl, + parseTarget, + projectRefFromUrl, + projectRefVerdict, + resolveTargetDatabaseUrl, + targetBanner, + targetHostVerdict, + targetMatches, }; -/** The migration whose presence proves this is the real, baselined database. */ -export const PRODUCTION_BASELINE_MIGRATION = "20260814000000_baseline_production"; - -/** - * `--target ` out of an argv array. Returns the name or an error string; - * never throws, so `main()` can print and exit rather than stack-trace. - */ -export function parseTarget(argv) { - const idx = argv.indexOf("--target"); - if (idx < 0) { - return { error: `--target is required: one of ${Object.keys(APPLY_TARGETS).join(", ")}.` }; - } - const name = argv[idx + 1]; - if (!name || !Object.prototype.hasOwnProperty.call(APPLY_TARGETS, name)) { - return { error: `Unknown --target ${JSON.stringify(name ?? null)}: expected one of ${Object.keys(APPLY_TARGETS).join(", ")}.` }; - } - return { name, target: APPLY_TARGETS[name] }; -} - -/** - * The URL this target is allowed to use. - * - * `env` and the two fs functions are parameters so the rule can be tested - * without a `.env.production.local` on the machine running the tests — and so - * the "ambient DATABASE_URL is ignored for prod" claim is checked rather than - * asserted. - */ -export function resolveTargetDatabaseUrl( - name, - { env = process.env, exists = fs.existsSync, read = file => fs.readFileSync(file, "utf8") } = {}, -) { - const target = APPLY_TARGETS[name]; - if (!target) return { error: `Unknown target ${name}.` }; - if (target.envFile) { - if (!exists(target.envFile)) { - return { error: `--target ${name} reads ${target.envFile}, which does not exist. Run: vercel env pull ${target.envFile}` }; - } - const match = String(read(target.envFile)).match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); - if (!match) return { error: `${target.envFile} has no DATABASE_URL.` }; - // Deliberately NOT falling back to the ambient value: for this target - // the file is the only authority, and a missing key is an error rather - // than a reason to use whatever is in the shell. - return { url: match[1], from: target.envFile }; - } - if (!env.DATABASE_URL) return { error: `--target ${name} needs DATABASE_URL in the environment.` }; - return { url: env.DATABASE_URL, from: "process.env.DATABASE_URL" }; -} - -/** - * Does the URL's HOST agree with what this target is? Checked on the URL and - * not on `inet_server_addr()`, because the latter is an IP address and "is - * this the pooler" is a question about the name we dialled. - */ -export function targetHostVerdict(name, url) { - const target = APPLY_TARGETS[name]; - if (!target) return `Unknown target ${name}.`; - let host; - try { - host = new URL(url).hostname; - } catch { - return `The resolved DATABASE_URL is not a valid URL.`; - } - if (target.hostMustMatch && !target.hostMustMatch.test(host)) { - return `--target ${name} expects ${target.hostDescription}, but the URL points at ${host}.`; - } - if (target.hostMustNotMatch && target.hostMustNotMatch.test(host)) { - return `--target ${name} must never point at ${host} — that is production.`; - } - return null; -} - -/** - * The Supabase PROJECT REF out of a connection URL, or null. - * - * The pooler username is `postgres.`; a direct connection uses a - * bare `postgres` with the ref in the HOST (`db..supabase.co`). Both are - * read, because a future change of connection style must not silently turn the - * check off. - */ -export function projectRefFromUrl(url) { - let parsed; - try { - parsed = new URL(url); - } catch { - return null; - } - const user = decodeURIComponent(parsed.username ?? ""); - const dotted = /^postgres\.([a-z0-9]+)$/i.exec(user); - if (dotted) return dotted[1]; - const host = /^db\.([a-z0-9]+)\.supabase\.co$/i.exec(parsed.hostname ?? ""); - return host ? host[1] : null; -} - -/** - * Is this the project the operator meant? `APPLY_EXPECT_PROJECT_REF` is the - * shared name every apply script uses, so setting it once covers all of them. - * - * UNSET IS A REFUSAL, not a skip. A guard that disables itself when its input - * is missing protects nothing on the machine that matters — the one where - * somebody is running this in a hurry. - */ -export function projectRefVerdict(name, url, env = process.env) { - const target = APPLY_TARGETS[name]; - if (!target?.requireProjectRef) return null; - const expected = (env.APPLY_EXPECT_PROJECT_REF ?? "").trim(); - if (!expected) { - return `--target ${name} requires APPLY_EXPECT_PROJECT_REF (the Supabase project ref, e.g. the value in postgres.). Set it and re-run.`; - } - const actual = projectRefFromUrl(url); - if (!actual) { - return `--target ${name} could not read a project ref from the connection URL — expected a postgres. username or a db..supabase.co host.`; - } - if (actual !== expected) { - return `REFUSING: this URL is for project ${actual}, not ${expected}. The pooler host and the database name are shared across projects in a region, so they cannot tell production from a staging clone.`; - } - return null; -} - -/** The one line printed before any DDL, with the credentials removed. */ -export function targetBanner(name, { url, from, db, host }) { - const ref = projectRefFromUrl(url); - return ( - `TARGET ${name}: db="${db}" server="${host || "(local socket)"}" ` + - `project="${ref ?? "(none)"}" url=${maskUrl(url)} (from ${from})` - ); -} - -export function maskUrl(url) { - return url.replace(/:[^:@]*@/, ":****@"); -} - function readFlagValue(flag) { const idx = process.argv.indexOf(flag); return idx >= 0 ? process.argv[idx + 1] : undefined; } -/** - * Pure comparison, exported for unit testing without a live DB. Compares BOTH - * database name and server host, and both EXACTLY — same rule and same reason - * as apply-receipt-intake.mjs: a guard that accepts a substring gets looser the - * shorter the operator's input is. - */ -export function targetMatches(actual, expectDb, expectHost) { - if (!actual || typeof actual !== "object") return false; - if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; - return String(actual.host ?? "") === String(expectHost ?? ""); -} - /** * The company zone the legacy re-anchor uses. Read from CompanySettings at run * time when it is there; the fallback matches src/lib/tz-date.ts. @@ -846,6 +680,11 @@ export const AMOUNT_TAX_GUARD_DROP_SQL = [ export const COMPATIBILITY_TRIGGERS = [ "probuild_expense_estimate_pair_guard", "probuild_expense_amount_tax_guard", + // The Drive-receipt bridge (round 48, item 1). Listed here so the verify + // pass reports it standing during the drain window and gone afterwards, + // like the other two -- a scaffolding trigger nobody checks for is one that + // outlives its window. + "probuild_expense_source_file_bridge", ]; /** @@ -1229,7 +1068,127 @@ export function toConcurrentIndexSql(sql) { * for either FK, and well before phase B's backfill — see the comment at * SPLIT_JOB_GUARD_SQL for why they must exist before the columns carry values. */ -export const TRIGGER_STATEMENTS = [...SPLIT_JOB_GUARD_SQL, ...AMOUNT_TAX_GUARD_SQL]; +/** + * THE DRAIN-WINDOW BRIDGE FOR DRIVE RECEIPTS (Codex round 48, item 1). + * + * THE FAILURE. During the rollout, old instances are still serving + * `/api/integrations/receipt-ingest`. Their Prisma client predates + * `sourceFileId`/`sourceGroupIndex`, so every row they insert carries NULL in + * both — including rows inserted AFTER the pre-deploy `SOURCE_FILE_ID_BACKFILL` + * has already run. The NEW route dedupes on `sourceFileId` alone, in both its + * fast path and its locked re-check, so it cannot see those rows: a delivery + * whose response was lost, retried against a new instance, inserts the whole + * receipt a second time. The `--post-deploy` backfill then stamps the legacy + * row's `sourceFileId` and the duplicate is permanent — and the partial unique + * index never objected, because it only covers rows where `sourceFileId` is NOT + * NULL, which the legacy row was not at insert time. + * + * THE BRIDGE. A BEFORE INSERT trigger, created pre-deploy and dropped by + * `--post-deploy` exactly like the amount/tax guard, that does two things for + * any insert which does not speak for itself: + * + * 1. derives `sourceFileId` from `receiptUrl` with the SAME expression + * `SOURCE_FILE_ID_BACKFILL` uses, so an old build's row is identified the + * moment it lands rather than minutes later, and + * 2. takes THE SAME per-file advisory lock the new route takes, with the same + * key expression, so an old-version insert and a new-version request for + * one file serialize against each other. Whichever runs second sees the + * first: the new route because its locked dedupe now finds a stamped row, + * the old route because the unique index refuses the second copy of a + * group. + * + * The group ordinal is assigned MAX+1 per file, under that lock, so a document + * the old build writes as N rows becomes N distinct keys rather than N + * conflicting NULLs. Rows already in the table when this trigger is created are + * the backfill's job, not the trigger's. + * + * WHY A TRIGGER AND NOT AN APPLICATION FIX. The instances that need fixing are + * running code that predates the columns. There is no application change that + * reaches them; the database is the only writer both versions go through. + */ +/** + * The advisory-lock key, byte-for-byte what the route computes. + * + * `src/app/api/integrations/receipt-ingest/route.ts` locks + * `hashtextextended('receipt-ingest:' || fileId, 0)`. A trigger that hashed + * anything else -- `hashtext` instead of `hashtextextended`, or the bare id + * without the prefix -- would take a DIFFERENT lock and serialize with nobody, + * which is the one way this bridge could look installed and do nothing. + * tests/apply-expense-attribution.test.ts pins the two against each other. + */ +export const SOURCE_FILE_BRIDGE_SQL = [ + `CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() + RETURNS trigger + LANGUAGE plpgsql + AS $bridge$ + DECLARE + derived TEXT; + next_index INT; + counter_key TEXT; + BEGIN + -- A row that already names its file speaks for itself: the new build + -- inserted it, and it has chosen its own group ordinal. + IF NEW."sourceFileId" IS NULL AND NEW."receiptUrl" IS NOT NULL THEN + derived := COALESCE( + substring(NEW."receiptUrl" from '/d/([A-Za-z0-9_-]+)'), + substring(NEW."receiptUrl" from '[?&]id=([A-Za-z0-9_-]+)') + ); + NEW."sourceFileId" := derived; + END IF; + + IF NEW."sourceFileId" IS NULL THEN + RETURN NEW; + END IF; + + -- THE SAME LOCK THE ROUTE TAKES, so an old-version insert and a + -- new-version request for one file cannot both believe they are first. + -- Transaction-scoped: released at COMMIT or ROLLBACK, nothing to leak. + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) + ); + + -- THE ORDINAL COUNTS WITHIN THIS TRANSACTION, NOT WITHIN THE TABLE. + -- + -- MAX(existing) + 1 was the obvious rule and it is the wrong one: it + -- makes a RE-DELIVERY of a document the table already holds land on + -- fresh ordinals and insert cleanly, which is the duplicate this whole + -- bridge exists to stop. Counting per transaction instead means the N + -- groups of one delivery get 0,1,2..., and a SECOND delivery of the + -- same file starts at 0 again -- colliding with the row already there + -- and aborting on the partial unique index + -- ("sourceFileId", "sourceGroupIndex"). An old instance that retries a + -- document therefore fails loudly instead of duplicating it, and its + -- Apps Script does not archive the file. + -- + -- set_config(..., true) is transaction-local, so the counter cannot + -- survive a COMMIT or leak into another session. The key is hashed + -- because a Drive file id is not a legal GUC name. + IF NEW."sourceGroupIndex" IS NULL THEN + counter_key := 'probuild.bridge_' || md5(NEW."sourceFileId"); + next_index := COALESCE(NULLIF(current_setting(counter_key, true), '')::int, -1) + 1; + PERFORM set_config(counter_key, next_index::text, true); + NEW."sourceGroupIndex" := next_index; + END IF; + + RETURN NEW; + END; + $bridge$`, + `DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge ON "Expense"`, + `CREATE TRIGGER probuild_expense_source_file_bridge + BEFORE INSERT ON "Expense" + FOR EACH ROW + EXECUTE FUNCTION probuild_expense_source_file_bridge()`, +]; + +export const SOURCE_FILE_BRIDGE_DROP_SQL = [ + // Dropped by --post-deploy with the other compatibility scaffolding: once + // every instance names its own file id, this only costs an advisory lock + // and a MAX() on every expense insert in the system. + `DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge ON "Expense"`, + `DROP FUNCTION IF EXISTS probuild_expense_source_file_bridge()`, +]; + +export const TRIGGER_STATEMENTS = [...SPLIT_JOB_GUARD_SQL, ...AMOUNT_TAX_GUARD_SQL, ...SOURCE_FILE_BRIDGE_SQL]; /** * ReceiptIntake is Phase 1's table, not Project, Estimate, or even Expense — @@ -1491,6 +1450,13 @@ export function postDeployTeardownStatements({ repairSplitJobs = false } = {}) { // rewrites `projectId` and `attributionAnchoredAt` and never touches // `amount`, so this trigger never sees it either way. ...AMOUNT_TAX_GUARD_DROP_SQL, + // ...and the Drive-receipt bridge (round 48, item 1). It comes out + // LAST, after SOURCE_FILE_ID_BACKFILL has run in this same + // --post-deploy pass: while it stands, an old instance's insert is + // still stamped and still serialized, so the backfill cannot race a + // straggler and leave the very duplicate this bridge exists to + // prevent. + ...SOURCE_FILE_BRIDGE_DROP_SQL, ]; } @@ -1735,62 +1701,33 @@ async function main() { process.exit(1); } - // The TARGET decides where the URL comes from — for `prod` that is - // `.env.production.local` and the ambient `DATABASE_URL` is ignored, which - // is the whole point: a local server in the shell must not be able to - // impersonate production. - const resolved = resolveTargetDatabaseUrl(chosen.name); - if (resolved.error) { - console.error(`REFUSING: ${resolved.error}`); - process.exit(1); - } - const { url, from } = resolved; - const hostProblem = targetHostVerdict(chosen.name, url); - if (hostProblem) { - console.error(`REFUSING: ${hostProblem}`); - process.exit(1); - } - // ...and WHICH project, not just which kind of host. Checked before the - // client is even constructed, so a wrong ref never opens a connection. - const refProblem = projectRefVerdict(chosen.name, url); - if (refProblem) { - console.error(refProblem.startsWith("REFUSING") ? refProblem : `REFUSING: ${refProblem}`); + // WHICH DATABASE, decided before a client exists (round 48, item 5). The + // target picks where the URL may come from — for `prod` that is + // `.env.production.local` and the ambient `DATABASE_URL` is ignored — and + // the host and project ref readable from that URL are checked here, so a + // wrong answer never opens a connection. Shared with the money backfill. + const targeted = resolveTargetOrRefuse(process.argv); + if (targeted.error) { + console.error(`REFUSING: ${targeted.error}`); process.exit(1); } + const { url, from } = targeted; const prisma = new PrismaClient({ datasources: { db: { url } } }); try { - const [actual] = await prisma.$queryRawUnsafe( - `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, - ); - // The REDACTED target line, before a single statement of phase A — - // so what the operator sees first is which database is about to be - // changed, with the credentials removed. - console.log(targetBanner(chosen.name, { url, from, db: actual.db, host: actual.host })); - if (!targetMatches(actual, expectDb, expectHost)) { - console.error(`REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to db="${actual.db}" host="${actual.host}".`); + // The server's OWN answer about what it is, plus the baseline-row + // proof for a target that demands one, plus the REDACTED identity + // line — one shared call, so this script and the money backfill can + // never check different things (round 48, item 5). + const identity = await verifyTargetIdentity(prisma, { + target: chosen.name, url, from, expectDb, expectHost, + }); + console.log(identity.banner); + if (!identity.ok) { + console.error(identity.error); process.exit(1); } - // AND THE DATABASE'S OWN IDENTITY, not just the one we dialled. The - // production baseline row is written once, by the deliberate - // `migrate resolve --applied` step documented in CLAUDE.md; a local - // database somebody built with `db push` does not have it, and neither - // does a fresh container built from a subset of the migrations. - if (APPLY_TARGETS[chosen.name].requireBaseline) { - const baseline = await prisma.$queryRawUnsafe( - `SELECT 1 AS present FROM "_prisma_migrations" - WHERE migration_name = $1 AND finished_at IS NOT NULL`, - PRODUCTION_BASELINE_MIGRATION, - ); - if (!baseline?.length) { - console.error( - `REFUSING: this database has no applied ${PRODUCTION_BASELINE_MIGRATION} row, ` + - `so it is not the baselined production database.`, - ); - process.exit(1); - } - console.log(`verified baseline ${PRODUCTION_BASELINE_MIGRATION} is applied here`); - } + for (const note of identity.notes ?? []) console.log(note); // The company zone, for the legacy re-anchor below. Read before the DDL // so a bad answer fails before anything is written. diff --git a/scripts/backfill-expense-attribution.ts b/scripts/backfill-expense-attribution.ts index 894349adb..2e642e3d4 100644 --- a/scripts/backfill-expense-attribution.ts +++ b/scripts/backfill-expense-attribution.ts @@ -52,6 +52,9 @@ */ import { PrismaClient } from "@prisma/client"; import { config } from "dotenv"; +// The SHARED target guard — the same module scripts/apply-expense-attribution.mjs +// imports, so the two scripts cannot check different things (round 48, item 5). +import { resolveTargetOrRefuse, verifyTargetIdentity } from "./lib/apply-target.mjs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { writeFileSync } from "node:fs"; @@ -118,11 +121,21 @@ export function scopedItemCostCodes(rows, items, allowedCodesByProject) { const item = items.get(row.itemId); if (!item || !item.costCodeId) continue; const projectId = resolveExpenseProjectId(row); - // Same two gates the writer applies: the link must not cross jobs, and - // the code must be a live phase of that job. + // The cross-job gate is unconditional: a link to another job's line + // item resolves to nothing on the variance page and must resolve to + // nothing here. if (!projectId || item.projectId !== projectId) continue; - const allowed = allowedCodesByProject.get(projectId); - if (!allowed || !allowed.has(item.costCodeId)) continue; + // The PHASE gate is optional (round 48, item 3). It is the WRITE rule + // — an active code carried by a committed estimate — and belongs to + // callers deciding what this script may write. A caller MEASURING what + // the variance report shows passes null, because that report keeps + // draft/archived attribution-only items and never checks `isActive`; + // applying the write rule there reported money as unattributed that the + // page itself attributes. + if (allowedCodesByProject) { + const allowed = allowedCodesByProject.get(projectId); + if (!allowed || !allowed.has(item.costCodeId)) continue; + } scoped.set(coverageKey(projectId, row.itemId), item.costCodeId); } return scoped; @@ -553,7 +566,27 @@ export async function runBackfill({ // already lives (`allowedCodesByProject`, built from committed estimates). // Scoped to the items expenses actually LINK to, so widening the filter // does not turn this into a full-table read. - const linkedItemIds = [...new Set(expenses.map(e => e.itemId).filter(Boolean))]; + // ...AND THE ITEMS ONLY TIME ENTRIES POINT AT (round 48, item 3). + // + // The labor half of the coverage metric resolves `TimeEntry.estimateItemId` + // through this very map, and the query below used to be scoped to the items + // EXPENSES link to. Labor whose phase comes only from its line item — + // exactly what the clock-in flow produces — therefore found no entry and + // was reported as unattributed, which understated the one metric the + // rollout is judged on. The time entries are read here, before the map, + // rather than in the middle of the report where they used to be. + const timeEntries = await db.timeEntry.findMany({ + where: { projectId: { in: scopedProjectIds } }, + select: { + costCodeId: true, estimateItemId: true, laborCost: true, burdenCost: true, + // Needed to scope the item fallback to the entry's OWN job. + projectId: true, + }, + }); + const linkedItemIds = [...new Set([ + ...expenses.map(e => e.itemId), + ...timeEntries.map(t => t.estimateItemId), + ].filter(Boolean))]; const itemRowsRaw = linkedItemIds.length ? await db.estimateItem.findMany({ where: { id: { in: linkedItemIds } }, @@ -571,6 +604,14 @@ export async function runBackfill({ // item, and every reader of `costCodeId` gets the same answer it got // when the query itself did the filtering. costCodeId: row.costCode?.isActive ? row.costCodeId : null, + // WHAT THE VARIANCE REPORT SEES (round 48, item 3). The report's + // attribution pool is every coded line item on the project's estimates + // — any status, and with no `costCode.isActive` filter (see + // `attributionOnlyItems` in src/lib/job-variance-db.ts). Measuring + // coverage through the WRITE-eligibility view above called those rows + // unattributed when the page itself attributes them, so the script and + // the page disagreed about the same seed. + reportedCostCodeId: row.costCodeId ?? null, estimateId: row.estimateId, projectId: row.estimate?.projectId ?? null, })); @@ -580,6 +621,14 @@ export async function runBackfill({ projectId: row.projectId, }])); const itemCostCodeById = new Map(itemRows.map(row => [row.id, row.costCodeId])); + // The same ownership, with the code the REPORT would use. Two maps rather + // than one flag, because every reader of `items` is a WRITE decision and + // every reader of this one is a MEASUREMENT. + const reportingItems = new Map(itemRows.map(row => [row.id, { + costCodeId: row.reportedCostCodeId, + estimateId: row.estimateId, + projectId: row.projectId, + }])); // THE PHASES EACH JOB ACTUALLY HAS — the same set the app itself uses. // @@ -646,11 +695,20 @@ export async function runBackfill({ estimate: { select: { projectId: true } }, }, }); - // An item whose code was cleared, or whose code was deactivated, is no - // longer a source of one — the same rule the snapshot query applies. - if (!row?.costCodeId || row.costCode?.isActive === false) return fresh; + // OWNERSHIP SURVIVES A MISSING CODE (round 48, item 4). + // + // This used to return an EMPTY map when the item had no code, or a + // retired one — which `planBackfill` cannot tell apart from "there is + // no such item". So a cross-job link whose code had just been cleared + // stopped being reported as `item-outside-estimate` and fell through to + // regex inference instead: the locked re-plan reintroduced, one + // transaction later, exactly the bug round 44 fixed in the snapshot. + // + // The two questions stay separate here, as they do in the snapshot: the + // row still says who OWNS the item, and an unusable code reads as null. + if (!row) return fresh; fresh.set(row.id, { - costCodeId: row.costCodeId, + costCodeId: row.costCode?.isActive ? row.costCodeId : null, estimateId: row.estimateId, projectId: row.estimate?.projectId ?? null, }); @@ -664,9 +722,13 @@ export async function runBackfill({ // ── the table ─────────────────────────────────────────────────────────── const scoped = new Set(scopedProjectIds); const inScopeExpenses = expenses.filter(e => scoped.has(resolveExpenseProjectId(e) ?? "")); - // Project-scoped and phase-validated, so a cross-job item link stays - // unattributed in the metric exactly as it does on the variance page. - const coverageItems = scopedItemCostCodes(inScopeExpenses, items, allowedCodesByProject); + // Project-scoped, exactly as the variance page is — but NOT phase-validated + // (round 48, item 3). Passing `allowedCodesByProject` here applied the + // WRITE rule (an active code on a committed estimate) to a MEASUREMENT of + // what the report shows, and the report keeps draft/archived + // attribution-only items and never checks `isActive`. The cross-job scope + // stays, because the page really does drop a link to another job's item. + const coverageItems = scopedItemCostCodes(inScopeExpenses, reportingItems, null); const before = measureCoverage(inScopeExpenses, coverageItems); const after = measureCoverage(projectedRows(inScopeExpenses, plan.codeFills), coverageItems); @@ -691,14 +753,6 @@ export async function runBackfill({ // The §1.6 headline is measured on the variance page's basis, which counts // LABOR as well. Reporting only the expense share would flatter the number, // because clock-in already requires a phase. - const timeEntries = await db.timeEntry.findMany({ - where: { projectId: { in: scopedProjectIds } }, - select: { - costCodeId: true, estimateItemId: true, laborCost: true, burdenCost: true, - // Needed to scope the item fallback to the entry's OWN job. - projectId: true, - }, - }); // PROJECT-SCOPED, exactly like the expense side. A time entry pointing at // another job's estimate item is unattributed on the variance page, so // resolving it through a global id->code map counted labor dollars as @@ -710,8 +764,8 @@ export async function runBackfill({ estimate: null, itemId: t.estimateItemId, })), - items, - allowedCodesByProject, + reportingItems, + null, ); const laborRows = timeEntries.map(t => ({ costCodeId: resolveExpenseCostCodeId( @@ -756,10 +810,21 @@ export async function runBackfill({ log(`wrote ${csvPath} (${plan.remainder.length} rows for Marge)`); } + // The measured numbers, RETURNED and not merely printed (round 48, item 3). + // tests/backfill-coverage-parity-db.test.ts re-computes the same figures + // with `computeProjectVariance` over the same seed and asserts they agree; + // a metric nobody can compare against its own source is a claim, not a + // measurement. + const coverage = { + expenses: { before, after }, + labor, + varianceBasis: { before: variancedBefore, after: variancedAfter, total: variancedTotal }, + }; + if (!apply) { log(""); log("DRY RUN — nothing written. Re-run with --apply once the table above is reviewed."); - return { plan, before, after, written: { projectIds: 0, costCodes: 0 } }; + return { plan, before, after, coverage, written: { projectIds: 0, costCodes: 0 } }; } // ── the writes ────────────────────────────────────────────────────────── @@ -853,9 +918,21 @@ export async function runBackfill({ // do not cover it. const plannedEstimateId = fill.expense?.estimateId ?? null; const plannedItemId = fill.expense?.itemId ?? null; + // THE ITEM OWNER IS PART OF THE ANSWER (round 48, item 4). + // + // `readItem` reads the item's estimate to decide whether the link + // crosses jobs. That estimate is not necessarily the expense's own, and + // it was not in the lock set — so the fact the verdict rests on could + // move while this transaction held everything else still. Named here, + // from the plan, so `lockAttributionParents` takes it in the canonical + // order with the rest; the re-read below then refuses if the ownership + // it locked is not the ownership that exists. + const plannedItemOwnerEstimateId = plannedItemId + ? (items.get(plannedItemId)?.estimateId ?? null) + : null; const result = await writeUnderAttributionLocks(db, { expenseId: fill.id, - estimateIds: [plannedEstimateId], + estimateIds: [plannedEstimateId, plannedItemOwnerEstimateId], estimateItemIds: [plannedItemId], phaseProjectId: fill.expectedProjectId ?? null, costCodeId: fill.costCodeId, @@ -914,6 +991,17 @@ export async function runBackfill({ // re-read here any more; see the proof below. const resolvedProjectId = current.projectId ?? current.estimate?.projectId ?? null; const freshItems = await readItem(tx, current.itemId); + // THE OWNERSHIP THIS WRITE LOCKED IS THE OWNERSHIP IT JUDGES. + // + // If the item has been moved onto a different estimate since the + // plan, the estimate whose `projectId` decides "does this link + // cross jobs" is one nothing here is holding. Skipped and counted, + // like every other moved-under-us case; a re-run plans it against + // the truth and locks what that truth rests on. + const freshOwner = current.itemId ? freshItems.get(current.itemId) : undefined; + if (current.itemId && (freshOwner?.estimateId ?? null) !== plannedItemOwnerEstimateId) { + return { count: 0 }; + } // THE RE-PLAN PROPOSES; THE LOCKED PROOF DECIDES. // // `planBackfill`'s phase gate is a set-membership test against a @@ -1015,19 +1103,40 @@ export async function runBackfill({ plan, before, after, + coverage, + // Skips are part of the outcome, not just a log line: a run that plans + // a write and then refuses it because the row moved is the correct + // behaviour, and a test has to be able to see the difference between + // that and a write that never happened. + skipped: { costCodes: costCodesSkipped }, written: { projectIds: projectIdsWritten, costCodes: costCodesWritten }, }; } +/** A flag's value from argv, read inside main() so import stays inert. */ +function expectFlag(flag: string): string | undefined { + const idx = process.argv.indexOf(flag); + return idx >= 0 ? process.argv[idx + 1] : undefined; +} + const HELP = `Backfill Expense attribution (Receipt Pipeline v2, Phase 3). node --import=tsx scripts/backfill-expense-attribution.ts # dry run node --import=tsx scripts/backfill-expense-attribution.ts --csv out.csv # + remainder CSV - node --import=tsx scripts/backfill-expense-attribution.ts --apply # write + node --import=tsx scripts/backfill-expense-attribution.ts --target prod \ + --expect-db --expect-host --apply # write Dry run is the DEFAULT. --apply writes; re-run dry afterwards and it must report zero planned changes. +--apply REQUIRES --target (prod or ci) plus --expect-db and --expect-host, the +same guard scripts/apply-expense-attribution.mjs uses and from the same shared +module. --target prod reads the URL from .env.production.local (an ambient +DATABASE_URL is IGNORED), requires the Supabase pooler host, requires +APPLY_EXPECT_PROJECT_REF to match the project ref in that URL, and requires the +production baseline migration to be applied. A redacted identity line is +printed before anything is written. + The --import=tsx loader is required: this script imports TypeScript from src/, and plain node only strips types on Node 22.6+.`; @@ -1049,8 +1158,54 @@ async function main() { const csvIdx = process.argv.indexOf("--csv"); const csvPath = csvIdx > -1 ? process.argv[csvIdx + 1] : null; - const prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL } } }); + // WHICH DATABASE (round 48, item 5). + // + // This script WRITES MONEY COLUMNS — `projectId`, `costCodeId`, + // `costCodeSource` — and until now it took whatever `DATABASE_URL` the + // shell happened to hold, after loading .env.local and .env. The DDL apply + // script had already grown a target guard, a project-ref check and a + // redacted identity line; this one had none of it, so the more dangerous of + // the two scripts was the less guarded. Same helper, same checks, imported + // rather than copied. + // + // A DRY RUN still needs no target: it only reads, and forcing a target on + // it would push people toward `--apply` to get any output at all. + let url = process.env.DATABASE_URL; + let target: string | null = null; + let from = "process.env.DATABASE_URL"; + if (apply) { + const targeted = resolveTargetOrRefuse(process.argv); + if (targeted.error) { + console.error(`REFUSING: ${targeted.error}`); + process.exitCode = 1; + return; + } + ({ target, url, from } = targeted as { target: string; url: string; from: string }); + if (!expectFlag("--expect-db") || !expectFlag("--expect-host")) { + console.error( + "REFUSING: --apply requires --expect-db and --expect-host as well as --target.", + ); + process.exitCode = 1; + return; + } + } + + const prisma = new PrismaClient({ datasources: { db: { url } } }); try { + if (apply && target) { + const identity = await verifyTargetIdentity(prisma, { + target, url, from, + expectDb: expectFlag("--expect-db"), + expectHost: expectFlag("--expect-host"), + }); + console.log(identity.banner); + if (!identity.ok) { + console.error(identity.error); + process.exitCode = 1; + return; + } + for (const note of identity.notes ?? []) console.log(note); + } await runBackfill({ db: prisma, apply, csvPath }); } finally { await prisma.$disconnect(); diff --git a/scripts/ci-apply-expense-attribution-e2e.mjs b/scripts/ci-apply-expense-attribution-e2e.mjs index c5a5d3ebd..81a22f331 100644 --- a/scripts/ci-apply-expense-attribution-e2e.mjs +++ b/scripts/ci-apply-expense-attribution-e2e.mjs @@ -81,4 +81,16 @@ run("node", [script, "--post-deploy", ...guard], env); // between the phases has to be safe to re-run from the top. console.log("\n=== pre-deploy, again (idempotency) ==="); run("node", [script, ...guard], env); +// ...AND THE MONEY BACKFILL, through the same guard (round 48, item 5). +// It writes `projectId`/`costCodeId`/`costCodeSource`, so its target guard is +// the one that most needs proving end to end: `--target ci` resolves the +// ambient URL, refuses a Supabase-looking host, and prints the redacted +// identity line before a single row is touched. The database is empty here, +// so the run is a no-op by construction — what is under test is that it LOADS +// under tsx, passes the guard, and reaches its own reporting. +const backfill = path.join("scripts", "backfill-expense-attribution.ts"); +console.log(""); +console.log("=== money backfill (--target ci) ==="); +run("node", ["--import=tsx", backfill, "--target", "ci", "--expect-db", DB, "--expect-host", host, "--apply"], env); + console.log("\napply script end-to-end: OK"); diff --git a/scripts/lib/apply-target.mjs b/scripts/lib/apply-target.mjs new file mode 100644 index 000000000..2159a44f6 --- /dev/null +++ b/scripts/lib/apply-target.mjs @@ -0,0 +1,277 @@ +// THE ONE PLACE THAT ANSWERS "WHICH DATABASE AM I ABOUT TO WRITE TO?" +// +// Shared by every script that can change production data — the DDL apply +// script and the money backfill both import it (Codex round 48, item 5). It +// was extracted rather than copied because the two had already drifted once: +// the apply script grew `--target`, a project-ref check and a redacted +// identity banner, while `backfill-expense-attribution.ts` still took whatever +// `DATABASE_URL` happened to be in the shell and wrote money columns with it. +// A second copy of a guard is a guard that will be half-updated. +// +// PURE, AND INERT ON IMPORT. There is no `main()` here and nothing runs at +// module scope: importing this file reads no environment, opens no connection +// and executes no SQL. `tests/apply-scripts-inert-on-import.test.ts` pins that +// (scripts/apply-*.mjs is checked for its entrypoint guard; this file is +// checked for having no entrypoint at all). +import fs from "node:fs"; + +/** + * WHICH DATABASE, SAID OUT LOUD (cross-PR rule, round 46). + * + * `resolveDatabaseUrl` above prefers an AMBIENT `DATABASE_URL`. That is the + * right default for a driver that hands the script a throwaway container, and + * the wrong one for a person: a developer with a local Postgres in their shell + * runs this, watches every "verified ..." line print, and merges believing + * production has the columns. Nothing in the output contradicts them — + * `--expect-db postgres --expect-host ...` can be satisfied by a local server + * as easily as by the real one, because the operator supplies both sides of + * that comparison. + * + * So the TARGET is now an explicit argument, and each target decides where the + * URL may come from: + * + * * `--target prod` reads `.env.production.local` and IGNORES the ambient + * `DATABASE_URL` entirely — the file Vercel writes is the only thing that + * can name production — and additionally requires the pooler host and the + * production baseline migration row. + * * `--target ci` is the throwaway container: ambient `DATABASE_URL`, no + * baseline row (a database built from `migrate deploy` in a fresh + * container has one, but a hand-rolled fixture may not), and it REFUSES a + * Supabase-looking URL so the CI path can never be pointed at prod. + * + * Both are named on the command line. There is deliberately no default: a + * missing `--target` is an error, not a guess. + */ +export const APPLY_TARGETS = { + prod: { + envFile: ".env.production.local", + allowAmbient: false, + requireBaseline: true, + hostMustMatch: /(^|\.)pooler\.supabase\.com$/i, + hostDescription: "the Supabase pooler", + // THE HOST IS NOT THE IDENTITY. Supabase's pooler hostnames are shared + // REGIONALLY — `aws-0-us-west-2.pooler.supabase.com` is every project + // in that region — and the database is called `postgres` in all of + // them. A migrated staging clone therefore matches the host, the + // database name AND the baseline row. What actually names the project + // is the URL's USERNAME: `postgres.`. + requireProjectRef: true, + }, + ci: { + envFile: null, + allowAmbient: true, + requireBaseline: false, + hostMustNotMatch: /supabase\.(co|com)$/i, + hostDescription: "a throwaway container", + // A container has no project ref, and requiring one would only mean + // inventing a fake to satisfy the check. + requireProjectRef: false, + }, +}; + +/** The migration whose presence proves this is the real, baselined database. */ +export const PRODUCTION_BASELINE_MIGRATION = "20260814000000_baseline_production"; + +/** + * `--target ` out of an argv array. Returns the name or an error string; + * never throws, so `main()` can print and exit rather than stack-trace. + */ +export function parseTarget(argv) { + const idx = argv.indexOf("--target"); + if (idx < 0) { + return { error: `--target is required: one of ${Object.keys(APPLY_TARGETS).join(", ")}.` }; + } + const name = argv[idx + 1]; + if (!name || !Object.prototype.hasOwnProperty.call(APPLY_TARGETS, name)) { + return { error: `Unknown --target ${JSON.stringify(name ?? null)}: expected one of ${Object.keys(APPLY_TARGETS).join(", ")}.` }; + } + return { name, target: APPLY_TARGETS[name] }; +} + +/** + * The URL this target is allowed to use. + * + * `env` and the two fs functions are parameters so the rule can be tested + * without a `.env.production.local` on the machine running the tests — and so + * the "ambient DATABASE_URL is ignored for prod" claim is checked rather than + * asserted. + */ +export function resolveTargetDatabaseUrl( + name, + { env = process.env, exists = fs.existsSync, read = file => fs.readFileSync(file, "utf8") } = {}, +) { + const target = APPLY_TARGETS[name]; + if (!target) return { error: `Unknown target ${name}.` }; + if (target.envFile) { + if (!exists(target.envFile)) { + return { error: `--target ${name} reads ${target.envFile}, which does not exist. Run: vercel env pull ${target.envFile}` }; + } + const match = String(read(target.envFile)).match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); + if (!match) return { error: `${target.envFile} has no DATABASE_URL.` }; + // Deliberately NOT falling back to the ambient value: for this target + // the file is the only authority, and a missing key is an error rather + // than a reason to use whatever is in the shell. + return { url: match[1], from: target.envFile }; + } + if (!env.DATABASE_URL) return { error: `--target ${name} needs DATABASE_URL in the environment.` }; + return { url: env.DATABASE_URL, from: "process.env.DATABASE_URL" }; +} + +/** + * Does the URL's HOST agree with what this target is? Checked on the URL and + * not on `inet_server_addr()`, because the latter is an IP address and "is + * this the pooler" is a question about the name we dialled. + */ +export function targetHostVerdict(name, url) { + const target = APPLY_TARGETS[name]; + if (!target) return `Unknown target ${name}.`; + let host; + try { + host = new URL(url).hostname; + } catch { + return `The resolved DATABASE_URL is not a valid URL.`; + } + if (target.hostMustMatch && !target.hostMustMatch.test(host)) { + return `--target ${name} expects ${target.hostDescription}, but the URL points at ${host}.`; + } + if (target.hostMustNotMatch && target.hostMustNotMatch.test(host)) { + return `--target ${name} must never point at ${host} — that is production.`; + } + return null; +} + +/** + * The Supabase PROJECT REF out of a connection URL, or null. + * + * The pooler username is `postgres.`; a direct connection uses a + * bare `postgres` with the ref in the HOST (`db..supabase.co`). Both are + * read, because a future change of connection style must not silently turn the + * check off. + */ +export function projectRefFromUrl(url) { + let parsed; + try { + parsed = new URL(url); + } catch { + return null; + } + const user = decodeURIComponent(parsed.username ?? ""); + const dotted = /^postgres\.([a-z0-9]+)$/i.exec(user); + if (dotted) return dotted[1]; + const host = /^db\.([a-z0-9]+)\.supabase\.co$/i.exec(parsed.hostname ?? ""); + return host ? host[1] : null; +} + +/** + * Is this the project the operator meant? `APPLY_EXPECT_PROJECT_REF` is the + * shared name every apply script uses, so setting it once covers all of them. + * + * UNSET IS A REFUSAL, not a skip. A guard that disables itself when its input + * is missing protects nothing on the machine that matters — the one where + * somebody is running this in a hurry. + */ +export function projectRefVerdict(name, url, env = process.env) { + const target = APPLY_TARGETS[name]; + if (!target?.requireProjectRef) return null; + const expected = (env.APPLY_EXPECT_PROJECT_REF ?? "").trim(); + if (!expected) { + return `--target ${name} requires APPLY_EXPECT_PROJECT_REF (the Supabase project ref, e.g. the value in postgres.). Set it and re-run.`; + } + const actual = projectRefFromUrl(url); + if (!actual) { + return `--target ${name} could not read a project ref from the connection URL — expected a postgres. username or a db..supabase.co host.`; + } + if (actual !== expected) { + return `REFUSING: this URL is for project ${actual}, not ${expected}. The pooler host and the database name are shared across projects in a region, so they cannot tell production from a staging clone.`; + } + return null; +} + +/** The one line printed before any DDL, with the credentials removed. */ +export function targetBanner(name, { url, from, db, host }) { + const ref = projectRefFromUrl(url); + return ( + `TARGET ${name}: db="${db}" server="${host || "(local socket)"}" ` + + `project="${ref ?? "(none)"}" url=${maskUrl(url)} (from ${from})` + ); +} + +export function maskUrl(url) { + return url.replace(/:[^:@]*@/, ":****@"); +} + +/** + * Pure comparison, exported for unit testing without a live DB. Compares BOTH + * database name and server host, and both EXACTLY — same rule and same reason + * as apply-receipt-intake.mjs: a guard that accepts a substring gets looser the + * shorter the operator's input is. + */ +export function targetMatches(actual, expectDb, expectHost) { + if (!actual || typeof actual !== "object") return false; + if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; + return String(actual.host ?? "") === String(expectHost ?? ""); +} + +/** + * THE WHOLE IDENTITY CHECK, in one call, so two scripts cannot check different + * things. + * + * Asks the SERVER what it is (`current_database()`, `inet_server_addr()`), + * compares both against what the operator said, and — for a target that + * demands it — proves the production baseline migration is applied here. + * Returns the banner to print rather than printing it, so the caller decides + * where its output goes and a test can read the string. + * + * `prisma` is any client with `$queryRawUnsafe`; the backfill passes its own. + */ +export async function verifyTargetIdentity(prisma, { target, url, from, expectDb, expectHost }) { + const [actual] = await prisma.$queryRawUnsafe( + `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, + ); + const banner = targetBanner(target, { url, from, db: actual?.db, host: actual?.host }); + if (!targetMatches(actual, expectDb, expectHost)) { + return { + ok: false, + banner, + error: + `REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to ` + + `db="${actual?.db}" host="${actual?.host}".`, + }; + } + if (APPLY_TARGETS[target]?.requireBaseline) { + const baseline = await prisma.$queryRawUnsafe( + `SELECT 1 AS present FROM "_prisma_migrations" + WHERE migration_name = $1 AND finished_at IS NOT NULL`, + PRODUCTION_BASELINE_MIGRATION, + ); + if (!baseline?.length) { + return { + ok: false, + banner, + error: + `REFUSING: this database has no applied ${PRODUCTION_BASELINE_MIGRATION} row, ` + + `so it is not the baselined production database.`, + }; + } + return { ok: true, banner, notes: [`verified baseline ${PRODUCTION_BASELINE_MIGRATION} is applied here`] }; + } + return { ok: true, banner, notes: [] }; +} + +/** + * Everything a caller must do BEFORE it constructs a client: name the target, + * resolve the URL the way that target allows, and check the host and project + * ref that are readable from the URL alone. Returns `{ error }` or + * `{ target, url, from }`. + */ +export function resolveTargetOrRefuse(argv, env = process.env) { + const chosen = parseTarget(argv); + if (chosen.error) return { error: chosen.error }; + const resolved = resolveTargetDatabaseUrl(chosen.name, { env }); + if (resolved.error) return { error: resolved.error }; + const hostProblem = targetHostVerdict(chosen.name, resolved.url); + if (hostProblem) return { error: hostProblem }; + const refProblem = projectRefVerdict(chosen.name, resolved.url, env); + if (refProblem) return { error: refProblem.replace(/^REFUSING: /, "") }; + return { target: chosen.name, url: resolved.url, from: resolved.from }; +} diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index ff6e4b951..0b3f532ff 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -94,8 +94,25 @@ export async function POST(req: Request) { // of a file whose Drive folder has since been renamed used to answer // `alreadyIngested`, and must keep doing so rather than suddenly // reporting `project-not-matched`. + // ...AND A LEGACY ROW IS STILL THIS FILE (round 48, item 1). + // + // During the rollout, old instances insert rows with `sourceFileId` NULL — + // their Prisma client predates the column — so a dedupe on that column + // alone cannot see them, and a delivery retried against a new instance + // inserts the whole receipt again. The drain-window trigger + // (`probuild_expense_source_file_bridge`) stamps those rows on INSERT and + // serializes them against this route's advisory lock; this OR is the second + // half of the same fix, for a row whose `receiptUrl` the trigger could not + // parse an id out of. `receiptUrl` is exact-matched, never `contains` — + // that substring test is what round 34 removed. + const alreadyIngestedWhere = { + OR: [ + { sourceFileId: body.fileId }, + { AND: [{ sourceFileId: null }, { receiptUrl }] }, + ], + }; const existing = await prisma.expense.findFirst({ - where: { sourceFileId: body.fileId }, + where: alreadyIngestedWhere, select: { id: true }, }); if (existing) { @@ -270,7 +287,12 @@ export async function POST(req: Request) { `${RECEIPT_INGEST_LOCK_PREFIX}${body.fileId}`, ); const alreadyIngested = await tx.expense.findFirst({ - where: { sourceFileId: body.fileId }, + // The same predicate as the fast path, including the legacy + // `receiptUrl` arm — this is the authoritative one, taken under + // the per-file advisory lock that the drain-window trigger + // takes too, so an old instance's in-flight insert is either + // already visible here or still blocking this read. + where: alreadyIngestedWhere, select: { id: true }, }); if (alreadyIngested) return null; diff --git a/src/lib/job-variance-db.ts b/src/lib/job-variance-db.ts index 3bb0f7978..42246f45a 100644 --- a/src/lib/job-variance-db.ts +++ b/src/lib/job-variance-db.ts @@ -214,6 +214,16 @@ export async function loadProjectVariance(projectIds?: string[]): Promise ({ costCodeId: e.costCodeId, + // ...AND IT IS PASSED THROUGH (round 48, item 2). The + // select above has carried this column since round 42 and + // this mapper dropped it, so every row reached + // `computeProjectVariance` with `costCodeSource: + // undefined` — which reads as "nobody has spoken" and runs + // the item fallback. A bookkeeper's explicit "no phase" + // (`manual-none`) therefore kept charging the phase its + // line item names, on the one report that decision exists + // to correct. Selecting a column is not using it. + costCodeSource: e.costCodeSource, itemId: e.itemId, amount: Number(e.amount ?? 0), })), diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 39b732bc0..aef15e7fc 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -52,6 +52,8 @@ import { reanchorSql, SOURCE_FILE_ID_BACKFILL, SPLIT_JOB_GUARD_DROP_SQL, + SOURCE_FILE_BRIDGE_DROP_SQL, + SOURCE_FILE_BRIDGE_SQL, SPLIT_JOB_GUARD_SQL, SPLIT_JOB_REPAIR, SPLIT_JOB_REPAIR_LOCK_PROJECTS, @@ -287,7 +289,7 @@ test("every statement is additive — nothing drops, renames, or rewrites data", statement.includes("Expense_estimateId_fkey") && statement.includes("ON DELETE SET NULL"); const isGuardTriggerReplace = - /^DROP TRIGGER IF EXISTS probuild_expense_(estimate_pair|amount_tax)_(guard|ack) ON "Expense"$/.test(statement.trim()); + /^DROP TRIGGER IF EXISTS probuild_expense_(estimate_pair_guard|amount_tax_(guard|ack)|source_file_bridge) ON "Expense"$/.test(statement.trim()); assert.ok( isConstraintReplace || isGuardTriggerReplace || isNullabilityWidening || isEstimateFkReplace || !/\bDROP\b/i.test(statement), `destructive statement: ${statement}`, @@ -1140,7 +1142,14 @@ test("the post-deploy pass takes BOTH guards back out", () => { `${name}'s function is never dropped`, ); } - assert.deepEqual(teardown, [...SPLIT_JOB_GUARD_DROP_SQL, ...AMOUNT_TAX_GUARD_DROP_SQL]); + // The Drive-receipt bridge comes out in the same pass, LAST (round 48, + // item 1): while it stands, a straggler instance is still stamped and + // still serialized, so the sourceFileId backfill above it cannot race one. + assert.deepEqual(teardown, [ + ...SPLIT_JOB_GUARD_DROP_SQL, + ...AMOUNT_TAX_GUARD_DROP_SQL, + ...SOURCE_FILE_BRIDGE_DROP_SQL, + ]); }); test("the amount/tax guard is a transcription of planExpenseUpdate, not a new policy", () => { @@ -1617,3 +1626,85 @@ test("the banner names the project, still redacted", () => { assert.match(line, /project="ghzdbzdnwjxazvmcefbh"/); assert.doesNotMatch(line, /sup3rs3cret/); }); + +// ── the Drive-receipt drain-window bridge (round 48, item 1) ─────────────── + +test("the bridge locks the SAME key the ingest route locks", () => { + // The one way this bridge could look installed and do nothing: hash a + // different string, take a different lock, serialize with nobody. The + // route's key is read out of the route rather than restated here. + const route = readFileSync( + path.resolve(__dirname, "..", "src", "app", "api", "integrations", "receipt-ingest", "route.ts"), + "utf8", + ); + const prefix = route.match(/RECEIPT_INGEST_LOCK_PREFIX = "([^"]+)"/)?.[1]; + assert.equal(prefix, "receipt-ingest:", "the route's lock prefix moved"); + assert.match(route, /pg_advisory_xact_lock\(hashtextextended\(\$1, 0\)\)/, + "the route hashes with hashtextextended"); + + const fn = SOURCE_FILE_BRIDGE_SQL[0]; + assert.match(fn, /pg_advisory_xact_lock\(/); + assert.match(fn, /hashtextextended\('receipt-ingest:' \|\| NEW\."sourceFileId", 0\)/, + "the trigger must hash the SAME prefixed id, with the SAME function"); + assert.doesNotMatch(fn, /hashtext\(/, "hashtext() is a different lock space"); +}); + +test("the bridge derives the file id the same way the backfill does", () => { + // Two extractors that disagree would stamp one id at INSERT and a + // different one at backfill time, which is a duplicate with extra steps. + for (const pattern of ["/d/([A-Za-z0-9_-]+)", "[?&]id=([A-Za-z0-9_-]+)"]) { + assert.ok(SOURCE_FILE_ID_BACKFILL.includes(pattern), `backfill lost ${pattern}`); + assert.ok(SOURCE_FILE_BRIDGE_SQL[0].includes(pattern), `bridge lost ${pattern}`); + } +}); + +test("the ordinal counts within the TRANSACTION, not within the table", () => { + // MAX(existing) + 1 is the obvious rule and it is the wrong one: a + // re-delivery would land on fresh ordinals and insert cleanly, which is + // exactly the duplicate this bridge exists to stop. Counting per + // transaction makes a second delivery collide with the rows already there. + const fn = SOURCE_FILE_BRIDGE_SQL[0]; + assert.match(fn, /set_config\(counter_key, next_index::text, true\)/, + "the counter must be TRANSACTION-local"); + assert.match(fn, /current_setting\(counter_key, true\)/); + assert.doesNotMatch(fn, /MAX\("sourceGroupIndex"\)/, + "a table-wide MAX lets a re-delivery insert cleanly"); +}); + +test("it fires BEFORE INSERT and only touches rows that stay silent", () => { + assert.match(SOURCE_FILE_BRIDGE_SQL[2], /BEFORE INSERT ON "Expense"/); + assert.match(SOURCE_FILE_BRIDGE_SQL[2], /FOR EACH ROW/); + // A row that names its own file id is the NEW build's; the trigger must + // not renumber it. + assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceFileId" IS NULL AND NEW\."receiptUrl" IS NOT NULL THEN/); + assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceGroupIndex" IS NULL THEN/); + // ...and an expense with no Drive url at all pays nothing. + assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceFileId" IS NULL THEN\s+RETURN NEW;/); +}); + +test("the bridge and its teardown are BOTH in the committed migration", () => { + // Same contract as the other two guards: a fresh CI/dev database replays + // this migration end to end and must finish in production's END state, + // with no scaffolding standing. + for (const sql of [...SOURCE_FILE_BRIDGE_SQL, ...SOURCE_FILE_BRIDGE_DROP_SQL]) { + assert.ok(normalizedMigration.includes(normalize(sql).replace(/;$/, "")), `migration.sql is missing:\n ${sql}`); + } + const create = migrationSql.indexOf("CREATE TRIGGER probuild_expense_source_file_bridge"); + const fill = migrationSql.indexOf('UPDATE "Expense" e SET "projectId" = locked."projectId"'); + const drop = migrationSql.lastIndexOf("DROP FUNCTION IF EXISTS probuild_expense_source_file_bridge"); + assert.ok(create > -1 && fill > -1 && drop > -1); + assert.ok(create < fill, "it has to stand before the backfill stamps ids"); + assert.ok(fill < drop, "and it comes out only after the backfill is done"); +}); + +test("--post-deploy drops the bridge AFTER it has stamped the stragglers", () => { + // Order is the whole argument: the backfill can only be safe if nothing + // is still inserting unstamped rows behind it. + const teardown = postDeployStatements("America/Los_Angeles") + .concat(postDeployTeardownStatements({})); + const backfillAt = teardown.findIndex(sql => sql.includes('SET "sourceFileId" = COALESCE')); + const dropAt = teardown.findIndex(sql => sql.includes("DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge")); + assert.ok(backfillAt > -1, "the sourceFileId backfill runs in --post-deploy"); + assert.ok(dropAt > -1, "and the bridge is dropped in the same pass"); + assert.ok(backfillAt < dropAt, "the stamping happens while the bridge still stands"); +}); diff --git a/tests/apply-scripts-inert-on-import.test.ts b/tests/apply-scripts-inert-on-import.test.ts index 5308df6fc..6a0c71a99 100644 --- a/tests/apply-scripts-inert-on-import.test.ts +++ b/tests/apply-scripts-inert-on-import.test.ts @@ -82,7 +82,16 @@ const ALLOWED_CALLEES: Record = { }; const ALLOWED_GLOBAL_CALLEES = new Set(["process.argv.includes", "process.argv.indexOf"]); /** The only modules an apply script may import. Anything else (a `data:` URL, `dotenv/config`, a driver) runs code on import. */ -const ALLOWED_IMPORTS = new Set(["@prisma/client", "dotenv", "node:fs", "fs", "node:url", "url", "node:path", "path", "node:crypto", "crypto"]); +/** + * ...plus the repo's OWN shared guard helper, which is on this list only + * because it is held to a STRICTER rule than the scripts that import it: it + * has no entrypoint at all, and the test at the bottom of this file runs the + * same AST scan over it and imports it in a child process to prove it opens no + * connection (round 48, item 5). Adding a local module here without that proof + * would reopen exactly the hole this allowlist exists to close. + */ +const SHARED_GUARD_MODULE = "./lib/apply-target.mjs"; +const ALLOWED_IMPORTS = new Set(["@prisma/client", "dotenv", "node:fs", "fs", "node:url", "url", "node:path", "path", "node:crypto", "crypto", SHARED_GUARD_MODULE]); /** Names a script may never declare itself (they would let a guard or helper be spoofed). */ const RESERVED_NAMES = new Set(["process", "import", "pathToFileURL", "fileURLToPath", "dirname", "join", "resolve", "isMainModule"]); @@ -619,3 +628,34 @@ for (const name of scripts) { ); }); } + +// ── the shared guard helper (round 48, item 5) ───────────────────────────── + +/** + * `scripts/lib/apply-target.mjs` is imported BY apply scripts, so it is inside + * the blast radius of every rule above. It is held to a stricter version of + * them: an apply script may have a `main()` behind a guard, this may not have + * an entrypoint at all. + */ +const SHARED_HELPER = path.join(scriptsDir, "lib", "apply-target.mjs"); + +test("the shared target helper has no side effects and no entrypoint", () => { + const text = readFileSync(SHARED_HELPER, "utf8"); + const report = analyse("lib/apply-target.mjs", text); + assert.deepEqual(report.violations, [], report.violations.join("; ")); + assert.equal(report.hasMain, false, "a shared helper must not declare main()"); + assert.equal(report.guardIfs, 0, "...and must not carry an entrypoint guard"); + // The one import it is allowed is node:fs, for reading .env.production.local + // when a caller asks it to. Nothing that runs code on import. + assert.match(text, /^import fs from "node:fs";$/m); + assert.doesNotMatch(text, /^import .*@prisma\/client/m, "it must not construct a client"); +}); + +test("runtime: importing the shared target helper opens no DB connection and exits 0", async () => { + const before = connections; + const result = await importInChild(pathToFileURL(SHARED_HELPER).href); + const seen = connections - before; + assert.equal(seen, 0, `import attempted ${seen} DB connection(s)`); + assert.equal(result.code, 0, `import did not exit cleanly: ${result.stderr}`); + assert.doesNotMatch(result.stdout + result.stderr, /applied|verified|Refusing|DATABASE_URL/i); +}); \ No newline at end of file diff --git a/tests/attribution-lock-order-db.test.ts b/tests/attribution-lock-order-db.test.ts index 0e74cb04c..7364b5413 100644 --- a/tests/attribution-lock-order-db.test.ts +++ b/tests/attribution-lock-order-db.test.ts @@ -46,7 +46,7 @@ import { type QboExpensePersistenceClient, } from "../src/lib/qbo-expense-sync"; import { createParsedReceiptExpense } from "../src/lib/receipt-parse-expense"; -import { runBackfill, writeUnderAttributionLocks } from "../scripts/backfill-expense-attribution"; +import { planBackfill, runBackfill, writeUnderAttributionLocks } from "../scripts/backfill-expense-attribution"; import { backfillStatements, DDL_STATEMENTS, @@ -1995,3 +1995,147 @@ test("CONTROL: two batches taking the same rows in opposite orders deadlock", { await cleanupBatch(); } }); + +/** + * THE LOCKED RE-READ MUST NOT FORGET WHO OWNS THE ITEM (round 48, item 4). + * + * `readItem` used to return an EMPTY map when the linked item had no cost + * code, or a retired one. `planBackfill` cannot tell that apart from "there is + * no such item", so the cross-job check it does FIRST never ran, and the row + * fell through to vendor-regex inference — the locked re-plan reintroducing, + * one transaction later, exactly the bug round 44 fixed in the snapshot. + * + * The interleaving that makes it bite: the item is on this job when the plan + * is made, and its estimate moves to another job before the write. + */ +const CODELESS_ITEM = `${PFX}-item-codeless`; +/** A SECOND estimate of job A carrying the same code, so the phase survives the move. */ +const KEEPER_ESTIMATE = `${PFX}-estimate-keeper`; +const KEEPER_ITEM = `${PFX}-item-keeper`; +const REGEX_EXPENSE = `${PFX}-expense-regex`; + +async function seedCodelessCrossJob() { + await seedTwoJobs(); + // The phase has to OUTLIVE the estimate move, or a later guard + // (provePhaseMembershipTx) refuses the write for its own good reason and + // this test proves nothing about the item read. A second committed + // estimate of job A carries the same code. + await writerDb!.estimate.create({ + data: { + id: KEEPER_ESTIMATE, title: "Keeper", code: `EST-${PFX}-keeper`, projectId: PROJECT, + status: "Approved", totalAmount: 100, balanceDue: 100, + }, + }); + await writerDb!.estimateItem.create({ + data: { id: KEEPER_ITEM, estimateId: KEEPER_ESTIMATE, name: "keeps the phase", costCodeId: CODE }, + }); + // A line item with NO cost code, on job A's estimate. + await writerDb!.estimateItem.create({ + data: { id: CODELESS_ITEM, estimateId: ESTIMATE, name: "uncoded line", costCodeId: null }, + }); + // An expense the VENDOR RULE can code on its own ("Summit Plumbing" -> + // 03-PLUMB, the seeded cost code), linked to that uncoded item. This is + // the row the pre-fix read would machine-code. + await writerDb!.expense.create({ + data: { + id: REGEX_EXPENSE, projectId: PROJECT, estimateId: ESTIMATE, itemId: CODELESS_ITEM, + costCodeId: null, amount: 250, vendor: "Summit Plumbing", status: "Pending", + }, + }); +} + +async function cleanupCodelessCrossJob() { + await writerDb!.expense.deleteMany({ where: { id: REGEX_EXPENSE } }); + await writerDb!.estimateItem.deleteMany({ where: { id: { in: [CODELESS_ITEM, KEEPER_ITEM] } } }); + await writerDb!.estimate.deleteMany({ where: { id: KEEPER_ESTIMATE } }); + await cleanupTwoJobs(); +} + +test("CONTROL: a codeless item read as MISSING gets machine-coded across jobs", { skip }, async () => { + // The pre-fix read, verbatim: an empty map. `planBackfill` is the single + // copy of the rules, so driving it with each map is the honest way to show + // what the two reads make it decide about the SAME row. + await seedCodelessCrossJob(); + try { + const expense = { + id: REGEX_EXPENSE, projectId: PROJECT, estimateId: ESTIMATE, itemId: CODELESS_ITEM, + costCodeId: null, costCodeSource: null, vendor: "Summit Plumbing", + description: null, amount: 250, estimate: { projectId: TARGET_PROJECT }, + }; + const args = { + costCodeIdByCode: new Map([["03-PLUMB", CODE]]), + scopedProjectIds: [PROJECT], + allowedCodesByProject: new Map([[PROJECT, new Set([CODE])]]), + }; + + const preFix = planBackfill({ expenses: [expense], items: new Map(), ...args }); + assert.equal(preFix.codeFills.length, 1, "the pre-fix read machine-codes it"); + assert.equal(preFix.codeFills[0].costCodeId, CODE); + assert.equal(preFix.remainder.length, 0, "...and never reports the corrupt link"); + + // The SHIPPED read: ownership survives the missing code, so the + // cross-job check runs and the row is reported instead of guessed at. + const postFix = planBackfill({ + expenses: [expense], + items: new Map([[CODELESS_ITEM, { + costCodeId: null, estimateId: ESTIMATE, projectId: TARGET_PROJECT, + }]]), + ...args, + }); + assert.equal(postFix.codeFills.length, 0, "nothing is written"); + assert.equal(postFix.remainder[0]?.reason, "item-outside-estimate"); + } finally { + await cleanupCodelessCrossJob(); + } +}); + +test("an estimate that moves between the plan and the write is REFUSED, not guessed", { skip }, async () => { + // End to end, through the real script. The move lands between the snapshot + // and the write transaction — the exact window the locked re-plan exists + // for — by intercepting the FIRST `$transaction` the write loop opens. + await seedCodelessCrossJob(); + try { + let moved = false; + const client = writerDb as unknown as Record; + const proxy = new Proxy(client, { + get(target, prop, receiver) { + if (prop === "$transaction") { + return async (...args: unknown[]) => { + if (!moved) { + moved = true; + await editorDb!.estimate.update({ + where: { id: ESTIMATE }, + data: { projectId: TARGET_PROJECT }, + }); + } + return (target as { $transaction: (...a: unknown[]) => Promise }) + .$transaction(...args); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + const outcome = await runBackfill({ + db: proxy, apply: true, log: () => {}, overheadProjectId: "no-such-project", + }); + + assert.ok(moved, "the estimate really did move between the plan and the write"); + // THIS row, not the run: the seed also holds an ordinary expense the + // vendor rule may legitimately code, and counting the whole run would + // make this test pass or fail for reasons that have nothing to do with + // the cross-job link. + assert.ok((outcome.skipped?.costCodes ?? 0) >= 1, "the planned write was skipped, not silently dropped"); + const row = await editorDb!.expense.findUnique({ + where: { id: REGEX_EXPENSE }, + select: { costCodeId: true, costCodeSource: true }, + }); + assert.deepEqual(row, { costCodeId: null, costCodeSource: null }, + "the row is left for a human, which is what item-outside-estimate means"); + } finally { + await editorDb!.estimate.update({ where: { id: ESTIMATE }, data: { projectId: PROJECT } }) + .catch(() => {}); + await cleanupCodelessCrossJob(); + } +}); diff --git a/tests/backfill-coverage-parity-db.test.ts b/tests/backfill-coverage-parity-db.test.ts new file mode 100644 index 000000000..93a1df44f --- /dev/null +++ b/tests/backfill-coverage-parity-db.test.ts @@ -0,0 +1,281 @@ +/** + * THE SCRIPT'S COVERAGE NUMBERS AND THE PAGE'S, ON ONE SEED (Codex round 48, + * items 2 and 3). + * + * The backfill prints a "VARIANCE-BASIS coverage" headline and the rollout is + * judged on it. That number is the script's own re-implementation of what + * `computeProjectVariance` does with the same rows, and the two had drifted in + * two directions at once: + * + * * the script loaded item metadata only for the items EXPENSES link to, and + * then resolved TIME ENTRIES through that same map — so labor whose phase + * comes only from `estimateItemId` counted as unattributed; and + * * it applied WRITE eligibility (an active cost code, on a committed + * estimate) to a MEASUREMENT, while the page keeps draft/archived + * attribution-only items and never looks at `costCode.isActive`. + * + * Both directions are invisible to a unit test with a hand-built fixture, + * because the fixture is written by whoever wrote the code. So this seeds a + * real database with exactly the shapes that used to disagree and asserts the + * two answers are the same number. + * + * It also covers the OTHER half of the same class of bug (item 2): the variance + * loader selected `costCodeSource` and then dropped it when building the rows + * it passes to the math, so a bookkeeper's explicit "no phase" (`manual-none`) + * silently kept charging the phase its line item names. + * + * Opt-in by design: it needs a THROWAWAY database and it writes rows. It runs + * in CI's migrations job and skips everywhere else, including anywhere + * DATABASE_URL looks like production. + */ +import test, { after, before } from "node:test"; +import assert from "node:assert/strict"; +import { PrismaClient } from "@prisma/client"; +import { runBackfill } from "../scripts/backfill-expense-attribution"; + +const url = + process.env.PHASE_INVARIANT_DB_TEST_URL ?? + process.env.RECEIPT_INTAKE_DB_TEST_URL ?? + process.env.MIGRATION_HISTORY_TEST_URL; +const looksLikeProd = !!url && /supabase\.(co|com)/i.test(url); +const skip = !url + ? "set PHASE_INVARIANT_DB_TEST_URL to a disposable PostgreSQL URL" + : looksLikeProd + ? "refusing to run against what looks like production" + : false; + +const db = url && !looksLikeProd ? new PrismaClient({ datasources: { db: { url } } }) : null; + +/** The REAL variance loader, on the singleton, pointed at this database. */ +let loadProjectVariance: typeof import("../src/lib/job-variance-db").loadProjectVariance; + +before(async () => { + if (!url || looksLikeProd) return; + const pooled = new URL(url); + pooled.searchParams.set("pgbouncer", "true"); + process.env.DATABASE_URL = pooled.toString(); + ({ loadProjectVariance } = await import("../src/lib/job-variance-db")); +}); + +const PFX = "cov-parity"; +const CLIENT = `${PFX}-client`; +const PROJECT = `${PFX}-project`; +const DRAFT_ESTIMATE = `${PFX}-draft-estimate`; +const LIVE_ESTIMATE = `${PFX}-live-estimate`; +const DRAFT_ITEM = `${PFX}-draft-item`; +const LIVE_ITEM = `${PFX}-live-item`; +const RETIRED_CODE = `${PFX}-retired-code`; +const LIVE_CODE = `${PFX}-live-code`; +/** Active, but carried ONLY by a draft estimate — so not a PHASE by the write rule. */ +const DRAFT_ONLY_CODE = `${PFX}-draft-code`; +const RETIRED_ITEM = `${PFX}-retired-item`; +const LABOR_ONLY_ITEM = `${PFX}-labor-item`; +const USER = `${PFX}-user`; + +async function cleanup() { + if (!db) return; + await db.timeEntry.deleteMany({ where: { projectId: PROJECT } }); + await db.expense.deleteMany({ where: { id: { startsWith: PFX } } }); + await db.estimateItem.deleteMany({ where: { id: { startsWith: PFX } } }); + await db.estimate.deleteMany({ where: { id: { startsWith: PFX } } }); + await db.project.deleteMany({ where: { id: PROJECT } }); + await db.user.deleteMany({ where: { id: USER } }); + await db.costCode.deleteMany({ where: { id: { startsWith: PFX } } }); + await db.client.deleteMany({ where: { id: CLIENT } }); +} + +/** + * Every shape the two readers used to disagree about, in one job: + * + * * a labor entry whose phase comes ONLY from its line item (the item the + * script's map did not load, because no expense points at it); + * * an expense linked to an item on a DRAFT estimate (attribution-only on the + * page, ineligible for a WRITE); and + * * an expense linked to an item carrying a RETIRED cost code (still + * attributed by the page, not writable by the script). + */ +async function seed() { + await cleanup(); + await db!.client.create({ data: { id: CLIENT, name: "Coverage Parity", initials: "CP" } }); + await db!.project.create({ + data: { id: PROJECT, name: "Coverage Parity", clientId: CLIENT, status: "In Progress" }, + }); + await db!.user.create({ + data: { id: USER, email: `${PFX}@example.test`, name: "Parity Crew", role: "FIELD_CREW" }, + }); + await db!.costCode.createMany({ + data: [ + { id: LIVE_CODE, code: `${PFX}-01`, name: "Live phase", isActive: true }, + { id: RETIRED_CODE, code: `${PFX}-02`, name: "Retired phase", isActive: false }, + { id: DRAFT_ONLY_CODE, code: `${PFX}-03`, name: "Draft-only phase", isActive: true }, + ], + }); + await db!.estimate.createMany({ + data: [ + { + id: LIVE_ESTIMATE, title: "Live", code: `EST-${PFX}-live`, projectId: PROJECT, + status: "Approved", totalAmount: 5000, balanceDue: 5000, + }, + { + id: DRAFT_ESTIMATE, title: "Draft", code: `EST-${PFX}-draft`, projectId: PROJECT, + status: "Draft", totalAmount: 1000, balanceDue: 1000, + }, + ], + }); + await db!.estimateItem.createMany({ + data: [ + { id: LIVE_ITEM, estimateId: LIVE_ESTIMATE, name: "live line", costCodeId: LIVE_CODE, total: 3000 }, + // A code the WRITE rule does not admit — it lives only on a draft + // estimate, so `resolveProjectPhaseCodes` never offers it — while the + // variance page attributes spend through it all the same. + { id: DRAFT_ITEM, estimateId: DRAFT_ESTIMATE, name: "draft line", costCodeId: DRAFT_ONLY_CODE, total: 500 }, + // ...and a RETIRED code on a committed estimate: same asymmetry, the + // other way round. + { id: RETIRED_ITEM, estimateId: LIVE_ESTIMATE, name: "retired line", costCodeId: RETIRED_CODE, total: 800 }, + { id: LABOR_ONLY_ITEM, estimateId: LIVE_ESTIMATE, name: "labor line", costCodeId: LIVE_CODE, total: 1200 }, + ], + }); + // Labor attributed ONLY through its line item — the case the script's item + // universe did not cover. + await db!.timeEntry.create({ + data: { + projectId: PROJECT, userId: USER, estimateItemId: LABOR_ONLY_ITEM, + startTime: new Date("2026-08-14T15:00:00.000Z"), + endTime: new Date("2026-08-14T23:00:00.000Z"), + durationHours: 8, laborCost: 400, burdenCost: 100, + }, + }); + await db!.expense.createMany({ + data: [ + // Attributed through an item on a DRAFT estimate. + { + id: `${PFX}-draft-expense`, projectId: PROJECT, estimateId: DRAFT_ESTIMATE, + itemId: DRAFT_ITEM, amount: 300, vendor: "Draft Supply", status: "Pending", + date: new Date("2026-08-14T19:00:00.000Z"), + }, + // Attributed by its own column, no item. + { + id: `${PFX}-coded-expense`, projectId: PROJECT, estimateId: LIVE_ESTIMATE, + costCodeId: LIVE_CODE, costCodeSource: "manual", amount: 200, + vendor: "Coded Supply", status: "Pending", + date: new Date("2026-08-14T19:00:00.000Z"), + }, + // Attributed through an item whose code has been RETIRED. The page + // still places it; the write rule would not. + { + id: `${PFX}-retired-expense`, projectId: PROJECT, estimateId: LIVE_ESTIMATE, + itemId: RETIRED_ITEM, amount: 150, vendor: "Retired Supply", status: "Pending", + date: new Date("2026-08-14T19:00:00.000Z"), + }, + // ...and money nobody has placed, so both sides report a share + // strictly between 0 and 1. A seed that is 100% attributed lets a + // broken denominator agree with a broken numerator. + { + id: `${PFX}-unattributed`, projectId: PROJECT, estimateId: LIVE_ESTIMATE, + amount: 100, vendor: "Unknown Supply", status: "Pending", + date: new Date("2026-08-14T19:00:00.000Z"), + }, + ], + }); +} + +/** + * What the PAGE says, on the same basis the script measures: ABSOLUTE dollars. + * + * `attributedShare` is `absAttributed / (absAttributed + absUnattributed)` and + * `unattributedGross` is `absUnattributed` — the two the script's percentage is + * built from, so they are the two to compare against. + */ +async function pageBasis() { + const [report] = await loadProjectVariance([PROJECT]); + assert.ok(report, "the seeded job must appear on the variance report"); + return { coverage: report.variance.coverage, variance: report.variance }; +} + +after(async () => { + if (!db) return; + await cleanup(); + await db.$disconnect(); +}); + +test("the script's variance-basis coverage equals the page's, on one seed", { skip }, async () => { + await seed(); + try { + const outcome = await runBackfill({ db, apply: false, log: () => {}, overheadProjectId: "no-such-project" }); + const { coverage } = await pageBasis(); + + const cents = (n: number) => Math.round(n * 100); + const script = outcome.coverage.varianceBasis; + const scriptUnattributed = script.total - script.before; + const scriptShare = script.total > 0 ? script.before / script.total : 1; + + // The seed is only meaningful if BOTH numbers are in play. + assert.ok(coverage.attributedShare > 0 && coverage.attributedShare < 1, + `the seed must be partly attributed, got ${coverage.attributedShare}`); + + assert.equal( + cents(scriptUnattributed), + cents(coverage.unattributedGross), + "the script and the page must agree on the dollars nobody placed", + ); + assert.ok( + Math.abs(scriptShare - coverage.attributedShare) < 1e-9, + `coverage disagrees: script ${scriptShare} vs page ${coverage.attributedShare}`, + ); + } finally { + await cleanup(); + } +}); + +test("labor attributed only through its line item is counted, not written off", { skip }, async () => { + // The specific regression: `TimeEntry.estimateItemId` pointing at an item no + // expense links to. The script loaded no metadata for it and reported the + // labor as unattributed, understating the headline. + await seed(); + try { + const outcome = await runBackfill({ db, apply: false, log: () => {}, overheadProjectId: "no-such-project" }); + assert.equal(Math.round(outcome.coverage.labor.total * 100), 50_000, "the seeded labor is $500"); + assert.equal( + Math.round(outcome.coverage.labor.attributed * 100), + 50_000, + "and all of it resolves through the line item", + ); + } finally { + await cleanup(); + } +}); + +test("a manual-none expense contributes to NO phase, through the real loader", { skip }, async () => { + // Item 2. The loader selected `costCodeSource` and then dropped it when + // building `VarianceExpense`, so `manual-none` arrived as undefined — which + // reads as "nobody has spoken" and runs the item fallback. The money went + // on charging the phase the bookkeeper had just cleared, on the one report + // that decision exists to correct. + await seed(); + try { + await db!.expense.create({ + data: { + id: `${PFX}-cleared`, projectId: PROJECT, estimateId: LIVE_ESTIMATE, + itemId: LIVE_ITEM, costCodeId: null, costCodeSource: "manual-none", + amount: 900, vendor: "Cleared Supply", status: "Pending", + date: new Date("2026-08-14T19:00:00.000Z"), + }, + }); + const { variance } = await pageBasis(); + const livePhase = variance.phases.find(p => p.costCodeId === LIVE_CODE); + + assert.ok(livePhase, "the live phase still exists on the report"); + assert.equal( + Math.round(Number(livePhase!.actualMaterial) * 100), + 20_000, + "only the $200 coded expense reaches the LIVE phase, not the $900 cleared one whose item names it", + ); + assert.equal( + Math.round(Number(variance.coverage.unattributedMaterial) * 100), + 100_000, + "the cleared $900 joins the $100 nobody placed, which is what 'no phase' means", + ); + } finally { + await cleanup(); + } +}); diff --git a/tests/receipt-ingest-attribution.test.ts b/tests/receipt-ingest-attribution.test.ts index 81a826bd5..6f770f850 100644 --- a/tests/receipt-ingest-attribution.test.ts +++ b/tests/receipt-ingest-attribution.test.ts @@ -154,15 +154,32 @@ const fakePrisma: any = { // regression back to matching a substring of the caller-supplied // `receiptUrl` throws rather than quietly passing the substring tests // below by doing the very thing they exist to forbid. + // + // TWO ARMS SINCE ROUND 48, ITEM 1: the file id, and a LEGACY row (one + // an old instance inserted with `sourceFileId` NULL) matched by its + // exact `receiptUrl`. Both are modelled as equality, and anything else + // still throws — a regression to `contains` on the caller-supplied url + // is what round 34 removed and it must not come back through this door. findFirst: async (args: any) => { - const wanted = args?.where?.sourceFileId; - if (typeof wanted !== "string") { + const arms = args?.where?.OR; + if (!Array.isArray(arms) || arms.length !== 2) { + throw new Error( + "receipt-ingest dedupe must ask for an exact sourceFileId OR a legacy receiptUrl, got " + + JSON.stringify(args?.where), + ); + } + const wanted = arms[0]?.sourceFileId; + const legacy = arms[1]?.AND; + const legacyUrl = legacy?.[1]?.receiptUrl; + if (typeof wanted !== "string" || legacy?.[0]?.sourceFileId !== null || typeof legacyUrl !== "string") { throw new Error( "receipt-ingest dedupe must ask for an exact sourceFileId, got " + JSON.stringify(args?.where), ); } - const hit = created.find(row => row.sourceFileId === wanted); + const hit = created.find(row => + row.sourceFileId === wanted || + (row.sourceFileId == null && row.receiptUrl === legacyUrl)); return hit ? { id: "exp-existing" } : null; }, create: async (args: { data: Record }) => { @@ -595,3 +612,52 @@ test("the date is judged BEFORE any group is inserted", async () => { assert.equal(res.status, 400); assert.deepEqual(created, []); }); + +// ── the mixed-version drain window (Codex round 48, item 1) ──────────────── + +test("a row an OLD instance inserted still counts as this file", async () => { + // The failure this closes: during the rollout, old instances insert with + // `sourceFileId` NULL (their Prisma client predates the column). A dedupe + // on that column alone cannot see them, so a delivery whose response was + // lost — retried against a NEW instance — inserted the whole receipt a + // second time, and the later backfill made the duplicate permanent. + created.push({ + sourceFileId: null, + sourceGroupIndex: null, + receiptUrl: "https://drive.google.com/file/d/drive-file-1/view", + amount: 120.5, + }); + const res = await post(PAYLOAD); + const json = await res.json(); + assert.deepEqual(json, { ok: true, alreadyIngested: true, created: 0 }); + assert.equal(created.length, 1, "nothing new was written"); +}); + +test("...and a legacy row for a DIFFERENT file does not block this one", async () => { + // The legacy arm is exact equality on the stored url, never a substring + // test — that is what round 34 removed and it must not come back. + created.push({ + sourceFileId: null, + sourceGroupIndex: null, + receiptUrl: "https://drive.google.com/file/d/some-other-file/view", + amount: 9.99, + }); + const res = await post(PAYLOAD); + const json = await res.json(); + assert.equal(json.created, 1, "this delivery still lands"); + assert.equal(created.length, 2); +}); + +test("a legacy row whose url the caller did not send is not matched by accident", async () => { + // A `fileUrl` the caller supplies is what gets stored, so the legacy arm + // compares against the url THIS delivery would write, not a guess. + created.push({ + sourceFileId: null, + sourceGroupIndex: null, + receiptUrl: "https://drive.google.com/uc?export=download&id=drive-file-1", + amount: 120.5, + }); + const res = await post({ ...PAYLOAD, fileUrl: "https://drive.google.com/uc?export=download&id=drive-file-1" }); + const json = await res.json(); + assert.deepEqual(json, { ok: true, alreadyIngested: true, created: 0 }); +}); diff --git a/tests/receipt-ingest-drain-window-db.test.ts b/tests/receipt-ingest-drain-window-db.test.ts new file mode 100644 index 000000000..430bba984 --- /dev/null +++ b/tests/receipt-ingest-drain-window-db.test.ts @@ -0,0 +1,352 @@ +/** + * THE MIXED-VERSION DRAIN WINDOW, AGAINST A REAL POSTGRES (Codex round 48, + * item 1). + * + * During the rollout, old instances are still serving + * `/api/integrations/receipt-ingest`. Their Prisma client predates + * `sourceFileId`/`sourceGroupIndex`, so every row they insert carries NULL in + * both — including rows inserted AFTER the pre-deploy `SOURCE_FILE_ID_BACKFILL` + * has already run. The NEW route dedupes on `sourceFileId`, so it cannot see + * those rows: a delivery whose response was lost, retried against a new + * instance, inserts the whole receipt a second time. The `--post-deploy` + * backfill then stamps the legacy row and the duplicate is permanent, and the + * partial unique index never objected because it only covers rows where + * `sourceFileId` is NOT NULL — which the legacy row was not, at insert time. + * + * Two halves of one fix are exercised here, and only a real server can show + * either: + * + * * the BRIDGE TRIGGER stamps an old-style insert and takes the same + * per-file advisory lock the route takes, so the two versions serialize; + * * the route's locked dedupe finds the stamped row and answers + * `alreadyIngested` instead of writing a second copy. + * + * Each test carries its CONTROL: the same interleaving with the trigger absent + * duplicates the receipt, which is what production would have done. + * + * Opt-in by design: it needs a THROWAWAY database and it writes rows. It runs + * in CI's migrations job and skips everywhere else, including anywhere + * DATABASE_URL looks like production. + */ +import test, { after, before } from "node:test"; +import assert from "node:assert/strict"; +import { PrismaClient } from "@prisma/client"; +import { + SOURCE_FILE_BRIDGE_DROP_SQL, + SOURCE_FILE_BRIDGE_SQL, + SOURCE_FILE_ID_BACKFILL, +} from "../scripts/apply-expense-attribution.mjs"; + +const url = + process.env.PHASE_INVARIANT_DB_TEST_URL ?? + process.env.RECEIPT_INTAKE_DB_TEST_URL ?? + process.env.MIGRATION_HISTORY_TEST_URL; +const looksLikeProd = !!url && /supabase\.(co|com)/i.test(url); +const skip = !url + ? "set PHASE_INVARIANT_DB_TEST_URL to a disposable PostgreSQL URL" + : looksLikeProd + ? "refusing to run against what looks like production" + : false; + +/** Two CONNECTIONS: the "old instance" and the "new instance" must be able to block on each other. */ +const oldBuild = url && !looksLikeProd ? new PrismaClient({ datasources: { db: { url } } }) : null; +const db = url && !looksLikeProd ? new PrismaClient({ datasources: { db: { url } } }) : null; + +/** The REAL route handler, on the real singleton, pointed at this database. */ +let POST: (req: Request) => Promise; + +const SECRET = "drain-window-test-secret"; + +before(async () => { + if (!url || looksLikeProd) return; + const pooled = new URL(url); + pooled.searchParams.set("pgbouncer", "true"); + process.env.DATABASE_URL = pooled.toString(); + process.env.RECEIPT_INGEST_SECRET = SECRET; + const mod = await import("../src/app/api/integrations/receipt-ingest/route"); + POST = mod.POST as unknown as (req: Request) => Promise; +}); + +const PFX = "drain-db"; +const CLIENT = `${PFX}-client`; +const PROJECT = `${PFX}-project`; +const ESTIMATE = `${PFX}-estimate`; +const CODE = `${PFX}-costcode`; +const ITEM = `${PFX}-item`; +const FILE = `${PFX}-drive-file`; +const FILE_URL = `https://drive.google.com/file/d/${FILE}/view`; +const PROJECT_NAME = "Drain Window Job"; + +function gate() { + let open!: () => void; + const reached = new Promise(resolve => (open = resolve)); + return { reached, open }; +} + +async function cleanup() { + if (!db) return; + // By ID PREFIX as well: the control test deliberately rewrites a row's + // receiptUrl, which takes it out of reach of the other two arms. + await db.expense.deleteMany({ + where: { OR: [{ sourceFileId: FILE }, { receiptUrl: FILE_URL }, { id: { startsWith: PFX } }] }, + }); + await db.estimateItem.deleteMany({ where: { id: ITEM } }); + await db.estimate.deleteMany({ where: { id: ESTIMATE } }); + await db.project.deleteMany({ where: { id: PROJECT } }); + await db.costCode.deleteMany({ where: { id: CODE } }); + await db.client.deleteMany({ where: { id: CLIENT } }); +} + +async function seed() { + await cleanup(); + await db!.client.create({ data: { id: CLIENT, name: "Drain Window", initials: "DW" } }); + await db!.project.create({ + data: { id: PROJECT, name: PROJECT_NAME, clientId: CLIENT, status: "In Progress" }, + }); + await db!.costCode.create({ data: { id: CODE, code: "03-PLUMB", name: "Plumbing", isActive: true } }); + await db!.estimate.create({ + data: { + id: ESTIMATE, title: "Drain Window", code: `EST-${PFX}`, projectId: PROJECT, + status: "Approved", totalAmount: 1000, balanceDue: 1000, + }, + }); + await db!.estimateItem.create({ + data: { id: ITEM, estimateId: ESTIMATE, name: "rough-in", costCodeId: CODE }, + }); +} + +async function installBridge() { + for (const sql of SOURCE_FILE_BRIDGE_SQL) await db!.$executeRawUnsafe(sql); +} +async function removeBridge() { + for (const sql of SOURCE_FILE_BRIDGE_DROP_SQL) await db!.$executeRawUnsafe(sql); +} + +/** + * An OLD instance's insert, verbatim: it names neither new column, because its + * client does not know they exist. Raw SQL for exactly that reason — a Prisma + * client generated from today's schema cannot express the old shape. + */ +const OLD_INSERT = + `INSERT INTO "Expense" (id, amount, vendor, description, status, date, "receiptUrl", "estimateId", "createdAt", "updatedAt") + VALUES ($1, $2, 'Home Depot', 'old build row', 'Pending', now(), $3, $4, now(), now())`; + +async function oldStyleInsert(client: PrismaClient, id: string, amount = 120.5) { + await client.$executeRawUnsafe(OLD_INSERT, id, amount, FILE_URL, ESTIMATE); +} + +function ingest(body: Record) { + return POST(new Request("https://probuild.test/api/integrations/receipt-ingest", { + method: "POST", + headers: { "x-ingest-key": SECRET, "content-type": "application/json" }, + body: JSON.stringify(body), + })); +} + +const PAYLOAD = { + projectName: PROJECT_NAME, + vendor: "Home Depot", + date: "2026-08-14", + fileId: FILE, + fileUrl: FILE_URL, + groups: [{ category: "Plumbing", amount: 120.5 }], +}; + +const rowsForFile = async () => + db!.expense.findMany({ + where: { OR: [{ sourceFileId: FILE }, { receiptUrl: FILE_URL }] }, + select: { id: true, sourceFileId: true, sourceGroupIndex: true, amount: true }, + orderBy: { id: "asc" }, + }); + +after(async () => { + if (!db) return; + await removeBridge().catch(() => {}); + await cleanup(); + await db.$disconnect(); + await oldBuild?.$disconnect(); +}); + +test("CONTROL: without the bridge, an old-build row is invisible and the receipt duplicates", { skip }, async () => { + await seed(); + try { + await removeBridge().catch(() => {}); + // The old instance lands its row. `sourceFileId` is NULL, because its + // client has never heard of the column. + await oldStyleInsert(db!, `${PFX}-legacy`); + const legacy = await rowsForFile(); + assert.equal(legacy.length, 1); + assert.equal(legacy[0].sourceFileId, null, "an old-build row names no file"); + + // ...and the pre-deploy backfill has ALREADY run, so nothing stamps it. + // The new route is then asked for the same document. The receipt-url + // arm of the dedupe is what catches it (round 48, item 1, second half), + // so this control isolates the trigger by deleting that row's url. + await db!.$executeRawUnsafe( + `UPDATE "Expense" SET "receiptUrl" = $1 WHERE id = $2`, + "https://drive.google.com/uc?export=view", `${PFX}-legacy`, + ); + const res = await ingest(PAYLOAD); + const json = await res.json() as { created?: number; alreadyIngested?: boolean }; + assert.equal(json.alreadyIngested, undefined, "the new build cannot see the old row"); + assert.equal(json.created, 1, "so it writes the receipt a second time"); + + // The duplicate the --post-deploy backfill would then make permanent. + await db!.$executeRawUnsafe(SOURCE_FILE_ID_BACKFILL); + const after = await db!.expense.findMany({ + where: { OR: [{ sourceFileId: FILE }, { id: `${PFX}-legacy` }] }, + select: { id: true }, + }); + assert.equal(after.length, 2, "two rows for one delivery — the failure this bridge exists to stop"); + } finally { + await cleanup(); + await installBridge(); + } +}); + +test("the bridge stamps an old-build insert on the way in", { skip }, async () => { + await seed(); + await installBridge(); + try { + await oldStyleInsert(db!, `${PFX}-stamped`); + const rows = await rowsForFile(); + assert.equal(rows.length, 1); + assert.equal(rows[0].sourceFileId, FILE, "derived from the Drive url, at INSERT time"); + assert.equal(rows[0].sourceGroupIndex, 0, "and given the first ordinal"); + } finally { + await cleanup(); + } +}); + +test("N groups of one old-build transaction get N distinct ordinals", { skip }, async () => { + await seed(); + await installBridge(); + try { + await db!.$transaction(async tx => { + const raw = tx as unknown as { $executeRawUnsafe(q: string, ...v: unknown[]): Promise }; + await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g0`, 10, FILE_URL, ESTIMATE); + await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g1`, 20, FILE_URL, ESTIMATE); + await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g2`, 30, FILE_URL, ESTIMATE); + }); + const rows = await rowsForFile(); + assert.deepEqual(rows.map(r => r.sourceGroupIndex), [0, 1, 2]); + } finally { + await cleanup(); + } +}); + +test("a SECOND old-build delivery of the same file is refused, not duplicated", { skip }, async () => { + // The ordinal counts within the TRANSACTION, so a re-delivery starts at 0 + // again and collides with the row already there. The unique index refuses + // it, the old instance sees an error, and its Apps Script does not archive + // the file — which is the correct outcome for a document already booked. + await seed(); + await installBridge(); + try { + await oldStyleInsert(db!, `${PFX}-first`); + let error: unknown = null; + await oldStyleInsert(db!, `${PFX}-second`).catch(caught => { error = caught; }); + assert.ok(error, "the second delivery must fail"); + // Postgres names the constraint and the colliding key; Prisma passes + // the message through with SQLSTATE 23505 (unique_violation). + assert.match(String((error as { message?: string })?.message ?? error), /23505|already exists/i); + assert.equal((await rowsForFile()).length, 1, "exactly one copy survives"); + } finally { + await cleanup(); + } +}); + +test("old-build insert FIRST, then the new route: one receipt, no duplicate", { skip }, async () => { + await seed(); + await installBridge(); + try { + await oldStyleInsert(db!, `${PFX}-legacy`); + const res = await ingest(PAYLOAD); + const json = await res.json() as { created?: number; alreadyIngested?: boolean }; + assert.equal(json.alreadyIngested, true, "the stamped row is visible to the new dedupe"); + assert.equal(json.created, 0); + assert.equal((await rowsForFile()).length, 1); + } finally { + await cleanup(); + } +}); + +test("...and they SERIALIZE: an in-flight old insert blocks the new route", { skip }, async () => { + // The lock is the half a sequential test cannot show. The old insert takes + // the per-file advisory lock inside its trigger and holds it until COMMIT; + // the route takes the same key before its authoritative dedupe, so it + // cannot read "nothing here" while the other transaction is mid-flight. + await seed(); + await installBridge(); + try { + const inserted = gate(); + let oldError: unknown = null; + const oldSide = (async () => { + try { + await oldBuild!.$transaction(async tx => { + const raw = tx as unknown as { $executeRawUnsafe(q: string, ...v: unknown[]): Promise }; + await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-inflight`, 120.5, FILE_URL, ESTIMATE); + inserted.open(); + // Long enough that the route has certainly reached its lock. + await new Promise(resolve => setTimeout(resolve, 750)); + }, { timeout: 30_000 }); + } catch (caught) { + oldError = caught; + } + })(); + + await inserted.reached; + const res = await ingest(PAYLOAD); + const json = await res.json() as { created?: number; alreadyIngested?: boolean }; + await oldSide; + + assert.equal(oldError, null, `the old insert failed: ${oldError}`); + assert.equal(json.alreadyIngested, true, "it waited, then saw the committed row"); + assert.equal(json.created, 0); + assert.equal((await rowsForFile()).length, 1, "one delivery, one row"); + } finally { + await cleanup(); + } +}); + +test("the NEW route first, then an old-build delivery of the same file", { skip }, async () => { + // The other order. The new build writes ordinal 0 for its single group; the + // old instance's retry starts its own count at 0 and collides, so the + // duplicate is refused by the index rather than landing beside it. + await seed(); + await installBridge(); + try { + const res = await ingest(PAYLOAD); + assert.equal((await res.json() as { created?: number }).created, 1); + + let error: unknown = null; + await oldStyleInsert(db!, `${PFX}-late`).catch(caught => { error = caught; }); + assert.ok(error, "the old build's second copy must be refused"); + assert.equal((await rowsForFile()).length, 1, "still exactly one row for this file"); + } finally { + await cleanup(); + } +}); + +test("an expense with no Drive url is untouched by the bridge", { skip }, async () => { + // Every expense insert in the system passes through this trigger. One that + // carries no receipt url must not be stamped, must not take a lock, and + // must not be given an ordinal. + await seed(); + await installBridge(); + try { + await db!.$executeRawUnsafe( + `INSERT INTO "Expense" (id, amount, vendor, status, date, "estimateId", "createdAt", "updatedAt") + VALUES ($1, 42, 'Cash', 'Pending', now(), $2, now(), now())`, + `${PFX}-plain`, ESTIMATE, + ); + const row = await db!.expense.findUnique({ + where: { id: `${PFX}-plain` }, + select: { sourceFileId: true, sourceGroupIndex: true }, + }); + assert.deepEqual(row, { sourceFileId: null, sourceGroupIndex: null }); + } finally { + await db!.expense.deleteMany({ where: { id: `${PFX}-plain` } }); + await cleanup(); + } +}); diff --git a/tests/scripts-runtime-smoke.test.ts b/tests/scripts-runtime-smoke.test.ts index 80f7eca21..856d0fceb 100644 --- a/tests/scripts-runtime-smoke.test.ts +++ b/tests/scripts-runtime-smoke.test.ts @@ -84,3 +84,50 @@ test("the report scopes by the canonical overhead id, not the name \"Shop\"", () assert.ok(source.includes("resolveExpenseProjectId")); assert.ok(source.includes("csvCell"), "OCR'd vendor text is formula-neutralized"); }); + +// ── the money backfill names its target (Codex round 48, item 5) ────────── + +test("--apply refuses an ambient DATABASE_URL with no --target, before any write", () => { + // This script writes `projectId`, `costCodeId` and `costCodeSource`. It + // used to load .env.local/.env and take whatever DATABASE_URL was in the + // shell — so the more dangerous of the two Phase 3 scripts was the less + // guarded one, while the DDL script already had a target guard, a + // project-ref check and a redacted identity line. + // + // The refusal happens before a PrismaClient is constructed, which is why + // this needs no database: a local URL is supplied and never dialled. + const ambient = { + ...process.env, + DATABASE_URL: "postgresql://probuild:probuild@localhost:5432/probuild", + }; + const attempt = (args: string[]) => { + try { + const stdout = execFileSync( + process.execPath, + ["--import=tsx", "scripts/backfill-expense-attribution.ts", ...args], + { cwd: ROOT, env: ambient, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ); + return { code: 0, output: stdout }; + } catch (error) { + const failure = error as { status?: number; stdout?: string; stderr?: string }; + return { code: failure.status ?? -1, output: `${failure.stdout ?? ""}${failure.stderr ?? ""}` }; + } + }; + + const noTarget = attempt(["--apply"]); + assert.notEqual(noTarget.code, 0, "it must not run"); + assert.match(noTarget.output, /REFUSING: --target is required/); + assert.doesNotMatch(noTarget.output, /planned writes|applied \d/, "nothing was planned or written"); + + // ...and naming prod does not rescue it: that target reads + // .env.production.local, which is not checked in and is not on CI. + const asProd = attempt(["--target", "prod", "--apply"]); + assert.notEqual(asProd.code, 0); + assert.match(asProd.output, /REFUSING/); + assert.doesNotMatch(asProd.output, /planned writes|applied \d/); + + // ...and a target WITHOUT the database/host assertions is refused too. + const noExpect = attempt(["--target", "ci", "--apply"]); + assert.notEqual(noExpect.code, 0); + assert.match(noExpect.output, /--expect-db and --expect-host/); +}); From f72ffa5cd63924a9a29b293b3ca2f0f7611a895c Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 3 Sep 2026 11:41:23 -0700 Subject: [PATCH 143/144] fix(expenses): pin the prod target guard against the REAL pooler URL (port and all) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex found on the sibling PR that `new URL(url).host` includes the PORT, so a `/pooler\.supabase\.com$/` test against `host` rejects every real transaction- pooler URL — and CI would never catch it, because CI only ever exercises `--target ci`. This branch's shared helper already reads `hostname`, so there was nothing to fix here; what was missing was the test that keeps it that way. `tests/apply-expense-attribution.test.ts` now drives the WHOLE composite guard (`resolveTargetOrRefuse`) against the exact documented production URL shape (`postgresql://postgres.:***@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true`) with `.env.production.local` faked in memory: it passes with a matching APPLY_EXPECT_PROJECT_REF and refuses with a wrong one, naming both refs. The composite takes an `io` parameter for exactly that, because the pieces can each be right while the wiring drops one. Mutating `hostname` back to `host` fails three tests. Co-Authored-By: Claude Fable 5.1 --- scripts/lib/apply-target.mjs | 8 +++- tests/apply-expense-attribution.test.ts | 52 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/lib/apply-target.mjs b/scripts/lib/apply-target.mjs index 2159a44f6..406a8da2a 100644 --- a/scripts/lib/apply-target.mjs +++ b/scripts/lib/apply-target.mjs @@ -264,10 +264,14 @@ export async function verifyTargetIdentity(prisma, { target, url, from, expectDb * ref that are readable from the URL alone. Returns `{ error }` or * `{ target, url, from }`. */ -export function resolveTargetOrRefuse(argv, env = process.env) { +export function resolveTargetOrRefuse(argv, env = process.env, io = {}) { const chosen = parseTarget(argv); if (chosen.error) return { error: chosen.error }; - const resolved = resolveTargetDatabaseUrl(chosen.name, { env }); + // `io` exists so a test can drive this WHOLE chain against the exact + // production URL shape without a .env.production.local on disk. The + // composite is what a caller uses, so the composite is what has to be + // tested: the pieces can each be right while the wiring drops one. + const resolved = resolveTargetDatabaseUrl(chosen.name, { env, ...io }); if (resolved.error) return { error: resolved.error }; const hostProblem = targetHostVerdict(chosen.name, resolved.url); if (hostProblem) return { error: hostProblem }; diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index aef15e7fc..17517f960 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -18,6 +18,7 @@ import test from "node:test"; import { readFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; import path from "node:path"; +import { resolveTargetOrRefuse } from "../scripts/lib/apply-target.mjs"; import { APPLY_TARGETS, PRODUCTION_BASELINE_MIGRATION, @@ -1708,3 +1709,54 @@ test("--post-deploy drops the bridge AFTER it has stamped the stragglers", () => assert.ok(dropAt > -1, "and the bridge is dropped in the same pass"); assert.ok(backfillAt < dropAt, "the stamping happens while the bridge still stands"); }); + +// ── the guard must ACCEPT the real production URL (P0 on the sibling PR) ── + +/** Byte-for-byte the shape CLAUDE.md documents for production. */ +const PROD_URL = + "postgresql://postgres.ghzdbzdnwjxazvmcefbh:s3cr3t@aws-0-us-west-2.pooler.supabase.com:6543/postgres?pgbouncer=true"; + +test("the prod guard ACCEPTS the real pooler URL, port and all", () => { + // The failure this pins was found on the sibling PR: `new URL(url).host` + // includes the PORT (`aws-0-us-west-2.pooler.supabase.com:6543`), so a + // `/pooler\.supabase\.com$/` test against `host` rejects every real + // transaction-pooler URL — and CI would never notice, because CI only + // ever exercises `--target ci`. This helper reads `hostname`; the test is + // here so it keeps doing that. + assert.equal(new URL(PROD_URL).host, "aws-0-us-west-2.pooler.supabase.com:6543"); + assert.equal(new URL(PROD_URL).hostname, "aws-0-us-west-2.pooler.supabase.com"); + assert.equal(targetHostVerdict("prod", PROD_URL), null, "the REAL prod URL must pass the host check"); +}); + +test("...and the whole chain passes with the right ref, refuses with a wrong one", () => { + // The composite a caller actually uses — target, url source, host, ref — + // driven against the production URL with the env file faked, because the + // pieces can each be right while the wiring drops one. + const files = { ".env.production.local": `DATABASE_URL="${PROD_URL}"\n` }; + const disk = { + exists: (file: unknown) => String(file) in files, + read: (file: unknown) => files[String(file) as keyof typeof files], + }; + const argv = ["node", "apply.mjs", "--target", "prod", "--yes"]; + + const right = resolveTargetOrRefuse( + argv, + { APPLY_EXPECT_PROJECT_REF: "ghzdbzdnwjxazvmcefbh" } as unknown as NodeJS.ProcessEnv, + disk, + ); + assert.equal(right.error, undefined, `the real production target must be accepted: ${right.error}`); + assert.equal(right.target, "prod"); + assert.equal(right.url, PROD_URL); + assert.equal(right.from, ".env.production.local"); + + const wrong = resolveTargetOrRefuse( + argv, + { APPLY_EXPECT_PROJECT_REF: "stagingprojectref" } as unknown as NodeJS.ProcessEnv, + disk, + ); + assert.match(wrong.error ?? "", /this URL is for project ghzdbzdnwjxazvmcefbh, not stagingprojectref/); + assert.equal(wrong.url, undefined, "and no URL is handed back to connect with"); + + // ...and the banner built from that URL still hides the password. + assert.doesNotMatch(targetBanner("prod", { url: PROD_URL, from: ".env.production.local", db: "postgres", host: "10.0.0.5" }), /s3cr3t/); +}); From b9e9537de9046aa749325ec7665cb16691ee4ab7 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 3 Sep 2026 12:29:25 -0700 Subject: [PATCH 144/144] =?UTF-8?q?fix(expenses):=20PR=20#442=20round-19?= =?UTF-8?q?=20gate=20=E2=80=94=20the=20drain-window=20bridge=20no=20longer?= =?UTF-8?q?=20loses=20multi-group=20receipts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P0] The bridge assigned ordinals from a TRANSACTION-local counter, which assumed the old handler writes a receipt's groups in one transaction. It does not: the deployed handler calls prisma.expense.create() once per group, each its own autocommit statement. So every group got ordinal 0, group two died on the partial unique index the moment group one committed, and a retry — seeing group one — answered alreadyIngested. The rest of the receipt was dropped for good. Two halves to the fix: * the ordinal is now MAX(committed) + 1 for that sourceFileId, read under the advisory lock, so three autocommit inserts become 0, 1, 2; and * the route RESUMES a partial delivery. `missingGroupIndexes()` compares the ordinals already present with the groups being offered and inserts only what is missing, answering alreadyIngested only when nothing is. A row with a NULL ordinal (it predates the trigger) still reads as "nothing missing", which is the conservative direction. What this gives up, deliberately and now stated in the code and the tests: a re-delivery of an already-complete document appends instead of colliding. That is what the old build does today with no bridge at all, it is guarded on both sides, and a duplicate a human can see beats money silently missing. [P1 #2] The bridge returned without taking any lock when the url carried no derivable Drive id (a shortened or re-hosted link), so the route's exact-url fallback could not see an in-flight old insert — only a committed one. The normalised url is now the second identity: the trigger locks hashtextextended('receipt-ingest-url:' || lower(btrim(receiptUrl)), 0) in that case, and the route takes BOTH keys in a fixed order (id, then url). The trigger only ever holds one of the two, so the pair cannot cycle against it. [P1 #4] One named three-trigger contract. The operator message counted COMPATIBILITY_TRIGGERS.length instead of saying "BOTH", the migration comment and the test names follow, and a test asserts the message counts the list rather than restating a number. [P2 #5] Two planning documents still advertised an `installedAtCustomer` default the code has not had since round 2 (the spec's intake contract and the v2 plan's decision 7). Corrected and pinned in the wording test. Co-Authored-By: Claude Fable 5.1 --- docs/plans/PHASE-3-ATTRIBUTION-SPEC.md | 3 +- docs/plans/RECEIPT-PIPELINE-V2-PLAN.md | 2 +- .../migration.sql | 90 +++++-- scripts/apply-expense-attribution.mjs | 89 +++++-- .../api/integrations/receipt-ingest/route.ts | 93 ++++++- tests/apply-expense-attribution.test.ts | 76 +++++- tests/phase3-spec-wording.test.ts | 18 ++ tests/receipt-ingest-attribution.test.ts | 45 +++- tests/receipt-ingest-drain-window-db.test.ts | 244 ++++++++++++++++-- 9 files changed, 546 insertions(+), 114 deletions(-) diff --git a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md index ab4db6144..040b46678 100644 --- a/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md +++ b/docs/plans/PHASE-3-ATTRIBUTION-SPEC.md @@ -333,7 +333,8 @@ backfill > null. Nothing but a human edit may change a row whose `costCodeSource and `costCodeSource`/`costCodeConfidence`: "capture" (confidence null) when a human chose `costCodeId` at capture; "ai" + `suggestedConfidence` when it fell back to `matchCostCode(suggestedPhaseCode)`. `POST /api/receipts/intake` accepts - `installedAtCustomer?: boolean` (defaulting per §5), and for drive/email/chat sources + `installedAtCustomer?: boolean` (**no default — see the TAX POSITION note below**; an + absent answer stays NULL and the excise report skips the row), and for drive/email/chat sources the read step sets `taxAtSource = taxCents > 0`. 3. **receipt-ingest v1** (`receipt-ingest/route.ts:108`): add `projectId: project.id`, `costCodeSource: costCode ? "ai" : null`, `costCodeConfidence: null` (the category is diff --git a/docs/plans/RECEIPT-PIPELINE-V2-PLAN.md b/docs/plans/RECEIPT-PIPELINE-V2-PLAN.md index 487c5aff4..836606aca 100644 --- a/docs/plans/RECEIPT-PIPELINE-V2-PLAN.md +++ b/docs/plans/RECEIPT-PIPELINE-V2-PLAN.md @@ -125,7 +125,7 @@ Tabs by state: Needs job, Needs review, Booking (with reason), Booked today, Mis ### Tax paid at source (decision 7) -- Gemini already extracts `taxAmount`. Store it. Crew/Marge flag `installedAtCustomer` (default true for job-folder receipts, false for Shop). +- Gemini already extracts `taxAmount`. Store it. Crew/Marge flag `installedAtCustomer` — **nothing defaults it** (superseded during PR #442: an earlier build defaulted it to true for any non-overhead job, which claims a deduction nobody looked at; WAC 458-20-102(12)(b) allows the cost of the articles actually RESOLD, and a receipt coded to a live job is just as likely to be consumables, tools, fuel or dump fees). A NULL is "nobody said" and the report skips it. - `/reports/tax-paid-at-source`: per period, per job, the deductible amount for the WA excise return "taxable amount for tax paid at source" line. This is the first bookkeeper task ProBuild absorbs. Others (month close, sales tax filing prep) get scoped after we list what Vanessa actually does each month. ### Gusto (decision 6) diff --git a/prisma/migrations/20260901120000_expense_attribution/migration.sql b/prisma/migrations/20260901120000_expense_attribution/migration.sql index 33d794b6f..2c5ab88d4 100644 --- a/prisma/migrations/20260901120000_expense_attribution/migration.sql +++ b/prisma/migrations/20260901120000_expense_attribution/migration.sql @@ -440,7 +440,8 @@ EXECUTE FUNCTION probuild_expense_amount_tax_guard(); -- serialize; the ordinal counts within the transaction, so a re-delivery lands -- on the ordinals already there and the partial unique index refuses it. -- --- Like the two guards above it is drain-window scaffolding: created here and +-- It is the THIRD of three drain-window triggers (the split-job pair guard, +-- the amount/tax guard, this bridge): created here and -- dropped at the end of this file, so a fresh database finishes in the shape -- production finishes in. CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() @@ -450,7 +451,6 @@ CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() DECLARE derived TEXT; next_index INT; - counter_key TEXT; BEGIN -- A row that already names its file speaks for itself: the new build -- inserted it, and it has chosen its own group ordinal. @@ -462,37 +462,77 @@ CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() NEW."sourceFileId" := derived; END IF; - IF NEW."sourceFileId" IS NULL THEN + -- AN IDENTITY THAT DOES NOT DEPEND ON PARSING THE URL (round 49, + -- item 2). + -- + -- This used to RETURN early when no id could be derived -- a + -- shortened link, a re-hosted copy, a /uc?export=download form -- so + -- the old instance's insert took no lock at all. The new route's + -- exact-url fallback can see such a row only once it has COMMITTED; + -- against an in-flight one it read "nothing here" and inserted the + -- receipt again. The lock is what makes the two versions wait for + -- each other, so it cannot be conditional on the url being + -- parseable: when there is no file id, the normalised url IS the + -- identity, and the route hashes the same string. + -- + -- An insert with no receipt url at all is not a Drive receipt: no + -- lock, no ordinal, nothing. + IF NEW."sourceFileId" IS NULL AND NEW."receiptUrl" IS NULL THEN RETURN NEW; END IF; -- THE SAME LOCK THE ROUTE TAKES, so an old-version insert and a - -- new-version request for one file cannot both believe they are first. - -- Transaction-scoped: released at COMMIT or ROLLBACK, nothing to leak. - PERFORM pg_advisory_xact_lock( - hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) - ); + -- new-version request for one document cannot both believe they are + -- first. Transaction-scoped: released at COMMIT or ROLLBACK, nothing + -- to leak. + IF NEW."sourceFileId" IS NOT NULL THEN + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) + ); + ELSE + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest-url:' || lower(btrim(NEW."receiptUrl")), 0) + ); + -- No id means no ordinal: the unique index is partial on + -- "sourceFileId" IS NOT NULL, so a number here would guard + -- nothing and would invite a reader to treat it as a group + -- position it cannot be. The row is still identified by its url, + -- which is what the route matches it on. + RETURN NEW; + END IF; - -- THE ORDINAL COUNTS WITHIN THIS TRANSACTION, NOT WITHIN THE TABLE. + -- THE ORDINAL COUNTS WHAT IS COMMITTED, NOT WHAT THIS TRANSACTION + -- HAS DONE (round 49, item 1 -- a P0 this replaces). + -- + -- The first version counted per TRANSACTION, on the theory that a + -- re-delivery would then collide with the rows already there and be + -- refused. That theory assumed the old handler writes its groups in + -- one transaction. It does not: the deployed handler calls + -- prisma.expense.create() once per group, each its own autocommit + -- statement (see the loop in the pre-Phase-3 + -- src/app/api/integrations/receipt-ingest/route.ts). So every group of + -- one receipt got ordinal 0, group two violated the unique index the + -- moment group one committed, and a retry -- seeing group one -- + -- answered alreadyIngested. The remaining groups were dropped for + -- good: money silently missing from a receipt the archive says was + -- imported. -- - -- MAX(existing) + 1 was the obvious rule and it is the wrong one: it - -- makes a RE-DELIVERY of a document the table already holds land on - -- fresh ordinals and insert cleanly, which is the duplicate this whole - -- bridge exists to stop. Counting per transaction instead means the N - -- groups of one delivery get 0,1,2..., and a SECOND delivery of the - -- same file starts at 0 again -- colliding with the row already there - -- and aborting on the partial unique index - -- ("sourceFileId", "sourceGroupIndex"). An old instance that retries a - -- document therefore fails loudly instead of duplicating it, and its - -- Apps Script does not archive the file. + -- MAX(committed) + 1, under the advisory lock above, is what the old + -- handler's actual boundaries need: three autocommit inserts become + -- 0, 1, 2 because each one sees its committed siblings. -- - -- set_config(..., true) is transaction-local, so the counter cannot - -- survive a COMMIT or leak into another session. The key is hashed - -- because a Drive file id is not a legal GUC name. + -- WHAT THIS GIVES UP, said plainly: a re-delivery of a document that + -- is already complete no longer collides -- it appends. That is the + -- behaviour the old build has today with no bridge at all, and it is + -- still guarded on both sides (the old handler's own url dedupe, and + -- the new route's alreadyIngestedWhere). Losing groups is worse + -- than a duplicate a human can see, and the resume path in the route + -- is what makes the retry land the missing groups instead of nothing. IF NEW."sourceGroupIndex" IS NULL THEN - counter_key := 'probuild.bridge_' || md5(NEW."sourceFileId"); - next_index := COALESCE(NULLIF(current_setting(counter_key, true), '')::int, -1) + 1; - PERFORM set_config(counter_key, next_index::text, true); + SELECT COALESCE(MAX("sourceGroupIndex"), -1) + 1 + INTO next_index + FROM "Expense" + WHERE "sourceFileId" = NEW."sourceFileId"; NEW."sourceGroupIndex" := next_index; END IF; diff --git a/scripts/apply-expense-attribution.mjs b/scripts/apply-expense-attribution.mjs index 5258b8321..53b534a7c 100644 --- a/scripts/apply-expense-attribution.mjs +++ b/scripts/apply-expense-attribution.mjs @@ -1124,7 +1124,6 @@ export const SOURCE_FILE_BRIDGE_SQL = [ DECLARE derived TEXT; next_index INT; - counter_key TEXT; BEGIN -- A row that already names its file speaks for itself: the new build -- inserted it, and it has chosen its own group ordinal. @@ -1136,37 +1135,77 @@ export const SOURCE_FILE_BRIDGE_SQL = [ NEW."sourceFileId" := derived; END IF; - IF NEW."sourceFileId" IS NULL THEN + -- AN IDENTITY THAT DOES NOT DEPEND ON PARSING THE URL (round 49, + -- item 2). + -- + -- This used to RETURN early when no id could be derived -- a + -- shortened link, a re-hosted copy, a /uc?export=download form -- so + -- the old instance's insert took no lock at all. The new route's + -- exact-url fallback can see such a row only once it has COMMITTED; + -- against an in-flight one it read "nothing here" and inserted the + -- receipt again. The lock is what makes the two versions wait for + -- each other, so it cannot be conditional on the url being + -- parseable: when there is no file id, the normalised url IS the + -- identity, and the route hashes the same string. + -- + -- An insert with no receipt url at all is not a Drive receipt: no + -- lock, no ordinal, nothing. + IF NEW."sourceFileId" IS NULL AND NEW."receiptUrl" IS NULL THEN RETURN NEW; END IF; -- THE SAME LOCK THE ROUTE TAKES, so an old-version insert and a - -- new-version request for one file cannot both believe they are first. - -- Transaction-scoped: released at COMMIT or ROLLBACK, nothing to leak. - PERFORM pg_advisory_xact_lock( - hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) - ); + -- new-version request for one document cannot both believe they are + -- first. Transaction-scoped: released at COMMIT or ROLLBACK, nothing + -- to leak. + IF NEW."sourceFileId" IS NOT NULL THEN + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) + ); + ELSE + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest-url:' || lower(btrim(NEW."receiptUrl")), 0) + ); + -- No id means no ordinal: the unique index is partial on + -- "sourceFileId" IS NOT NULL, so a number here would guard + -- nothing and would invite a reader to treat it as a group + -- position it cannot be. The row is still identified by its url, + -- which is what the route matches it on. + RETURN NEW; + END IF; - -- THE ORDINAL COUNTS WITHIN THIS TRANSACTION, NOT WITHIN THE TABLE. + -- THE ORDINAL COUNTS WHAT IS COMMITTED, NOT WHAT THIS TRANSACTION + -- HAS DONE (round 49, item 1 -- a P0 this replaces). + -- + -- The first version counted per TRANSACTION, on the theory that a + -- re-delivery would then collide with the rows already there and be + -- refused. That theory assumed the old handler writes its groups in + -- one transaction. It does not: the deployed handler calls + -- prisma.expense.create() once per group, each its own autocommit + -- statement (see the loop in the pre-Phase-3 + -- src/app/api/integrations/receipt-ingest/route.ts). So every group of + -- one receipt got ordinal 0, group two violated the unique index the + -- moment group one committed, and a retry -- seeing group one -- + -- answered alreadyIngested. The remaining groups were dropped for + -- good: money silently missing from a receipt the archive says was + -- imported. -- - -- MAX(existing) + 1 was the obvious rule and it is the wrong one: it - -- makes a RE-DELIVERY of a document the table already holds land on - -- fresh ordinals and insert cleanly, which is the duplicate this whole - -- bridge exists to stop. Counting per transaction instead means the N - -- groups of one delivery get 0,1,2..., and a SECOND delivery of the - -- same file starts at 0 again -- colliding with the row already there - -- and aborting on the partial unique index - -- ("sourceFileId", "sourceGroupIndex"). An old instance that retries a - -- document therefore fails loudly instead of duplicating it, and its - -- Apps Script does not archive the file. + -- MAX(committed) + 1, under the advisory lock above, is what the old + -- handler's actual boundaries need: three autocommit inserts become + -- 0, 1, 2 because each one sees its committed siblings. -- - -- set_config(..., true) is transaction-local, so the counter cannot - -- survive a COMMIT or leak into another session. The key is hashed - -- because a Drive file id is not a legal GUC name. + -- WHAT THIS GIVES UP, said plainly: a re-delivery of a document that + -- is already complete no longer collides -- it appends. That is the + -- behaviour the old build has today with no bridge at all, and it is + -- still guarded on both sides (the old handler's own url dedupe, and + -- the new route's alreadyIngestedWhere). Losing groups is worse + -- than a duplicate a human can see, and the resume path in the route + -- is what makes the retry land the missing groups instead of nothing. IF NEW."sourceGroupIndex" IS NULL THEN - counter_key := 'probuild.bridge_' || md5(NEW."sourceFileId"); - next_index := COALESCE(NULLIF(current_setting(counter_key, true), '')::int, -1) + 1; - PERFORM set_config(counter_key, next_index::text, true); + SELECT COALESCE(MAX("sourceGroupIndex"), -1) + 1 + INTO next_index + FROM "Expense" + WHERE "sourceFileId" = NEW."sourceFileId"; NEW."sourceGroupIndex" := next_index; END IF; @@ -1776,7 +1815,7 @@ async function main() { ? [...postDeployStatements(companyTimeZone), ...postDeployTeardownStatements({ repairSplitJobs })] : backfillStatements(companyTimeZone); if (postDeployOnly) { - console.log("--post-deploy: the three backfills, then BOTH compatibility guards come out (see PROJECT_ID_BACKFILL and AMOUNT_TAX_GUARD_SQL)."); + console.log(`--post-deploy: the three backfills, then ALL ${COMPATIBILITY_TRIGGERS.length} compatibility triggers come out: ${COMPATIBILITY_TRIGGERS.join(", ")}.`); console.log( repairSplitJobs ? "--repair-split-jobs: ALSO re-deriving projectId from the estimate for QBO-synced rows whose pair disagrees. Read SPLIT_JOB_REPAIR before trusting this on a database where humans have re-attributed expenses." diff --git a/src/app/api/integrations/receipt-ingest/route.ts b/src/app/api/integrations/receipt-ingest/route.ts index 0b3f532ff..4f96818f7 100644 --- a/src/app/api/integrations/receipt-ingest/route.ts +++ b/src/app/api/integrations/receipt-ingest/route.ts @@ -16,6 +16,22 @@ export const maxDuration = 60; * different scopes, and two of them can be held at once. */ const RECEIPT_INGEST_LOCK_PREFIX = "receipt-ingest:"; +/** + * The SECOND identity, for a document whose url carries no Drive id (round 49, + * item 2). + * + * The drain-window trigger cannot always derive a file id — a shortened link, a + * re-hosted copy, a `/uc?export=download` form — and until it had this key it + * simply took no lock in that case, which left an old instance's in-flight + * insert invisible to this route rather than merely uncommitted. Both sides + * hash the same normalised string, so the same document serialises whether or + * not anyone can parse an id out of it. + */ +const RECEIPT_INGEST_URL_LOCK_PREFIX = "receipt-ingest-url:"; +/** Normalised exactly as the trigger normalises it: `lower(btrim(url))`. */ +function urlLockKey(receiptUrl: string): string { + return `${RECEIPT_INGEST_URL_LOCK_PREFIX}${receiptUrl.trim().toLowerCase()}`; +} /** * Receipt/check ingest from the "GOLDEN TOUCH — RECEIPT + CHECK AUTOMATION" @@ -46,6 +62,40 @@ interface IngestPayload { groups: IngestGroup[]; } +/** + * WHICH GROUPS OF THIS DOCUMENT ARE NOT IN THE TABLE YET (round 49, item 1). + * + * A partial delivery is a real state, not a theoretical one: the deployed old + * handler writes one autocommit INSERT per group, so a crash — or a lost + * response — after group one leaves exactly one row behind. The old rule + * ("any row for this file means the document is done") then answered + * `alreadyIngested` to the retry and the remaining groups were dropped for + * good, which is money missing from a receipt the archive says was imported. + * + * So the answer is per GROUP, keyed on the ordinal the row carries: the + * trigger numbers an old handler's rows 0, 1, 2... in arrival order, and this + * route writes its own group index, so the two agree by construction. + * + * A row with a NULL ordinal is the one shape that cannot be reasoned about + * positionally — it predates the trigger, and nothing says which group it was. + * That case answers "nothing is missing", which is the conservative direction: + * a possible duplicate is refused at the cost of a possible gap, and the gap is + * visible to a human on the expense list while a duplicate quietly doubles a + * job's cost. + */ +export function missingGroupIndexes( + rows: { sourceGroupIndex: number | null }[], + groupCount: number, +): number[] { + if (rows.some(row => row.sourceGroupIndex === null)) return []; + const taken = new Set(rows.map(row => row.sourceGroupIndex)); + const missing: number[] = []; + for (let index = 0; index < groupCount; index++) { + if (!taken.has(index)) missing.push(index); + } + return missing; +} + export async function POST(req: Request) { const secret = process.env.RECEIPT_INGEST_SECRET; if (!secret || req.headers.get("x-ingest-key") !== secret) { @@ -111,11 +161,11 @@ export async function POST(req: Request) { { AND: [{ sourceFileId: null }, { receiptUrl }] }, ], }; - const existing = await prisma.expense.findFirst({ + const existing = await prisma.expense.findMany({ where: alreadyIngestedWhere, - select: { id: true }, + select: { sourceGroupIndex: true }, }); - if (existing) { + if (existing.length && missingGroupIndexes(existing, body.groups.length).length === 0) { return NextResponse.json({ ok: true, alreadyIngested: true, created: 0 }); } @@ -282,20 +332,35 @@ export async function POST(req: Request) { // lock. A violation aborts the whole transaction — the document is // written whole or not at all — and the retry then reads the // winner's committed rows and answers `alreadyIngested`. - await raw.$queryRawUnsafe( - "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))::text AS lock_result", - `${RECEIPT_INGEST_LOCK_PREFIX}${body.fileId}`, - ); - const alreadyIngested = await tx.expense.findFirst({ + // BOTH IDENTITIES, IN A FIXED ORDER (round 49, item 2): the file + // id first, then the normalised url. The trigger takes exactly one + // of the two — whichever the row it is stamping can be identified + // by — so it can never hold one and wait for the other, and this + // pair cannot cycle against it. Taking both here is what makes an + // old insert of a SHORTENED url block this read instead of being + // invisible to it. + for (const key of [`${RECEIPT_INGEST_LOCK_PREFIX}${body.fileId}`, urlLockKey(receiptUrl)]) { + await raw.$queryRawUnsafe( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))::text AS lock_result", + key, + ); + } + const present = await tx.expense.findMany({ // The same predicate as the fast path, including the legacy // `receiptUrl` arm — this is the authoritative one, taken under - // the per-file advisory lock that the drain-window trigger - // takes too, so an old instance's in-flight insert is either - // already visible here or still blocking this read. + // the advisory locks the drain-window trigger takes too, so an + // old instance's in-flight insert is either already visible + // here or still blocking this read. where: alreadyIngestedWhere, - select: { id: true }, + select: { sourceGroupIndex: true }, }); - if (alreadyIngested) return null; + // RESUME, DO NOT REPORT DONE. A partial old-handler delivery leaves + // some of the groups behind; this delivery lands exactly the ones + // that are missing, and answers `alreadyIngested` only when there + // is genuinely nothing left to write. + const missing = present.length ? missingGroupIndexes(present, body.groups.length) : null; + if (missing !== null && missing.length === 0) return null; + const wanted = missing === null ? null : new Set(missing); // THE WHOLE LOCK SET, IN THE CANONICAL ORDER, BEFORE THE LOOP // (round 37, item 3): Project -> Estimate -> EstimateItem, then @@ -312,6 +377,8 @@ export async function POST(req: Request) { const outcomes: GroupResult[] = []; for (const [groupIndex, group] of body.groups.entries()) { + // Already landed by the delivery this one is resuming. + if (wanted && !wanted.has(groupIndex)) continue; // Finite and non-zero by the validation above, which refuses // the whole document rather than letting this loop skip a // group out of a transaction that then reports success. diff --git a/tests/apply-expense-attribution.test.ts b/tests/apply-expense-attribution.test.ts index 17517f960..31070a02d 100644 --- a/tests/apply-expense-attribution.test.ts +++ b/tests/apply-expense-attribution.test.ts @@ -1127,7 +1127,7 @@ test("the guard is idempotent: it drops its own trigger before creating it", () assert.ok(dropIndex < createIndex, "drop must come before create"); }); -test("the post-deploy pass takes BOTH guards back out", () => { +test("the post-deploy pass takes ALL THREE compatibility triggers back out", () => { // They are compatibility scaffolding for ONE deploy. Left standing, the // split-job guard would overrule a future writer that legitimately moves // an estimate, and the amount/tax guard would re-open a review on every @@ -1248,7 +1248,7 @@ test("the amount/tax guard is idempotent to re-create", () => { assert.match(AMOUNT_TAX_GUARD_SQL[0], /CREATE OR REPLACE FUNCTION/); }); -test("both guards go in BEFORE the projectId backfill", () => { +test("every compatibility trigger goes in BEFORE the projectId backfill", () => { // From the moment the columns carry values, an old instance can damage // them. A guard created after the fill leaves exactly that window open. const fillAt = (statements as string[]).indexOf(PROJECT_ID_BACKFILL); @@ -1659,17 +1659,45 @@ test("the bridge derives the file id the same way the backfill does", () => { } }); -test("the ordinal counts within the TRANSACTION, not within the table", () => { - // MAX(existing) + 1 is the obvious rule and it is the wrong one: a - // re-delivery would land on fresh ordinals and insert cleanly, which is - // exactly the duplicate this bridge exists to stop. Counting per - // transaction makes a second delivery collide with the rows already there. +test("the ordinal counts what is COMMITTED, not what this transaction did", () => { + // ROUND 49, ITEM 1 — a P0 this replaces. The first version counted per + // TRANSACTION, which assumed the old handler writes its groups in one. It + // does not: the deployed handler calls prisma.expense.create() once per + // group, each its own autocommit statement, so every group got ordinal 0, + // group two violated the unique index the moment group one committed, and + // a retry answered alreadyIngested with the rest of the receipt missing. const fn = SOURCE_FILE_BRIDGE_SQL[0]; - assert.match(fn, /set_config\(counter_key, next_index::text, true\)/, - "the counter must be TRANSACTION-local"); - assert.match(fn, /current_setting\(counter_key, true\)/); - assert.doesNotMatch(fn, /MAX\("sourceGroupIndex"\)/, - "a table-wide MAX lets a re-delivery insert cleanly"); + assert.match(fn, /SELECT COALESCE\(MAX\("sourceGroupIndex"\), -1\) \+ 1/, + "the ordinal comes from the committed rows for this file"); + assert.match(fn, /WHERE "sourceFileId" = NEW\."sourceFileId"/); + assert.doesNotMatch(fn, /set_config\('probuild\.bridge_/, + "a transaction-local counter cannot survive the old handler's boundaries"); + assert.doesNotMatch(fn, /current_setting\(counter_key/); + // ...and the read happens UNDER the advisory lock, or two concurrent old + // inserts would compute the same next ordinal. + assert.ok( + fn.indexOf("pg_advisory_xact_lock") < fn.indexOf('MAX("sourceGroupIndex")'), + "the lock has to be held before the MAX is read", + ); +}); + +test("an unparseable url still gets an identity to lock on", () => { + // ROUND 49, ITEM 2. The bridge used to RETURN early when no Drive id could + // be derived, so a shortened or re-hosted link took no lock at all and an + // old instance's in-flight insert was invisible to the new route rather + // than merely uncommitted. + const fn = SOURCE_FILE_BRIDGE_SQL[0]; + assert.match(fn, /hashtextextended\('receipt-ingest-url:' \|\| lower\(btrim\(NEW\."receiptUrl"\)\), 0\)/, + "the normalised url is the second identity"); + // The route must hash the SAME string, normalised the same way. + const route = readFileSync( + path.resolve(__dirname, "..", "src", "app", "api", "integrations", "receipt-ingest", "route.ts"), + "utf8", + ); + assert.match(route, /RECEIPT_INGEST_URL_LOCK_PREFIX = "receipt-ingest-url:"/); + assert.match(route, /receiptUrl\.trim\(\)\.toLowerCase\(\)/); + // Only a row with NO url at all escapes without a lock. + assert.match(fn, /IF NEW\."sourceFileId" IS NULL AND NEW\."receiptUrl" IS NULL THEN\s+RETURN NEW;/); }); test("it fires BEFORE INSERT and only touches rows that stay silent", () => { @@ -1679,8 +1707,8 @@ test("it fires BEFORE INSERT and only touches rows that stay silent", () => { // not renumber it. assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceFileId" IS NULL AND NEW\."receiptUrl" IS NOT NULL THEN/); assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceGroupIndex" IS NULL THEN/); - // ...and an expense with no Drive url at all pays nothing. - assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceFileId" IS NULL THEN\s+RETURN NEW;/); + // ...and an expense with no receipt url at all pays nothing. + assert.match(SOURCE_FILE_BRIDGE_SQL[0], /IF NEW\."sourceFileId" IS NULL AND NEW\."receiptUrl" IS NULL THEN\s+RETURN NEW;/); }); test("the bridge and its teardown are BOTH in the committed migration", () => { @@ -1760,3 +1788,23 @@ test("...and the whole chain passes with the right ref, refuses with a wrong one // ...and the banner built from that URL still hides the password. assert.doesNotMatch(targetBanner("prod", { url: PROD_URL, from: ".env.production.local", db: "postgres", host: "10.0.0.5" }), /s3cr3t/); }); + +test("the operator message names the same triggers the teardown drops", () => { + // Round 49, item 4. The list grew to three and the message still said + // "BOTH compatibility guards", so an operator watching a --post-deploy + // run was told to expect two removals and saw three. One list, read in + // both places, is the only version of this that cannot drift again. + const source = readFileSync( + path.resolve(__dirname, "..", "scripts", "apply-expense-attribution.mjs"), + "utf8", + ); + assert.equal(COMPATIBILITY_TRIGGERS.length, 3); + assert.match(source, /ALL \$\{COMPATIBILITY_TRIGGERS\.length\} compatibility triggers come out/, + "the message must COUNT the list rather than restate a number"); + assert.doesNotMatch(source, /BOTH compatibility guards/, "the two-trigger wording is gone"); + // ...and the teardown really does drop exactly that list. + const teardown = postDeployTeardownStatements(); + for (const name of COMPATIBILITY_TRIGGERS) { + assert.ok(teardown.some(sql => sql.includes(`DROP TRIGGER IF EXISTS ${name}`)), `${name} is never dropped`); + } +}); diff --git a/tests/phase3-spec-wording.test.ts b/tests/phase3-spec-wording.test.ts index ff003791b..7106889a5 100644 --- a/tests/phase3-spec-wording.test.ts +++ b/tests/phase3-spec-wording.test.ts @@ -101,3 +101,21 @@ test("the report's filter is documented as it is implemented", () => { assert.match(report, /taxAmount: \{ not: 0 \}/); assert.match(report, /needsTaxReview: false/); }); + +test("no document still advertises an installedAtCustomer DEFAULT", () => { + // Round 49, item 5. The rule shipped in round 2 — nothing defaults this + // field — and two planning documents went on describing the behaviour it + // replaced: the intake contract said "defaulting per §5" and the v2 plan + // said "default true for job-folder receipts, false for Shop". A spec that + // contradicts the code invites someone to implement the spec. + assert.doesNotMatch(SPEC, /\(defaulting per §5\)/); + assert.match(SPEC, /\*\*no default — see the TAX POSITION note below\*\*/); + assert.match(SPEC, /`installedAtCustomer` has no default/); + + const plan = readFileSync( + path.resolve(__dirname, "..", "docs", "plans", "RECEIPT-PIPELINE-V2-PLAN.md"), + "utf8", + ); + assert.doesNotMatch(plan, /default true for job-folder receipts/); + assert.match(plan, /\*\*nothing defaults it\*\*/); +}); diff --git a/tests/receipt-ingest-attribution.test.ts b/tests/receipt-ingest-attribution.test.ts index 6f770f850..d57bc2d7a 100644 --- a/tests/receipt-ingest-attribution.test.ts +++ b/tests/receipt-ingest-attribution.test.ts @@ -160,7 +160,10 @@ const fakePrisma: any = { // exact `receiptUrl`. Both are modelled as equality, and anything else // still throws — a regression to `contains` on the caller-supplied url // is what round 34 removed and it must not come back through this door. - findFirst: async (args: any) => { + findMany: async (args: any) => { + // A findMany since round 49, item 1: the route needs the ORDINALS, + // not just "is there a row", so it can resume a partial delivery + // instead of calling the document done. const arms = args?.where?.OR; if (!Array.isArray(arms) || arms.length !== 2) { throw new Error( @@ -177,10 +180,11 @@ const fakePrisma: any = { JSON.stringify(args?.where), ); } - const hit = created.find(row => - row.sourceFileId === wanted || - (row.sourceFileId == null && row.receiptUrl === legacyUrl)); - return hit ? { id: "exp-existing" } : null; + return created + .filter(row => + row.sourceFileId === wanted || + (row.sourceFileId == null && row.receiptUrl === legacyUrl)) + .map(row => ({ sourceGroupIndex: row.sourceGroupIndex ?? null })); }, create: async (args: { data: Record }) => { created.push(args.data); @@ -345,12 +349,32 @@ test("two concurrent deliveries of the same file ingest it exactly ONCE", async assert.equal(second.status, 200); }); -test("the ingest lock is keyed on the DRIVE FILE, not on something coarser", async () => { +test("the ingest lock is keyed on the DRIVE FILE, and on its url as well", async () => { // A lock on the project (or a constant) would serialise unrelated // receipts and still not identify this document; a lock on the whole // payload would change with every retry and guard nothing. + // + // TWO keys since round 49, item 2, in a fixed order: the drain-window + // trigger cannot always parse a Drive id out of the url it is handed (a + // shortened link, a re-hosted copy), and it locks the NORMALISED URL in + // that case. This route takes both, so it waits for such an insert instead + // of reading past it. The trigger only ever holds one of the two, so the + // pair cannot cycle against it. await post(PAYLOAD); - assert.deepEqual(lockKeys, ["receipt-ingest:drive-file-1"]); + assert.deepEqual(lockKeys, [ + "receipt-ingest:drive-file-1", + "receipt-ingest-url:https://drive.google.com/file/d/drive-file-1/view", + ]); +}); + +test("the url key is NORMALISED the same way the trigger normalises it", async () => { + // `lower(btrim(url))` in SQL, `trim().toLowerCase()` here. A mismatch of a + // single character means two different locks and no serialisation at all. + await post({ ...PAYLOAD, fileUrl: " HTTPS://Drive.Google.com/OPEN-SHORT/AbCdEf " }); + assert.deepEqual(lockKeys, [ + "receipt-ingest:drive-file-1", + "receipt-ingest-url:https://drive.google.com/open-short/abcdef", + ]); }); test("two deliveries of DIFFERENT files do not block each other", async () => { @@ -358,7 +382,12 @@ test("two deliveries of DIFFERENT files do not block each other", async () => { assert.equal(created.length, 2); assert.deepEqual( [...lockKeys].sort(), - ["receipt-ingest:drive-file-1", "receipt-ingest:drive-file-2"], + [ + "receipt-ingest-url:https://drive.google.com/file/d/drive-file-1/view", + "receipt-ingest-url:https://drive.google.com/file/d/drive-file-2/view", + "receipt-ingest:drive-file-1", + "receipt-ingest:drive-file-2", + ], ); }); diff --git a/tests/receipt-ingest-drain-window-db.test.ts b/tests/receipt-ingest-drain-window-db.test.ts index 430bba984..9bb331fc3 100644 --- a/tests/receipt-ingest-drain-window-db.test.ts +++ b/tests/receipt-ingest-drain-window-db.test.ts @@ -143,6 +143,54 @@ function ingest(body: Record) { })); } +/** A url with no derivable Drive id: a shortened link, as Drive sometimes returns. */ +const SHORT_URL = "https://drive.google.com/open-short/abcdef"; + +/** Three groups of one receipt, the shape the old handler writes one row at a time. */ +const THREE_GROUPS = [ + { category: "Plumbing", amount: 10 }, + { category: "Plumbing", amount: 20 }, + { category: "Plumbing", amount: 30 }, +]; + +/** + * THE PRE-FIX BRIDGE, kept verbatim so the control is the real thing rather + * than a description of it. Transaction-local counter: three autocommit + * inserts each start at 0. + */ +const PRE_FIX_BRIDGE_FUNCTION = ` + CREATE OR REPLACE FUNCTION probuild_expense_source_file_bridge() + RETURNS trigger + LANGUAGE plpgsql + AS $bridge$ + DECLARE + derived TEXT; + next_index INT; + counter_key TEXT; + BEGIN + IF NEW."sourceFileId" IS NULL AND NEW."receiptUrl" IS NOT NULL THEN + derived := COALESCE( + substring(NEW."receiptUrl" from '/d/([A-Za-z0-9_-]+)'), + substring(NEW."receiptUrl" from '[?&]id=([A-Za-z0-9_-]+)') + ); + NEW."sourceFileId" := derived; + END IF; + IF NEW."sourceFileId" IS NULL THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock( + hashtextextended('receipt-ingest:' || NEW."sourceFileId", 0) + ); + IF NEW."sourceGroupIndex" IS NULL THEN + counter_key := 'probuild.bridge_' || md5(NEW."sourceFileId"); + next_index := COALESCE(NULLIF(current_setting(counter_key, true), '')::int, -1) + 1; + PERFORM set_config(counter_key, next_index::text, true); + NEW."sourceGroupIndex" := next_index; + END IF; + RETURN NEW; + END; + $bridge$`; + const PAYLOAD = { projectName: PROJECT_NAME, vendor: "Home Depot", @@ -218,39 +266,177 @@ test("the bridge stamps an old-build insert on the way in", { skip }, async () = } }); -test("N groups of one old-build transaction get N distinct ordinals", { skip }, async () => { +test("a 3-group delivery at the OLD HANDLER'S boundaries lands all three", { skip }, async () => { + // ROUND 49, ITEM 1 — the P0. The deployed handler does NOT wrap its groups + // in a transaction: it calls prisma.expense.create() once per group, each + // its own autocommit statement. Wrapping them in one transaction, as the + // first version of this test did, tested a shape production never + // produces — and hid that the transaction-local counter gave every group + // ordinal 0, so group two died on the unique index and was lost. await seed(); await installBridge(); try { - await db!.$transaction(async tx => { - const raw = tx as unknown as { $executeRawUnsafe(q: string, ...v: unknown[]): Promise }; - await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g0`, 10, FILE_URL, ESTIMATE); - await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g1`, 20, FILE_URL, ESTIMATE); - await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-g2`, 30, FILE_URL, ESTIMATE); - }); + await oldStyleInsert(db!, `${PFX}-g0`, 10); + await oldStyleInsert(db!, `${PFX}-g1`, 20); + await oldStyleInsert(db!, `${PFX}-g2`, 30); + + const rows = await rowsForFile(); + assert.equal(rows.length, 3, "all three groups are in the table"); + assert.deepEqual(rows.map(r => r.sourceGroupIndex).sort((a, b) => (a ?? 0) - (b ?? 0)), [0, 1, 2]); + assert.deepEqual( + rows.map(r => Number(r.amount)).sort((a, b) => a - b), + [10, 20, 30], + "and the money is all of it, not just the first group", + ); + } finally { + await cleanup(); + } +}); + +test("CONTROL: a transaction-local ordinal loses groups two and three", { skip }, async () => { + // The pre-fix bridge, verbatim, installed for this test only: the counter + // lives in a transaction-local GUC, so three autocommit inserts each start + // at 0. This is what production would have done to a three-group receipt. + await seed(); + await removeBridge().catch(() => {}); + try { + await db!.$executeRawUnsafe(PRE_FIX_BRIDGE_FUNCTION); + await db!.$executeRawUnsafe(`DROP TRIGGER IF EXISTS probuild_expense_source_file_bridge ON "Expense"`); + await db!.$executeRawUnsafe( + `CREATE TRIGGER probuild_expense_source_file_bridge + BEFORE INSERT ON "Expense" FOR EACH ROW + EXECUTE FUNCTION probuild_expense_source_file_bridge()`, + ); + + await oldStyleInsert(db!, `${PFX}-g0`, 10); + let second: unknown = null; + await oldStyleInsert(db!, `${PFX}-g1`, 20).catch(caught => { second = caught; }); + let third: unknown = null; + await oldStyleInsert(db!, `${PFX}-g2`, 30).catch(caught => { third = caught; }); + + assert.ok(second, "group two collides on ordinal 0"); + assert.ok(third, "and so does group three"); + assert.equal((await rowsForFile()).length, 1, "two thirds of the receipt is gone"); + + // ...and the retry against the new route then calls it done, which is + // how the loss becomes permanent. + const res = await ingest({ ...PAYLOAD, groups: THREE_GROUPS }); + const json = await res.json() as { alreadyIngested?: boolean; created?: number }; + assert.equal(json.alreadyIngested, undefined, "the SHIPPED route resumes rather than reporting done"); + assert.equal(json.created, 2, "it lands the two groups the old handler lost"); + } finally { + await removeBridge().catch(() => {}); + await cleanup(); + await installBridge(); + } +}); + +test("a retry after a PARTIAL old delivery lands the missing groups, not nothing", { skip }, async () => { + // The other half of the P0: a crash (or a lost response) after group one + // leaves exactly one row. The old rule — any row for this file means the + // document is done — answered alreadyIngested and the rest of the receipt + // was dropped for good. + await seed(); + await installBridge(); + try { + await oldStyleInsert(db!, `${PFX}-partial`, 10); + assert.equal((await rowsForFile()).length, 1, "the crash left one group behind"); + + const res = await ingest({ ...PAYLOAD, groups: THREE_GROUPS }); + const json = await res.json() as { created?: number; alreadyIngested?: boolean }; + assert.equal(json.alreadyIngested, undefined, "not 'done' — there are two groups still missing"); + assert.equal(json.created, 2); + const rows = await rowsForFile(); - assert.deepEqual(rows.map(r => r.sourceGroupIndex), [0, 1, 2]); + assert.equal(rows.length, 3, "the document is whole"); + assert.deepEqual( + rows.map(r => r.sourceGroupIndex).sort((a, b) => (a ?? 0) - (b ?? 0)), + [0, 1, 2], + "distinct ordinals, no duplicate of group one", + ); + + // ...and a SECOND retry of the complete document adds nothing. + const again = await ingest({ ...PAYLOAD, groups: THREE_GROUPS }); + assert.deepEqual(await again.json(), { ok: true, alreadyIngested: true, created: 0 }); + assert.equal((await rowsForFile()).length, 3); + } finally { + await cleanup(); + } +}); + +test("an UNPARSEABLE url still serializes an old insert against the new route", { skip }, async () => { + // ROUND 49, ITEM 2. No /d/ and no ?id=, so the bridge can derive + // nothing — and used to return without taking any lock, which left the new + // route reading "nothing here" while the old insert was still in flight. + // The normalised url is the identity both sides hash instead. + await seed(); + await installBridge(); + try { + const inserted = gate(); + let oldError: unknown = null; + const oldSide = (async () => { + try { + await oldBuild!.$transaction(async tx => { + const raw = tx as unknown as { $executeRawUnsafe(q: string, ...v: unknown[]): Promise }; + await raw.$executeRawUnsafe(OLD_INSERT, `${PFX}-short`, 120.5, SHORT_URL, ESTIMATE); + inserted.open(); + await new Promise(resolve => setTimeout(resolve, 750)); + }, { timeout: 30_000 }); + } catch (caught) { + oldError = caught; + } + })(); + + await inserted.reached; + const res = await ingest({ ...PAYLOAD, fileUrl: SHORT_URL }); + const json = await res.json() as { created?: number; alreadyIngested?: boolean }; + await oldSide; + + assert.equal(oldError, null, `the old insert failed: ${oldError}`); + assert.equal(json.alreadyIngested, true, "it waited for the old insert, then saw it"); + assert.equal(json.created, 0); + const rows = await db!.expense.findMany({ + where: { receiptUrl: SHORT_URL }, select: { id: true, sourceFileId: true }, + }); + assert.equal(rows.length, 1, "one delivery, one row"); + assert.equal(rows[0].sourceFileId, null, "and no fake file id was invented for it"); } finally { + await db!.expense.deleteMany({ where: { receiptUrl: SHORT_URL } }); await cleanup(); } }); -test("a SECOND old-build delivery of the same file is refused, not duplicated", { skip }, async () => { - // The ordinal counts within the TRANSACTION, so a re-delivery starts at 0 - // again and collides with the row already there. The unique index refuses - // it, the old instance sees an error, and its Apps Script does not archive - // the file — which is the correct outcome for a document already booked. +test("THE TRADE: a second OLD delivery appends, and the new route still refuses one", { skip }, async () => { + // Stated plainly, because round 49 changed it. With ordinals counted per + // TRANSACTION, a re-delivery restarted at 0 and died on the unique index — + // which looked like a free dedupe and was actually the P0: the old handler + // writes each group in its own autocommit, so that same rule killed group + // TWO of a first delivery and lost it. + // + // Counting COMMITTED rows fixes the loss and gives up the accidental + // dedupe: a second old-handler delivery appends. That is exactly what the + // old build does today with no bridge at all, it is guarded by the old + // handler's own url dedupe, and a duplicate a human can see on the expense + // list is a smaller failure than money silently missing from a receipt the + // archive says was imported. + // + // What this PR is responsible for — the NEW route never adding a copy — + // still holds, and is asserted here rather than assumed. await seed(); await installBridge(); try { await oldStyleInsert(db!, `${PFX}-first`); - let error: unknown = null; - await oldStyleInsert(db!, `${PFX}-second`).catch(caught => { error = caught; }); - assert.ok(error, "the second delivery must fail"); - // Postgres names the constraint and the colliding key; Prisma passes - // the message through with SQLSTATE 23505 (unique_violation). - assert.match(String((error as { message?: string })?.message ?? error), /23505|already exists/i); - assert.equal((await rowsForFile()).length, 1, "exactly one copy survives"); + await oldStyleInsert(db!, `${PFX}-second`); + const rows = await rowsForFile(); + assert.equal(rows.length, 2, "the old build appends, as it always did"); + assert.deepEqual(rows.map(r => r.sourceGroupIndex).sort((a, b) => (a ?? 0) - (b ?? 0)), [0, 1], + "with distinct ordinals, so neither insert is lost"); + + // The new route, asked for the same one-group document, adds nothing: + // group 0 is present, so there is nothing missing to resume. + const res = await ingest(PAYLOAD); + assert.deepEqual(await res.json(), { ok: true, alreadyIngested: true, created: 0 }); + assert.equal((await rowsForFile()).length, 2, "no third copy"); } finally { await cleanup(); } @@ -310,19 +496,23 @@ test("...and they SERIALIZE: an in-flight old insert blocks the new route", { sk }); test("the NEW route first, then an old-build delivery of the same file", { skip }, async () => { - // The other order. The new build writes ordinal 0 for its single group; the - // old instance's retry starts its own count at 0 and collides, so the - // duplicate is refused by the index rather than landing beside it. + // The other order. The new build writes ordinal 0; a straggler old instance + // appends at 1 (see THE TRADE above — its own url dedupe is what stops it + // in production, and this PR does not change that handler). What must hold + // is that nothing is LOST and that the new route does not then add a third. await seed(); await installBridge(); try { const res = await ingest(PAYLOAD); assert.equal((await res.json() as { created?: number }).created, 1); - let error: unknown = null; - await oldStyleInsert(db!, `${PFX}-late`).catch(caught => { error = caught; }); - assert.ok(error, "the old build's second copy must be refused"); - assert.equal((await rowsForFile()).length, 1, "still exactly one row for this file"); + await oldStyleInsert(db!, `${PFX}-late`); + const rows = await rowsForFile(); + assert.deepEqual(rows.map(r => r.sourceGroupIndex).sort((a, b) => (a ?? 0) - (b ?? 0)), [0, 1]); + + const again = await ingest(PAYLOAD); + assert.deepEqual(await again.json(), { ok: true, alreadyIngested: true, created: 0 }); + assert.equal((await rowsForFile()).length, 2, "the new route added nothing"); } finally { await cleanup(); }