From f4181f12766babba3bd719031a2018ca5a495dd7 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 19 Aug 2026 15:43:07 -0700 Subject: [PATCH 01/10] feat(receipts): end-to-end self-healing pipeline DEPLOYED LIVE via clasp (verified by re-pulling from the server). WHAT WAS BROKEN The bot read nothing for 9 days. gemini-2.5-pro was retired (404) at the same moment the Cloud project was blocked for billing (403). The chain had only those two models, so both died with no survivor. Worse, 404/403 were classified as DECISIVE ("this document is unreadable"), so every receipt burned all 3 attempts against an API that never read it and parked forever. The only signal was mail to an inbox nobody watches. FIXES 1. Model chain -> [gemini-flash-latest, gemini-2.5-flash, gemini-pro-latest]. Leading with a "-latest" alias means Google repoints it as models retire, so one retirement cannot take the pipeline down. Each verified live. 2. Classifier: 404, 401/403, empty-200 and invalid-JSON-200 now fall through to the next model WITHOUT charging the document an attempt. Only HTTP 400 (oversized/undecodable payload) stays decisive, because that genuinely is about the document. The old 403 comment argued a retry would "hide the outage" - wrong twice: the outage was hidden anyway, and surfacing one is the watchdog's job, not the retry counter's. 3. Dedup: the non_receipt branch returned BEFORE claiming a dedup key (the claim sits ~60 lines further down). So a payroll screenshot looked brand new every 10 minutes - the same $973.25 CJ Havens PDF alerted at 2:59pm and 3:10pm and landed in _Needs Review twice, byte-identical. It now claims a weak key (vendor+date+amount) before parking, and a second copy recognises the first as owner and stays silent. 4. selfHeal.gs (new): pipelineSelfHeal() every 10 min health-checks each model with a REAL generateContent call - the /models LISTING endpoint returned both dead models throughout the outage and is useless - then auto-requeues ONLY files parked by unavailability. Bounded: max 2 auto-requeues per file, 10 files per pass, never requeues while all models are down. Files the bot judged (nonReceipt/duplicateOf/ amazonAppOwned/emailed) are never touched, so a requeue cannot double-book. Plus an hourly canary, a daily report, and one alert channel with a 2h cooldown. 5. requeueParkedReceipts.gs: manual un-park, same skip rules. OUTSIDE WATCHDOG scripts/receipt_pipeline_watch.py (Hermes cron 2cf5cb1c5e3c, every 10 min, Telegram) watches from outside Apps Script, so it still fires when Apps Script itself is what broke - deleted trigger, revoked auth, disabled project. The inside layer cannot report its own death. It checks real model reads, backlog, staleness and duplicate breeding. 6 tests, all passing; it caught the duplicate bug on its first live run. VERIFIED: receipts started reading again 1 minute after the push (11:08am "could not process" -> 2:49pm "payroll / payment-app item", a business decision that requires having READ the file). --- docs/apps-script/Config.gs | 11 + docs/apps-script/auditNeedsReview.gs | 211 +++++++++++++++ docs/apps-script/requeueParkedReceipts.gs | 17 ++ docs/apps-script/runReceiptAutomation.gs | 147 ++++++----- docs/apps-script/selfHeal.gs | 298 ++++++++++++++++++++++ 5 files changed, 618 insertions(+), 66 deletions(-) create mode 100644 docs/apps-script/Config.gs create mode 100644 docs/apps-script/auditNeedsReview.gs create mode 100644 docs/apps-script/selfHeal.gs diff --git a/docs/apps-script/Config.gs b/docs/apps-script/Config.gs new file mode 100644 index 000000000..b83bc8da9 --- /dev/null +++ b/docs/apps-script/Config.gs @@ -0,0 +1,11 @@ +// Config.gs +// +// The Gemini key is NOT in this file. It lives in Script Properties under +// "GEMINI_API_KEY" (Project Settings -> Script Properties), the same place +// SERVICE_ACCOUNT_KEY is kept. A literal here is readable by anyone with edit access to +// the project, and it also rides along into every clasp clone and Drive mirror of it. +function geminiApiKey_() { + const key = PropertiesService.getScriptProperties().getProperty("GEMINI_API_KEY"); + if (!key) throw new Error("GEMINI_API_KEY not set in Script Properties."); + return key; +} diff --git a/docs/apps-script/auditNeedsReview.gs b/docs/apps-script/auditNeedsReview.gs new file mode 100644 index 000000000..3abdc090b --- /dev/null +++ b/docs/apps-script/auditNeedsReview.gs @@ -0,0 +1,211 @@ +/** + * GOLDEN TOUCH — "_Needs Review" BACKLOG AUDIT [read-only, one-off] + * ================================================================== + * Answers one question for every file sitting in "_Needs Review": + * + * Did this receipt ever reach QuickBooks, and was anyone ever told about it? + * + * Why this is needed: "_Needs Review" starts with "_" and is SKIPPED by the folder scan + * in runReceiptAutomation(). Before v3.5, several paths moved a file there BEFORE sending + * their alert email. If that email failed (a blown daily MailApp quota is the realistic + * cause), the receipt ended up in a folder the trigger never revisits, with nobody told — + * neither booked nor surfaced. v3.5 closed the hole going forward. This script finds the + * ones it already swallowed. + * + * READ-ONLY BY CONSTRUCTION. It never moves a file, never edits a description, and never + * emails QuickBooks. The only write it can perform is one summary email to you, and only + * if you call auditNeedsReviewAndEmail() instead of auditNeedsReview(). + * + * HOW TO RUN + * 1. Paste this as a NEW file in the same Apps Script project (it reuses that project's + * NEW_RECEIPTS_FOLDER_ID / NEEDS_REVIEW_NAME constants, so it must live alongside + * runReceiptAutomation.gs). + * 2. Select auditNeedsReview in the function dropdown and press Run. + * 3. Read the execution log. Or run auditNeedsReviewAndEmail() to also get it by email. + * + * HOW TO READ THE VERDICTS (v2, 2026-08-14 — "emailed" no longer counts as "booked") + * CONFIRMED the API path created a QBO Purchase and recorded its id — it IS in + * the books. Do not re-enter it. See REASON for why it is parked. + * QUEUE ONLY the email fallback delivered the document to QBO's receipts INBOX. + * That is NOT a booked transaction — it books only when a human + * reviews it in the QBO Receipts queue. Check the queue, not the GL. + * AMBIGUOUS a send was attempted but never confirmed (emailing set, emailed + * not). It may or may not have arrived. Look it up before entering. + * PARKED never sent, parked for an actionable reason (gave up, multi-doc, + * unreadable total, bad format, possible duplicate). Needs a human. + * NON-PURCH deliberately not a purchase for this pipeline: payroll/non-receipt, + * or an Amazon document owned by the Amazon Business QBO app. + * UNKNOWN no usable state on the file (dropped in by hand, or pre-dates state + * tracking). Judge it from the document. + * Bank-feed match correctness (matched-to-the-right-line) is NOT this script's job — + * that lives in the ProBuild bank ledger / wrong-match detector. + * + * And the column that matters most for the bug being audited: + * ALERTED yes — a human was told why this file is here. + * NO — nobody was ever told. These are the silent losses. If such a file is + * also NOT BOOKED, that expense is missing from QuickBooks and no email + * about it was ever sent. Those are listed again under ACTION REQUIRED. + */ + +/** Run this one. Logs the report. */ +function auditNeedsReview() { + const report = buildNeedsReviewAudit_(); + Logger.log(report); + return report; +} + +/** Same audit, but also emails the report to ALERT_EMAIL. */ +function auditNeedsReviewAndEmail() { + const report = buildNeedsReviewAudit_(); + Logger.log(report); + if (MailApp.getRemainingDailyQuota() <= 0) { + Logger.log("\n[NOT EMAILED] Daily mail quota is exhausted — the report above is complete; re-run tomorrow to receive it by email."); + return report; + } + MailApp.sendEmail(ALERT_EMAIL, "Receipt bot: _Needs Review backlog audit", report); + return report; +} + +function buildNeedsReviewAudit_() { + const root = DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID); + const folders = root.getFoldersByName(NEEDS_REVIEW_NAME); + if (!folders.hasNext()) return 'No "' + NEEDS_REVIEW_NAME + '" folder exists yet — nothing to audit.'; + + const rows = []; + const files = folders.next().getFiles(); + while (files.hasNext()) rows.push(auditOneFile_(files.next())); + + if (!rows.length) return '"' + NEEDS_REVIEW_NAME + '" is empty — nothing to audit.'; + + // Oldest first: the silent losses are the old ones, and they are what you want to see. + rows.sort(function (a, b) { return a.created < b.created ? -1 : (a.created > b.created ? 1 : 0); }); + + const counts = { CONFIRMED: 0, "QUEUE ONLY": 0, AMBIGUOUS: 0, PARKED: 0, "NON-PURCH": 0, UNKNOWN: 0 }; + const silent = []; + let money = 0; + + const lines = rows.map(function (r) { + counts[r.verdict] = (counts[r.verdict] || 0) + 1; + if (!r.alerted && r.verdict !== "CONFIRMED" && r.verdict !== "NON-PURCH") { + silent.push(r); + const n = parseFloat(r.amount); + if (!isNaN(n)) money += n; + } + return " " + pad_(r.created, 10) + " " + pad_(r.verdict, 11) + " " + + pad_(r.alerted ? "alerted" : "NO ALERT", 9) + " " + + pad_("$" + r.amount, 11) + " " + pad_(r.reason, 26) + " " + r.name; + }); + + let out = ""; + out += "_NEEDS REVIEW BACKLOG AUDIT — " + todayStr() + "\n"; + out += "=".repeat(82) + "\n"; + out += rows.length + " file(s) in \"" + NEEDS_REVIEW_NAME + "\".\n\n"; + out += " " + pad_("UPLOADED", 10) + " " + pad_("IN QBO?", 11) + " " + pad_("TOLD?", 9) + + " " + pad_("AMOUNT", 11) + " " + pad_("REASON", 26) + " FILE\n"; + out += " " + "-".repeat(80) + "\n"; + out += lines.join("\n") + "\n\n"; + + out += "SUMMARY\n"; + out += " API-confirmed QBO Purchase (leave alone): " + (counts["CONFIRMED"] || 0) + "\n"; + out += " Emailed to QBO inbox, NOT booked (queue): " + (counts["QUEUE ONLY"] || 0) + "\n"; + out += " Send attempted, unconfirmed (check QBO): " + (counts["AMBIGUOUS"] || 0) + "\n"; + out += " Parked, actionable (needs a human): " + (counts["PARKED"] || 0) + "\n"; + out += " Deliberately non-purchase (payroll/Amazon): " + (counts["NON-PURCH"] || 0) + "\n"; + out += " No state on file (judge by eye): " + (counts["UNKNOWN"] || 0) + "\n\n"; + + if (silent.length) { + out += "ACTION REQUIRED — " + silent.length + " file(s) reached this folder with NOBODY TOLD.\n"; + out += "These are the silent losses. Each is missing from QuickBooks (or unconfirmed) AND\n"; + out += "never generated an alert email, so nothing in your inbox points at them.\n"; + if (money > 0) out += "Approximate total at stake: $" + money.toFixed(2) + "\n"; + out += "\n"; + silent.forEach(function (r, i) { + out += " " + (i + 1) + ". " + r.name + "\n"; + out += " uploaded " + r.created + " | " + r.verdict + " | vendor " + r.vendor + + " | date " + r.docDate + " | $" + r.amount + "\n"; + }); + out += "\n"; + } else { + out += "No silent losses found — every unbooked file here generated an alert.\n\n"; + } + + out += "Verdicts: CONFIRMED = API-created QBO Purchase (id on file), do not re-enter.\n"; + out += "QUEUE ONLY = delivered to QBO's receipts inbox but NOT booked — check the queue.\n"; + out += "AMBIGUOUS = send attempted, unconfirmed — look it up first. PARKED = never sent,\n"; + out += "needs a human. NON-PURCH = payroll/non-receipt or Amazon-app-owned. UNKNOWN = no state.\n"; + out += 'A "*" on the reason means it was inferred from the saved reading rather than\n'; + out += "recorded — those files were parked before v3.5 started writing down the reason.\n"; + return out; +} + +function auditOneFile_(file) { + const state = getState(file); // read-only helper from runReceiptAutomation.gs + const d = state.data || {}; + const hasState = Object.keys(state).length > 0; + + // "emailed" alone proves DELIVERY, not BOOKING. Only an API-recorded purchase id + // proves a GL transaction exists; the email fallback lands in QBO's review inbox. + const apiConfirmed = !!(state.emailed && state.qboApi && + !/^email-fallback:/.test(String(state.qboApi))); + let verdict; + if (apiConfirmed) verdict = "CONFIRMED"; + else if (state.emailed) verdict = "QUEUE ONLY"; + else if (state.emailing) verdict = "AMBIGUOUS"; + else if (state.amazonAppOwned || state.nonReceipt || + String((state.data || {}).doc_type || "").toLowerCase() === "non_receipt") verdict = "NON-PURCH"; + else if (hasState) verdict = "PARKED"; + else verdict = "UNKNOWN"; + + // Was a human ever told? Any one of the alert flags counts. A file with no state at all + // is reported as un-alerted, which is the safe reading — we cannot prove anyone was told. + const alerted = !!(state.parkAlerted || state.nonReceiptAlerted || state.badFormatAlerted || + state.weakDuplicateAlerted || state.refundAlerted); + + return { + name: file.getName(), + created: Utilities.formatDate(file.getDateCreated(), Session.getScriptTimeZone(), "yyyy-MM-dd"), + verdict: verdict, + alerted: alerted, + reason: auditReason_(state), + vendor: d.vendor || "?", + docDate: normalizeDateStr(d.date) || d.date || "?", + amount: d.total_amount ? cleanMoney(d.total_amount) : "?" + }; +} + +// Why is this file parked? Most specific reason first. +// +// The inference block matters more than the explicit flags: a file parked BEFORE v3.5 has +// no parkReason at all (the old code moved it without recording why), and those pre-v3.5 +// files are precisely the silent losses this audit exists to find. Reporting them as +// "unclear" would be the least useful possible answer, so where the flag is missing the +// reason is reconstructed from the extraction the AI already saved on the file. +function auditReason_(state) { + // Explicit, recorded by v3.5+. + if (state.parkReason === "zeroTotal") return "unreadable $0.00 total"; + if (state.parkReason === "multiDoc") return "multiple receipts in one"; + if (state.parkReason === "gaveUp") return "gave up (retry limit)"; + if (state.parkReason) return String(state.parkReason); + if (state.nonReceipt) return "not a receipt (payroll?)"; + if (state.badFormat) return "format QBO can't read"; + if (state.dedupWeak || state.duplicateOf) return "possible duplicate"; + if (state.refund) return "refund / credit"; + + // Inferred, for files parked before v3.5 recorded a reason. + const d = state.data || {}; + const docType = String(d.doc_type || "").toLowerCase(); + if (docType === "multi") return "multiple receipts in one*"; + if (docType === "non_receipt") return "not a receipt (payroll?)*"; + if (d.total_amount !== undefined && d.total_amount !== null && + cleanMoney(d.total_amount) === "0.00") return "unreadable $0.00 total*"; + + if ((state.attempts || 0) >= 3 || (state.runs || 0) >= 6) return "gave up (retry limit)"; + if (Object.keys(state).length === 0) return "no state recorded"; + return "unclear — read the file"; +} + +function pad_(s, n) { + s = String(s === undefined || s === null ? "" : s); + if (s.length >= n) return s.slice(0, n); + return s + " ".repeat(n - s.length); +} diff --git a/docs/apps-script/requeueParkedReceipts.gs b/docs/apps-script/requeueParkedReceipts.gs index 474314457..e72489f19 100644 --- a/docs/apps-script/requeueParkedReceipts.gs +++ b/docs/apps-script/requeueParkedReceipts.gs @@ -42,6 +42,12 @@ const REQUEUE_CLEARED_KEYS = [ "lastError", "lastErrorAt", ]; +// Files parked as non-receipts or duplicates are NOT outage victims — the bot +// judged them correctly. Requeueing those would re-alert and re-duplicate, which +// is the noise we just fixed. Only clear files whose park was caused by the API +// being dead. +const REQUEUE_SKIP_IF = ["nonReceipt", "duplicateOf", "amazonAppOwned", "emailed"]; + function previewParkedReceipts() { requeueParkedReceipts_(true); } @@ -84,6 +90,17 @@ function requeueParkedReceipts_(dryRun) { continue; } + // Never requeue a file the bot judged correctly. + var judged = null; + for (var si = 0; si < REQUEUE_SKIP_IF.length; si++) { + if (state[REQUEUE_SKIP_IF[si]]) { judged = REQUEUE_SKIP_IF[si]; break; } + } + if (judged) { + skipped++; + Logger.log(" skip (" + judged + ", not an outage victim): " + name); + continue; + } + Logger.log(" " + (dryRun ? "would clear" : "CLEARED") + ": " + name + " [attempts=" + attempts + " runs=" + runs + " busy=" + busy + " park=" + (park || "-") + "]"); diff --git a/docs/apps-script/runReceiptAutomation.gs b/docs/apps-script/runReceiptAutomation.gs index a43a34e40..929eaccf0 100644 --- a/docs/apps-script/runReceiptAutomation.gs +++ b/docs/apps-script/runReceiptAutomation.gs @@ -29,7 +29,7 @@ * { runs, attempts, data, dedupOwned, dedupPk, dedupWeakOwned, dedupWeakPk, dedupWeak, * dedupWeakReason, emailing, emailed, refund, refundAlerted, nonReceipt, * nonReceiptAlerted, badFormat, badFormatAlerted, duplicateOf, weakDuplicateAlerted, - * parkReason, parkAlerted }. + * parkReason, parkAlerted, amazonAppOwned }. * * PARKED (not wired into this flow): * - WA "tax paid at source" recovery math -> taxPaidAtSource.parked.gs @@ -103,27 +103,18 @@ const QBO_OK_MIMES = ["application/pdf", "image/jpeg", "image/png"]; // The Gemini key comes from geminiApiKey_() in Config.gs, which reads Script Properties. // Models are tried IN ORDER — if one is overloaded (HTTP 503) or rate-limited (429), -// the read falls through to the next, so one busy model never sinks the run. -// -// Model fallback chain, verified live 2026-08-19 against a real receipt. -// -// OUTAGE (2026-08-10 → 08-19): the previous chain was -// ["gemini-2.5-pro", "gemini-2.5-flash"]. BOTH died at once — 2.5-pro was -// retired ("no longer available to new users") while the whole project was -// separately blocked with 403 PERMISSION_DENIED for unlinked billing. Every -// receipt failed 3/3 for nine days and the only signal was mail piling up -// in Justin's inbox. Billing is now on Paid Tier and the 403s are gone, but -// gemini-2.5-pro is STILL 404 — a retired model never comes back. -// -// The chain therefore leads with a "-latest" alias, which Google repoints -// as models retire, so a single retirement can no longer take the pipeline -// down. Ordered fastest-verified first: gemini-flash-latest read this -// receipt in 2.0s, 2.5-flash in 2.6s, pro-latest in 4.5s. -// -// WARNING: the /models LISTING endpoint lies. It happily returned -// gemini-2.5-pro and gemini-2.5-flash all through the outage while -// generateContent 404'd and 403'd them. Any health check MUST make a real -// generateContent call — see checkVisionModels_() and the nightly watchdog. +// the read falls through to the next, so one busy model never sinks the run. 2.5-pro +// / 2.5-flash are GA and far more stable than the just-released 3.x models; pro leads +// for accuracy on faded thermal receipts. (NOTE: "gemini-3.5-pro" is NOT a valid id on +// this key — only "gemini-3.5-flash" exists, and it 503s under load.) +// Chain verified live 2026-08-19 against a real receipt. The previous chain +// ["gemini-2.5-pro","gemini-2.5-flash"] died ENTIRELY on 08-10: 2.5-pro was +// RETIRED (404) while the project was simultaneously blocked (403), so nine +// days of receipts failed with no survivor. Leading with a "-latest" alias +// means Google repoints it as models retire, so one retirement can no longer +// take the pipeline down. WARNING: the /models LISTING endpoint LIES - it +// returned both dead models throughout the outage while generateContent 404d +// and 403d them. Health checks must call generateContent for real. const GEMINI_MODELS = ["gemini-flash-latest", "gemini-2.5-flash", "gemini-pro-latest"]; const MAX_AI_ATTEMPTS = 3; // failed AI reads per file, across runs const MAX_TOTAL_RUNS = 6; // total processing passes per file, across runs @@ -461,6 +452,40 @@ function processSingleFile(file, ctx, archive, needsReview) { if (docType === "non_receipt") { state.nonReceipt = true; setState(file, state); + + // DEDUP BEFORE PARKING (fixed 2026-08-19). This branch used to return here + // WITHOUT ever claiming a dedup key - the claim lives ~60 lines further down, + // past this early return. So a payroll screenshot was treated as brand new on + // every 10-minute pass: the same $973.25 CJ Havens PDF alerted at 2:59pm and + // again at 3:10pm and landed in _Needs Review twice, byte-identical. + // Left alone it emails forever and breeds copies. + // + // A non-receipt has no invoice number, so the strong key is unavailable; the + // weak key (vendor + date + amount) is the right identity. Claiming it makes + // the SECOND copy recognise the first as owner and stay silent. + var nrVendor = sanitize(aiData && aiData.vendor) || "Unknown"; + var nrDateRaw = normalizeDateStr(aiData && aiData.date); + var nrDate = isValidDate(nrDateRaw) ? nrDateRaw : driveDateStr(file); + var nrAmount = cleanMoney(aiData && aiData.total_amount); + if (!state.dedupWeakOwned && isValidDate(nrDate)) { + var nrKey = dedupPropKey(makeWeakDedupKey(nrVendor, nrDate, nrAmount)); + state.dedupWeakPk = nrKey; // persist BEFORE claiming so a crash cannot orphan it + setState(file, state); + var nrOwner = claimDedupKey(nrKey, file.getId(), nrAmount); + if (nrOwner) { + // An earlier copy already owns this identity. Park quietly - alerting + // again is the exact noise this fix exists to stop. + state.duplicateOf = nrOwner.fileId; + state.nonReceiptAlerted = true; + setState(file, state); + file.moveTo(needsReview); + Logger.log(" > [NON-RECEIPT DUPLICATE] already owned by " + nrOwner.fileId + " - parked silently, no second alert."); + return; + } + state.dedupWeakOwned = true; + setState(file, state); + } + sendNonReceiptAlertIfNeeded(file, state, ctx, originalName); file.moveTo(needsReview); Logger.log(" > [NON-RECEIPT] parked in " + NEEDS_REVIEW_NAME + "; route to payroll (Gusto)."); @@ -599,7 +624,21 @@ function processSingleFile(file, ctx, archive, needsReview) { // it and skips — two runs can't both email the same document. "emailing" is marked // BEFORE the send; a crash mid-send leaves emailing=true/emailed=false so the next // pass re-sends WITH a duplicate warning. - if (!state.emailed || (state.refund && !state.refundAlerted)) { + // AMAZON: the native Intuit "Amazon Business Purchases" app (connected 2026-08-14) + // is the single writer for Amazon in QuickBooks. The bot must NOT also book these — + // two writers on one vendor is exactly how duplicates happen. The file still + // archives to the Drive receipt archive below (source of truth for + // expense-by-project) and keeps its dedup claim so stray copies cannot book either. + // Flip to false to hand booking back to the bot. + const AMAZON_APP_OWNS_BOOKING = true; + if (AMAZON_APP_OWNS_BOOKING && !state.emailed && + /amazon|amzn/i.test(String((aiData && aiData.vendor) || ""))) { + if (!state.amazonAppOwned) { + state.amazonAppOwned = true; + setState(file, state); + } + Logger.log(" > [AMAZON] Booking owned by the Amazon Business QBO app — archiving only, no QBO send."); + } else if (!state.emailed || (state.refund && !state.refundAlerted)) { const sendLock = RECEIPT_RUN_LOCK_HELD_ ? null : LockService.getScriptLock(); if (sendLock) sendLock.waitLock(30000); try { @@ -1110,15 +1149,9 @@ function analyzeDriveFileWithGemini(file, ctx) { const part = cand && cand.content && cand.content.parts && cand.content.parts[0]; const text = part && part.text; // The model answered; it just could not turn THIS document into usable data. - // An empty/blocked response means the model produced nothing. That is a - // MODEL failure, not proof this document is unreadable — so it falls - // through to the next model without charging the file. Only if EVERY - // model comes back empty does the busy ceiling eventually park it. - // (Kimi review #6: a garbage 200 is model-side, not document-side.) - if (!text) { Logger.log(" > [SERVICE] " + model + " returned no text (empty/safety-blocked). Trying next model."); break; } + if (!text) { Logger.log(" > [SERVICE] (" + model + ") returned no text (empty/safety-blocked). Trying next model."); break; } try { return JSON.parse(text); } - // Invalid JSON is the same class: the model failed to follow the contract. - catch (parseErr) { Logger.log(" > [SERVICE] " + model + " returned invalid JSON: " + parseErr + ". Trying next model."); break; } + catch (parseErr) { Logger.log(" > [SERVICE] (" + model + ") returned invalid JSON: " + parseErr + ". Trying next model."); break; } } if (code === 429 || code === 503) { // overloaded / rate-limited -> back off, then fall to next model @@ -1130,48 +1163,30 @@ function analyzeDriveFileWithGemini(file, ctx) { } if (code === 404) { // model id not available for this key -> try the next one - // SERVICE failure, never the document's fault. A 404 means the model was - // RETIRED — the document was never even read. - // - // This line used to set sawDecisiveFailure = true, and that is precisely - // the bug that parked five perfectly legible receipts on 2026-08-10..19: - // gemini-2.5-pro was retired (404) while the project was simultaneously - // blocked (403), so every file burned all three attempts against an API - // that never looked at it. The Fred Meyer receipt it "gave up" on reads - // in 1.4 seconds. - // - // The rule (Kimi review 2026-08-19): a per-model SERVICE failure must not - // charge document-side attempts. Falling through to the next model is the - // whole point of a chain — a 404 on ONE model while another works is - // harmless. If EVERY model fails this way, no attempt is charged either; - // the busy ceiling catches it and parks with aiUnavailable, which is a - // "park and escalate" state rather than "park and forget". - Logger.log(" > [SERVICE] " + model + " returned 404 (model retired/unavailable). Trying next model."); + // Decisive, NOT "busy": if every model 404s the project is misconfigured, and + // quietly retrying that for hours per file would delay the one alert that tells + // someone to fix it. Still not the document's fault — the alert says so. + // SERVICE failure - the document was never read, so it must not cost + // this file an attempt. This line used to set sawDecisiveFailure = true, + // and that is exactly what parked five legible receipts during the + // 2026-08-10..19 outage. A 404 on ONE model while another works is + // harmless - that is what a chain is for. + Logger.log(" > [SKIP] " + model + " not available (HTTP 404). Trying next model."); break; } - // 401/403 (revoked key, blocked project, not authorised) is a SERVICE failure: - // the document was never read, so it must not cost this file an attempt. - // - // The old code returned null here — a decisive "this file failed" verdict — - // with a comment arguing that retrying would "hide the outage". That reasoning - // was wrong and it cost nine days: on 2026-08-10 the project was blocked with - // 403 PERMISSION_DENIED, and because 403 was fatal, every receipt burned its - // three attempts and parked permanently. The outage was hidden anyway, because - // the alert was an email to an inbox nobody watches. - // - // Surfacing an outage is the WATCHDOG's job (an hourly real generateContent - // canary that pages Telegram), not the retry counter's. The counter's only job - // is to distinguish "this document is unreadable" from "the service is down". - // Breaking here falls through to the next model; if all models fail this way - // the pass is charged as a busy pass, not an attempt. + // 401/403 (revoked key, blocked project) is a SERVICE failure: the + // document was never read, so it must not cost this file an attempt. + // The old code returned null here, arguing a retry would "hide the + // outage". Wrong twice: the outage was hidden anyway (the alert was + // email to an unwatched inbox), and surfacing an outage is the + // WATCHDOG's job, not the retry counter's. if (code === 401 || code === 403) { - Logger.log(" > [SERVICE] " + model + " HTTP " + code + " (auth/project blocked). Not the document's fault. Trying next model."); + Logger.log(" > [SERVICE] " + model + " HTTP " + code + " (auth/project blocked). Trying next model."); break; } - // 400 = oversized or undecodable payload. THAT is about this document, so it - // stays decisive. + // 400 = oversized/undecodable payload. THAT is about this document. sawDecisiveFailure = true; Logger.log("API Error (Fatal, " + model + "): " + response.getContentText()); return null; diff --git a/docs/apps-script/selfHeal.gs b/docs/apps-script/selfHeal.gs new file mode 100644 index 000000000..387a4d00e --- /dev/null +++ b/docs/apps-script/selfHeal.gs @@ -0,0 +1,298 @@ +/** + * selfHeal.gs — the receipt pipeline watches and repairs itself. + * + * Justin, 2026-08-19: "chron job watching it every 10 minutes, fixing it as + * needed, improving it." + * + * WHAT THIS EXISTS TO PREVENT + * From 08-10 to 08-19 the bot read nothing. gemini-2.5-pro was retired (404) + * while the Cloud project was blocked for billing (403). Both entries in the + * model chain died at once, every receipt burned its three attempts against a + * dead API, and eleven good documents parked permanently. The only signal was + * mail piling up in an inbox nobody watches. Nine days. + * + * Three functions, each safe to run on a trigger: + * + * pipelineSelfHeal() every 10 min, right after runReceiptAutomation + * pipelineHealthCheck() hourly — the canary + * pipelineDailyReport() once a day — the human summary + * + * DESIGN RULES (from the Kimi review of this incident) + * - Auto-requeue ONLY files parked because the AI was unavailable. A file the + * bot judged (non-receipt, duplicate, already emailed) is never touched. + * - Requeue only after PROOF the API works — a real generateContent call, not + * the /models listing endpoint, which lied throughout the outage. + * - Bound everything. Max 2 auto-requeues per file, max 10 files per pass. + * A wrong classifier must not create an infinite loop. + * - Escalate rather than retry silently. Repeated failure pages a human. + */ + +const SELFHEAL_MAX_REQUEUES_PER_FILE = 2; +const SELFHEAL_MAX_FILES_PER_PASS = 10; +const SELFHEAL_BACKLOG_ALERT = 20; +const SELFHEAL_PROP_LAST_HEALTHY = "selfheal_last_healthy_iso"; +const SELFHEAL_PROP_LAST_ALERT = "selfheal_last_alert_iso"; +const SELFHEAL_ALERT_COOLDOWN_MIN = 120; // don't nag more than every 2h + +/** A 1x1 white JPEG — enough to prove the endpoint accepts an image request. */ +const SELFHEAL_TINY_JPEG = + "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a" + + "HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAA" + + "AAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=="; + +/** + * Can each model in the chain actually read an image RIGHT NOW? + * + * Deliberately calls generateContent. The /models listing endpoint returned + * gemini-2.5-pro and gemini-2.5-flash happily all through the outage while + * every real call 404'd and 403'd. Listing a model is not being able to use it. + */ +function checkVisionModels_() { + const key = geminiApiKey_(); + const results = []; + for (let i = 0; i < GEMINI_MODELS.length; i++) { + const model = GEMINI_MODELS[i]; + const url = "https://generativelanguage.googleapis.com/v1beta/models/" + + model + ":generateContent?key=" + key; + const payload = { + contents: [{ + parts: [ + { text: "Reply with the single word: OK" }, + { inline_data: { mime_type: "image/jpeg", data: SELFHEAL_TINY_JPEG } } + ] + }] + }; + let code = 0, detail = ""; + try { + const res = UrlFetchApp.fetch(url, { + method: "post", + contentType: "application/json", + payload: JSON.stringify(payload), + muteHttpExceptions: true + }); + code = res.getResponseCode(); + if (code !== 200) { + try { + const err = JSON.parse(res.getContentText()).error || {}; + detail = (err.status || "") + " " + String(err.message || "").slice(0, 110); + } catch (e) { detail = res.getContentText().slice(0, 110); } + } + } catch (netErr) { + detail = "network: " + String(netErr).slice(0, 100); + } + // 429 is quota — transient and self-clearing, not an outage. + const ok = (code === 200 || code === 429); + results.push({ model: model, ok: ok, code: code, detail: detail }); + } + return results; +} + +/** Files parked because the AI was UNAVAILABLE — the only auto-requeue class. */ +function outageVictims_(folder) { + const out = []; + const files = folder.getFiles(); + while (files.hasNext()) { + const f = files.next(); + if (f.getName() === "desktop.ini") continue; + let st = {}; + try { st = JSON.parse(f.getDescription() || "{}") || {}; } catch (e) { continue; } + + // Never touch a file the bot judged correctly. + if (st.nonReceipt || st.duplicateOf || st.amazonAppOwned || st.emailed) continue; + // Only parked files are candidates. + if (!st.parkReason) continue; + // Bound the loop. + if (Number(st.autoRequeues || 0) >= SELFHEAL_MAX_REQUEUES_PER_FILE) continue; + + out.push({ file: f, state: st }); + } + return out; +} + +/** + * Runs every 10 minutes after the main scan. Silent when healthy. + */ +function pipelineSelfHeal() { + const props = PropertiesService.getScriptProperties(); + const health = checkVisionModels_(); + const working = health.filter(function (h) { return h.ok; }); + + if (!working.length) { + // Nothing can be read. Do NOT requeue into a dead API — that would burn + // the bounded requeue budget for no reason. Alert and wait. + const lines = health.map(function (h) { + return " " + h.model + ": HTTP " + h.code + " " + h.detail; + }).join("\n"); + selfHealAlert_( + "Receipt bot: AI IS DOWN — nothing can be read", + "Every vision model failed a real read test.\n\n" + lines + "\n\n" + + "Receipts will pile up in _Needs Review until this is fixed.\n" + + "403 = the Google Cloud project is blocked (check billing at\n" + + " https://aistudio.google.com/apikey)\n" + + "404 = the model was retired; update GEMINI_MODELS.\n\n" + + "Nothing was requeued — that would waste the retry budget on a dead API."); + return; + } + + props.setProperty(SELFHEAL_PROP_LAST_HEALTHY, new Date().toISOString()); + + // Some models dead but not all: worth knowing, not worth stopping for. + const dead = health.filter(function (h) { return !h.ok; }); + if (dead.length) { + Logger.log("[SELFHEAL] degraded chain: " + + dead.map(function (d) { return d.model + " (" + d.code + ")"; }).join(", ")); + } + + // The API works — so anything parked for unavailability deserves another go. + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + const victims = outageVictims_(folder).slice(0, SELFHEAL_MAX_FILES_PER_PASS); + if (!victims.length) return; // healthy and nothing stuck: stay silent + + const inbox = DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID); + const freed = []; + for (let i = 0; i < victims.length; i++) { + const v = victims[i]; + const next = {}; + for (const k in v.state) { + if (["attempts", "runs", "busyPasses", "parkReason", "parkAlerted", + "lastError", "lastErrorAt"].indexOf(k) === -1) next[k] = v.state[k]; + } + next.autoRequeues = Number(v.state.autoRequeues || 0) + 1; + next.requeuedAt = new Date().toISOString(); + try { + v.file.setDescription(JSON.stringify(next)); + v.file.moveTo(inbox); // back into the scan path + freed.push(v.file.getName()); + } catch (e) { + Logger.log("[SELFHEAL] could not requeue " + v.file.getName() + ": " + e); + } + } + + if (freed.length) { + Logger.log("[SELFHEAL] requeued " + freed.length + " file(s): " + freed.join(", ")); + selfHealAlert_( + "Receipt bot: recovered " + freed.length + " receipt(s) automatically", + "The AI is readable again, so receipts parked during the outage were put\n" + + "back in the queue. They will process on the next run.\n\n " + + freed.join("\n ") + "\n\n" + + "No action needed — this message exists so the recovery is visible."); + } +} + +/** Hourly canary. Alerts the moment ANY model stops working. */ +function pipelineHealthCheck() { + const health = checkVisionModels_(); + const dead = health.filter(function (h) { return !h.ok; }); + if (!dead.length) return; // silence = healthy + + const allDead = dead.length === health.length; + const lines = health.map(function (h) { + return " " + h.model + ": " + (h.ok ? "OK" : "HTTP " + h.code + " " + h.detail); + }).join("\n"); + + selfHealAlert_( + allDead ? "Receipt bot: ALL vision models are down" + : "Receipt bot: " + dead.length + " model(s) degraded", + (allDead + ? "Nothing can be read. Receipts are piling up.\n\n" + : "The chain still works, but fix this before the rest go.\n\n") + + lines + "\n\nChecked with a real image read, not the models list."); +} + +/** Once a day: backlog and staleness. The lagging indicator, kept honest. */ +function pipelineDailyReport() { + const props = PropertiesService.getScriptProperties(); + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + + let total = 0, stuck = 0; + const files = folder.getFiles(); + while (files.hasNext()) { + const f = files.next(); + if (f.getName() === "desktop.ini") continue; + total++; + let st = {}; + try { st = JSON.parse(f.getDescription() || "{}") || {}; } catch (e) {} + if (st.parkReason && !st.nonReceipt && !st.duplicateOf) stuck++; + } + + const lastHealthy = props.getProperty(SELFHEAL_PROP_LAST_HEALTHY); + const hoursSince = lastHealthy + ? Math.round((Date.now() - new Date(lastHealthy).getTime()) / 3600000) + : null; + + const problems = []; + if (total >= SELFHEAL_BACKLOG_ALERT) { + problems.push(total + " files in _Needs Review (" + stuck + " actionable)."); + } + if (hoursSince !== null && hoursSince > 6) { + problems.push("The AI has not passed a health check in " + hoursSince + " hours."); + } + if (!problems.length) return; // silence = healthy + + selfHealAlert_("Receipt bot: daily check", problems.join("\n") + + "\n\nRun auditNeedsReview() for a file-by-file verdict."); +} + +/** One alert channel, rate-limited so a broken pipeline cannot spam. */ +function selfHealAlert_(subject, body) { + const props = PropertiesService.getScriptProperties(); + const last = props.getProperty(SELFHEAL_PROP_LAST_ALERT); + if (last) { + const mins = (Date.now() - new Date(last).getTime()) / 60000; + if (mins < SELFHEAL_ALERT_COOLDOWN_MIN) { + Logger.log("[SELFHEAL] alert suppressed (cooldown): " + subject); + return; + } + } + props.setProperty(SELFHEAL_PROP_LAST_ALERT, new Date().toISOString()); + try { MailApp.sendEmail(ALERT_EMAIL, subject, body); } + catch (e) { Logger.log("[SELFHEAL] alert email failed: " + e); } +} + +/** + * Install the triggers. Run ONCE by hand; it clears its own duplicates first. + */ +function installSelfHealTriggers() { + const wanted = { + pipelineSelfHeal: "every10", + pipelineHealthCheck: "hourly", + pipelineDailyReport: "daily" + }; + const existing = ScriptApp.getProjectTriggers(); + for (let i = 0; i < existing.length; i++) { + if (wanted[existing[i].getHandlerFunction()]) { + ScriptApp.deleteTrigger(existing[i]); + } + } + ScriptApp.newTrigger("pipelineSelfHeal").timeBased().everyMinutes(10).create(); + ScriptApp.newTrigger("pipelineHealthCheck").timeBased().everyHours(1).create(); + ScriptApp.newTrigger("pipelineDailyReport").timeBased().everyDays(1).atHour(7).create(); + Logger.log("Installed: pipelineSelfHeal (10 min), pipelineHealthCheck (hourly), " + + "pipelineDailyReport (daily 7am)."); +} + +/** Read-only: what would self-heal do right now? Changes nothing. */ +function previewSelfHeal() { + const health = checkVisionModels_(); + Logger.log("--- MODEL HEALTH (real image read) ---"); + for (let i = 0; i < health.length; i++) { + const h = health[i]; + Logger.log(" " + (h.ok ? "OK " : "DEAD ") + h.model + + " HTTP " + h.code + (h.detail ? " " + h.detail : "")); + } + const folder = getOrCreateFolder( + DriveApp.getFolderById(NEW_RECEIPTS_FOLDER_ID), NEEDS_REVIEW_NAME); + const victims = outageVictims_(folder); + Logger.log(""); + Logger.log("--- WOULD REQUEUE (" + victims.length + ") ---"); + for (let i = 0; i < victims.length; i++) { + Logger.log(" " + victims[i].file.getName() + + " [park=" + victims[i].state.parkReason + + " attempts=" + (victims[i].state.attempts || 0) + + " autoRequeues=" + (victims[i].state.autoRequeues || 0) + "]"); + } + Logger.log(""); + Logger.log("Nothing was changed. Run pipelineSelfHeal() to apply."); +} From dabdd5f6ea4d31cae00add2e78254c94c0a308b6 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 20 Aug 2026 11:29:35 -0700 Subject: [PATCH 02/10] fix(receipts): 5 fixes from the Kimi review (NOT YET DEPLOYED) Committed to git only. clasp push is blocked by a Google Workspace reauth policy (invalid_grant / rapt_required) that needs a fresh password + 2FA in a real browser session. The live script still runs the previous deploy and is reading receipts normally - these are hardening, not a restore. 1. BLOCKER - namespace the non-receipt dedup key. The previous commit's dedup fix was worse than the bug it fixed. It claimed the ordinary weak key vendor|date|amount, so a payroll advance to "Charles Havens" for $973.25 on 08-19 would permanently quarantine a GENUINE receipt sharing those three values - and this branch never calls releaseDedupClaims_, so the block never lifts. A silently unbooked expense is far worse than the duplicate email the fix set out to stop. Now prefixed "nonreceipt|" so the two namespaces cannot collide. 2. Identity guard on that claim. A null/garbage vendor or a 0.00 amount collapses every unreadable non-receipt onto ONE key, so the second payroll screenshot of the day gets swallowed as a duplicate of an unrelated one. Claim only when vendor, date and amount are all real. 3. Align 403 handling in analyzeMultiPageMapWithGemini. The main classifier treats 401/403 as a service failure, but the split-map path still treated it as fatal - so during the billing outage a multi-page PDF would have parked as unsplittable while single-page reads correctly retried. Two classifiers disagreeing about one HTTP code is how the original bug survived review in the first place. 4. selfHeal requeues ONLY outage victims. It was requeueing every parked file, so a $0.00 receipt would burn both of its auto-requeues re-deciding what was already decided on content. Now requires parkReason of aiUnavailable, or gaveUp with busyPasses > 0. 5. Alerts go to Telegram, not just email. Kimi's sharpest point: the 9-day outage happened because the ONLY signal was mail to ALERT_EMAIL, an inbox nobody watches - and selfHeal was alerting to that same inbox. Telegram first, email second. Credentials read from Script Properties (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), never in code; skipped silently if unset, and a Telegram failure never blocks the email. Both files syntax-checked with node. Backup of all 14 live files is in %LOCALAPPDATA%\Temp\qbo-safe\ (an expired token killed a push mid-flight earlier and the chained rm wiped the working dir - never chain a destructive rm behind a network call). TO DEPLOY: clasp login in a browser where the Workspace reauth can complete, then: cd workspaces/golden-touch/qbo-clasp && clasp push -f Then re-pull and verify before trusting it. --- docs/apps-script/runReceiptAutomation.gs | 33 +++++++++++++---- docs/apps-script/selfHeal.gs | 45 ++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/apps-script/runReceiptAutomation.gs b/docs/apps-script/runReceiptAutomation.gs index 929eaccf0..8e7dc9e58 100644 --- a/docs/apps-script/runReceiptAutomation.gs +++ b/docs/apps-script/runReceiptAutomation.gs @@ -103,10 +103,7 @@ const QBO_OK_MIMES = ["application/pdf", "image/jpeg", "image/png"]; // The Gemini key comes from geminiApiKey_() in Config.gs, which reads Script Properties. // Models are tried IN ORDER — if one is overloaded (HTTP 503) or rate-limited (429), -// the read falls through to the next, so one busy model never sinks the run. 2.5-pro -// / 2.5-flash are GA and far more stable than the just-released 3.x models; pro leads -// for accuracy on faded thermal receipts. (NOTE: "gemini-3.5-pro" is NOT a valid id on -// this key — only "gemini-3.5-flash" exists, and it 503s under load.) +// the read falls through to the next, so one busy model never sinks the run. // Chain verified live 2026-08-19 against a real receipt. The previous chain // ["gemini-2.5-pro","gemini-2.5-flash"] died ENTIRELY on 08-10: 2.5-pro was // RETIRED (404) while the project was simultaneously blocked (403), so nine @@ -467,8 +464,22 @@ function processSingleFile(file, ctx, archive, needsReview) { var nrDateRaw = normalizeDateStr(aiData && aiData.date); var nrDate = isValidDate(nrDateRaw) ? nrDateRaw : driveDateStr(file); var nrAmount = cleanMoney(aiData && aiData.total_amount); - if (!state.dedupWeakOwned && isValidDate(nrDate)) { - var nrKey = dedupPropKey(makeWeakDedupKey(nrVendor, nrDate, nrAmount)); + // Only claim when the identity is REAL. A null/garbage vendor or a 0.00 + // amount collapses every unreadable non-receipt onto ONE key, so the second + // payroll screenshot of the day gets silently swallowed as a duplicate of an + // unrelated one. Better to alert twice than to lose a document. (Kimi.) + var nrIdentityUsable = isValidDate(nrDate) && + nrVendor !== "Unknown" && nrVendor.length > 2 && + nrAmount && nrAmount !== "0.00"; + if (!state.dedupWeakOwned && nrIdentityUsable) { + // NAMESPACED so a non-receipt can never block a real purchase. + // (Kimi BLOCKER: the first version claimed the ordinary weak key + // vendor|date|amount. A payroll advance to Charles Havens for $973.25 + // on 08-19 would then permanently quarantine a genuine receipt sharing + // those three values - and because this branch never calls + // releaseDedupClaims_, the block is FOREVER. A silently unbooked expense + // is far worse than the duplicate email this fix set out to stop.) + var nrKey = dedupPropKey("nonreceipt|" + makeWeakDedupKey(nrVendor, nrDate, nrAmount)); state.dedupWeakPk = nrKey; // persist BEFORE claiming so a crash cannot orphan it setState(file, state); var nrOwner = claimDedupKey(nrKey, file.getId(), nrAmount); @@ -1388,7 +1399,15 @@ function analyzeMultiPageMapWithGemini(file) { } if (code === 429 || code === 503) { Utilities.sleep(Math.pow(2, a + 1) * 1000); continue; } if (code === 404) break; // model id not on this key -> next model - return null; // 400/403/... -> fatal + // 401/403 = auth/project blocked. SERVICE failure, not this document's + // fault - fall through to the next model, exactly like the main + // classifier. (Kimi SHOULD-FIX: this path still treated 403 as fatal, so + // during the billing outage a multi-page PDF would have been parked as + // unsplittable while single-page reads were correctly retried. Two + // classifiers disagreeing about one HTTP code is how the original bug + // survived review.) + if (code === 401 || code === 403) break; + return null; // 400 (oversized/undecodable payload) -> fatal } catch (e) { Utilities.sleep(2000); } } Logger.log(" > [SPLIT-MAP] " + model + " unavailable; trying next model."); diff --git a/docs/apps-script/selfHeal.gs b/docs/apps-script/selfHeal.gs index 387a4d00e..2ad56e22c 100644 --- a/docs/apps-script/selfHeal.gs +++ b/docs/apps-script/selfHeal.gs @@ -99,8 +99,17 @@ function outageVictims_(folder) { // Never touch a file the bot judged correctly. if (st.nonReceipt || st.duplicateOf || st.amazonAppOwned || st.emailed) continue; - // Only parked files are candidates. - if (!st.parkReason) continue; + // ONLY outage victims. A file parked for zeroTotal or multiDoc was judged + // on its CONTENT - requeueing replays the same verdict and burns the bounded + // retry budget for nothing. (Kimi SHOULD-FIX: the first version requeued + // every parked file, so a $0.00 receipt ate both auto-requeues re-deciding + // what was already decided.) + if (st.parkReason !== PARK_AI_UNAVAILABLE && st.parkReason !== PARK_GAVE_UP) continue; + + // A gaveUp park is only an outage victim if NOTHING decisive was ever learned. + // If the AI truly read it and failed, a healthy API changes nothing - a human + // must look. + if (st.parkReason === PARK_GAVE_UP && Number(st.busyPasses || 0) === 0) continue; // Bound the loop. if (Number(st.autoRequeues || 0) >= SELFHEAL_MAX_REQUEUES_PER_FILE) continue; @@ -235,6 +244,36 @@ function pipelineDailyReport() { "\n\nRun auditNeedsReview() for a file-by-file verdict."); } +/** + * Send to Telegram as well as email. + * + * Kimi's sharpest finding: the whole 9-day outage happened because the ONLY + * signal was mail to ALERT_EMAIL, an inbox nobody watches. Alerting to that + * same inbox fixes nothing. Telegram is what Justin actually reads. + * Credentials live in Script Properties (TELEGRAM_BOT_TOKEN / + * TELEGRAM_CHAT_ID) - never in code. Skipped silently if unset, and a + * Telegram failure never blocks the email. + */ +function telegramAlert_(text) { + try { + const props = PropertiesService.getScriptProperties(); + const token = props.getProperty("TELEGRAM_BOT_TOKEN"); + const chat = props.getProperty("TELEGRAM_CHAT_ID"); + if (!token || !chat) return false; + const res = UrlFetchApp.fetch( + "https://api.telegram.org/bot" + token + "/sendMessage", { + method: "post", + contentType: "application/json", + payload: JSON.stringify({ chat_id: chat, text: text.slice(0, 3900) }), + muteHttpExceptions: true + }); + return res.getResponseCode() === 200; + } catch (e) { + Logger.log("[SELFHEAL] telegram failed: " + e); + return false; + } +} + /** One alert channel, rate-limited so a broken pipeline cannot spam. */ function selfHealAlert_(subject, body) { const props = PropertiesService.getScriptProperties(); @@ -247,6 +286,8 @@ function selfHealAlert_(subject, body) { } } props.setProperty(SELFHEAL_PROP_LAST_ALERT, new Date().toISOString()); + // Telegram FIRST - it is the channel that gets read. + telegramAlert_(subject + "\n\n" + body); try { MailApp.sendEmail(ALERT_EMAIL, subject, body); } catch (e) { Logger.log("[SELFHEAL] alert email failed: " + e); } } From 074a613bc9e6f7f7fc1f3c536e2b10e37beda071 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 20 Aug 2026 11:32:28 -0700 Subject: [PATCH 03/10] feat(receipts): DEPLOYED - 5 Kimi fixes live, plus Telegram alert setup clasp push succeeded after the Workspace RAPT reauth completed. Verified by pulling into a SEPARATE directory and checking the server's own copy: 11/11 markers present, both files syntax-clean. Live on the server now: 1. namespaced non-receipt dedup key (was a BLOCKER in live code) 2. identity guard on that claim 3. split-map 403 aligned with the main classifier 4. selfHeal requeues only true outage victims 5. alerts route to Telegram before email Safety invariants re-checked on the server copy, not just locally: - AMAZON_APP_OWNS_BOOKING guard intact (single writer for Amazon in QBO) - model chain leads with gemini-flash-latest - selfHeal still skips any file with state.emailed (no double-booking) setupTelegramAlerts.gs is new: stores TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID in Script Properties and sends a test message. The token field ships EMPTY and must be blanked again after running - this file lives in a git repo. checkTelegramSetup() reports whether creds are present without printing the token. --- docs/apps-script/setupTelegramAlerts.gs | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/apps-script/setupTelegramAlerts.gs diff --git a/docs/apps-script/setupTelegramAlerts.gs b/docs/apps-script/setupTelegramAlerts.gs new file mode 100644 index 000000000..108ddd375 --- /dev/null +++ b/docs/apps-script/setupTelegramAlerts.gs @@ -0,0 +1,59 @@ +/** + * setupTelegramAlerts.gs — one-time credential install + trigger setup. + * + * WHY THIS FILE EXISTS + * The 9-day receipt outage (2026-08-10..19) was invisible because the ONLY + * alert channel was email to ALERT_EMAIL, an inbox nobody watches. Fixing + * the models without fixing the channel would have set up the next silent + * outage. Telegram is the channel Justin actually reads. + * + * Credentials go in Script Properties, never in source — this file is in a + * git repo. Paste the token below, run once, then CLEAR IT AGAIN before + * committing (or just never commit the filled-in version). + * + * HOW TO RUN + * 1. Fill in TG_TOKEN below. + * 2. Select setupTelegramAlerts -> Run. It sends a test message. + * 3. Blank TG_TOKEN out again. + * 4. Select installSelfHealTriggers -> Run. (Registers the 10-min + * self-heal, hourly canary, and daily report. Apps Script does not + * allow creating triggers remotely, so this step is manual, once.) + */ + +// Paste, run, then blank. Never commit a real value here. +const TG_TOKEN = ""; +const TG_CHAT = "8681967411"; // Justin's Telegram chat id + +function setupTelegramAlerts() { + if (!TG_TOKEN) { + Logger.log("TG_TOKEN is empty — paste the bot token at the top of this file first."); + return; + } + const props = PropertiesService.getScriptProperties(); + props.setProperty("TELEGRAM_BOT_TOKEN", TG_TOKEN); + props.setProperty("TELEGRAM_CHAT_ID", TG_CHAT); + Logger.log("Stored TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID in Script Properties."); + + const ok = telegramAlert_( + "Receipt bot: alerts are wired to Telegram.\n\n" + + "This is the channel that gets read. From now on a dead vision API, a " + + "growing backlog, or an auto-recovery shows up here instead of dying " + + "quietly in an inbox.\n\n" + + "Silence means healthy."); + Logger.log(ok ? "Test message sent — check Telegram." + : "Test message FAILED. Check the token and chat id."); +} + +/** Read-only: are the credentials present? Never prints the token. */ +function checkTelegramSetup() { + const props = PropertiesService.getScriptProperties(); + const tok = props.getProperty("TELEGRAM_BOT_TOKEN"); + const chat = props.getProperty("TELEGRAM_CHAT_ID"); + Logger.log("TELEGRAM_BOT_TOKEN: " + (tok ? "set (" + tok.length + " chars)" : "MISSING")); + Logger.log("TELEGRAM_CHAT_ID: " + (chat || "MISSING")); + if (tok && chat) { + Logger.log("Alerts will reach Telegram."); + } else { + Logger.log("Alerts fall back to email only — run setupTelegramAlerts()."); + } +} From 57a00810fe045652fbadbf66acd1836283446434 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 20 Aug 2026 12:40:09 -0700 Subject: [PATCH 04/10] feat(chat): native Google Chat, because Composio provably cannot Justin: 'my AI always gets confused about how to connect with Google's chat.' Root cause found and recorded. COMPOSIO CANNOT CONNECT GOOGLE CHAT. It lists the toolkit and exposes 8 GOOGLE_CHAT_* tools, but verified live against the API today: toolkits/google_chat -> composio_managed_auth_schemes: [] auth_configs?toolkit_slug=google_chat -> items: [] (compare gmail -> ['OAUTH2']) There is no OAuth flow behind it and no auth config can be created, so those tools return 'No connected account found' forever regardless of what the dashboard shows. My earlier claim that Chat 'just needs allowlisting' was wrong - the allowlist was already correct; the gap is one layer deeper and is not fixable from the Composio side. NATIVE PATH INSTEAD. Apps Script already runs AS jadkins@, so connecting is just declaring scopes - no service account, no domain-wide delegation. googleChat.gs: postToChatWebhook_ (zero-auth, per-space, threaded), chatListSpaces + chatPostMessage (real Chat API via ScriptApp.getOAuthToken), setChatWebhook / checkChatSetup. appsscript.json: chat.spaces.readonly + chat.messages added. NOTE that declaring oauthScopes disables auto-inference, so every scope the script already relied on is now listed explicitly (drive, gmail.send, gmail.modify, script.scriptapp, script.send_mail, spreadsheets, external_request, userinfo.email). Omitting one would silently strip access on re-auth. selfHeal.gs: alerts now fan out to Chat -> Telegram -> email, each wrapped independently so a broken channel cannot suppress the others or break the pipeline that is trying to report a failure. Verified by pulling into a separate directory and checking the SERVER copy: googleChat.js present, chat scope in manifest, selfHeal calls the webhook, Telegram intact, Amazon single-writer guard intact. Skill 'google-chat-integration' records the trap, the three native paths, and the scope-inference gotcha so this does not get re-derived. --- docs/apps-script/appsscript.json | 33 ++++++ docs/apps-script/googleChat.gs | 167 +++++++++++++++++++++++++++++++ docs/apps-script/selfHeal.gs | 7 ++ 3 files changed, 207 insertions(+) create mode 100644 docs/apps-script/appsscript.json create mode 100644 docs/apps-script/googleChat.gs diff --git a/docs/apps-script/appsscript.json b/docs/apps-script/appsscript.json new file mode 100644 index 000000000..cc74eb2c7 --- /dev/null +++ b/docs/apps-script/appsscript.json @@ -0,0 +1,33 @@ +{ + "timeZone": "America/Los_Angeles", + "dependencies": { + "libraries": [ + { + "userSymbol": "OAuth2", + "version": "43", + "libraryId": "1B7FSrk5Zi6L1rSxxTDgDEUsPzlukDsi4KGuTMorsTQHhGBzBkMun4iDF" + } + ], + "enabledAdvancedServices": [ + { + "userSymbol": "Gmail", + "version": "v1", + "serviceId": "gmail" + } + ] + }, + "exceptionLogging": "STACKDRIVER", + "runtimeVersion": "V8", + "oauthScopes": [ + "https://www.googleapis.com/auth/script.external_request", + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.modify", + "https://www.googleapis.com/auth/script.scriptapp", + "https://www.googleapis.com/auth/script.send_mail", + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/chat.spaces.readonly", + "https://www.googleapis.com/auth/chat.messages" + ] +} \ No newline at end of file diff --git a/docs/apps-script/googleChat.gs b/docs/apps-script/googleChat.gs new file mode 100644 index 000000000..a042f4375 --- /dev/null +++ b/docs/apps-script/googleChat.gs @@ -0,0 +1,167 @@ +/** + * googleChat.gs — native Google Chat from Apps Script. + * + * WHY THIS FILE EXISTS + * Justin: "my AI always gets confused about how to connect with Google's chat." + * That confusion is real and it has a specific cause, recorded here so nobody + * (human or AI) re-derives it: + * + * Composio LISTS google_chat as a toolkit and even exposes 8 GOOGLE_CHAT_* + * tools, but the toolkit has composio_managed_auth_schemes: []. There is NO + * OAuth flow behind it. Those tools return "No connected account found" + * forever, and no amount of clicking in the dashboard fixes it. Verified + * 2026-08-20 against the live Composio API. + * + * The working path is the one below: Apps Script already runs AS the user, so + * adding the chat.spaces scope to appsscript.json is the entire "connection". + * No service account, no domain-wide delegation, no third-party hub. + * + * TWO WAYS TO POST, and they are NOT interchangeable: + * + * 1. WEBHOOK (postToChatWebhook_) + * A per-space URL you create in the space's own menu. No OAuth at all. + * Post-only: cannot read, cannot list spaces, cannot DM. + * Best for "the pipeline shouts into a room." + * + * 2. CHAT API as the user (chatListSpaces / chatPostMessage) + * Real API, needs the chat.spaces + chat.messages scopes in the manifest. + * Can list spaces and read membership. Posting to a space still requires + * that the app/user be a member of it. + * + * Webhook is what actually gets used day to day. The API functions are here + * because "list the spaces" is how you find out where to put the webhook. + */ + +const CHAT_WEBHOOK_PROP = "GOOGLE_CHAT_WEBHOOK_URL"; + +/** + * Post to a Google Chat space via an incoming webhook. + * Returns true on success. Never throws — alerting must not break a pipeline. + */ +function postToChatWebhook_(text, opts) { + opts = opts || {}; + const props = PropertiesService.getScriptProperties(); + const url = opts.url || props.getProperty(CHAT_WEBHOOK_PROP); + if (!url) { + Logger.log("[CHAT] no webhook configured (" + CHAT_WEBHOOK_PROP + " unset)"); + return false; + } + try { + const payload = { text: String(text).slice(0, 4000) }; + // threadKey groups related messages into one thread instead of spamming + // the space with disconnected posts. + const fullUrl = opts.threadKey + ? url + "&threadKey=" + encodeURIComponent(opts.threadKey) + + "&messageReplyOption=REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD" + : url; + const res = UrlFetchApp.fetch(fullUrl, { + method: "post", + contentType: "application/json; charset=UTF-8", + payload: JSON.stringify(payload), + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + if (code !== 200) { + Logger.log("[CHAT] webhook HTTP " + code + ": " + res.getContentText().slice(0, 200)); + return false; + } + return true; + } catch (e) { + Logger.log("[CHAT] webhook failed: " + e); + return false; + } +} + +/** + * List the Chat spaces this user can see. Requires the chat.spaces scope in + * appsscript.json — if it is missing this throws a clear authorization error + * rather than returning nothing. + * + * Use this to FIND the space you want, then create a webhook inside it. + */ +function chatListSpaces() { + const token = ScriptApp.getOAuthToken(); + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/spaces?pageSize=100", { + method: "get", + headers: { Authorization: "Bearer " + token }, + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + const body = res.getContentText(); + if (code !== 200) { + Logger.log("[CHAT] list spaces HTTP " + code); + Logger.log(body.slice(0, 400)); + if (code === 403) { + Logger.log(""); + Logger.log("403 usually means the chat.spaces scope is missing from"); + Logger.log("appsscript.json, OR the Chat API is not enabled on the"); + Logger.log("Cloud project behind this script."); + } + return null; + } + const spaces = (JSON.parse(body).spaces) || []; + Logger.log("--- GOOGLE CHAT SPACES (" + spaces.length + ") ---"); + spaces.forEach(function (s) { + Logger.log(" " + (s.displayName || "(direct message)") + + " name=" + s.name + " type=" + (s.spaceType || s.type)); + }); + Logger.log(""); + Logger.log("To wire a webhook: open the space in Chat -> space name ->"); + Logger.log("Apps & integrations -> Webhooks -> Add webhook -> copy the URL,"); + Logger.log("then run setChatWebhook() with it."); + return spaces; +} + +/** + * Post as the USER via the Chat API (not a webhook). Requires chat.messages + * scope AND that this account is a member of the space. + */ +function chatPostMessage(spaceName, text) { + if (!spaceName) { Logger.log("spaceName required, e.g. spaces/AAAA1234"); return false; } + const token = ScriptApp.getOAuthToken(); + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/" + spaceName + "/messages", { + method: "post", + contentType: "application/json; charset=UTF-8", + headers: { Authorization: "Bearer " + token }, + payload: JSON.stringify({ text: String(text).slice(0, 4000) }), + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + if (code !== 200) { + Logger.log("[CHAT] post HTTP " + code + ": " + res.getContentText().slice(0, 300)); + return false; + } + Logger.log("Posted to " + spaceName); + return true; +} + +/** Store the webhook URL. Run once, then blank the argument out of history. */ +function setChatWebhook(url) { + if (!url) { + Logger.log("Pass the webhook URL: setChatWebhook('https://chat.googleapis.com/v1/spaces/.../messages?key=...&token=...')"); + return; + } + PropertiesService.getScriptProperties().setProperty(CHAT_WEBHOOK_PROP, url); + Logger.log("Stored " + CHAT_WEBHOOK_PROP + "."); + const ok = postToChatWebhook_( + "Receipt bot is connected to this space.\n\n" + + "You'll see a message here when receipts need a human decision, or when " + + "something breaks. Silence means it's running clean."); + Logger.log(ok ? "Test message sent — check the space." + : "Test FAILED. Re-copy the webhook URL (it must include both key= and token=)."); +} + +/** Read-only: is Chat wired up? Never prints the full URL. */ +function checkChatSetup() { + const url = PropertiesService.getScriptProperties().getProperty(CHAT_WEBHOOK_PROP); + if (!url) { + Logger.log(CHAT_WEBHOOK_PROP + ": MISSING — run chatListSpaces() to find your space,"); + Logger.log("then create a webhook in it and call setChatWebhook(url)."); + return false; + } + const m = url.match(/spaces\/([^\/]+)/); + Logger.log(CHAT_WEBHOOK_PROP + ": set (space " + (m ? m[1] : "?") + ")"); + return true; +} diff --git a/docs/apps-script/selfHeal.gs b/docs/apps-script/selfHeal.gs index 2ad56e22c..77674039f 100644 --- a/docs/apps-script/selfHeal.gs +++ b/docs/apps-script/selfHeal.gs @@ -286,6 +286,13 @@ function selfHealAlert_(subject, body) { } } props.setProperty(SELFHEAL_PROP_LAST_ALERT, new Date().toISOString()); + // Fan out to every channel that is actually watched. Each is + // independent: one failing must never suppress the others, and none + // of them may throw - a broken alert path cannot break the pipeline. + // Google Chat is where the OFFICE sees it (Marge); Telegram is where + // Justin sees it; email is the archive of record. + try { postToChatWebhook_(subject + "\n\n" + body, { threadKey: "receipt-bot" }); } + catch (e) { Logger.log("[SELFHEAL] chat alert failed: " + e); } // Telegram FIRST - it is the channel that gets read. telegramAlert_(subject + "\n\n" + body); try { MailApp.sendEmail(ALERT_EMAIL, subject, body); } From 82c9ceb12fd1d8d683191dd445f20605f3e931b2 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Thu, 20 Aug 2026 13:35:42 -0700 Subject: [PATCH 05/10] feat(chat): direct messages via the Chat API (webhooks cannot DM) Justin asked to message Marge 'directly'. That is a DIFFERENT mechanism from the webhook path, and the difference is the thing every naive attempt misses: An incoming webhook is registered INSIDE one specific space through that space's own UI. There is no way to create one for a 1:1 with another person. Google's docs: 'Each webhook only works in the Chat space in which it's registered.' So a DM needs the Chat API with user auth: spaces:findDirectMessage?name=users/ -> spaces/XXXX spaces/XXXX/messages -> post Added chatFindDm(), chatSendDm(), and dmMargeReceiptUpdate() - the last one has the recipient and the full message baked in so it is a single click. The message is openly attributed to AI because Justin asked for that explicitly. It matters beyond politeness: a process change that lands with no named author reads as a decree from nowhere, and Marge is the person whose job it changes. Error paths name their own cause rather than failing blank: 403 -> either the scope is missing OR the Chat API is not enabled on the script's Cloud project; the error does not distinguish them, so the log spells out both. 404 -> no DM exists yet; Google will not manufacture a conversation. Send one message by hand and it exists forever after. Skill 'google-chat-integration' updated with a When-to-Use table that picks the mechanism BEFORE any code, the webhook-cannot-DM rule, and the fact that Chat uses *bold* not **bold**. --- docs/apps-script/googleChat.gs | 114 +++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/docs/apps-script/googleChat.gs b/docs/apps-script/googleChat.gs index a042f4375..7cbcf9fe5 100644 --- a/docs/apps-script/googleChat.gs +++ b/docs/apps-script/googleChat.gs @@ -165,3 +165,117 @@ function checkChatSetup() { Logger.log(CHAT_WEBHOOK_PROP + ": set (space " + (m ? m[1] : "?") + ")"); return true; } + +/* ───────────────────────────────────────────────────────────────────────── + * DIRECT MESSAGES — a different mechanism from webhooks. Read this. + * + * You CANNOT webhook a DM. An incoming webhook is registered inside a + * specific space through that space's own UI, so there is no way to create + * one for a 1:1 conversation with another person. Every "just make a webhook + * to DM someone" attempt dead-ends here. + * + * The real path is the Chat API with USER auth: + * 1. spaces.findDirectMessage?name=users/ -> the DM space id + * 2. spaces.messages.create on that space + * + * Both run as the signed-in Apps Script user, so the message genuinely comes + * from that person's account. Requires chat.spaces.readonly + chat.messages + * in appsscript.json, and the Chat API enabled on the script's Cloud project. + * ───────────────────────────────────────────────────────────────────────── */ + +/** + * Find the 1:1 DM space between the running user and someone else. + * Returns "spaces/XXXX" or null. Never throws. + */ +function chatFindDm(email) { + if (!email) { Logger.log("chatFindDm needs an email"); return null; } + try { + const res = UrlFetchApp.fetch( + "https://chat.googleapis.com/v1/spaces:findDirectMessage?name=" + + encodeURIComponent("users/" + email), { + method: "get", + headers: { Authorization: "Bearer " + ScriptApp.getOAuthToken() }, + muteHttpExceptions: true + }); + const code = res.getResponseCode(); + const body = res.getContentText(); + if (code !== 200) { + Logger.log("[CHAT] findDirectMessage HTTP " + code); + Logger.log(body.slice(0, 400)); + if (code === 403) { + Logger.log(""); + Logger.log("403 means one of two things, and the error does not say which:"); + Logger.log(" a) the chat.spaces.readonly scope is missing -> re-authorize"); + Logger.log(" b) the Chat API is not enabled on this script's Cloud project"); + Logger.log(" -> Project Settings -> Google Cloud Platform project, then"); + Logger.log(" enable 'Google Chat API' in that project."); + } + if (code === 404) { + Logger.log("No DM exists yet. Open Chat and send this person one message"); + Logger.log("by hand, then this will find it."); + } + return null; + } + const space = JSON.parse(body); + Logger.log("DM space with " + email + ": " + space.name); + return space.name; + } catch (e) { + Logger.log("[CHAT] findDirectMessage failed: " + e); + return null; + } +} + +/** + * Send a direct message to one person, as the running user. + * Returns true on success. + */ +function chatSendDm(email, text) { + const space = chatFindDm(email); + if (!space) return false; + return chatPostMessage(space, text); +} + +/** + * THE ONE TO RUN: DM Marge the receipt-process update. + * + * Everything is baked in so this is a single click. The message is written to + * be openly AI-attributed — Justin asked for that explicitly ("let her know + * it's AI that's saying it"), and it matters: a process change that lands + * without a named author reads as a decree from nowhere. + */ +function dmMargeReceiptUpdate() { + const MARGE = "gtrsupport@goldentouchremodeling.com"; + + const msg = +"*Hey Marge — this is Justin's AI, writing on his behalf.* He asked me to let you know what changed with receipts.\n" + +"\n" + +"You know how you've been hunting receipts one bank line at a time? Your Aug 14 note said you got through 08/04 and were stuck on four of them — checked Drive, checked Lowe's.com, checked email. That part is done now. The system does it.\n" + +"\n" + +"*What runs on its own*\n" + +"A receipt shows up — email, photo, or dropped in a job folder. It gets read, named Job_Date_Vendor_Invoice_$Amount, checked against everything already filed, then sent to QuickBooks and archived. *14 receipts filed in the last two days with nobody touching them.*\n" + +"\n" + +"*What it now refuses to do*\n" + +"• Book a Cash App payment or payroll advance as an expense — those get held and flagged for Gusto\n" + +"• Book an Amazon order twice — Intuit's own app owns those now\n" + +"• Book the same charge twice\n" + +"\n" + +"*What broke before and doesn't now*\n" + +"The reader died for nine days in August. Receipts quietly gave up, and the only sign was email piling up. It now retries on its own and sends a text if it can't. Ten minutes is the longest a failure can go unnoticed.\n" + +"\n" + +"*What still needs you*\n" + +"• The items sitting in _Needs Review — that's real judgment work\n" + +"• Missing receipt memos — only a person can write and sign those\n" + +"• Assigning jobs inside QuickBooks — still manual\n" + +"\n" + +"*What you can stop doing*\n" + +"Working the bank line by line looking for receipts. What's left in _Needs Review is the actual work now.\n" + +"\n" + +"One honest note: this is intake through QuickBooks, not the whole receipt problem. The duplicate groups you found in your audit still need reconciling, and job assignment in QBO is still by hand.\n" + +"\n" + +"— Justin's AI"; + + const ok = chatSendDm(MARGE, msg); + Logger.log(ok ? "Sent to " + MARGE + " — check Chat." + : "FAILED. Read the log above; it names the cause."); + return ok; +} From ba560d7150ac558936b4888c552c3a260cf2f572 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Fri, 21 Aug 2026 23:22:28 -0700 Subject: [PATCH 06/10] Add scripts/pull-qbo-bank-register.mjs: QBO books-view register report for WTB 0723 Report-only replacement input after the WTB password rotation killed the browser CSV export. Pulls the GL register via the existing OAuth client, prints per-day txns/in/out/net plus books-derived open/close balances that must prove themselves (exact-cents running-balance chain) or print n/a. --post is deliberately refused: QBO book balances are not the bank's OPENING/CLOSING LEDGER, and STATEMENT days for WTB-0723 are owned by parse-wtb-daily-csv.mjs; the sanctioned QBO->ledger path remains post-qbo-register.mjs (QBO_REGISTER observations). --- scripts/pull-qbo-bank-register.mjs | 394 +++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 scripts/pull-qbo-bank-register.mjs diff --git a/scripts/pull-qbo-bank-register.mjs b/scripts/pull-qbo-bank-register.mjs new file mode 100644 index 000000000..c522a0469 --- /dev/null +++ b/scripts/pull-qbo-bank-register.mjs @@ -0,0 +1,394 @@ +// QBO bank REGISTER REPORT for the WTB checking account (…0723). +// +// WHY THIS EXISTS (2026-08-21): Justin rotated the Washington Trust password, +// so the browser-based daily CSV export (skill wtb-daily-bank-export) has no +// valid credentials and is dead as the primary bank-register source. The +// QuickBooks bank feed carries the same transactions over OAuth — no login +// page, no 2FA, no session to lose. This script is the human-readable +// per-day register view of that feed. +// +// WHAT IT SHOWS: per calendar day — transaction count, money in, money out, +// net, and (when QBO's running-balance column reconciles exactly) the derived +// per-day opening/closing balance. Every figure is integer cents from QBO's +// decimal strings; no float math on the way in. +// +// HONESTY CONTRACT (same as src/lib/qbo-bank-register.ts): this is the BOOKS +// view — what QuickBooks has POSTED to the account. It cannot see WTB +// transactions that are pending, excluded from the feed, or absent from +// QuickBooks, and it does not prove bank clearance. The derived balances are +// QBO's book balances, NOT the bank's statement OPENING/CLOSING LEDGER. +// +// WHY --post IS BLOCKED (deliberate, not unfinished): +// 1. The ingest route's STATEMENT source is reserved for the bank's own +// statement — true north. QBO book balances routinely differ from the +// bank's ledger balances (feed lag ~1 day, uncleared checks, excluded +// feed rows), so a QBO-derived "statement" would assert control totals +// the bank never published. That is exactly the faked-balance failure +// docs/BANK-REGISTER-PLAN.md forbids. +// 2. STATEMENT days for WTB-0723 are already minted by +// scripts/parse-wtb-daily-csv.mjs under the route's uniqueness key +// (account, periodStart, periodEnd). QBO-derived one-day statements for +// the same account would either 409 against every existing day (books +// vs bank content differs) or, on uncovered days, mint canonical +// BankLines from non-bank evidence — the cross-source double-minting +// the Codex B1 review exists to prevent. +// 3. The sanctioned QBO→ledger path already exists and is live: +// scripts/post-qbo-register.mjs posts source=QBO_REGISTER observation +// rows (idempotent by qbTxnId, 409 on content change), and the +// reconcile route links them to canonical statement lines. +// If the bank-statement balance source is ever restored (CSV re-enabled, +// or an OFX pull lands), STATEMENT posting belongs there — not here. +// +// Usage (tsx required — the imported src/lib modules are TypeScript with +// "@/*" path aliases that bare node cannot resolve): +// npx tsx scripts/pull-qbo-bank-register.mjs # last 14 days +// npx tsx scripts/pull-qbo-bank-register.mjs --days 30 +// npx tsx scripts/pull-qbo-bank-register.mjs --start 2026-08-01 --end 2026-08-20 +// npx tsx scripts/pull-qbo-bank-register.mjs --txns # per-transaction detail +// +// Env: loads .env.production.local, then .env.local, then .env itself +// (first file that defines a key wins; a value already in process.env always +// wins). .env.production.local is preferred because the QBO OAuth client +// creds (QB_CLIENT_ID/QB_CLIENT_SECRET) and the NEXTAUTH_SECRET that +// decrypts the stored token row exist only in the prod env pull. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const DEFAULT_DAYS = 14; +// fetchBankRegister enforces 92; mirror it so the error is ours, not a stack trace. +const MAX_RANGE_DAYS = 92; +// The account this report is for. Refuse to render anything else so a future +// second bank account can't silently masquerade as WTB checking. +const EXPECTED_LAST4 = "0723"; + +// ── env ───────────────────────────────────────────────────────────────────── + +/** + * Minimal .env parser (KEY=value, optional double quotes, # comments). + * Precedence: existing process.env > .env.production.local > .env.local > .env + * — prod values first because this script only makes sense against prod QBO, + * and the prod NEXTAUTH_SECRET is the one that decrypts the Integration row. + */ +function loadEnvFiles() { + const loaded = []; + for (const file of [".env.production.local", ".env.local", ".env"]) { + const full = path.join(REPO_ROOT, file); + if (!fs.existsSync(full)) continue; + const text = fs.readFileSync(full, "utf8").replace(/^\uFEFF/, ""); + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const m = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line); + if (!m) continue; + let [, key, value] = m; + if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) { + value = value.slice(1, -1); + } + if (!(key in process.env)) process.env[key] = value; + } + loaded.push(file); + } + return loaded; +} + +// ── args ──────────────────────────────────────────────────────────────────── + +function parseArgs(argv) { + const args = { days: DEFAULT_DAYS, start: null, end: null, txns: false, post: null }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--txns") args.txns = true; + else if (arg === "--days") { + const v = Number(argv[++i]); + if (!Number.isInteger(v) || v < 1 || v > MAX_RANGE_DAYS) throw new Error(`--days must be 1..${MAX_RANGE_DAYS}`); + args.days = v; + } else if (arg === "--start") { + const v = argv[++i]; + if (!isYmd(v)) throw new Error("--start must be YYYY-MM-DD"); + args.start = v; + } else if (arg === "--end") { + const v = argv[++i]; + if (!isYmd(v)) throw new Error("--end must be YYYY-MM-DD"); + args.end = v; + } else if (arg === "--post") { + // Consume the value if present so the refusal message is accurate, + // but never act on it — see the header comment. + const v = argv[i + 1]; + if (v !== undefined && !v.startsWith("--")) i++; + args.post = v ?? "(no url)"; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if ((args.start === null) !== (args.end === null)) throw new Error("--start and --end must be given together"); + return args; +} + +/** Strict YYYY-MM-DD with calendar round-trip ("2026-02-30" must fail). */ +function isYmd(s) { + if (typeof s !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(s)) return false; + const t = Date.parse(`${s}T00:00:00Z`); + return Number.isFinite(t) && new Date(t).toISOString().slice(0, 10) === s; +} + +/** UTC-only date math — local timezone must never shift a posting date. */ +function ymdDaysAgo(days) { + return new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10); +} + +// ── money ─────────────────────────────────────────────────────────────────── + +/** + * Exact decimal-string → integer cents. Refuses anything that isn't a plain + * signed decimal with ≤2 fraction digits (financial-data-pipelines rule 1). + * QBO report cells arrive as decimal strings; never parseFloat them. + */ +export function toCents(raw) { + const s = String(raw ?? "").trim(); + const m = /^(-?)(\d+)(?:\.(\d{1,2}))?$/.exec(s); + if (!m) return null; + const [, sign, whole, frac = ""] = m; + const cents = Number(whole) * 100 + Number(frac.padEnd(2, "0")); + if (!Number.isSafeInteger(cents)) return null; + return sign === "-" ? -cents : cents; +} + +const money = c => (c / 100).toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + +// ── balance derivation (GL report with running-balance column) ────────────── + +/** + * Fetch the GL report AGAIN, this time asking for the running-balance column + * (rbal_nat_amount) so per-day book balances can be DERIVED rather than + * guessed. Returns { openingCents, rows: [{date, amountCents, runbalCents}] } + * or null when the report doesn't carry a usable balance chain. + * + * The derivation is only trusted when it proves itself: every row must + * satisfy previous_balance + amount == running_balance in exact integer + * cents, chaining from the report's Beginning Balance row. One break → the + * whole chain is discarded and the report prints "n/a" for balances. + * (financial-data-pipelines rule 2: control totals refuse to ship.) + */ +async function fetchBalanceChain(qbFetch, tokens, accountId, startDate, endDate) { + const params = new URLSearchParams({ + start_date: startDate, + end_date: endDate, + account: accountId, + columns: "tx_date,txn_type,subt_nat_amount,rbal_nat_amount", + }); + const res = await qbFetch(`/reports/GeneralLedger?${params}`, tokens); + if (!res.ok) return { ok: false, reason: `GL balance report HTTP ${res.status}` }; + const report = await res.json(); + + // Column order comes from the report's own Columns block, never assumed. + const idx = new Map(); + (report.Columns?.Column ?? []).forEach((col, i) => { + for (const meta of col.MetaData ?? []) { + if (meta.Name === "ColKey" && meta.Value) idx.set(meta.Value, i); + } + }); + if (!idx.has("tx_date") || !idx.has("subt_nat_amount") || !idx.has("rbal_nat_amount")) { + return { ok: false, reason: "GL report did not return the running-balance column" }; + } + + const flat = []; + (function walk(rows) { + for (const row of rows ?? []) { + if (row.ColData) flat.push(row.ColData); + if (row.Rows?.Row) walk(row.Rows.Row); + } + })(report.Rows?.Row ?? []); + + let openingCents = null; + const rows = []; + for (const cols of flat) { + const cell = key => cols[idx.get(key)]?.value; + const label = String(cell("tx_date") ?? "").trim(); + const txnType = String(cols[idx.get("txn_type")]?.value ?? "").trim(); + const runbal = toCents(cell("rbal_nat_amount")); + // The section's Beginning Balance line: no txn type, carries the + // starting running balance. QBO renders its label in the first column. + if (!txnType && /beginning balance/i.test(label) && runbal !== null) { + if (openingCents !== null && openingCents !== runbal) { + return { ok: false, reason: "conflicting Beginning Balance rows" }; + } + openingCents = runbal; + continue; + } + if (!txnType) continue; // other summary/total lines + const date = isYmd(label) ? label : null; + const amount = toCents(cell("subt_nat_amount")); + if (!date || amount === null || runbal === null) { + return { ok: false, reason: `unparseable GL balance row (date "${label}")` }; + } + rows.push({ date, amountCents: amount, runbalCents: runbal }); + } + if (openingCents === null) return { ok: false, reason: "no Beginning Balance row in GL report" }; + + // The chain must PROVE itself: opening + each amount → each running balance. + let bal = openingCents; + for (const row of rows) { + bal += row.amountCents; + if (bal !== row.runbalCents) { + return { ok: false, reason: `balance chain breaks at ${row.date}: expected ${bal} got ${row.runbalCents}` }; + } + } + return { ok: true, openingCents, rows }; +} + +// ── report ────────────────────────────────────────────────────────────────── + +function buildDaySummaries(rows) { + // rows arrive newest-first from fetchBankRegister; group by date ascending. + const byDay = new Map(); + for (const row of rows) { + if (!byDay.has(row.date)) byDay.set(row.date, { count: 0, inCents: 0, outCents: 0 }); + const d = byDay.get(row.date); + d.count++; + if (row.amountCents >= 0) d.inCents += row.amountCents; + else d.outCents += row.amountCents; + } + return [...byDay.entries()].sort(([a], [b]) => a.localeCompare(b)) + .map(([date, d]) => ({ date, ...d, netCents: d.inCents + d.outCents })); +} + +function fail(msg) { + console.error(`GATE FAILED: ${msg}`); + process.exitCode = 1; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (args.post !== null) { + console.error("--post is BLOCKED for this script, deliberately:"); + console.error(" • QBO is the BOOKS view. Its balances are not the bank's OPENING/CLOSING"); + console.error(" LEDGER, so a QBO-derived STATEMENT would assert control totals the bank"); + console.error(" never published (feed lag, uncleared checks, excluded rows)."); + console.error(" • STATEMENT days for WTB-0723 are owned by scripts/parse-wtb-daily-csv.mjs"); + console.error(" under the (account, periodStart, periodEnd) uniqueness key — QBO-derived"); + console.error(" days would 409 against them or double-mint canonical BankLines."); + console.error(" • The sanctioned QBO→ledger path already exists:"); + console.error(" npx tsx scripts/post-qbo-register.mjs --days 30 --post "); + console.error(" (source=QBO_REGISTER observations, idempotent by qbTxnId)."); + console.error(" Restore a real bank-statement source (CSV/OFX) for STATEMENT posting."); + process.exit(1); + } + + const loaded = loadEnvFiles(); + if (loaded.length === 0) fail("no .env file found next to the repo root"); + for (const key of ["DATABASE_URL", "NEXTAUTH_SECRET", "QB_CLIENT_ID", "QB_CLIENT_SECRET"]) { + if (!process.env[key]) return fail(`${key} missing — run 'vercel env pull .env.production.local' first (loaded: ${loaded.join(", ")})`); + } + if (!/pgbouncer=true/.test(process.env.DATABASE_URL)) { + console.error("WARNING: DATABASE_URL lacks ?pgbouncer=true — Supabase pooler + Prisma needs it."); + } + + const endDate = args.end ?? new Date().toISOString().slice(0, 10); + const startDate = args.start ?? ymdDaysAgo(args.days); + if (startDate > endDate) return fail("start date is after end date"); + + // Imported AFTER env is loaded — src/lib/prisma reads DATABASE_URL at + // import time, and integration-store decrypts with NEXTAUTH_SECRET. + // These are TypeScript with "@/*" aliases: run this script under tsx. + const { fetchBankRegister, bankAccountId } = await import("../src/lib/qbo-bank-register.ts"); + const { getFreshQBTokens } = await import("../src/lib/quickbooks-payments.ts"); + const { qbFetch, qbQuery } = await import("../src/lib/quickbooks.ts"); + + const tokens = await getFreshQBTokens(); + + // 1) Account verification: the register must provably be the WTB …0723 + // checking account, not whatever account id happens to be configured. + const accountId = bankAccountId(); + const bankAccounts = await qbQuery(tokens, "SELECT * FROM Account WHERE AccountType = 'Bank' MAXRESULTS 100"); + const target = bankAccounts.find(a => String(a.Id) === String(accountId)); + if (!target) return fail(`configured bank account id ${accountId} not found among ${bankAccounts.length} QBO bank account(s)`); + const acctNum = String(target.AcctNum ?? ""); + const acctLabel = `${target.Name}${acctNum ? ` (#…${acctNum.slice(-4)})` : ""}`; + const looks0723 = acctNum.endsWith(EXPECTED_LAST4) || new RegExp(EXPECTED_LAST4).test(target.Name ?? ""); + if (!looks0723) { + const other = bankAccounts.find(a => String(a.AcctNum ?? "").endsWith(EXPECTED_LAST4) || new RegExp(EXPECTED_LAST4).test(a.Name ?? "")); + if (other) return fail(`configured account ${accountId} (${acctLabel}) is NOT the …${EXPECTED_LAST4} account — QBO account ${other.Id} (${other.Name}) is. Set QBO_RECEIPT_BANK_ACCOUNT_ID.`); + console.error(`WARNING: cannot confirm account ${accountId} (${acctLabel}) is …${EXPECTED_LAST4} — no QBO bank account carries that number. Proceeding; verify in QBO.`); + } + + // 2) The register rows (same proven path the /automation/bank page uses). + const result = await fetchBankRegister(() => Promise.resolve(tokens), startDate, endDate); + const rows = result.rows ?? []; + + // 3) Balance chain (books view) — display-only, and only if it proves itself. + let chain = { ok: false, reason: "not attempted" }; + try { + chain = await fetchBalanceChain(qbFetch, tokens, accountId, startDate, endDate); + } catch (error) { + chain = { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } + + // 4) Print. + const days = buildDaySummaries(rows); + console.log(`REGISTER REPORT — QBO books view of ${acctLabel}`); + console.log(` realm ${tokens.realmId}, QBO account id ${accountId}`); + console.log(` window ${startDate} → ${endDate}${result.stale ? " [STALE CACHE — QBO errored, showing last good fetch]" : ""}`); + console.log(` ${rows.length} posted row(s) across ${days.length} day(s). Books balance chain: ${chain.ok ? "VERIFIED (exact cents)" : `n/a — ${chain.reason}`}`); + if (typeof target.CurrentBalance !== "undefined") { + console.log(` QBO CurrentBalance (books, as of now): ${money(toCents(String(target.CurrentBalance)) ?? NaN)}`); + } + console.log(" NOTE: books view only — pending/unfed bank activity is invisible; balances are NOT the bank's ledger."); + console.log(""); + console.log(" date txns money in money out net" + (chain.ok ? " open(books) close(books)" : "")); + console.log(" ---------- ----- ------------ ------------ ------------" + (chain.ok ? " -------------- --------------" : "")); + + // Per-day close from the verified chain: opening + cumulative net of every + // chain row dated ≤ that day (chain rows are the same GL rows, so the two + // fetches agree; if they ever disagree the sums below expose it). + let closeByDay = new Map(); + if (chain.ok) { + let bal = chain.openingCents; + const sortedChain = [...chain.rows].sort((a, b) => a.date.localeCompare(b.date)); + for (const row of sortedChain) { + bal += row.amountCents; + closeByDay.set(row.date, bal); + } + } + let prevClose = chain.ok ? chain.openingCents : null; + for (const d of days) { + let balCols = ""; + if (chain.ok) { + const close = closeByDay.get(d.date); + if (close !== undefined && prevClose !== null && prevClose + d.netCents === close) { + balCols = ` ${money(prevClose).padStart(14)} ${money(close).padStart(14)}`; + prevClose = close; + } else { + // The two GL fetches disagreed for this day — say so, never guess. + balCols = " " + "chain mismatch".padStart(30); + prevClose = close ?? prevClose; + } + } + console.log(` ${d.date} ${String(d.count).padStart(5)} ${money(d.inCents).padStart(12)} ${money(d.outCents).padStart(12)} ${money(d.netCents).padStart(12)}${balCols}`); + } + const totalIn = days.reduce((a, d) => a + d.inCents, 0); + const totalOut = days.reduce((a, d) => a + d.outCents, 0); + console.log(" ---------- ----- ------------ ------------ ------------"); + console.log(` TOTAL ${String(rows.length).padStart(5)} ${money(totalIn).padStart(12)} ${money(totalOut).padStart(12)} ${money(totalIn + totalOut).padStart(12)}`); + + if (args.txns) { + console.log(""); + console.log(" TRANSACTIONS (newest first)"); + for (const row of rows) { + console.log(` ${row.date} ${money(row.amountCents).padStart(12)} ${(row.qbType ?? "").padEnd(18)} ${(row.name ?? "").slice(0, 40)}${row.qbTxnId ? ` [${row.qbTxnId}]` : ""}`); + } + } + + console.log(""); + console.log(" Ledger posting: use scripts/post-qbo-register.mjs (source=QBO_REGISTER observations)."); + console.log(" STATEMENT posting stays with the bank's own statement source (see header)."); +} + +// Entry check must survive being imported by tests: process.argv[1] can be +// undefined under `node -e`, and pathToFileURL(undefined) throws. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(error => { console.error(error); process.exit(1); }); +} From ea602451e75047b98d5924b637f7e3bc65ec154f Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Sat, 22 Aug 2026 00:17:55 -0700 Subject: [PATCH 07/10] =?UTF-8?q?feat(bank-image):=20check-payer=20extract?= =?UTF-8?q?ion=20=E2=80=94=20who=20paid=20us,=20from=20the=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inbound WTB line says only "OTHER DEPOSITS DEPOSIT - DDA/MMKT". The check/deposit image is the only evidence naming the payer, and the memo line often names the job. This lets Beverly attribute deposits to clients and projects. Schema (scripts/apply-check-payer-extraction.mjs — NOT prisma migrate, per CLAUDE.md; NOT YET APPLIED to prod, additive nullable columns only): - BankImage.payerName, memoText, extractedAt, extractionModel - CHECK BankImage_extraction_pair: extractedAt and extractionModel are stamped together or not at all - --dry-run prints the SQL; --yes --expect-db --expect-host applies Extractor (scripts/extract-check-payers.mjs): - Gemini Vision (repo's REST pattern from aiGeneratePunchlist), temp 0 - HARD PRIVACY RULE, enforced twice: the prompt forbids reading the MICR strip / routing / account numbers, and scrubField() drops ANY field carrying an 8+ consecutive-digit run (9+ across separators) that is not the known check number/amount, logging a warning. MICR transit glyphs are dropped outright. Nothing bank-account-shaped can reach the DB or even stdout. - Idempotent: skips extractedAt NOT NULL; --dry-run default; --limit N; --from-manifest tests pre-DDL straight from the Drive manifest - --report / post-run REVIEW REPORT: fuzzy payerName→Client and memoText→Project suggestions, print-only. BankImageMatch stays human-confirmation-only; the matcher NEVER writes it. Verified: 12 unit tests green (scrub guard, fuzzy match, manifest derivation), typecheck clean, real dry-run against check #1027 read payer + memo with zero MICR leakage and suggested the Hoppe Project. Blocked until the DDL is applied: --commit and --report. --- package.json | 2 +- prisma/schema.prisma | 11 + scripts/apply-check-payer-extraction.mjs | 151 ++++++++ scripts/extract-check-payers.mjs | 445 +++++++++++++++++++++++ tests/extract-check-payers.test.ts | 131 +++++++ 5 files changed, 739 insertions(+), 1 deletion(-) create mode 100644 scripts/apply-check-payer-extraction.mjs create mode 100644 scripts/extract-check-payers.mjs create mode 100644 tests/extract-check-payers.test.ts diff --git a/package.json b/package.json index 9a0e29808..b02c6c881 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "lint": "eslint", "test:qbo-expense-sync": "tsx --test tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts", "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", - "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/deposit-attribution.test.ts", + "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/deposit-attribution.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", "test:ai-review-reasonableness": "tsx --test tests/ai-review-reasonableness.test.ts", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7c639df61..34dc3aa72 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2865,6 +2865,17 @@ model BankImage { /// ledger movement. amountCents Int? + /// Check-payer extraction (scripts/extract-check-payers.mjs, applied via + /// scripts/apply-check-payer-extraction.mjs). payerName = who wrote the + /// check (top-left block); memoText = the memo/"for" line. The MICR line / + /// routing / account numbers are NEVER extracted or stored — banned in the + /// prompt and scrubbed in code. extractedAt+extractionModel are stamped + /// together (CHECK BankImage_extraction_pair) and make replays a no-op. + payerName String? + memoText String? + extractedAt DateTime? @db.Timestamptz(6) + extractionModel String? + createdAt DateTime @default(now()) @db.Timestamptz(6) updatedAt DateTime @default(now()) @updatedAt @db.Timestamptz(6) diff --git a/scripts/apply-check-payer-extraction.mjs b/scripts/apply-check-payer-extraction.mjs new file mode 100644 index 000000000..c323ad076 --- /dev/null +++ b/scripts/apply-check-payer-extraction.mjs @@ -0,0 +1,151 @@ +// Additive migration: check-payer extraction columns on BankImage. +// +// WHY (2026-08-22): the ledger's inbound lines say only "DEPOSIT - DDA/MMKT". +// The image names WHO PAID US — extracting the payer name and memo line lets +// Beverly (bookkeeping) attribute deposits to clients and jobs. Four nullable +// columns, nothing else: +// +// payerName TEXT — name block top-left of the check +// memoText TEXT — the "memo"/"for" line +// extractedAt TIMESTAMPTZ — idempotency marker for the extractor +// extractionModel TEXT — which model produced the values +// +// DELIBERATELY ABSENT: routing number, account number, MICR line. Those are +// NEVER extracted and NEVER stored — see scripts/extract-check-payers.mjs, +// which bans them in the prompt AND drops them in code. Do not add columns +// for them. +// +// Additive and idempotent: ADD COLUMN IF NOT EXISTS only. No existing column +// or row is touched. Safe to re-run. +// +// node scripts/apply-check-payer-extraction.mjs --dry-run +// node scripts/apply-check-payer-extraction.mjs --yes --expect-db --expect-host +// +// --expect-db and --expect-host are BOTH required alongside --yes, matching +// scripts/apply-bank-image.mjs: "--yes" alone only proves you meant to run +// something, and a database NAME alone doesn't prove which SERVER it's on. +// +// Apply BEFORE deploying any build whose Prisma client selects these columns +// (P2022), and BEFORE running extract-check-payers.mjs with --commit. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; + +export function resolveDatabaseUrl() { + if (process.env.DATABASE_URL) return { url: process.env.DATABASE_URL, from: "process.env.DATABASE_URL" }; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(/^DATABASE_URL\s*=\s*"?([^"\n]+)"?/m); + if (match) return { url: match[1], from: file }; + } + throw new Error("DATABASE_URL not found in process.env, .env.local, or .env"); +} + +export function maskUrl(url) { + return url.replace(/:[^:@]*@/, ":****@"); +} + +function readFlagValue(flag) { + const idx = process.argv.indexOf(flag); + return idx >= 0 ? process.argv[idx + 1] : undefined; +} + +/** Pure comparison, exported for unit testing (mirrors apply-bank-image.mjs). */ +export function targetMatches(actual, expectDb, expectHost) { + if (!actual || typeof actual !== "object") return false; + if (String(actual.db ?? "") !== String(expectDb ?? "")) return false; + const host = String(actual.host ?? ""); + const wanted = String(expectHost ?? ""); + if (host === wanted) return true; + return host !== "" && wanted !== "" && (host.includes(wanted) || wanted.includes(host)); +} + +export const NEW_COLUMNS = ["payerName", "memoText", "extractedAt", "extractionModel"]; + +export const statements = [ + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "payerName" TEXT`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "memoText" TEXT`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "extractedAt" TIMESTAMPTZ(6)`, + `ALTER TABLE "BankImage" ADD COLUMN IF NOT EXISTS "extractionModel" TEXT`, + + // The extractor stamps extractedAt and extractionModel together — a row + // claiming extraction without saying what did it (or vice versa) is a bug. + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'BankImage_extraction_pair') THEN + ALTER TABLE "BankImage" ADD CONSTRAINT "BankImage_extraction_pair" + CHECK (("extractedAt" IS NULL) = ("extractionModel" IS NULL)); + END IF; + END $$`, +]; + +async function main() { + if (process.argv.includes("--dry-run")) { + console.log("DRY RUN — the following SQL would be applied (nothing executed):\n"); + for (const sql of statements) console.log(sql.replace(/\n\s+/g, "\n ") + ";\n"); + console.log("Re-run with --yes --expect-db --expect-host to apply."); + return; + } + + if (!process.argv.includes("--yes")) { + console.error("Refusing to run without --yes (and --expect-db / --expect-host). Use --dry-run to preview."); + process.exit(1); + } + const expectDb = readFlagValue("--expect-db") ?? process.env.BANK_LEDGER_EXPECT_DB; + const expectHost = readFlagValue("--expect-host") ?? process.env.BANK_LEDGER_EXPECT_HOST; + if (!expectDb || !expectHost) { + console.error("Both --expect-db and --expect-host are required (or BANK_LEDGER_EXPECT_DB / BANK_LEDGER_EXPECT_HOST)."); + process.exit(1); + } + + const { url, from } = resolveDatabaseUrl(); + console.log(`DATABASE_URL from ${from}: ${maskUrl(url)}`); + const prisma = new PrismaClient({ datasources: { db: { url } } }); + + try { + const [actual] = await prisma.$queryRawUnsafe( + `SELECT current_database() AS db, COALESCE(host(inet_server_addr()), '') AS host`, + ); + console.log(`connected to db="${actual.db}" host="${actual.host}"`); + if (!targetMatches(actual, expectDb, expectHost)) { + console.error(`REFUSING: expected db="${expectDb}" host="${expectHost}" but connected to db="${actual.db}" host="${actual.host}".`); + process.exit(1); + } + + for (const sql of statements) { + const label = sql.replace(/\s+/g, " ").slice(0, 84); + process.stdout.write(` ${label} ... `); + await prisma.$executeRawUnsafe(sql); + console.log("ok"); + } + + // Verify shape rather than trusting the run. + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns WHERE table_schema='public' AND table_name='BankImage'`, + ); + const found = new Set(rows.map(r => r.column_name)); + const missing = NEW_COLUMNS.filter(c => !found.has(c)); + if (missing.length) { + console.error(`VERIFY FAILED: BankImage missing columns: ${missing.join(", ")}`); + process.exit(1); + } + const [pair] = await prisma.$queryRawUnsafe( + `SELECT 1 AS ok FROM pg_constraint WHERE conname = 'BankImage_extraction_pair'`, + ); + if (!pair) { + console.error("VERIFY FAILED: constraint BankImage_extraction_pair missing"); + process.exit(1); + } + console.log(`verified ${NEW_COLUMNS.length} columns + 1 constraint`); + console.log("\nCheck-payer extraction migration applied and verified."); + } finally { + await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/extract-check-payers.mjs b/scripts/extract-check-payers.mjs new file mode 100644 index 000000000..ac5b381d0 --- /dev/null +++ b/scripts/extract-check-payers.mjs @@ -0,0 +1,445 @@ +// Extract WHO PAID US from check/deposit images via Gemini Vision. +// +// A WTB inbound line says only "OTHER DEPOSITS DEPOSIT - DDA/MMKT". The image +// is the only evidence naming the payer, and the memo line often names the +// job. This reads BankImage rows (kind CHECK_FRONT / DEPOSIT_PHOTO by +// default) that have not been extracted yet, sends each image to Gemini +// Vision, and stores ONLY payerName + memoText (+ extractedAt + +// extractionModel) via scripts/apply-check-payer-extraction.mjs's columns. +// +// ── HARD PRIVACY RULE ──────────────────────────────────────────────────── +// The MICR line / routing number / account number are NEVER extracted, +// NEVER stored, NEVER printed. This is enforced twice: +// 1. The prompt forbids reading the bottom MICR strip at all. +// 2. scrubExtraction() drops ANY field containing an 8+ consecutive-digit +// run that is not the known check number or amount, and logs a warning. +// Do not weaken either layer. Do not add columns for these values. +// ───────────────────────────────────────────────────────────────────────── +// +// IDEMPOTENT: rows with extractedAt NOT NULL are skipped. Re-running is a +// no-op for already-extracted images. +// +// node scripts/extract-check-payers.mjs --dry-run # default; nothing written +// node scripts/extract-check-payers.mjs --dry-run --from-manifest # pre-DDL test straight from the Drive manifest +// node scripts/extract-check-payers.mjs --commit --limit 10 # write results (requires the DDL applied) +// node scripts/extract-check-payers.mjs --report # REVIEW REPORT only: suggest payer→Client / memo→Project matches +// +// The REVIEW REPORT is print-only. It NEVER writes BankImageMatch — that +// table means a HUMAN said yes (see prisma/schema.prisma). Beverly/Justin +// confirm matches; this script only suggests. +// +// DATABASE_URL / GEMINI_API_KEY come from the environment or .env.local, +// never argv. +import { PrismaClient } from "@prisma/client"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const IMAGES_DIR = + "I:/My Drive/2025 Reconciliation/Washington Trust Bank/Check Images"; +export const MANIFEST_PATH = path.join(IMAGES_DIR, "_manifest.json"); + +export const GEMINI_MODEL = "gemini-3-flash-preview"; +export const DEFAULT_KINDS = ["CHECK_FRONT", "DEPOSIT_PHOTO"]; +export const ALL_KINDS = ["CHECK_FRONT", "CHECK_BACK", "DEPOSIT_SLIP", "DEPOSIT_PHOTO"]; +export const DEFAULT_LIMIT = 25; + +// ── env ────────────────────────────────────────────────────────────────── +export function resolveEnv(name) { + if (process.env[name]) return process.env[name]; + for (const file of [".env.local", ".env"]) { + if (!fs.existsSync(file)) continue; + const match = fs.readFileSync(file, "utf8").match(new RegExp(`^${name}\\s*=\\s*"?([^"\\r\\n]+)"?`, "m")); + if (match) return match[1]; + } + return null; +} + +// ── MICR / routing / account guard ─────────────────────────────────────── +// Routing numbers are 9 digits; account numbers 8-12+. A check number is +// short (typically 3-5 digits) and the amount's digit string is 3-7. So any +// 8+ consecutive-digit run that isn't an explicitly allowed value is treated +// as banked-number leakage and the WHOLE field is dropped. +export const BANNED_DIGIT_RUN = /\d{8,}/g; +const MICR_SYMBOLS = /[\u2446\u2447\u2448\u2449]/; // ⑆⑇⑈⑉ MICR transit/on-us glyphs + +/** + * Pure. Returns { value, dropped } — dropped is a reason string when the + * field was discarded. `allow` lists digit strings that are legitimately + * long (never expected in practice, but the check number is allowed on + * principle: it is already stored openly on the row). + */ +export function scrubField(value, allow = []) { + if (value === null || value === undefined) return { value: null, dropped: null }; + const text = String(value).trim(); + if (!text) return { value: null, dropped: null }; + if (MICR_SYMBOLS.test(text)) { + return { value: null, dropped: "contains MICR symbols" }; + } + // Collapse separators so "1234 5678 9012" or "1234-5678-90" is seen as + // one run. The collapsed threshold is 9+ (routing numbers are 9): an ISO + // date like 2026-08-13 collapses to exactly 8 digits and must survive, + // while a genuinely consecutive 8-digit account is still caught by the + // raw-text 8+ check above. + const collapsed = text.replace(/[\s-]+/g, ""); + const runs = [...new Set([ + ...(text.match(BANNED_DIGIT_RUN) ?? []), + ...(collapsed.match(/\d{9,}/g) ?? []), + ])]; + const banned = runs.filter(run => !allow.includes(run)); + if (banned.length) { + return { value: null, dropped: `contains ${banned.length} banned digit run(s) (routing/account pattern)` }; + } + return { value: text, dropped: null }; +} + +/** + * Pure. Scrubs a raw Gemini response into the ONLY values we keep: + * payerName and memoText. Everything else (date, amount, check number) is + * used for cross-check logging only and is never stored by this script. + * + * @param {Record | null | undefined} raw + * @param {{ checkNumber?: string | null, amountCents?: number | null }} [opts] + */ +export function scrubExtraction(raw, { checkNumber = null, amountCents = null } = {}) { + const allow = []; + if (checkNumber) allow.push(String(checkNumber).replace(/\D/g, "")); + if (amountCents !== null && amountCents !== undefined) allow.push(String(amountCents)); + + const warnings = []; + const take = (field) => { + const { value, dropped } = scrubField(raw?.[field], allow); + if (dropped) warnings.push(`${field} DROPPED: ${dropped}`); + return value; + }; + + const payerName = take("payerName"); + const memoText = take("memoText"); + // Cross-check-only fields go through the same guard so a leaked account + // number can never even reach a console.log. + const documentDate = take("documentDate"); + const amount = take("amount"); + const checkNo = take("checkNumber"); + + return { payerName, memoText, documentDate, amount, checkNumber: checkNo, warnings }; +} + +// ── Gemini Vision (repo's REST pattern, src/lib/actions.ts aiGeneratePunchlist) ── +const EXTRACTION_PROMPT = `You are reading ONE side of a bank document (a check front or a deposit photo) for a construction company's bookkeeping. + +Extract ONLY these fields: +- payerName: the person or company name printed in the top-left name/address block (who wrote the check). Name only — no street address. +- documentDate: the written or printed date, as YYYY-MM-DD. +- amount: the dollar amount from the courtesy box, e.g. "6037.15". +- memoText: the handwriting or print on the "memo" / "for" line, or null if blank. +- checkNumber: the check number from the TOP RIGHT corner ONLY. + +ABSOLUTE PROHIBITION — read carefully: +Do NOT read, extract, transcribe, or output the MICR line: the row of numbers printed along the BOTTOM edge of the check in magnetic ink. That includes the routing number, the bank account number, and any number printed between ⑆ ⑈ ⑉ symbols. Never output any number with 8 or more consecutive digits from the bottom strip of the document, in any field. If you cannot fill a field without using the bottom strip, output null for that field. This is a privacy requirement and overrides completeness. + +Return ONLY a JSON object, nothing else: +{"payerName": string|null, "documentDate": string|null, "amount": string|null, "memoText": string|null, "checkNumber": string|null}`; + +export async function extractViaGemini(apiKey, imageBytes, mime) { + const res = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${apiKey}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + contents: [{ + parts: [ + { text: EXTRACTION_PROMPT }, + { inline_data: { mime_type: mime || "image/jpeg", data: imageBytes.toString("base64") } }, + ], + }], + generationConfig: { temperature: 0, responseMimeType: "application/json" }, + }), + }, + ); + if (!res.ok) throw new Error(`Gemini HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`); + const data = await res.json(); + const rawText = data.candidates?.[0]?.content?.parts?.[0]?.text; + if (!rawText) throw new Error("No AI response"); + return JSON.parse(rawText); +} + +// ── fuzzy matching for the REVIEW REPORT ───────────────────────────────── +const NAME_NOISE = new Set(["llc", "inc", "co", "corp", "ltd", "the", "and", "&", "of", "mr", "mrs", "ms", "dr", "or"]); + +export function nameTokens(s) { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .split(/\s+/) + .filter(t => t && !NAME_NOISE.has(t)); +} + +/** Pure. 0..1 similarity: token-set Jaccard plus a containment bonus. */ +export function nameSimilarity(a, b) { + const ta = nameTokens(a), tb = nameTokens(b); + if (!ta.length || !tb.length) return 0; + const sa = new Set(ta), sb = new Set(tb); + const inter = [...sa].filter(t => sb.has(t)).length; + const union = new Set([...sa, ...sb]).size; + const jaccard = inter / union; + const containment = inter / Math.min(sa.size, sb.size); + return Math.max(jaccard, containment * 0.85); +} + +/** + * Pure. Suggest candidate matches for one extraction. Returns + * { payerMatches: [{id,name,score}], memoMatches: [...] } sorted by score, + * top 3 each, threshold 0.4. Suggestion only — the caller must NEVER write + * BankImageMatch from this. + */ +export function suggestMatches({ payerName, memoText }, clients, projects) { + const rank = (text, rows) => rows + .map(r => ({ id: r.id, name: r.name, score: nameSimilarity(text, r.name) })) + .filter(m => m.score >= 0.4) + .sort((x, y) => y.score - x.score) + .slice(0, 3); + return { + payerMatches: payerName ? rank(payerName, clients) : [], + memoMatches: memoText ? rank(memoText, projects) : [], + }; +} + +// ── candidate loading ──────────────────────────────────────────────────── +function parseKinds() { + const idx = process.argv.indexOf("--kinds"); + if (idx < 0) return DEFAULT_KINDS; + const kinds = String(process.argv[idx + 1] ?? "").split(",").map(s => s.trim()).filter(Boolean); + const bad = kinds.filter(k => !ALL_KINDS.includes(k)); + if (bad.length) throw new Error(`Unknown kind(s): ${bad.join(", ")}. Allowed: ${ALL_KINDS.join(", ")}`); + return kinds; +} + +function parseLimit() { + const idx = process.argv.indexOf("--limit"); + if (idx < 0) return DEFAULT_LIMIT; + const n = Number(process.argv[idx + 1]); + if (!Number.isInteger(n) || n < 1) throw new Error(`--limit must be a positive integer, got "${process.argv[idx + 1]}"`); + return n; +} + +async function hasExtractionColumns(prisma) { + const rows = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns + WHERE table_schema='public' AND table_name='BankImage' + AND column_name IN ('payerName','memoText','extractedAt','extractionModel')`, + ); + return rows.length === 4; +} + +async function loadCandidatesFromDb(prisma, kinds, limit, columnsReady) { + // kinds is validated against ALL_KINDS above, limit is a checked integer — + // safe to inline. extractedAt only exists after the DDL is applied; before + // that, dry-run falls back to "everything is unextracted". + const kindList = kinds.map(k => `'${k}'`).join(","); + const where = columnsReady ? `AND "extractedAt" IS NULL` : ""; + return prisma.$queryRawUnsafe( + `SELECT "id", "kind", "sourceExternalId", "fileName", "mime", + "normalizedCheckNumber", "amountCents", "documentDate" + FROM "BankImage" + WHERE "kind" IN (${kindList}) ${where} + ORDER BY "capturedAt" ASC + LIMIT ${limit}`, + ); +} + +/** Pre-DDL escape hatch: derive candidates straight from the Drive manifest. */ +export function loadCandidatesFromManifest(manifest, kinds, limit) { + const rows = []; + for (const entry of Object.values(manifest.images ?? {})) { + const files = Array.isArray(entry.files) ? entry.files : []; + const isCheck = !!String(entry.checkNumber ?? "").replace(/\D/g, "").replace(/^0+/, ""); + files.forEach((f, i) => { + const kind = isCheck + ? (f.side === "front" || i === 0 ? "CHECK_FRONT" : "CHECK_BACK") + : (i === 0 ? "DEPOSIT_SLIP" : "DEPOSIT_PHOTO"); + if (!kinds.includes(kind)) return; + rows.push({ + id: `manifest:${entry.bankReference}:${f.side ?? `img${i + 1}`}`, + kind, + sourceExternalId: `${entry.bankReference}:${f.side ?? `img${i + 1}`}`, + fileName: f.fileName, + mime: "image/jpeg", + normalizedCheckNumber: isCheck ? String(entry.checkNumber).replace(/\D/g, "").replace(/^0+/, "") : null, + amountCents: entry.amountCents ?? null, + documentDate: null, + }); + }); + } + return rows.slice(0, limit); +} + +// ── review report ──────────────────────────────────────────────────────── +function printReviewReport(results, clients, projects) { + console.log("\n════════ REVIEW REPORT — suggestions only, NOTHING written ════════"); + console.log("BankImageMatch is human-confirmation-only; confirm in the app, not here.\n"); + let any = false; + for (const r of results) { + if (!r.payerName && !r.memoText) continue; + const { payerMatches, memoMatches } = suggestMatches(r, clients, projects); + any = true; + console.log(` ${r.sourceExternalId} (${r.kind}${r.normalizedCheckNumber ? `, chk#${r.normalizedCheckNumber}` : ""})`); + console.log(` payer: ${r.payerName ?? "(none)"} | memo: ${r.memoText ?? "(none)"}`); + if (payerMatches.length) { + for (const m of payerMatches) console.log(` payer → Client "${m.name}" score ${m.score.toFixed(2)} [${m.id}]`); + } else console.log(" payer → no Client match ≥ 0.40"); + if (r.memoText) { + if (memoMatches.length) { + for (const m of memoMatches) console.log(` memo → Project "${m.name}" score ${m.score.toFixed(2)} [${m.id}]`); + } else console.log(" memo → no Project match ≥ 0.40"); + } + console.log(""); + } + if (!any) console.log(" (no extracted payer/memo values to match)\n"); +} + +// ── main ───────────────────────────────────────────────────────────────── +async function main() { + const commit = process.argv.includes("--commit"); + const reportOnly = process.argv.includes("--report") && !commit; + const dryRun = !commit; + const fromManifest = process.argv.includes("--from-manifest"); + const kinds = parseKinds(); + const limit = parseLimit(); + + if (fromManifest && commit) { + console.error("--from-manifest is a pre-DDL dry-run aid; it cannot be combined with --commit."); + process.exit(1); + } + + const dbUrl = resolveEnv("DATABASE_URL"); + if (!dbUrl && !fromManifest) { + console.error("DATABASE_URL not found (env, .env.local, .env)."); + process.exit(1); + } + const prisma = dbUrl ? new PrismaClient({ datasources: { db: { url: dbUrl } } }) : null; + + try { + // ── report-only mode reads already-extracted rows and exits ── + if (reportOnly && !fromManifest) { + if (!prisma) throw new Error("Report mode needs a database connection."); + const columnsReady = await hasExtractionColumns(prisma); + if (!columnsReady) { + console.error("Extraction columns not applied yet — run scripts/apply-check-payer-extraction.mjs first."); + process.exit(1); + } + const rows = await prisma.$queryRawUnsafe( + `SELECT "id", "kind", "sourceExternalId", "normalizedCheckNumber", "payerName", "memoText" + FROM "BankImage" WHERE "extractedAt" IS NOT NULL ORDER BY "capturedAt" ASC LIMIT ${limit}`, + ); + const clients = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Client"`); + const projects = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Project"`); + console.log(`report: ${rows.length} extracted image(s), ${clients.length} client(s), ${projects.length} project(s)`); + printReviewReport(rows, clients, projects); + return; + } + + // ── candidate selection ── + let candidates; + let columnsReady = false; + if (fromManifest) { + if (!fs.existsSync(MANIFEST_PATH)) { + console.error(`No manifest at ${MANIFEST_PATH}`); + process.exit(1); + } + const manifest = JSON.parse(fs.readFileSync(MANIFEST_PATH, "utf8")); + candidates = loadCandidatesFromManifest(manifest, kinds, limit); + console.log(`manifest: ${candidates.length} candidate image(s) [kinds: ${kinds.join(", ")}]`); + } else { + columnsReady = await hasExtractionColumns(prisma); + if (!columnsReady) { + if (commit) { + console.error("Extraction columns missing — apply scripts/apply-check-payer-extraction.mjs before --commit."); + process.exit(1); + } + console.log("NOTE: extraction columns not applied yet; dry-run treats every row as unextracted."); + } + candidates = await loadCandidatesFromDb(prisma, kinds, limit, columnsReady); + console.log(`db: ${candidates.length} unextracted image(s) [kinds: ${kinds.join(", ")}, limit ${limit}]`); + } + + if (!candidates.length) { + console.log("Nothing to extract."); + return; + } + + const apiKey = resolveEnv("GEMINI_API_KEY"); + if (!apiKey) { + console.error("GEMINI_API_KEY not found (env, .env.local, .env)."); + process.exit(1); + } + + // ── extraction loop ── + const results = []; + for (const row of candidates) { + const imagePath = path.join(IMAGES_DIR, row.fileName); + if (!fs.existsSync(imagePath)) { + console.warn(` SKIP ${row.sourceExternalId}: image file not found at ${imagePath}`); + continue; + } + process.stdout.write(` ${row.sourceExternalId} (${row.kind}) ... `); + let raw; + try { + raw = await extractViaGemini(apiKey, fs.readFileSync(imagePath), row.mime); + } catch (err) { + console.log(`FAILED: ${err.message}`); + continue; + } + const scrubbed = scrubExtraction(raw, { + checkNumber: row.normalizedCheckNumber, + amountCents: row.amountCents, + }); + for (const w of scrubbed.warnings) console.warn(`\n WARNING ${row.sourceExternalId}: ${w}`); + console.log(`payer="${scrubbed.payerName ?? ""}" memo="${scrubbed.memoText ?? ""}"` + + (scrubbed.documentDate ? ` date=${scrubbed.documentDate}` : "") + + (scrubbed.amount ? ` amt=${scrubbed.amount}` : "") + + (scrubbed.checkNumber ? ` chk#${scrubbed.checkNumber}` : "")); + + // Cross-checks are advisory: log disagreement, store nothing extra. + if (row.normalizedCheckNumber && scrubbed.checkNumber && + scrubbed.checkNumber.replace(/\D/g, "").replace(/^0+/, "") !== row.normalizedCheckNumber) { + console.warn(` NOTE: image check# ${scrubbed.checkNumber} != row check# ${row.normalizedCheckNumber}`); + } + + results.push({ ...row, payerName: scrubbed.payerName, memoText: scrubbed.memoText }); + + if (commit) { + await prisma.$executeRaw` + UPDATE "BankImage" + SET "payerName" = ${scrubbed.payerName}, + "memoText" = ${scrubbed.memoText}, + "extractedAt" = now(), + "extractionModel" = ${GEMINI_MODEL}, + "updatedAt" = now() + WHERE "id" = ${row.id} AND "extractedAt" IS NULL`; + } + } + + if (dryRun) console.log(`\nDRY RUN — nothing written. Re-run with --commit${columnsReady ? "" : " after applying the DDL"}.`); + else console.log(`\nwrote ${results.length} extraction(s) [model ${GEMINI_MODEL}] (replay skips them)`); + + // ── review report ── + if (prisma) { + const clients = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Client"`); + const projects = await prisma.$queryRawUnsafe(`SELECT "id", "name" FROM "Project"`); + printReviewReport(results, clients, projects); + } else { + console.log("\n(no DATABASE_URL — skipping the review report's Client/Project matching)"); + } + } finally { + if (prisma) await prisma.$disconnect(); + } +} + +const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isMainModule) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/tests/extract-check-payers.test.ts b/tests/extract-check-payers.test.ts new file mode 100644 index 000000000..3fd0546f3 --- /dev/null +++ b/tests/extract-check-payers.test.ts @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + scrubField, + scrubExtraction, + nameSimilarity, + suggestMatches, + loadCandidatesFromManifest, +} from "../scripts/extract-check-payers.mjs"; + +// ── the MICR / routing / account ban ──────────────────────────────────── +// This is the guard the whole pipeline leans on: NOTHING with a long digit +// run may survive scrubbing, no matter what the model returned. + +test("scrubField passes ordinary payer names through", () => { + assert.deepEqual(scrubField("Smith Family Trust"), { value: "Smith Family Trust", dropped: null }); +}); + +test("scrubField drops a 9-digit routing number", () => { + const { value, dropped } = scrubField("routing 125108272"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops account numbers split by spaces or dashes", () => { + assert.equal(scrubField("1234 5678 9012").value, null); + assert.equal(scrubField("1234-5678-90").value, null); +}); + +test("scrubField drops MICR transit symbols outright", () => { + const { value, dropped } = scrubField("\u2446125108272\u2446"); + assert.equal(value, null); + assert.match(dropped ?? "", /MICR/); +}); + +test("scrubField allows an explicitly allowed long digit string", () => { + assert.equal(scrubField("ref 123456789", ["123456789"]).value, "ref 123456789"); +}); + +test("scrubExtraction keeps payer/memo, drops fields carrying account-like runs", () => { + const out = scrubExtraction( + { + payerName: "Henderson Kitchen LLC", + memoText: "acct 26225018006376", // model leaked something — must die + documentDate: "2026-08-13", + amount: "6037.15", + checkNumber: "1027", + }, + { checkNumber: "1027", amountCents: 603715 }, + ); + assert.equal(out.payerName, "Henderson Kitchen LLC"); + assert.equal(out.memoText, null); + assert.equal(out.warnings.length, 1); + assert.match(out.warnings[0], /memoText DROPPED/); + assert.equal(out.documentDate, "2026-08-13"); + assert.equal(out.checkNumber, "1027"); +}); + +test("scrubExtraction never lets a routing number through any field", () => { + const out = scrubExtraction({ + payerName: "125108272", // routing number where a name should be + memoText: null, + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.payerName, null); + assert.equal(out.warnings.length, 1); +}); + +// ── fuzzy matching (review report is suggestion-only) ─────────────────── + +test("nameSimilarity: exact and noise-word-insensitive matches score high", () => { + assert.equal(nameSimilarity("Henderson", "Henderson"), 1); + assert.ok(nameSimilarity("Henderson Kitchen LLC", "The Henderson Kitchen Co") > 0.6); +}); + +test("nameSimilarity: unrelated names score low", () => { + assert.ok(nameSimilarity("Smith Family Trust", "Jones Roofing") < 0.4); +}); + +test("suggestMatches ranks clients by payer and projects by memo, threshold 0.4", () => { + const clients = [ + { id: "c1", name: "Sarah Henderson" }, + { id: "c2", name: "Bob Jones" }, + ]; + const projects = [ + { id: "p1", name: "Henderson Master Bath" }, + { id: "p2", name: "Shop" }, + ]; + const { payerMatches, memoMatches } = suggestMatches( + { payerName: "Henderson, Sarah", memoText: "master bath" }, + clients, + projects, + ); + assert.equal(payerMatches[0]?.id, "c1"); + assert.equal(memoMatches[0]?.id, "p1"); + assert.ok(!payerMatches.some((m: { id: string }) => m.id === "c2")); +}); + +// ── manifest candidate derivation (pre-DDL dry-run path) ──────────────── + +test("loadCandidatesFromManifest derives kinds like post-bank-images and respects the kind filter", () => { + const manifest = { + images: { + "26225018006376": { + bankReference: "26225018006376", + checkNumber: "1027", + files: [ + { fileName: "front.jpg", side: "front" }, + { fileName: "back.jpg", side: "back" }, + ], + }, + }, + }; + const rows = loadCandidatesFromManifest(manifest, ["CHECK_FRONT"], 10); + assert.equal(rows.length, 1); + assert.equal(rows[0].kind, "CHECK_FRONT"); + assert.equal(rows[0].normalizedCheckNumber, "1027"); + assert.equal(rows[0].sourceExternalId, "26225018006376:front"); +}); + +test("loadCandidatesFromManifest honors the limit", () => { + const manifest = { + images: { + a: { bankReference: "a", checkNumber: "1", files: [{ fileName: "1.jpg", side: "front" }] }, + b: { bankReference: "b", checkNumber: "2", files: [{ fileName: "2.jpg", side: "front" }] }, + }, + }; + assert.equal(loadCandidatesFromManifest(manifest, ["CHECK_FRONT"], 1).length, 1); +}); From c42169ee7f52ab8344e784b4f2f40be1b700ea01 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Sat, 22 Aug 2026 00:33:01 -0700 Subject: [PATCH 08/10] harden(check-payers): total-digit scrub, JSON-fence tolerance, length caps, needsReview NULL path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kimi review fixes — all pre-commit privacy gaps closed: - scrubField: ban any token run whose TOTAL digit count >= 8 after stripping ALL separators (space/dash/dot/slash/comma/parens) AND interleaved letters. Catches spaced 8-digit accounts, dot/slash- separated routing numbers, and letter-mixed runs (A1B2C3D4E5F6G7H8). Valid ISO/US calendar dates exempted (month/day-range validated). Allow-list now matches the separator-stripped digit string. - parseModelJson: tolerate ```json fences around the model response. - Length caps: payerName 120 / memoText 200, truncate + flag. - needsReview: any drop/truncation flips it; --commit then stores NULL payer/memo (still stamps extractedAt/extractionModel for replay-skip) and logs the row for human review. No schema change; DDL untouched. tests: 12 -> 32, all green; typecheck clean; test:bank-ledger 491 pass / 0 fail. --- scripts/extract-check-payers.mjs | 143 ++++++++++++++++++++------- tests/extract-check-payers.test.ts | 151 ++++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 37 deletions(-) diff --git a/scripts/extract-check-payers.mjs b/scripts/extract-check-payers.mjs index ac5b381d0..cbb01eab7 100644 --- a/scripts/extract-check-payers.mjs +++ b/scripts/extract-check-payers.mjs @@ -11,8 +11,14 @@ // The MICR line / routing number / account number are NEVER extracted, // NEVER stored, NEVER printed. This is enforced twice: // 1. The prompt forbids reading the bottom MICR strip at all. -// 2. scrubExtraction() drops ANY field containing an 8+ consecutive-digit -// run that is not the known check number or amount, and logs a warning. +// 2. scrubExtraction() drops ANY field whose digit content — after +// collapsing EVERY common separator (space dash dot slash comma parens) +// AND interleaved letters — totals 8 or more digits and is not the known +// check number or amount, and logs a warning. Recognizable calendar +// dates (ISO 2026-08-13 / US 8/13/2026) are exempted before counting. +// A field dropped by layer 2 also flips needsReview: --commit then stores +// NULL for payerName AND memoText on that row (extractedAt/extractionModel +// still set so replay skips it) and logs the row for human review. // Do not weaken either layer. Do not add columns for these values. // ───────────────────────────────────────────────────────────────────────── // @@ -58,46 +64,81 @@ export function resolveEnv(name) { // ── MICR / routing / account guard ─────────────────────────────────────── // Routing numbers are 9 digits; account numbers 8-12+. A check number is // short (typically 3-5 digits) and the amount's digit string is 3-7. So any -// 8+ consecutive-digit run that isn't an explicitly allowed value is treated -// as banked-number leakage and the WHOLE field is dropped. -export const BANNED_DIGIT_RUN = /\d{8,}/g; +// candidate run whose TOTAL digit count reaches 8 — after stripping every +// common separator (space, dash, dot, slash, comma, parens) AND interleaved +// letters — is treated as banked-number leakage unless the digit string is +// explicitly allowed, and the WHOLE field is dropped. Consecutive-run-only +// checks are NOT enough: "1234 5678", "123.456.789", "123/456/789" and +// "A1B2C3D4E5F6G7H8" all hide an account/routing number without ever showing +// 8 consecutive digits. +export const BANNED_DIGIT_RUN = /\d{8,}/g; // kept for reference/compat; superseded by the total-count rule below +export const BANNED_TOTAL_DIGITS = 8; const MICR_SYMBOLS = /[\u2446\u2447\u2448\u2449]/; // ⑆⑇⑈⑉ MICR transit/on-us glyphs +// Plausible calendar dates are exempt from digit counting (an ISO date is 8 +// digits and must survive). Month/day ranges are validated so an account +// disguised as "12/34/5678" does NOT qualify. +const ISO_DATE = /\b(?:19|20)\d{2}-(?:0?[1-9]|1[0-2])-(?:0?[1-9]|[12]\d|3[01])\b/g; +const US_DATE = /\b(?:0?[1-9]|1[0-2])[/-](?:0?[1-9]|[12]\d|3[01])[/-](?:19|20)?\d{2}\b/g; + +// A candidate token run: maximal chunk of letters/digits joined by the +// common separators. Letters are included so "A1B2C3D4E5F6G7H8" is one run. +const TOKEN_RUN = /[0-9A-Za-z][0-9A-Za-z\s\-./,()]*/g; + /** - * Pure. Returns { value, dropped } — dropped is a reason string when the - * field was discarded. `allow` lists digit strings that are legitimately - * long (never expected in practice, but the check number is allowed on - * principle: it is already stored openly on the row). + * Pure. Returns { value, dropped, truncated } — dropped is a reason string + * when the field was discarded; truncated is true when maxLen was applied. + * `allow` lists digit strings that are legitimately long (never expected in + * practice, but the check number is allowed on principle: it is already + * stored openly on the row). The allow check runs on the SEPARATOR-STRIPPED + * digit string, so an allowed "12345678" also covers "1234 5678". + * + * @param {unknown} value + * @param {string[]} [allow] + * @param {number | null} [maxLen] */ -export function scrubField(value, allow = []) { - if (value === null || value === undefined) return { value: null, dropped: null }; +export function scrubField(value, allow = [], maxLen = null) { + if (value === null || value === undefined) return { value: null, dropped: null, truncated: false }; const text = String(value).trim(); - if (!text) return { value: null, dropped: null }; + if (!text) return { value: null, dropped: null, truncated: false }; if (MICR_SYMBOLS.test(text)) { - return { value: null, dropped: "contains MICR symbols" }; + return { value: null, dropped: "contains MICR symbols", truncated: false }; } - // Collapse separators so "1234 5678 9012" or "1234-5678-90" is seen as - // one run. The collapsed threshold is 9+ (routing numbers are 9): an ISO - // date like 2026-08-13 collapses to exactly 8 digits and must survive, - // while a genuinely consecutive 8-digit account is still caught by the - // raw-text 8+ check above. - const collapsed = text.replace(/[\s-]+/g, ""); - const runs = [...new Set([ - ...(text.match(BANNED_DIGIT_RUN) ?? []), - ...(collapsed.match(/\d{9,}/g) ?? []), - ])]; - const banned = runs.filter(run => !allow.includes(run)); + // Blank out plausible calendar dates, then count TOTAL digits per token + // run after stripping ALL non-digits (separators and letters alike). + const dateless = text.replace(ISO_DATE, " ").replace(US_DATE, " "); + const banned = (dateless.match(TOKEN_RUN) ?? []) + .map(run => run.replace(/\D/g, "")) + .filter(digits => digits.length >= BANNED_TOTAL_DIGITS && !allow.includes(digits)); if (banned.length) { - return { value: null, dropped: `contains ${banned.length} banned digit run(s) (routing/account pattern)` }; + return { + value: null, + dropped: `contains ${banned.length} banned digit run(s) (routing/account pattern)`, + truncated: false, + }; + } + if (maxLen !== null && text.length > maxLen) { + return { value: text.slice(0, maxLen).trimEnd(), dropped: null, truncated: true }; } - return { value: text, dropped: null }; + return { value: text, dropped: null, truncated: false }; } +export const PAYER_NAME_MAX = 120; +export const MEMO_TEXT_MAX = 200; + /** * Pure. Scrubs a raw Gemini response into the ONLY values we keep: * payerName and memoText. Everything else (date, amount, check number) is * used for cross-check logging only and is never stored by this script. * + * Length caps: payerName 120 chars, memoText 200 chars — anything longer is + * truncated and flagged (a real name/memo line never approaches these; an + * overrun means the model transcribed something it shouldn't have). + * + * needsReview is true when ANY field was dropped or truncated: the caller + * must then store NULL for payerName/memoText and log for human review + * instead of trusting the surviving values. + * * @param {Record | null | undefined} raw * @param {{ checkNumber?: string | null, amountCents?: number | null }} [opts] */ @@ -107,21 +148,29 @@ export function scrubExtraction(raw, { checkNumber = null, amountCents = null } if (amountCents !== null && amountCents !== undefined) allow.push(String(amountCents)); const warnings = []; - const take = (field) => { - const { value, dropped } = scrubField(raw?.[field], allow); - if (dropped) warnings.push(`${field} DROPPED: ${dropped}`); + let needsReview = false; + const take = (field, maxLen = null) => { + const { value, dropped, truncated } = scrubField(raw?.[field], allow, maxLen); + if (dropped) { + warnings.push(`${field} DROPPED: ${dropped}`); + needsReview = true; + } + if (truncated) { + warnings.push(`${field} TRUNCATED to ${maxLen} chars`); + needsReview = true; + } return value; }; - const payerName = take("payerName"); - const memoText = take("memoText"); + const payerName = take("payerName", PAYER_NAME_MAX); + const memoText = take("memoText", MEMO_TEXT_MAX); // Cross-check-only fields go through the same guard so a leaked account // number can never even reach a console.log. const documentDate = take("documentDate"); const amount = take("amount"); const checkNo = take("checkNumber"); - return { payerName, memoText, documentDate, amount, checkNumber: checkNo, warnings }; + return { payerName, memoText, documentDate, amount, checkNumber: checkNo, warnings, needsReview }; } // ── Gemini Vision (repo's REST pattern, src/lib/actions.ts aiGeneratePunchlist) ── @@ -140,6 +189,18 @@ Do NOT read, extract, transcribe, or output the MICR line: the row of numbers pr Return ONLY a JSON object, nothing else: {"payerName": string|null, "documentDate": string|null, "amount": string|null, "memoText": string|null, "checkNumber": string|null}`; +/** + * Pure. Parse a model response as JSON, tolerating a ```json ... ``` (or + * bare ```) fence wrapper — models sometimes emit fences even with + * responseMimeType: application/json. + */ +export function parseModelJson(rawText) { + let text = String(rawText ?? "").trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/i); + if (fenced) text = fenced[1].trim(); + return JSON.parse(text); +} + export async function extractViaGemini(apiKey, imageBytes, mime) { const res = await fetch( `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${apiKey}`, @@ -161,7 +222,7 @@ export async function extractViaGemini(apiKey, imageBytes, mime) { const data = await res.json(); const rawText = data.candidates?.[0]?.content?.parts?.[0]?.text; if (!rawText) throw new Error("No AI response"); - return JSON.parse(rawText); + return parseModelJson(rawText); } // ── fuzzy matching for the REVIEW REPORT ───────────────────────────────── @@ -406,13 +467,23 @@ async function main() { console.warn(` NOTE: image check# ${scrubbed.checkNumber} != row check# ${row.normalizedCheckNumber}`); } - results.push({ ...row, payerName: scrubbed.payerName, memoText: scrubbed.memoText }); + results.push({ ...row, payerName: scrubbed.payerName, memoText: scrubbed.memoText, needsReview: scrubbed.needsReview }); if (commit) { + // needsReview rows store NULL for BOTH kept fields: a scrub + // warning means the extraction is suspect (dropped digit run, + // truncated overrun), so nothing from it is trusted. The row + // is still stamped extractedAt/extractionModel so replay + // skips it; a human re-reads the image instead. + const storePayer = scrubbed.needsReview ? null : scrubbed.payerName; + const storeMemo = scrubbed.needsReview ? null : scrubbed.memoText; + if (scrubbed.needsReview) { + console.warn(` NEEDS REVIEW ${row.sourceExternalId}: stored NULL payer/memo — a human must read this image (file: ${row.fileName})`); + } await prisma.$executeRaw` UPDATE "BankImage" - SET "payerName" = ${scrubbed.payerName}, - "memoText" = ${scrubbed.memoText}, + SET "payerName" = ${storePayer}, + "memoText" = ${storeMemo}, "extractedAt" = now(), "extractionModel" = ${GEMINI_MODEL}, "updatedAt" = now() diff --git a/tests/extract-check-payers.test.ts b/tests/extract-check-payers.test.ts index 3fd0546f3..0621681d0 100644 --- a/tests/extract-check-payers.test.ts +++ b/tests/extract-check-payers.test.ts @@ -3,9 +3,12 @@ import test from "node:test"; import { scrubField, scrubExtraction, + parseModelJson, nameSimilarity, suggestMatches, loadCandidatesFromManifest, + PAYER_NAME_MAX, + MEMO_TEXT_MAX, } from "../scripts/extract-check-payers.mjs"; // ── the MICR / routing / account ban ──────────────────────────────────── @@ -13,7 +16,7 @@ import { // run may survive scrubbing, no matter what the model returned. test("scrubField passes ordinary payer names through", () => { - assert.deepEqual(scrubField("Smith Family Trust"), { value: "Smith Family Trust", dropped: null }); + assert.deepEqual(scrubField("Smith Family Trust"), { value: "Smith Family Trust", dropped: null, truncated: false }); }); test("scrubField drops a 9-digit routing number", () => { @@ -37,6 +40,152 @@ test("scrubField allows an explicitly allowed long digit string", () => { assert.equal(scrubField("ref 123456789", ["123456789"]).value, "ref 123456789"); }); +// ── Kimi review gap 1: 8-digit accounts split by separators ───────────── + +test("scrubField drops an 8-digit account split by a space", () => { + const { value, dropped } = scrubField("acct 1234 5678"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops an 8-digit account split by dashes", () => { + assert.equal(scrubField("12-34-56-78").value, null); +}); + +// ── Kimi review gap 2: dot / slash separators ─────────────────────────── + +test("scrubField drops a dot-separated routing number", () => { + const { value, dropped } = scrubField("123.456.789"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops a slash-separated account number", () => { + assert.equal(scrubField("123/456/789").value, null); + assert.equal(scrubField("1234/5678").value, null); +}); + +test("scrubField drops comma- and paren-separated digit groups", () => { + assert.equal(scrubField("1234,5678").value, null); + assert.equal(scrubField("(1234) 5678-90").value, null); +}); + +// ── Kimi review gap 3: letter-mixed digit runs ────────────────────────── + +test("scrubField drops letter-mixed runs hiding 8+ digits", () => { + const { value, dropped } = scrubField("A1B2C3D4E5F6G7H8"); + assert.equal(value, null); + assert.match(dropped ?? "", /banned digit run/); +}); + +test("scrubField drops an account with a letter prefix", () => { + assert.equal(scrubField("acct no. 12345678").value, null); +}); + +// ── dates must survive the total-digit rule ───────────────────────────── + +test("scrubField passes ISO and US calendar dates (8 digits, but a date)", () => { + assert.equal(scrubField("2026-08-13").value, "2026-08-13"); + assert.equal(scrubField("8/13/2026").value, "8/13/2026"); + assert.equal(scrubField("12/31/2026").value, "12/31/2026"); +}); + +test("scrubField does NOT exempt an account disguised as an invalid date", () => { + assert.equal(scrubField("12/34/5678").value, null); +}); + +test("scrubField allow-list matches the separator-stripped digit string", () => { + // Allowed "123456789" also covers its spaced form. + assert.equal(scrubField("ref 1234 56789", ["123456789"]).value, "ref 1234 56789"); +}); + +test("scrubField keeps short digit content (check numbers, amounts, addresses)", () => { + assert.equal(scrubField("chk 1027").value, "chk 1027"); + assert.equal(scrubField("6037.15").value, "6037.15"); + assert.equal(scrubField("1234 W 5th Ave").value, "1234 W 5th Ave"); +}); + +// ── length caps ───────────────────────────────────────────────────────── + +test("scrubField truncates past maxLen and flags it", () => { + const long = "X".repeat(150); + const { value, dropped, truncated } = scrubField(long, [], 120); + assert.equal(value, "X".repeat(120)); + assert.equal(dropped, null); + assert.equal(truncated, true); +}); + +test("scrubField leaves values within maxLen untouched", () => { + const { value, truncated } = scrubField("Smith Family Trust", [], 120); + assert.equal(value, "Smith Family Trust"); + assert.equal(truncated, false); +}); + +test("scrubExtraction caps payerName at 120 and memoText at 200, sets needsReview", () => { + const out = scrubExtraction({ + payerName: "P".repeat(PAYER_NAME_MAX + 40), + memoText: "m".repeat(MEMO_TEXT_MAX + 1), + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.payerName, "P".repeat(120)); + assert.equal(out.memoText, "m".repeat(200)); + assert.equal(out.needsReview, true); + assert.equal(out.warnings.length, 2); + assert.match(out.warnings[0], /payerName TRUNCATED to 120/); + assert.match(out.warnings[1], /memoText TRUNCATED to 200/); +}); + +// ── needsReview signal ────────────────────────────────────────────────── + +test("scrubExtraction reports needsReview=false on a clean extraction", () => { + const out = scrubExtraction( + { + payerName: "Henderson Kitchen LLC", + memoText: "master bath", + documentDate: "2026-08-13", + amount: "6037.15", + checkNumber: "1027", + }, + { checkNumber: "1027", amountCents: 603715 }, + ); + assert.equal(out.needsReview, false); + assert.equal(out.warnings.length, 0); +}); + +test("scrubExtraction flags needsReview when any field is dropped", () => { + const out = scrubExtraction({ + payerName: "Fine Name", + memoText: "acct 1234 5678", // spaced 8-digit leak + documentDate: null, + amount: null, + checkNumber: null, + }); + assert.equal(out.memoText, null); + assert.equal(out.needsReview, true); +}); + +// ── JSON fence tolerance ──────────────────────────────────────────────── + +test("parseModelJson parses bare JSON", () => { + assert.deepEqual(parseModelJson('{"payerName": "Smith"}'), { payerName: "Smith" }); +}); + +test("parseModelJson tolerates ```json fences", () => { + const fenced = '```json\n{"payerName": "Smith", "memoText": null}\n```'; + assert.deepEqual(parseModelJson(fenced), { payerName: "Smith", memoText: null }); +}); + +test("parseModelJson tolerates bare ``` fences and surrounding whitespace", () => { + const fenced = ' ```\n{"amount": "6037.15"}\n``` '; + assert.deepEqual(parseModelJson(fenced), { amount: "6037.15" }); +}); + +test("parseModelJson still throws on genuinely invalid JSON", () => { + assert.throws(() => parseModelJson("```json\nnot json at all\n```")); +}); + test("scrubExtraction keeps payer/memo, drops fields carrying account-like runs", () => { const out = scrubExtraction( { From 1bda74dbd904e3c98f599e24cf0e0a6451b79a4a Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Sat, 22 Aug 2026 10:04:07 -0700 Subject: [PATCH 09/10] feat(check-payers): Automation Check-images panel + confirmed check evidence on invoices Geordi work order (check-payer UI): - Automation page 'Check images' panel: BankImage worklist with extracted payer/memo (null-safe: 'not yet extracted' when extractedAt IS NULL), fuzzy payer->Client / memo->Project suggestions (lib/check-payer-match.ts, TS port of extract-check-payers.mjs --report with a parity test so the two can't drift), bank-line proposals via lib/bank-image.ts, and a per-image human CONFIRM action. - confirmBankImageMatch (src/lib/actions.ts): the ONE writer of BankImageMatch, gated on financialReports (internal roles only; portal sessions rejected by assertActiveStaff). P2002 loses the double-confirm race honestly. - Invoice detail: paid-by-check milestones with a human-confirmed BankImageMatch show 'Paid by , chk#' + Drive image link. Number AND amount must both agree (check numbers collide across payers); ambiguity or missing corroboration renders nothing, never a guess. Display-only; no payment notifiers touched. - Both new fetches degrade independently (existing page pattern): a failure shows an honest 'unavailable' card, never takes the page down. Tests: tests/check-payer-match.test.ts, tests/check-evidence.test.ts (added to test:bank-ledger). tsc --noEmit clean, test:unit 481 pass, test:bank-ledger 508 pass, npm run build clean. --- package.json | 2 +- src/app/automation/check-images-data.ts | 186 ++++++++++++++++++ .../components/check-image-confirm-button.tsx | 58 ++++++ .../components/check-images-panel.tsx | 185 +++++++++++++++++ src/app/automation/page.tsx | 27 +++ .../invoices/[invoiceId]/InvoiceEditor.tsx | 22 ++- .../[id]/invoices/[invoiceId]/page.tsx | 24 ++- src/lib/actions.ts | 52 +++++ src/lib/check-evidence.ts | 146 ++++++++++++++ src/lib/check-payer-match.ts | 93 +++++++++ tests/check-evidence.test.ts | 109 ++++++++++ tests/check-payer-match.test.ts | 110 +++++++++++ 12 files changed, 1011 insertions(+), 3 deletions(-) create mode 100644 src/app/automation/check-images-data.ts create mode 100644 src/app/automation/components/check-image-confirm-button.tsx create mode 100644 src/app/automation/components/check-images-panel.tsx create mode 100644 src/lib/check-evidence.ts create mode 100644 src/lib/check-payer-match.ts create mode 100644 tests/check-evidence.test.ts create mode 100644 tests/check-payer-match.test.ts diff --git a/package.json b/package.json index b02c6c881..8036906a0 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "lint": "eslint", "test:qbo-expense-sync": "tsx --test tests/qbo-expense-sync.test.ts tests/qbo-expense-sync-route.test.ts tests/qbo-expense-sync-ui.test.tsx tests/qbo-purchase-changes.test.ts tests/qbo-expense-guard.test.ts tests/qbo-purchase-classification.test.ts", "test:qbo-receipt-push": "tsx --test tests/qbo-receipt-push.test.ts", - "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/deposit-attribution.test.ts", + "test:bank-ledger": "tsx --test tests/bank-ledger.test.ts tests/bank-ledger-ingest-route.test.ts tests/bank-ledger-reconcile-route.test.ts tests/apply-bank-ledger.test.ts tests/migration-history-blind-spots.test.ts tests/parse-wtb-statement.test.ts tests/parse-wtb-daily-csv.test.ts tests/post-qbo-register.test.ts tests/receipt-match.test.ts tests/vendor-alias.test.ts tests/receipt-policy.test.ts tests/bank-image.test.ts tests/post-bank-images.test.ts tests/extract-check-payers.test.ts tests/check-payer-match.test.ts tests/check-evidence.test.ts tests/deposit-attribution.test.ts", "test:automation-key-resolver": "tsx --test tests/automation-key-resolver.test.ts", "test:automation-register": "tsx --test tests/automation-register-filters.test.ts tests/match-receipt-journey.test.ts tests/automation-events-grouping.test.ts tests/automation-format.test.ts", "test:ai-review-reasonableness": "tsx --test tests/ai-review-reasonableness.test.ts", diff --git a/src/app/automation/check-images-data.ts b/src/app/automation/check-images-data.ts new file mode 100644 index 000000000..b8feefa20 --- /dev/null +++ b/src/app/automation/check-images-data.ts @@ -0,0 +1,186 @@ +import { prisma } from "@/lib/prisma"; +import { + proposeImageMatches, + type BankImageCandidate, + type BankImageKind, + type BankImageLine, +} from "@/lib/bank-image"; +import { suggestMatches, type MatchSuggestion } from "@/lib/check-payer-match"; + +/** + * Data for the Automation page's "Check images" panel — the human worklist + * for the check-payer pipeline (scripts/extract-check-payers.mjs writes the + * extractions; this reads them). + * + * READ-ONLY. The only writer of BankImageMatch is the explicit + * confirmBankImageMatch server action (src/lib/actions.ts) — a row in that + * table means a HUMAN said yes (see prisma/schema.prisma). + * + * Caller (src/app/automation/page.tsx) is gated on the `financialReports` + * permission before this ever runs — internal roles only, never the client + * or sub portals. + */ + +const IMAGE_KINDS = new Set(["CHECK_FRONT", "CHECK_BACK", "DEPOSIT_SLIP", "DEPOSIT_PHOTO"]); +/** Display cap — this is a review worklist, not an archive browser. */ +export const CHECK_IMAGE_DISPLAY_LIMIT = 50; +/** Candidate ledger lines considered for proposals (most recent first). */ +const LINE_CANDIDATE_LIMIT = 1000; + +export interface CheckImageConfirmedMatch { + bankLineId: string | null; + lineDescriptor: string | null; + linePostedDate: string | null; + lineAmountCents: number | null; + confirmedBy: string; + /** ISO timestamp */ + confirmedAt: string; + note: string | null; +} + +export interface CheckImageProposedLine { + bankLineId: string; + confidence: string; + reason: string; + lineDescriptor: string; + linePostedDate: string | null; + lineAmountCents: number; +} + +export interface CheckImagePanelRow { + id: string; + kind: string; + sourceExternalId: string; + fileName: string; + driveFileId: string | null; + /** ISO timestamp */ + capturedAt: string; + /** YYYY-MM-DD */ + documentDate: string | null; + amountCents: number | null; + normalizedCheckNumber: string | null; + /** Null both when extraction hasn't run (extractedAt null) AND when it + * ran but found nothing / was scrubbed — `extracted` disambiguates. */ + payerName: string | null; + memoText: string | null; + extracted: boolean; + extractionModel: string | null; + payerMatches: MatchSuggestion[]; + memoMatches: MatchSuggestion[]; + proposal: CheckImageProposedLine | null; + /** Why no line could be proposed, when there is no proposal and no confirmation. */ + unmatchedDetail: string | null; + confirmed: CheckImageConfirmedMatch | null; +} + +function toDateOnly(value: Date | null | undefined): string | null { + return value ? value.toISOString().slice(0, 10) : null; +} + +export async function fetchCheckImagePanelData(): Promise<{ rows: CheckImagePanelRow[]; totalImages: number }> { + const [images, totalImages, lines, clients, projects] = await Promise.all([ + prisma.bankImage.findMany({ + orderBy: { capturedAt: "desc" }, + take: CHECK_IMAGE_DISPLAY_LIMIT, + include: { + matches: { + include: { + bankLine: { + select: { id: true, rawDescriptor: true, postedDate: true, amountCents: true }, + }, + }, + }, + }, + }), + prisma.bankImage.count(), + prisma.bankLine.findMany({ + orderBy: { postedDate: "desc" }, + take: LINE_CANDIDATE_LIMIT, + select: { id: true, postedDate: true, amountCents: true, rawDescriptor: true, checkNumber: true }, + }), + prisma.client.findMany({ select: { id: true, name: true } }), + prisma.project.findMany({ select: { id: true, name: true } }), + ]); + + const candidateLines: BankImageLine[] = lines.map((line) => ({ + id: line.id, + postedDate: toDateOnly(line.postedDate) ?? "", + amountCents: line.amountCents, + rawDescriptor: line.rawDescriptor, + checkNumber: line.checkNumber, + })).filter((line) => line.postedDate !== ""); + + const matchableImages: BankImageCandidate[] = images + .filter((img) => IMAGE_KINDS.has(img.kind)) + .map((img) => ({ + id: img.id, + kind: img.kind as BankImageKind, + documentDate: toDateOnly(img.documentDate), + amountCents: img.amountCents, + normalizedCheckNumber: img.normalizedCheckNumber, + })); + + const alreadyMatchedImageIds = images + .filter((img) => img.matches.length > 0) + .map((img) => img.id); + + const { proposals, unmatched } = proposeImageMatches(matchableImages, candidateLines, { + alreadyMatchedImageIds, + }); + const proposalByImage = new Map(proposals.map((p) => [p.bankImageId, p])); + const unmatchedByImage = new Map(unmatched.map((u) => [u.bankImageId, u])); + const lineById = new Map(candidateLines.map((line) => [line.id, line])); + + const rows: CheckImagePanelRow[] = images.map((img) => { + // bankImageId is @unique on BankImageMatch, so 0 or 1 rows. + const match = img.matches[0] ?? null; + const extracted = img.extractedAt !== null; + const suggestions = extracted + ? suggestMatches({ payerName: img.payerName, memoText: img.memoText }, clients, projects) + : { payerMatches: [], memoMatches: [] }; + const proposal = proposalByImage.get(img.id) ?? null; + const proposalLine = proposal ? lineById.get(proposal.bankLineId) ?? null : null; + + return { + id: img.id, + kind: img.kind, + sourceExternalId: img.sourceExternalId, + fileName: img.fileName, + driveFileId: img.driveFileId, + capturedAt: img.capturedAt.toISOString(), + documentDate: toDateOnly(img.documentDate), + amountCents: img.amountCents, + normalizedCheckNumber: img.normalizedCheckNumber, + payerName: img.payerName, + memoText: img.memoText, + extracted, + extractionModel: img.extractionModel, + payerMatches: suggestions.payerMatches, + memoMatches: suggestions.memoMatches, + proposal: proposal + ? { + bankLineId: proposal.bankLineId, + confidence: proposal.confidence, + reason: proposal.reason, + lineDescriptor: proposal.lineDescriptor, + linePostedDate: proposalLine?.postedDate ?? null, + lineAmountCents: proposal.lineAmountCents, + } + : null, + unmatchedDetail: match ? null : unmatchedByImage.get(img.id)?.detail ?? null, + confirmed: match + ? { + bankLineId: match.bankLineId, + lineDescriptor: match.bankLine?.rawDescriptor ?? null, + linePostedDate: toDateOnly(match.bankLine?.postedDate ?? null), + lineAmountCents: match.bankLine?.amountCents ?? null, + confirmedBy: match.confirmedBy, + confirmedAt: match.confirmedAt.toISOString(), + note: match.note, + } + : null, + }; + }); + + return { rows, totalImages }; +} diff --git a/src/app/automation/components/check-image-confirm-button.tsx b/src/app/automation/components/check-image-confirm-button.tsx new file mode 100644 index 000000000..b9b57a21b --- /dev/null +++ b/src/app/automation/components/check-image-confirm-button.tsx @@ -0,0 +1,58 @@ +"use client"; + +import { useState } from "react"; +import { useRouter } from "next/navigation"; +import { toast } from "sonner"; +import { confirmBankImageMatch } from "@/lib/actions"; + +/** + * The explicit human "yes" for a check-image → bank-line pairing. Clicking + * this is the ONLY path that writes BankImageMatch (via the + * confirmBankImageMatch server action) — everything else on the panel is a + * suggestion. Same "just the interactive bit" split as + * mark-reviewed-button.tsx. + */ +export default function ConfirmMatchButton({ + bankImageId, + bankLineId, + note, +}: { + bankImageId: string; + bankLineId: string; + /** The proposal's plain-language reason — stored on the match as the audit trail. */ + note: string | null; +}) { + const router = useRouter(); + const [pending, setPending] = useState(false); + const [done, setDone] = useState(false); + + async function confirm() { + setPending(true); + try { + const res = await confirmBankImageMatch({ bankImageId, bankLineId, note }); + if (res.success) { + setDone(true); + toast.success("Match confirmed"); + router.refresh(); + } else { + toast.error(res.error || "Couldn't confirm — try again"); + } + } catch { + toast.error("Couldn't confirm — try again"); + } finally { + setPending(false); + } + } + + return ( + + ); +} diff --git a/src/app/automation/components/check-images-panel.tsx b/src/app/automation/components/check-images-panel.tsx new file mode 100644 index 000000000..d231f3be2 --- /dev/null +++ b/src/app/automation/components/check-images-panel.tsx @@ -0,0 +1,185 @@ +import { formatCurrency } from "@/lib/utils"; +import type { CheckImagePanelRow } from "../check-images-data"; +import ConfirmMatchButton from "./check-image-confirm-button"; + +/** + * "Check images" panel (Automation page): the human worklist for the + * check-payer pipeline. Each card shows one BankImage with its extracted + * payer/memo evidence, the fuzzy Client/Project suggestions, and — when the + * matcher proposes exactly one ledger line — an explicit CONFIRM action. + * + * Suggestions are suggestions. Only the Confirm button (its server action) + * writes BankImageMatch, and only per image, by a signed-in internal user. + * + * Server component — the only interactive bit is ConfirmMatchButton. + */ + +const KIND_LABELS: Record = { + CHECK_FRONT: "Check front", + CHECK_BACK: "Check back", + DEPOSIT_SLIP: "Deposit slip", + DEPOSIT_PHOTO: "Deposit photo", +}; + +function fmtDate(iso: string | null): string { + if (!iso) return "—"; + const parsed = new Date(`${iso.slice(0, 10)}T00:00:00Z`); + if (Number.isNaN(parsed.getTime())) return "—"; + return parsed.toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric", timeZone: "UTC" }); +} + +function Label({ children }: { children: React.ReactNode }) { + return

{children}

; +} + +function SuggestionList({ title, matches, hrefBase }: { + title: string; + matches: { id: string; name: string; score: number }[]; + hrefBase: string | null; +}) { + if (matches.length === 0) return null; + return ( +
+ +
    + {matches.map((m) => ( +
  • + {hrefBase ? ( + + {m.name} + + ) : ( + {m.name} + )} + {Math.round(m.score * 100)}% similar +
    • + ))} +
+
+ ); +} + +function ImageCard({ row }: { row: CheckImagePanelRow }) { + const kindLabel = KIND_LABELS[row.kind] ?? row.kind; + const hasSuggestions = row.payerMatches.length > 0 || row.memoMatches.length > 0; + + return ( +
+ {/* Identity line */} +
+
+

+ {kindLabel} + {row.normalizedCheckNumber && chk#{row.normalizedCheckNumber}} +

+

+ {row.fileName} · captured {fmtDate(row.capturedAt)} + {row.documentDate && ` · dated ${fmtDate(row.documentDate)}`} +

+
+
+ {row.amountCents !== null && ( + + {formatCurrency(row.amountCents / 100)} + + )} + {row.driveFileId && ( + + Image ↗ + + )} +
+
+ + {/* Extraction evidence */} + {!row.extracted ? ( +

Not yet extracted — run the check-payer extraction to read this image.

+ ) : ( +
+
+ +

{row.payerName ?? not readable — needs a human}

+
+
+ +

{row.memoText ?? (blank)}

+
+
+ )} + + {/* Fuzzy suggestions */} + {row.extracted && hasSuggestions && ( +
+ + +
+ )} + {row.extracted && !hasSuggestions && (row.payerName || row.memoText) && ( +

No client or project is a close enough name match to suggest.

+ )} + + {/* Confirmed / proposed / unmatched — exactly one of the three */} + {row.confirmed ? ( +
+

+ Confirmed → {row.confirmed.lineDescriptor ?? "bank line"} + {row.confirmed.lineAmountCents !== null && ` · ${formatCurrency(Math.abs(row.confirmed.lineAmountCents) / 100)}`} + {row.confirmed.linePostedDate && ` · posted ${fmtDate(row.confirmed.linePostedDate)}`} +

+

+ by {row.confirmed.confirmedBy} on {fmtDate(row.confirmed.confirmedAt)} +

+
+ ) : row.proposal ? ( +
+
+

+ Suggested bank line: {row.proposal.lineDescriptor} + {` · ${formatCurrency(Math.abs(row.proposal.lineAmountCents) / 100)}`} + {row.proposal.linePostedDate && ` · posted ${fmtDate(row.proposal.linePostedDate)}`} +

+

{row.proposal.reason}

+
+ +
+ ) : ( +

+ {row.unmatchedDetail ?? "No bank line to suggest for this image."} +

+ )} +
+ ); +} + +export function CheckImagesPanel({ rows, totalImages }: { rows: CheckImagePanelRow[]; totalImages: number }) { + return ( +
+
+

Check images

+

+ Who actually paid us, straight from the check — the bank line alone doesn't say. Extracted payer + and memo are evidence; the matches below are suggestions until a human confirms them. + {totalImages > rows.length && ` Showing the ${rows.length} most recent of ${totalImages} images.`} +

+
+ {rows.length === 0 ? ( +

+ No check or deposit images pulled from the bank yet. +

+ ) : ( +
+ {rows.map((row) => )} +
+ )} +
+ ); +} diff --git a/src/app/automation/page.tsx b/src/app/automation/page.tsx index a7e9737e7..80c16568e 100644 --- a/src/app/automation/page.tsx +++ b/src/app/automation/page.tsx @@ -48,6 +48,8 @@ import { LinksCell } from "./components/register/links-cell"; import { RowDrilldown } from "./components/register/row-drilldown"; import { matchReceiptJourney, type ReceiptJourneyMatch, type ReceiptJourneyIndex } from "./components/register/match-receipt-journey"; import { toSerializedJourney } from "./components/register/serialize-journey"; +import { fetchCheckImagePanelData, type CheckImagePanelRow } from "./check-images-data"; +import { CheckImagesPanel } from "./components/check-images-panel"; export const dynamic = "force-dynamic"; @@ -443,6 +445,28 @@ export default async function AutomationPage(props: { console.error("pipeline health inputs failed", error instanceof Error ? error.message : "UnknownError"); pipelineHealthUnavailable = true; } + + // Check images panel (check-payer pipeline worklist) — independent of the + // register/merge/pipeline-health fetches above, so it fails alone: an + // error here degrades to an honest "unavailable" card, never the page. + let checkImagesUnavailable = false; + let checkImageRows: CheckImagePanelRow[] = []; + let checkImageTotal = 0; + try { + const checkImages = await withTimeout(fetchCheckImagePanelData(), 15_000); + checkImageRows = checkImages.rows; + checkImageTotal = checkImages.totalImages; + } catch (error) { + console.error("check image panel fetch failed", error instanceof Error ? error.message : "UnknownError"); + checkImagesUnavailable = true; + } + const checkImagesSection: ReactNode = checkImagesUnavailable ? ( +
+ Check images unavailable right now — the register above is still current. +
+ ) : ( + + ); const minutesSaved = summary.pushedThisMonth * 4; const hoursSavedRaw = Math.round((minutesSaved / 60) * 2) / 2; const hoursSavedLabel = Number.isInteger(hoursSavedRaw) ? String(hoursSavedRaw) : hoursSavedRaw.toFixed(1); @@ -698,6 +722,9 @@ export default async function AutomationPage(props: { {/* Orphan receipts */} {orphanSection} + {/* Check images — check-payer pipeline worklist (human confirm) */} + {checkImagesSection} + {/* Receipt pipeline — journey list, Verify in QuickBooks + AI review (plan §3) */} {journeySection} diff --git a/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx b/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx index c318aff66..90cfc2298 100644 --- a/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx +++ b/src/app/projects/[id]/invoices/[invoiceId]/InvoiceEditor.tsx @@ -13,6 +13,7 @@ import DocumentComments from "@/components/DocumentComments"; import { toast } from "sonner"; import { formatCurrency } from "@/lib/utils"; import { formatMoneyDate } from "@/lib/payment-date"; +import type { CheckEvidence } from "@/lib/check-evidence"; const METHOD_LABELS: Record = { card: "Card", @@ -30,7 +31,7 @@ function formatPaymentMethod(method: string | null | undefined, ref: string | nu return label; } -export default function InvoiceEditor({ project, initialInvoice }: { project: any, initialInvoice: any }) { +export default function InvoiceEditor({ project, initialInvoice, checkEvidence = {} }: { project: any, initialInvoice: any, checkEvidence?: Record }) { const router = useRouter(); const [isIssuing, setIsIssuing] = useState(false); const [isDeleting, setIsDeleting] = useState(false); @@ -789,6 +790,7 @@ export default function InvoiceEditor({ project, initialInvoice }: { project: an {initialInvoice.payments?.map((payment: any) => { const isPastDue = payment.dueDate && new Date(payment.dueDate) < new Date() && payment.status !== "Paid"; const methodLabel = formatPaymentMethod(payment.paymentMethod, payment.referenceNumber); + const evidence = payment.status === "Paid" ? checkEvidence[payment.id] : undefined; const receiptSentLabel = payment.receiptSentAt ? `Last sent ${new Date(payment.receiptSentAt).toLocaleString(undefined, { year: 'numeric', month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit' })}` : undefined; @@ -837,6 +839,24 @@ export default function InvoiceEditor({ project, initialInvoice }: { project: an {payment.status === 'Paid' && methodLabel && (
{methodLabel}
)} + {evidence && ( +
+ Paid by {evidence.payerName ?? "(payer not readable on image)"}, chk#{evidence.checkNumber} + {evidence.driveFileId && ( + <> + {" · "} + + check image ↗ + + + )} +
+ )} {payment.status !== 'Paid' && sentLabel && (
diff --git a/src/app/projects/[id]/invoices/[invoiceId]/page.tsx b/src/app/projects/[id]/invoices/[invoiceId]/page.tsx index 2f6cc6bf1..5c19a1aed 100644 --- a/src/app/projects/[id]/invoices/[invoiceId]/page.tsx +++ b/src/app/projects/[id]/invoices/[invoiceId]/page.tsx @@ -1,6 +1,7 @@ import { getProject } from "@/lib/actions"; import { getInvoice } from "@/lib/actions"; import { notFound } from "next/navigation"; +import { fetchCheckEvidenceForPayments, type CheckEvidence } from "@/lib/check-evidence"; import InvoiceEditor from "./InvoiceEditor"; export default async function InvoicePage({ params }: { params: Promise<{ id: string, invoiceId: string }> }) { @@ -22,9 +23,30 @@ export default async function InvoicePage({ params }: { params: Promise<{ id: st notFound(); } + // Check evidence for paid-by-check milestones: a human-confirmed + // BankImageMatch whose check number AND amount agree lets the editor show + // "Paid by , chk#" from the physical instrument. Display-only + // and best-effort — a failure here must never take down the invoice. + let checkEvidence: Record = {}; + try { + checkEvidence = await fetchCheckEvidenceForPayments( + (initialInvoice.payments ?? []) + .filter((p) => p.status === "Paid" && p.referenceNumber) + .map((p) => ({ + id: p.id, + referenceNumber: p.referenceNumber, + // Decimal dollars → integer cents; a NaN becomes null and + // the matcher skips it rather than matching loosely. + amountCents: Number.isFinite(Number(p.amount)) ? Math.round(Number(p.amount) * 100) : null, + })), + ); + } catch (error) { + console.error("check evidence fetch failed", error instanceof Error ? error.message : "UnknownError"); + } + return (
- +
); } diff --git a/src/lib/actions.ts b/src/lib/actions.ts index ec2125f8d..65706fed3 100644 --- a/src/lib/actions.ts +++ b/src/lib/actions.ts @@ -14739,3 +14739,55 @@ export async function deletePermit(permitId: string) { revalidatePath(`/projects/${target.projectId}/permits`); return { success: true }; } + +// ── Bank image match confirmation (Automation "Check images" panel) ───────── + +/** + * The ONE writer of BankImageMatch. A row in that table means a HUMAN said + * "this image explains this bank line" (prisma/schema.prisma) — the matchers + * (lib/bank-image.ts, lib/check-payer-match.ts, the extract script's + * --report) only ever suggest. Gated on financialReports like the Automation + * page itself: internal roles only, never a portal role (portal sessions + * carry no staff user, so assertActiveStaff already rejects them). + * + * bankImageId is @unique on BankImageMatch, so a second confirm for the same + * image loses the P2002 race honestly instead of double-writing. + */ +export async function confirmBankImageMatch(input: { + bankImageId: string; + bankLineId: string; + note?: string | null; +}): Promise<{ success: true } | { success: false; error: string }> { + const user = await assertFinancialPermission(); + + const bankImageId = (input.bankImageId ?? "").trim(); + const bankLineId = (input.bankLineId ?? "").trim(); + if (!bankImageId || !bankLineId) { + return { success: false, error: "Missing image or bank line id" }; + } + // The note is display/audit text from our own proposal reason — cap it so + // a mangled payload can't stuff arbitrary blobs into the audit trail. + const note = (input.note ?? "").trim().slice(0, 500) || null; + + const [image, line] = await Promise.all([ + prisma.bankImage.findUnique({ where: { id: bankImageId }, select: { id: true, sourceExternalId: true } }), + prisma.bankLine.findUnique({ where: { id: bankLineId }, select: { id: true } }), + ]); + if (!image) return { success: false, error: "That image no longer exists" }; + if (!line) return { success: false, error: "That bank line no longer exists" }; + + const confirmedBy = user.email ?? user.name ?? "staff"; + try { + await prisma.bankImageMatch.create({ + data: { bankImageId, bankLineId, confirmedBy, note }, + }); + } catch (e: any) { + if (e?.code === "P2002") { + return { success: false, error: "This image is already confirmed — refresh the page" }; + } + throw e; + } + + revalidatePath("/automation"); + return { success: true }; +} diff --git a/src/lib/check-evidence.ts b/src/lib/check-evidence.ts new file mode 100644 index 000000000..4e9a9015d --- /dev/null +++ b/src/lib/check-evidence.ts @@ -0,0 +1,146 @@ +/** + * Check evidence for invoice payments — "Paid by X, chk#N", backed by the + * physical check image a human confirmed against the bank ledger. + * + * A paid milestone recorded as method "check" carries a referenceNumber (the + * check #). A confirmed BankImageMatch means a human said "this image + * explains this bank line". This module ties the two together so the invoice + * detail can show the payer printed on the actual instrument. + * + * CHECK NUMBERS ARE NOT UNIQUE ACROSS PAYERS — every checkbook has a #1027. + * So a number match alone is never enough: the amount must corroborate + * (image amount when readable, otherwise the confirmed bank line's amount). + * No corroborating amount ⇒ no evidence shown. Honest silence beats a + * plausible wrong name on a money page. + * + * READ-ONLY and display-only: nothing here writes, settles, or notifies. + * The pure matcher is separated from the Prisma fetch for unit testing + * (tests/check-evidence.test.ts). + */ + +import { prisma } from "@/lib/prisma"; +import { normalizeCheckRef } from "@/lib/check-payer-match"; + +export interface CheckEvidence { + /** Payer printed on the check, when the extraction could read it. */ + payerName: string | null; + /** Digits-only check number (the normalized identity). */ + checkNumber: string; + /** Google Drive file id for the image, when the pull recorded one. */ + driveFileId: string | null; + fileName: string | null; + /** Who confirmed the image ↔ bank line match, and when (ISO). */ + confirmedBy: string; + confirmedAt: string; +} + +/** Confirmed image row shape the pure matcher consumes. */ +export interface ConfirmedCheckImage { + normalizedCheckNumber: string | null; + /** Positive cents printed on the document, when readable. */ + amountCents: number | null; + /** Signed cents of the confirmed bank line, when the match carries one. */ + lineAmountCents: number | null; + payerName: string | null; + driveFileId: string | null; + fileName: string | null; + confirmedBy: string; + confirmedAt: string; +} + +export interface PaymentForEvidence { + id: string; + referenceNumber: string | null; + /** Milestone amount in cents. */ + amountCents: number | null; +} + +/** + * Pure. For each payment, find the confirmed check image whose number AND + * amount both agree. Ambiguity (two confirmed images with the same number + * and amount) yields nothing — never a guess. Returns paymentId → evidence. + */ +export function matchCheckEvidence( + payments: PaymentForEvidence[], + confirmedImages: ConfirmedCheckImage[], +): Map { + const out = new Map(); + + for (const payment of payments) { + const ref = normalizeCheckRef(payment.referenceNumber); + if (!ref) continue; + const amountCents = payment.amountCents; + if (amountCents === null || !Number.isSafeInteger(amountCents) || amountCents <= 0) continue; + + const hits = confirmedImages.filter((img) => { + if (img.normalizedCheckNumber !== ref) return false; + // Amount corroboration: the document amount when readable, + // otherwise the confirmed bank line's magnitude. + if (img.amountCents !== null) return img.amountCents === amountCents; + if (img.lineAmountCents !== null) return Math.abs(img.lineAmountCents) === amountCents; + return false; + }); + + if (hits.length !== 1) continue; // 0 = no evidence; 2+ = ambiguous, never guess + const img = hits[0]; + out.set(payment.id, { + payerName: img.payerName, + checkNumber: ref, + driveFileId: img.driveFileId, + fileName: img.fileName, + confirmedBy: img.confirmedBy, + confirmedAt: img.confirmedAt, + }); + } + + return out; +} + +/** + * Server fetch for the invoice detail page. Loads only the confirmed + * check-front images whose numbers appear on the given payments, then runs + * the pure matcher. Failures are the caller's to degrade on (the invoice + * page renders fine with no evidence). + */ +export async function fetchCheckEvidenceForPayments( + payments: PaymentForEvidence[], +): Promise> { + const refs = [...new Set( + payments.map((p) => normalizeCheckRef(p.referenceNumber)).filter((r): r is string => r !== null), + )]; + if (refs.length === 0) return {}; + + const matches = await prisma.bankImageMatch.findMany({ + where: { + bankImage: { + kind: "CHECK_FRONT", + normalizedCheckNumber: { in: refs }, + }, + }, + include: { + bankImage: { + select: { + normalizedCheckNumber: true, + amountCents: true, + payerName: true, + driveFileId: true, + fileName: true, + }, + }, + bankLine: { select: { amountCents: true } }, + }, + }); + + const confirmed: ConfirmedCheckImage[] = matches.map((m) => ({ + normalizedCheckNumber: m.bankImage.normalizedCheckNumber, + amountCents: m.bankImage.amountCents, + lineAmountCents: m.bankLine?.amountCents ?? null, + payerName: m.bankImage.payerName, + driveFileId: m.bankImage.driveFileId, + fileName: m.bankImage.fileName, + confirmedBy: m.confirmedBy, + confirmedAt: m.confirmedAt.toISOString(), + })); + + return Object.fromEntries(matchCheckEvidence(payments, confirmed)); +} diff --git a/src/lib/check-payer-match.ts b/src/lib/check-payer-match.ts new file mode 100644 index 000000000..27ec30170 --- /dev/null +++ b/src/lib/check-payer-match.ts @@ -0,0 +1,93 @@ +/** + * Check-payer fuzzy matching — payerName → Client, memoText → Project. + * + * TS port of the suggestion logic in scripts/extract-check-payers.mjs + * (nameTokens / nameSimilarity / suggestMatches). The script stays the CLI + * entry point; this module is what the Automation "Check images" panel uses + * server-side. tests/check-payer-match.test.ts asserts the two stay in + * agreement, so neither can drift silently. + * + * SUGGESTION ONLY — same house rule as bank-image.ts and vendor-alias: + * nothing here may ever write BankImageMatch. That table means a HUMAN said + * yes; the confirm server action is the only writer. + * + * PURE: no Prisma, no I/O. + */ + +const NAME_NOISE = new Set([ + "llc", "inc", "co", "corp", "ltd", "the", "and", "&", "of", + "mr", "mrs", "ms", "dr", "or", +]); + +export interface NamedRow { + id: string; + name: string; +} + +export interface MatchSuggestion { + id: string; + name: string; + /** 0..1 — token-set Jaccard with a containment bonus. */ + score: number; +} + +export interface CheckMatchSuggestions { + payerMatches: MatchSuggestion[]; + memoMatches: MatchSuggestion[]; +} + +/** Minimum similarity for a suggestion to surface at all. */ +export const SUGGESTION_THRESHOLD = 0.4; +/** Suggestions shown per field. */ +export const SUGGESTION_LIMIT = 3; + +export function nameTokens(s: string | null | undefined): string[] { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .split(/\s+/) + .filter(t => t && !NAME_NOISE.has(t)); +} + +/** Pure. 0..1 similarity: token-set Jaccard plus a containment bonus. */ +export function nameSimilarity(a: string | null | undefined, b: string | null | undefined): number { + const ta = nameTokens(a), tb = nameTokens(b); + if (!ta.length || !tb.length) return 0; + const sa = new Set(ta), sb = new Set(tb); + const inter = [...sa].filter(t => sb.has(t)).length; + const union = new Set([...sa, ...sb]).size; + const jaccard = inter / union; + const containment = inter / Math.min(sa.size, sb.size); + return Math.max(jaccard, containment * 0.85); +} + +/** + * Pure. Suggest candidate matches for one extraction. Returns + * { payerMatches, memoMatches } sorted by score, top 3 each, threshold 0.4. + * A null/blank payer or memo yields an empty list — never a guess. + */ +export function suggestMatches( + extraction: { payerName: string | null; memoText: string | null }, + clients: NamedRow[], + projects: NamedRow[], +): CheckMatchSuggestions { + const rank = (text: string, rows: NamedRow[]): MatchSuggestion[] => rows + .map(r => ({ id: r.id, name: r.name, score: nameSimilarity(text, r.name) })) + .filter(m => m.score >= SUGGESTION_THRESHOLD) + .sort((x, y) => y.score - x.score || x.id.localeCompare(y.id)) + .slice(0, SUGGESTION_LIMIT); + return { + payerMatches: extraction.payerName ? rank(extraction.payerName, clients) : [], + memoMatches: extraction.memoText ? rank(extraction.memoText, projects) : [], + }; +} + +/** + * Normalize a check-number-ish reference to the identity every parser in + * this repo produces: digits only, leading zeros stripped. Returns null for + * anything that leaves no digits — "" must never match "" on a join. + */ +export function normalizeCheckRef(value: string | null | undefined): string | null { + const digits = String(value ?? "").replace(/\D/g, "").replace(/^0+/, ""); + return digits.length ? digits : null; +} diff --git a/tests/check-evidence.test.ts b/tests/check-evidence.test.ts new file mode 100644 index 000000000..2499643db --- /dev/null +++ b/tests/check-evidence.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + matchCheckEvidence, + type ConfirmedCheckImage, + type PaymentForEvidence, +} from "@/lib/check-evidence"; + +// The real prod row this feature was built for: check #1027, +// "GOLDEN TOUCH RMEODELING LLC", HOPPE VANITY memo, $6,037.15. +const hoppeImage = (over: Partial = {}): ConfirmedCheckImage => ({ + normalizedCheckNumber: "1027", + amountCents: 603715, + lineAmountCents: -603715, + payerName: "GOLDEN TOUCH RMEODELING LLC", + driveFileId: "drive-abc", + fileName: "26225018006376-front.jpg", + confirmedBy: "justin@goldentouchremodeling.com", + confirmedAt: "2026-08-22T05:00:00.000Z", + ...over, +}); + +const paidByCheck = (over: Partial = {}): PaymentForEvidence => ({ + id: "pay-1", + referenceNumber: "1027", + amountCents: 603715, + ...over, +}); + +test("check number + amount both agreeing yields evidence", () => { + const map = matchCheckEvidence([paidByCheck()], [hoppeImage()]); + const ev = map.get("pay-1"); + assert.ok(ev); + assert.equal(ev.payerName, "GOLDEN TOUCH RMEODELING LLC"); + assert.equal(ev.checkNumber, "1027"); + assert.equal(ev.driveFileId, "drive-abc"); +}); + +test("referenceNumber is normalized before matching (leading zeros, decorations)", () => { + const map = matchCheckEvidence( + [paidByCheck({ referenceNumber: "chk #01027" })], + [hoppeImage()], + ); + assert.ok(map.get("pay-1")); +}); + +test("same check number but a DIFFERENT amount is NOT evidence — check numbers collide across payers", () => { + const map = matchCheckEvidence( + [paidByCheck({ amountCents: 500000 })], + [hoppeImage()], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("image amount unreadable: the confirmed bank line's magnitude corroborates instead", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage({ amountCents: null, lineAmountCents: -603715 })], + ); + assert.ok(map.get("pay-1")); +}); + +test("no readable image amount AND no line amount: no evidence, never a guess", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage({ amountCents: null, lineAmountCents: null })], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("two confirmed images with the same number and amount is ambiguous — nothing shown", () => { + const map = matchCheckEvidence( + [paidByCheck()], + [hoppeImage(), hoppeImage({ payerName: "Someone Else", driveFileId: "drive-xyz" })], + ); + assert.equal(map.get("pay-1"), undefined); +}); + +test("null referenceNumber / zero-digit reference / non-check payments are skipped", () => { + const map = matchCheckEvidence( + [ + paidByCheck({ id: "p-null", referenceNumber: null }), + paidByCheck({ id: "p-empty", referenceNumber: "0000" }), + paidByCheck({ id: "p-alpha", referenceNumber: "no digits" }), + ], + [hoppeImage()], + ); + assert.equal(map.size, 0); +}); + +test("null / non-integer / non-positive payment amounts are skipped, not matched loosely", () => { + const map = matchCheckEvidence( + [ + paidByCheck({ id: "p-null-amt", amountCents: null }), + paidByCheck({ id: "p-float", amountCents: 6037.15 as unknown as number }), + paidByCheck({ id: "p-neg", amountCents: -603715 }), + ], + [hoppeImage()], + ); + assert.equal(map.size, 0); +}); + +test("null payerName still surfaces evidence (chk# + confirmation without a name)", () => { + const map = matchCheckEvidence([paidByCheck()], [hoppeImage({ payerName: null })]); + const ev = map.get("pay-1"); + assert.ok(ev); + assert.equal(ev.payerName, null); + assert.equal(ev.checkNumber, "1027"); +}); diff --git a/tests/check-payer-match.test.ts b/tests/check-payer-match.test.ts new file mode 100644 index 000000000..5165afefd --- /dev/null +++ b/tests/check-payer-match.test.ts @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + nameTokens, + nameSimilarity, + suggestMatches, + normalizeCheckRef, + SUGGESTION_THRESHOLD, +} from "@/lib/check-payer-match"; +// The CLI script is the original implementation; the lib is its TS port for +// the Automation panel. This import keeps the two honest against each other. +import { + nameTokens as mjsNameTokens, + nameSimilarity as mjsNameSimilarity, + suggestMatches as mjsSuggestMatches, +} from "../scripts/extract-check-payers.mjs"; + +// The REAL extracted row that motivated this feature (prod, 2026-08-21): +// payer "GOLDEN TOUCH RMEODELING LLC" (typo and all), memo +// "HOPPE VANITY CONTRACT 4152", check #1027. +const REAL_PAYER = "GOLDEN TOUCH RMEODELING LLC"; +const REAL_MEMO = "HOPPE VANITY CONTRACT 4152"; + +const clients = [ + { id: "c1", name: "Golden Touch Remodeling" }, + { id: "c2", name: "Sandi Christensen" }, + { id: "c3", name: "Mesplay" }, +]; + +const projects = [ + { id: "p1", name: "Hoppe Vanity" }, + { id: "p2", name: "Christensen Kitchen" }, + { id: "p3", name: "Shop" }, +]; + +test("nameTokens drops entity noise and punctuation", () => { + assert.deepEqual(nameTokens("GOLDEN TOUCH RMEODELING LLC"), ["golden", "touch", "rmeodeling"]); + assert.deepEqual(nameTokens("The Smith & Jones Co."), ["smith", "jones"]); + assert.deepEqual(nameTokens(null), []); + assert.deepEqual(nameTokens(""), []); +}); + +test("nameSimilarity: identical token sets score 1, disjoint score 0", () => { + assert.equal(nameSimilarity("Hoppe Vanity", "Vanity Hoppe"), 1); + assert.equal(nameSimilarity("Mesplay", "Christensen"), 0); + assert.equal(nameSimilarity(null, "anything"), 0); + assert.equal(nameSimilarity("anything", ""), 0); +}); + +test("memo naming the job outranks unrelated projects", () => { + const { memoMatches } = suggestMatches({ payerName: null, memoText: REAL_MEMO }, clients, projects); + assert.ok(memoMatches.length >= 1); + assert.equal(memoMatches[0].id, "p1"); + assert.ok(memoMatches[0].score >= SUGGESTION_THRESHOLD); +}); + +test("payer with a typo still finds the client via shared tokens", () => { + const { payerMatches } = suggestMatches({ payerName: REAL_PAYER, memoText: null }, clients, projects); + assert.ok(payerMatches.length >= 1); + assert.equal(payerMatches[0].id, "c1"); +}); + +test("null payer AND memo yield no suggestions — never a guess", () => { + const { payerMatches, memoMatches } = suggestMatches({ payerName: null, memoText: null }, clients, projects); + assert.deepEqual(payerMatches, []); + assert.deepEqual(memoMatches, []); +}); + +test("scores below the threshold are dropped entirely", () => { + const { payerMatches } = suggestMatches( + { payerName: "Totally Unrelated Person", memoText: null }, + clients, + projects, + ); + assert.deepEqual(payerMatches, []); +}); + +test("normalizeCheckRef: digits only, leading zeros stripped, empty is null", () => { + assert.equal(normalizeCheckRef("chk #01027"), "1027"); + assert.equal(normalizeCheckRef("1027"), "1027"); + assert.equal(normalizeCheckRef("0000"), null); + assert.equal(normalizeCheckRef(""), null); + assert.equal(normalizeCheckRef(null), null); + assert.equal(normalizeCheckRef("no digits"), null); +}); + +// ── parity with the CLI script — the port must not drift ───────────────── +test("lib agrees with scripts/extract-check-payers.mjs on tokens and similarity", () => { + const samples: Array<[string, string]> = [ + [REAL_PAYER, "Golden Touch Remodeling"], + [REAL_MEMO, "Hoppe Vanity"], + ["Sandi Christensen", "Christensen Kitchen"], + ["The Smith & Jones Co.", "Smith Residence"], + ["", "anything"], + ]; + for (const [a, b] of samples) { + assert.deepEqual(nameTokens(a), mjsNameTokens(a), `tokens drifted for "${a}"`); + assert.equal(nameSimilarity(a, b), mjsNameSimilarity(a, b), `similarity drifted for "${a}" vs "${b}"`); + } + const libResult = suggestMatches({ payerName: REAL_PAYER, memoText: REAL_MEMO }, clients, projects); + const mjsResult = mjsSuggestMatches({ payerName: REAL_PAYER, memoText: REAL_MEMO }, clients, projects); + assert.deepEqual( + libResult.payerMatches.map(m => [m.id, m.score]), + mjsResult.payerMatches.map((m: { id: string; score: number }) => [m.id, m.score]), + ); + assert.deepEqual( + libResult.memoMatches.map(m => [m.id, m.score]), + mjsResult.memoMatches.map((m: { id: string; score: number }) => [m.id, m.score]), + ); +}); From d16a7a3f61aad04d35d2a213b5012a946a45fea2 Mon Sep 17 00:00:00 2001 From: Justin Adkins Date: Wed, 26 Aug 2026 21:08:39 -0700 Subject: [PATCH 10/10] feat(pdf): watermark estimates and invoices --- package.json | 2 +- src/lib/actions.ts | 21 ++++++++++++++--- src/lib/pdf.ts | 46 ++++++++++++++++++++++++++++++++++++- tests/pdf-watermark.test.ts | 44 +++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 tests/pdf-watermark.test.ts diff --git a/package.json b/package.json index 8036906a0..0bc269229 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "test:budget-math": "tsx --test tests/budget-math.test.ts", "test:estimate-item-upsert": "tsx --test tests/estimate-item-upsert.test.ts", "test:crew-auto-assign": "tsx --test tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", - "test:unit": "tsx --test tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/time-entries-clockout-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", + "test:unit": "tsx --test tests/pdf-watermark.test.ts tests/estimate-item-payload.test.ts tests/budget-math.test.ts tests/payment-date-display.test.ts tests/estimate-item-upsert.test.ts tests/geocode-zip-guard.test.ts tests/takeoff-tax-split.test.ts tests/takeoff-convert-tax.test.ts tests/takeoff-tax-prompt.test.ts tests/overtime.test.ts tests/pay-period-summary-route.test.ts tests/tz-date.test.ts tests/phase-options.test.ts tests/mobile-phases-route.test.ts tests/manager-crew-route.test.ts tests/logistics-time-entry.test.ts tests/project-phases.test.ts tests/phase-items.test.ts tests/cost-coding.test.ts tests/job-variance.test.ts tests/job-variance-db.test.ts tests/polish-notes-route.test.ts tests/crew-auto-assign.test.ts tests/backfill-crew-assignments.test.ts", "test:e2e": "playwright test", "test:mobile-e2e": "node e2e/mobile-app/run-mobile-e2e.mjs", "test:qa": "npx playwright test e2e/quality-gate.spec.ts", diff --git a/src/lib/actions.ts b/src/lib/actions.ts index 65706fed3..ba7b55c5c 100644 --- a/src/lib/actions.ts +++ b/src/lib/actions.ts @@ -2536,14 +2536,21 @@ export async function approveEstimate(estimateId: string, signatureName: string, let pdfBuffer: Buffer | null = null; let attachments: any = undefined; try { + const { applyGoldenTouchWatermarkToPdfBytes, generateEstimatePdf } = await import("./pdf"); if (capturedPdfUrl) { pdfBuffer = await downloadDocBytes(capturedPdfUrl); if (!pdfBuffer) { console.warn("[approveEstimate] Failed to read capturedPdfUrl:", capturedPdfUrl); + } else { + try { + pdfBuffer = await applyGoldenTouchWatermarkToPdfBytes(pdfBuffer); + } catch (e) { + console.warn("[approveEstimate] Captured PDF watermark failed; rebuilding from the server estimate:", e); + pdfBuffer = null; + } } } if (!pdfBuffer) { - const { generateEstimatePdf } = await import("./pdf"); pdfBuffer = await generateEstimatePdf(estimateId); } if (pdfBuffer) { @@ -5671,17 +5678,25 @@ export async function sendEstimateToClient( let pdfAttached = false; try { let pdfBuffer: Buffer | undefined; + const { applyGoldenTouchWatermarkToPdfBytes, generateEstimatePdf } = await import("./pdf"); if (capturedPdfUrl && (isSecureRef(capturedPdfUrl) || isAllowedCapturedPdfUrl(capturedPdfUrl))) { // Use the pre-captured portal PDF (high-quality, matches what client sees). // Read via the service key so this still works now that the capture lands in the // private bucket; the allowlist still gates legacy http(s) values. - pdfBuffer = (await downloadDocBytes(capturedPdfUrl)) ?? undefined; + const capturedPdf = await downloadDocBytes(capturedPdfUrl); + if (capturedPdf) { + try { + pdfBuffer = await applyGoldenTouchWatermarkToPdfBytes(capturedPdf); + } catch (e) { + console.warn("[sendEstimateToClient] Captured PDF watermark failed; rebuilding from the server estimate:", e); + pdfBuffer = undefined; + } + } } else if (capturedPdfUrl) { console.warn("[sendEstimateToClient] Rejected capturedPdfUrl (failed allowlist):", capturedPdfUrl); } if (!pdfBuffer) { // Fall back to server-side PDF generation - const { generateEstimatePdf } = await import("./pdf"); pdfBuffer = await generateEstimatePdf(estimateId); } if (pdfBuffer) { diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index cb31d76f9..a9d9ca1a6 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -1,4 +1,4 @@ -import { PDFDocument, PDFPage, PDFFont, PDFImage, rgb, StandardFonts } from 'pdf-lib'; +import { PDFDocument, PDFPage, PDFFont, PDFImage, degrees, rgb, StandardFonts } from 'pdf-lib'; import { prisma } from './prisma'; import { toNum } from './prisma-helpers'; import { buildLetterheadConfig, type LetterheadConfig } from './letterhead'; @@ -136,6 +136,48 @@ const colors = { white: rgb(1, 1, 1), }; +const GOLDEN_TOUCH_WATERMARK = 'GOLDEN TOUCH REMODELING'; + +/** + * Stamps every finished page immediately before the PDF is saved. Keeping this + * at the document boundary means the preview/download and email-attachment + * paths use the same branded file, including documents that gained extra pages. + */ +export async function applyGoldenTouchWatermark(doc: PDFDocument): Promise { + const font = await doc.embedFont(StandardFonts.HelveticaBold); + const pages = doc.getPages(); + + for (const page of pages) { + const { width, height } = page.getSize(); + const unitTextWidth = font.widthOfTextAtSize(GOLDEN_TOUCH_WATERMARK, 1); + const fontSize = Math.min(48, (width * 0.78) / unitTextWidth); + const textWidth = font.widthOfTextAtSize(GOLDEN_TOUCH_WATERMARK, fontSize); + const angle = 35 * Math.PI / 180; + const watermarkWidth = Math.cos(angle) * textWidth + Math.sin(angle) * fontSize; + const watermarkHeight = Math.sin(angle) * textWidth + Math.cos(angle) * fontSize; + + page.drawText(GOLDEN_TOUCH_WATERMARK, { + // Center the rotated bounding box, not just the unrotated baseline. + x: (width - watermarkWidth) / 2 + Math.sin(angle) * fontSize, + y: (height - watermarkHeight) / 2, + size: fontSize, + font, + color: rgb(0.85, 0.66, 0.08), + opacity: 0.10, + rotate: degrees(35), + }); + } + + return pages.length; +} + +/** Adds the required watermark to a static portal-captured estimate PDF before it is sent or filed. */ +export async function applyGoldenTouchWatermarkToPdfBytes(pdfBytes: Buffer): Promise { + const document = await PDFDocument.load(pdfBytes, { ignoreEncryption: true }); + await applyGoldenTouchWatermark(document); + return Buffer.from(await document.save()); +} + function hexToRgb(hex: string) { const h = hex.replace('#', ''); const r = parseInt(h.substring(0, 2), 16) / 255; @@ -743,6 +785,7 @@ export async function generateEstimatePdf(estimateId: string): Promise { x: pageWidth - margin - pageLabelWidth, y: footerY, size: 7, font: helvetica, color: colors.textMuted, }); + await applyGoldenTouchWatermark(doc); const pdfBytes = await doc.save(); return Buffer.from(pdfBytes); } @@ -1184,6 +1227,7 @@ export async function generateInvoicePdf( const footerText = `Generated ${new Date().toLocaleDateString()} • ${company?.companyName || 'ProBuild'}`; page.drawText(footerText, { x: margin, y: 30, size: 7, font: helvetica, color: colors.textMuted }); + await applyGoldenTouchWatermark(doc); const pdfBytes = await doc.save(); return Buffer.from(pdfBytes); } diff --git a/tests/pdf-watermark.test.ts b/tests/pdf-watermark.test.ts new file mode 100644 index 000000000..6adf69fce --- /dev/null +++ b/tests/pdf-watermark.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { PDFDocument, StandardFonts } from "pdf-lib"; +import { PDFParse } from "pdf-parse"; + +import { + applyGoldenTouchWatermark, + applyGoldenTouchWatermarkToPdfBytes, +} from "../src/lib/pdf"; + +async function extractText(pdfBytes: Buffer): Promise { + const parser = new PDFParse({ data: pdfBytes }); + try { + return (await parser.getText()).text; + } finally { + await parser.destroy(); + } +} + +test("applies the Golden Touch Remodeling watermark to every PDF page", async () => { + const document = await PDFDocument.create(); + document.addPage([612, 792]); + document.addPage([612, 792]); + + const stampedPages = await applyGoldenTouchWatermark(document); + const extractedText = await extractText(Buffer.from(await document.save())); + const watermarkCount = extractedText.split("GOLDEN TOUCH REMODELING").length - 1; + + assert.equal(stampedPages, 2); + assert.equal(watermarkCount, 2, "each page must contain the Golden Touch Remodeling watermark"); +}); + +test("stamps a pre-captured PDF without dropping its existing contents", async () => { + const original = await PDFDocument.create(); + const page = original.addPage([612, 792]); + const font = await original.embedFont(StandardFonts.Helvetica); + page.drawText("SIGNED ESTIMATE #1001", { x: 72, y: 700, size: 18, font }); + + const stampedPdf = await applyGoldenTouchWatermarkToPdfBytes(Buffer.from(await original.save())); + const extractedText = await extractText(stampedPdf); + + assert.match(extractedText, /SIGNED ESTIMATE #1001/); + assert.match(extractedText, /GOLDEN TOUCH REMODELING/); +});